From a4a82f57053fadc67cdddbd05934c1ae18ad4a33 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 03:00:59 +0100 Subject: [PATCH] Give each portable copy its own setup and refresh changed asset groups --- .gitignore | 1 + README.md | 4 +- crates/skate-game/src/setup.rs | 45 ++++--- docs/installation.md | 39 +++++- docs/updates.md | 5 +- scripts/Build-Release.ps1 | 7 +- tools/asset_pipeline/install.py | 184 +++++++++++++++----------- tools/asset_pipeline/test_versions.py | 72 ++++++++++ tools/asset_pipeline/versions.py | 69 ++++++++++ tools/setup.py | 18 ++- 10 files changed, 339 insertions(+), 105 deletions(-) create mode 100644 tools/asset_pipeline/test_versions.py create mode 100644 tools/asset_pipeline/versions.py diff --git a/.gitignore b/.gitignore index ede2d0ae..18d1029f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /logs/ /.local/ /settings/ +/data/ __pycache__/ *.pyc /assets/private/ diff --git a/README.md b/README.md index 7ab9fd0a..4fc732d7 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,9 @@ Use an XInput controller to play. Escape opens graphics, difficulty and map settings. Maps can be switched without restarting the game. **Skate 3 assets are not included.** Your converted files stay in -`%LOCALAPPDATA%/Skate3RustEngine`. +the `data` folder beside your executable. Each freshly unpacked copy runs its +own setup; it does not adopt another installation. In-place updates refresh +only changed asset groups. ## Build diff --git a/crates/skate-game/src/setup.rs b/crates/skate-game/src/setup.rs index 88dbebd8..008e8f78 100644 --- a/crates/skate-game/src/setup.rs +++ b/crates/skate-game/src/setup.rs @@ -1,11 +1,7 @@ -//! Select local installed data before any stock-data or Bevy initialization. +//! Each portable copy owns its installation; explicit --assets is for development. use std::{path::{Path, PathBuf}, process::Command}; -fn base() -> Result { - Ok(PathBuf::from(std::env::var_os("LOCALAPPDATA") - .ok_or("LOCALAPPDATA is unavailable; use --assets DIRECTORY")?).join("Skate3RustEngine")) -} -fn installed(base: &Path) -> Result, String> { +fn installed(base: &Path) -> Result, String> { let bytes = match std::fs::read(base.join("installation.json")) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -19,28 +15,45 @@ fn installed(base: &Path) -> Result, String> { } let assets = base.join(relative).join("assets"); if !assets.join("private/game.json").is_file() { return Ok(None); } - Ok(Some(assets)) + Ok(Some((assets, marker))) } pub(crate) fn asset_root() -> Result { - let explicit = std::env::args_os().any(|arg| arg == "--assets"); - if explicit { return Ok(PathBuf::from("assets")); } - // Existing developer launches retain their explicit local assets. - if Path::new("assets/private/game.json").is_file() { return Ok(PathBuf::from("assets")); } - let base = base()?; - if let Some(assets) = installed(&base)? { return Ok(assets); } + if std::env::args_os().any(|arg| arg == "--assets") { return Ok(PathBuf::from("assets")); } let exe = std::env::current_exe().map_err(|e| e.to_string())?; - let setup = exe.parent().ok_or("No executable directory")?.join("support/skate3setup.exe"); + let root = exe.parent().ok_or("No executable directory")?; + let base = root.join("data"); + let expected = match std::fs::read(root.join("release.json")) { + Ok(bytes) => { + let text = std::str::from_utf8(&bytes).map_err(|e| e.to_string())?.trim_start_matches('\u{feff}'); + let release: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?; + release.get("asset_pipelines").cloned() + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e.to_string()), + }; + let existing = installed(&base)?; + if let Some((assets, marker)) = &existing { + if expected.as_ref().is_none_or(|versions| marker.get("pipelines") == Some(versions)) { + return Ok(assets.clone()); + } + } + let setup = root.join("support/skate3setup.exe"); if !setup.is_file() { - return Err("No installed assets. Use the complete Windows release package, or --assets DIRECTORY for a prepared development asset set.".into()); + return Err("This copy has not been set up. Use the complete Windows package, or --assets DIRECTORY for development.".into()); } let mut command = Command::new(setup); command.arg("--base").arg(&base).arg("--game-exe").arg(&exe); + if existing.is_some() { command.arg("--refresh"); } #[cfg(windows)] { use std::os::windows::process::CommandExt; command.creation_flags(0x08000000); } let status = command.status().map_err(|e| format!("Could not start setup: {e}"))?; if !status.success() { return Err("Setup was cancelled or did not complete".into()); } - installed(&base)?.ok_or_else(|| "Setup did not publish a complete installation".into()) + let (assets, marker) = installed(&base)?.ok_or("Setup did not publish a complete installation")?; + if expected.as_ref().is_some_and(|versions| marker.get("pipelines") != Some(versions)) { + return Err("Setup helper does not match this release's asset extractors. Unpack the complete package.".into()); + } + Ok(assets) } diff --git a/docs/installation.md b/docs/installation.md index 278e364e..ad18e739 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,6 +1,6 @@ # Windows installation -`Build-Release.ps1` produces `target/skate3rust-windows-x64.zip`. Keep its +`scripts/Build-Release.ps1` produces `target/skate3rust-windows-x64.zip`. Keep its `support` directory beside `skate3rust.exe`. The game links Bevy and the MSVC runtime statically; the setup helper bundles Python, NumPy, Pillow and Tcl/Tk. It also bundles a small Rust RefPack decoder. Users need no compiler. @@ -84,3 +84,40 @@ scheduled first. Per-map conversion and load logs remain in the installation. Decoded map geometry, material data, texture pixels, native collision archives, rail payloads and the character GLB were compared with the earlier outputs. Final map storage uses a different compression level, without reducing quality. + +## Separate portable installations and asset refresh + +Each copy uses `data/installation.json` beside its executable. A freshly unpacked +ZIP has no record and always asks for an Xbox source. Startup does not search the +working directory, another package, or `%LOCALAPPDATA%` for prepared game assets. +Once that copy is set up, ordinary launches reuse its own data. `--assets` remains +an explicit developer override and bypasses setup and asset-refresh management. +Do not ship a local `data` folder in release ZIPs. + +The release manifest includes fingerprints for core data, HUDs, character, +environment and maps. They are derived from the packaged extractor sources and +shared dependencies, not the game build number. An in-place program update keeps +`data` intact. Next startup offers an asset refresh only when these fingerprints +change. A HUD extractor edit refreshes HUDs without converting maps; changes to +core VLT/animation decoding invalidate dependent groups too. Shared map/material +parsers can invalidate both maps and environment assets. New converter modules +must be assigned to their consuming groups in `asset_pipeline/versions.py`. + +Refresh reuses the Xbox source path saved for this copy, asking for its location +if it moved. ISO sources must be unpacked again when a refresh needs disc files; +unchanged asset groups are copied, not reconverted. The chosen Xbox executable +must match the original edition. Source paths stay in local installation records +and are never included in published release metadata. + +Updates create a new generation within this copy's data directory, copy unchanged +assets/maps/settings, and run only affected conversion stages. No hardlinks are +used, so rebuilding an output cannot modify the previous generation. Runtime +asset checks must pass before the installation record is atomically replaced. +Cancellation/failure leaves the previous record and data intact. Previous data +is retained for recovery; staging needs additional disk space. A copied stale +record without fingerprints triggers a complete refresh once. + +Installations from releases that used the old global asset store need one setup +in the new per-copy layout. There is deliberately no automatic global migration. +Manually unpacking a ZIP over the same folder retains that folder's `data` and +behaves like an in-place update; unpack into a new folder for a fresh setup. diff --git a/docs/updates.md b/docs/updates.md index eee6e4ad..4c4c8d01 100644 --- a/docs/updates.md +++ b/docs/updates.md @@ -19,7 +19,10 @@ The helper replaces only `skate3rust.exe`, `support/skate3setup.exe`, `support/skate3update.exe`, and `release.json`. Assets, maps, settings, mods, character libraries and setup's installation marker are never replaced. The same working directory and arguments are used for restart; existing asset selection -continues to work. Packaged releases are required; developer binaries do not +continues to work. Each package owns its `data` installation. On the next normal +launch, changed extractor fingerprints trigger an update of affected asset groups; +unchanged groups are retained. Freshly unpacked copies run setup independently. +Explicit `--assets` developer launches bypass this management. Packaged releases are required; developer binaries do not silently adopt a downloaded build. # Package protocol and release maintenance diff --git a/scripts/Build-Release.ps1 b/scripts/Build-Release.ps1 index d7c6b295..525a3a2b 100644 --- a/scripts/Build-Release.ps1 +++ b/scripts/Build-Release.ps1 @@ -40,7 +40,7 @@ try { $toolsRoot = Join-Path $ProjectRoot 'tools' foreach ($source in Get-ChildItem -LiteralPath $toolsRoot -File -Recurse) { if ($source.FullName -match '[\\/]__pycache__[\\/]') { continue } - if ($source.Extension -notin '.py','.json','.txt','.md','.toml' -and $source.Name -ne 'LICENSE') { continue } + if ($source.Extension -notin '.py','.json','.txt','.md','.toml','.rs' -and $source.Name -ne 'LICENSE') { continue } $relative = [IO.Path]::GetRelativePath($toolsRoot, $source.FullName) $portableName = $relative.Replace('\','/') if ($portableName -match '(^|/)blender[^/]*(/|$)' -or @@ -66,13 +66,16 @@ try { # deliberately retain identity; publish a new run/tag for a new eligible build. $build = if ($env:GITHUB_RUN_NUMBER) { [long]$env:GITHUB_RUN_NUMBER } else { 0 } $tag = if ($env:RELEASE_TAG) { $env:RELEASE_TAG } else { 'development' } + $assetPipelinesJson = & $packagePython "$sourceStage/tools/asset_pipeline/versions.py" --tools "$sourceStage/tools" + if ($LASTEXITCODE -ne 0) { throw 'Could not identify asset extractors' } + $assetPipelines = $assetPipelinesJson | ConvertFrom-Json $files = @{} foreach ($name in @('skate3rust.exe', 'support/skate3setup.exe', 'support/skate3update.exe')) { $files[$name] = (Get-FileHash -LiteralPath (Join-Path $stage $name) -Algorithm SHA256).Hash.ToLower() } @{ schema = 1; repository = 'SK8-ENGINE/skate-3-rust-engine'; target = 'windows-x64' - build = $build; tag = $tag; revision = (& git rev-parse HEAD).Trim(); files = $files + build = $build; tag = $tag; revision = (& git rev-parse HEAD).Trim(); files = $files; asset_pipelines = $assetPipelines } | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath "$stage/release.json" -Encoding utf8 Copy-Item -LiteralPath "$stage/release.json" -Destination (Join-Path $ProjectRoot 'target/release.json') Copy-Item -LiteralPath README.md,docs/THIRD_PARTY_NOTICES.md -Destination $stage diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index b318ba0b..d8ae17e3 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -152,7 +152,15 @@ def convert_map(archive,work,maps,stage,game_exe,log,report): return entry -def install(iso,base,game_exe,report,game_root=None): +def install(iso,base,game_exe,report,game_root=None,refresh=False): + from .versions import fingerprints, changed_groups, installed, GROUPS + target_versions=fingerprints() + previous=installed(base) if refresh else None + groups=changed_groups(previous[1].get('pipelines',{}),target_versions) if previous else set(GROUPS) + if not groups: + report('Game assets are already current') + return previous[0] + source=str((iso if iso is not None else game_root).resolve()) if game_root is None and iso is not None: selected=iso.resolve() if selected.is_dir():game_root=selected @@ -169,8 +177,13 @@ def install(iso,base,game_exe,report,game_root=None): install_id=uuid.uuid4().hex stage=base/'installations'/install_id stage.mkdir(parents=True) - private=stage/'assets/private';private.mkdir(parents=True) - maps=stage/'maps';maps.mkdir() + if previous: + report('Preparing an asset update; keeping the previous installation until it succeeds') + for name in ('assets','maps','settings'): + if (previous[0]/name).is_dir():shutil.copytree(previous[0]/name,stage/name) + shutil.copy2(previous[0]/'maps.json',stage/'maps.json') + private=stage/'assets/private';private.mkdir(parents=True,exist_ok=True) + maps=stage/'maps';maps.mkdir(exist_ok=True) work=stage/'conversion';work.mkdir() with (stage/'setup.log').open('w',encoding='utf-8') as log: if game_root is None: @@ -184,89 +197,104 @@ def install(iso,base,game_exe,report,game_root=None): for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big', 'data/content/createacharacter.big','data/content/worldDIST_University.big']: if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required) - report('Extracting animation banks, graphs and gameplay inputs') + source_hash=digest(game_root/'default.xex') + if previous and previous[1].get('source_hash',source_hash)!=source_hash: + raise RuntimeError('Select the same Xbox game edition used to set up this copy') stock=private/'stock' - extract(game_root/'data/big/miscload.big',stock) - extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg') - # Some disc banks are loose files rather than members of miscload. - loose=game_root/'data/anim' - if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True) - report('Converting physics and difficulty settings') - database=work/'database' - extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in { - 'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'}) - names=(TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines() - converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names) - (stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8') - skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin') - (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8') - report('Preparing original scoring and session-marker HUD assets') - run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root, - '--assets', stage/'assets', '--work', work/'hud'), log, report) - report('Preparing the skater model and textures') - manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text()) - needed=set() - for c in manifest['components']: - needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower()) - needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values()) - extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed) - character=work/'character' - run(task(TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character, - '--private-root',private/'default_skater','--utt-root',TOOLS/'vendor/utt'),log,report) - report('Building the skater model and rig') - from .character_glb import convert as write_character - write_character(character/'selected/models',private,manifest) - from .character_lighting import convert as write_character_lighting - write_character_lighting(character/'selected/models',private,converted) - game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000', - 'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph', - 'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'} - (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8') - report('Preparing retail sky domes') - from .sky import convert as write_skies - write_skies(game_root,stage/'assets',converted) - from .render_parameters import convert as write_render_parameters - write_render_parameters(stage/'assets',converted) - report('Extracting original travel destinations and location names') - from .teleports import convert as write_teleports - write_teleports(game_root,stage/'assets',converted) - report('Preparing global foliage backdrops') - from .backdrop import convert as write_backdrops - write_backdrops(game_root,stage/'assets',converted) - report('Preparing authored movable-object models') - from .dynamic_props import prepare_catalog - prepare_catalog(game_root,work/'dmo') + if 'core' in groups: + report('Extracting animation banks, graphs and gameplay inputs') + extract(game_root/'data/big/miscload.big',stock) + extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg') + # Some disc banks are loose files rather than members of miscload. + loose=game_root/'data/anim' + if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True) + report('Converting physics and difficulty settings') + database=work/'database' + extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in { + 'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'}) + names=(TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines() + converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names) + (stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8') + skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin') + (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8') + else: + converted=json.loads((stock/'skater-collections.json').read_text(encoding='utf-8')) + if 'hud' in groups: + for folder in (private/'hud',private/'session-marker'): + if folder.is_dir():remove_intermediate(folder,stage) + report('Preparing original scoring and session-marker HUD assets') + run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root, + '--assets', stage/'assets', '--work', work/'hud'), log, report) + if 'character' in groups: + report('Preparing the skater model and textures') + manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text()) + needed=set() + for c in manifest['components']: + needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower()) + needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values()) + extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed) + character=work/'character' + run(task(TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character, + '--private-root',private/'default_skater','--utt-root',TOOLS/'vendor/utt'),log,report) + report('Building the skater model and rig') + from .character_glb import convert as write_character + write_character(character/'selected/models',private,manifest) + from .character_lighting import convert as write_character_lighting + write_character_lighting(character/'selected/models',private,converted) + game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000', + 'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph', + 'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'} + (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8') + if 'environment' in groups: + report('Preparing retail sky domes') + from .sky import convert as write_skies + write_skies(game_root,stage/'assets',converted) + from .render_parameters import convert as write_render_parameters + write_render_parameters(stage/'assets',converted) + report('Extracting original travel destinations and location names') + from .teleports import convert as write_teleports + write_teleports(game_root,stage/'assets',converted) + report('Preparing global foliage backdrops') + from .backdrop import convert as write_backdrops + write_backdrops(game_root,stage/'assets',converted) + if 'maps' in groups: + report('Preparing authored movable-object models') + from .dynamic_props import prepare_catalog + prepare_catalog(game_root,work/'dmo') report('Validating skater, input and animation data') run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) - archives=list((game_root/'data/content').glob('worldDIST_*.big')) - archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower())) - workers=map_workers() - report(f'Converting {len(archives)} maps with {workers} workers') - def map_job(archive): - result=work/(archive.stem+'.json') - with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log: - run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage, - '--game-exe',game_exe,'--result',result),map_log,report) - return json.loads(result.read_text(encoding='utf-8')) - completed={} - with ThreadPoolExecutor(max_workers=workers) as pool: - # Start the expensive districts together so one does not - # remain queued behind a string of small parks. - futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)} - for future in as_completed(futures): - archive=futures[future];completed[archive.name]=future.result() - report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}") - catalog=[completed[a.name] for a in archives] - if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted') + if 'maps' in groups: + archives=list((game_root/'data/content').glob('worldDIST_*.big')) + archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower())) + workers=map_workers() + report(f'Converting {len(archives)} maps with {workers} workers') + def map_job(archive): + result=work/(archive.stem+'.json') + with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log: + run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage, + '--game-exe',game_exe,'--result',result),map_log,report) + return json.loads(result.read_text(encoding='utf-8')) + completed={} + with ThreadPoolExecutor(max_workers=workers) as pool: + # Start the expensive districts together so one does not + # remain queued behind a string of small parks. + futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)} + for future in as_completed(futures): + archive=futures[future];completed[archive.name]=future.result() + report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}") + catalog=[completed[a.name] for a in archives] + if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted') report('Validating installed runtime inputs') run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) - settings=stage/'settings';settings.mkdir() - (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8') - (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8') + settings=stage/'settings';settings.mkdir(exist_ok=True) + if not previous: + (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8') + if 'maps' in groups: + (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8') remove_intermediate(work,stage) # Publish only after every conversion and the runtime's own load succeeds. marker=base/'installation.json.new' - marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id}),encoding='utf-8') + marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id,'source':source,'source_hash':source_hash,'pipelines':target_versions}),encoding='utf-8') marker.replace(base/'installation.json') report('Setup complete') return stage diff --git a/tools/asset_pipeline/test_versions.py b/tools/asset_pipeline/test_versions.py new file mode 100644 index 00000000..09466a1f --- /dev/null +++ b/tools/asset_pipeline/test_versions.py @@ -0,0 +1,72 @@ +"""Installation selection and refresh checks using synthetic files only.""" +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from tools.asset_pipeline import versions as v +from tools.asset_pipeline.install import install + + +class AssetVersions(unittest.TestCase): + def test_hud_change_does_not_reconvert_maps(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + (root/'prepare_hud.py').write_text('before') + before=v.fingerprints(root) + (root/'prepare_hud.py').write_text('after') + self.assertEqual(v.changed_groups(before,v.fingerprints(root)), {'hud'}) + (root/'asset_pipeline').mkdir() + (root/'asset_pipeline/vlt.py').write_text('new decoder') + self.assertEqual(v.changed_groups(before,v.fingerprints(root)),set(v.GROUPS)) + + def test_fresh_copy_never_adopts_other_installation(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + (root/'installation.json').write_text(json.dumps({'version':1,'directory':'../old'})) + with self.assertRaises(ValueError):v.installed(root) + self.assertIsNone(v.installed(root/'new-copy/data')) + + def fixture(self, root): + base=root/'copy/data';old=base/('installations/'+'a'*32) + private=old/'assets/private';(private/'stock').mkdir(parents=True) + (private/'game.json').write_text('{}');(private/'stock/skater-collections.json').write_text('{}') + (private/'hud').mkdir();(private/'hud/old.txt').write_text('old') + (old/'maps').mkdir();(old/'maps/University.skate').write_bytes(b'unchanged map') + (old/'settings').mkdir();(old/'settings/default-map.json').write_text('"custom.skate"') + (old/'maps.json').write_text('[{"name":"University"}]') + current={group:'new' for group in v.GROUPS} + marker={'version':1,'directory':'installations/'+'a'*32,'pipelines':{**current,'hud':'old'}} + (base/'installation.json').write_text(json.dumps(marker)) + source=root/'disc' + for name in ('default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big', + 'data/content/createacharacter.big','data/content/worldDIST_University.big'): + p=source/name;p.parent.mkdir(parents=True,exist_ok=True);p.touch() + return base,old,source,current,marker + + def test_refresh_preserves_maps_and_publishes_only_after_validation(self): + with tempfile.TemporaryDirectory() as temp: + base,old,source,current,marker=self.fixture(Path(temp)) + calls=[] + def run(args,*unused): + calls.append([str(a) for a in args]) + with patch.object(v,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run',side_effect=run): + new=install(source,base,Path('unused.exe'),lambda _:None,refresh=True) + self.assertEqual((new/'maps/University.skate').read_bytes(),b'unchanged map') + self.assertEqual((new/'settings/default-map.json').read_text(),'"custom.skate"') + self.assertTrue((old/'assets/private/hud/old.txt').is_file()) + self.assertFalse((new/'assets/private/hud/old.txt').exists()) + self.assertEqual(v.installed(base)[1]['pipelines'],current) + self.assertTrue(any(any('prepare_runtime_huds.py' in a for a in args) for args in calls)) + self.assertFalse(any(any('map_job.py' in a for a in args) for args in calls)) + + def test_failed_refresh_keeps_previous_record(self): + with tempfile.TemporaryDirectory() as temp: + base,old,source,current,marker=self.fixture(Path(temp)) + with patch.object(v,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run',side_effect=RuntimeError('failed')): + with self.assertRaises(RuntimeError):install(source,base,Path('unused.exe'),lambda _:None,refresh=True) + self.assertEqual(v.installed(base)[1],marker) + self.assertFalse((base/'setup.lock').exists()) + + +if __name__=='__main__':unittest.main() diff --git a/tools/asset_pipeline/versions.py b/tools/asset_pipeline/versions.py new file mode 100644 index 00000000..49ebcda5 --- /dev/null +++ b/tools/asset_pipeline/versions.py @@ -0,0 +1,69 @@ +"""Content identities for independently refreshable owned-disc asset groups.""" +import argparse +import hashlib +import json +from pathlib import Path +import re + +GROUPS = ('core', 'hud', 'character', 'environment', 'maps') +COMMON = ('owned_game/**/*.py', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/install.py', + 'asset_pipeline/versions.py', 'asset_pipeline/refpack_native.rs') +PARSERS = ('vendor/utt/**/*.py', 'vendor/university/**/*.py', 'vendor/utt/**/*.json', 'vendor/university/**/*.json') +SOURCES = { + 'core': ('asset_pipeline/vlt.py', 'asset_pipeline/names.txt', 'asset_pipeline/physics_skeleton.py'), + 'hud': ('prepare_hud.py', 'prepare_runtime_huds.py', 'install_prepared_hud.py', + 'extract_session_marker.py', 'vendor/skate3_ui/**/*.py', 'vendor/skate3_ui/**/*.json'), + 'character': PARSERS + ('extract_default_skater.py', 'default_skater_retail_manifest.json', + 'asset_pipeline/character*.py'), + 'environment': PARSERS + ('asset_pipeline/sky.py', 'asset_pipeline/backdrop.py', + 'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py', + 'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py', + 'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py'), + 'maps': PARSERS + ('asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py', + 'asset_pipeline/environment.py', 'asset_pipeline/irradiance.py', + 'asset_pipeline/retail_material.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/sky.py'), +} + + +def fingerprints(tools=None): + tools = tools or Path(__file__).resolve().parents[1] + result = {} + for group, patterns in SOURCES.items(): + paths = {p for pattern in COMMON + patterns for p in tools.glob(pattern) + if p.is_file() and not p.name.startswith('test_') and '__pycache__' not in p.parts} + digest = hashlib.sha256() + for path in sorted(paths, key=lambda p: p.relative_to(tools).as_posix()): + digest.update(path.relative_to(tools).as_posix().encode() + b'\0') + digest.update(hashlib.sha256(path.read_bytes()).digest()) + result[group] = digest.hexdigest() + return result + + +def changed_groups(previous, current): + changed = {name for name in GROUPS if previous.get(name) != current[name]} + # Other exports consume the core VLT/skeleton/animation data. + if 'core' in changed: + changed.update(GROUPS) + return changed + + +def installed(base): + marker = base/'installation.json' + if not marker.is_file(): + return None + value = json.loads(marker.read_text(encoding='utf-8-sig')) + directory = value.get('directory', '') + if value.get('version') != 1 or not re.fullmatch(r'installations/[0-9a-f]{32}', directory): + raise ValueError('Invalid installation record') + root = (base/directory).resolve() + if not root.is_relative_to(base.resolve()): + raise ValueError('Installation escapes its package data directory') + if not (root/'assets/private/game.json').is_file(): + return None + return root, value + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--tools', type=Path) + print(json.dumps(fingerprints(parser.parse_args().tools))) diff --git a/tools/setup.py b/tools/setup.py index 913a200e..122c0070 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -18,33 +18,39 @@ def main(): parser=argparse.ArgumentParser() parser.add_argument('--base',type=Path,required=True) parser.add_argument('--game-exe',type=Path,required=True) + parser.add_argument('--refresh',action='store_true') args=parser.parse_args() import tkinter as tk from tkinter import filedialog,messagebox,ttk from tools.asset_pipeline.install import install + from tools.asset_pipeline.versions import installed, fingerprints, changed_groups + previous=installed(args.base) if args.refresh else None + changed=changed_groups(previous[1].get('pipelines',{}),fingerprints()) if previous else set() + updating=previous is not None window=tk.Tk() window.title('Skate 3 Rust Engine setup') window.geometry('700x420');window.resizable(False,False) icon=ROOT/'docs/images/skating-crab.ico' if icon.is_file():window.iconbitmap(str(icon)) frame=ttk.Frame(window,padding=24);frame.pack(fill='both',expand=True) - ttk.Label(frame,text='Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16)) - ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.', + ttk.Label(frame,text='Update game assets' if updating else 'Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16)) + ttk.Label(frame,text=('Changed asset groups: '+', '.join(sorted(changed))+'.\nOnly these groups will be extracted again.\nYour previous installation is kept until the update succeeds.\nUse Update game assets to reuse your selected Xbox source,\nor choose its new location if it has moved.') if updating else 'Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.', font=('Segoe UI',11),justify='left').pack(anchor='w') status=tk.StringVar(value='Choose your game to begin.') ttk.Label(frame,textvariable=status,wraplength=600).pack(anchor='w',pady=(18,8)) progress=ttk.Progressbar(frame,mode='indeterminate');progress.pack(fill='x') messages=queue.Queue();running=False;success=False - def start(folder=False): + def start(folder=False,reuse=False): nonlocal running - iso=(filedialog.askdirectory(parent=window,title='Select the Skate 3 folder containing default.xex') if folder else + saved=previous[1].get('source') if previous else None + iso=saved if reuse and saved and Path(saved).exists() else (filedialog.askdirectory(parent=window,title='Select the Skate 3 folder containing default.xex') if folder else filedialog.askopenfilename(parent=window,title='Select your Skate 3 Xbox 360 game', filetypes=[('Xbox 360 game','*.iso *.xex'),('Xbox 360 ISO','*.iso'),('Xbox executable','*.xex')])) if not iso:return button.config(state='disabled');folder_button.config(state='disabled');running=True;progress.start() def work(): try: - install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text))) + install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating) messages.put(('done','Ready')) except Exception as error: args.base.mkdir(parents=True,exist_ok=True) @@ -57,7 +63,7 @@ def main(): else:window.destroy() buttons=ttk.Frame(frame);buttons.pack(anchor='e',pady=18) folder_button=ttk.Button(buttons,text='Select extracted folder',command=lambda:start(True));folder_button.pack(side='left',padx=(0,8)) - button=ttk.Button(buttons,text='Select ISO or default.xex',command=start);button.pack(side='left') + button=ttk.Button(buttons,text='Update game assets' if updating else 'Select ISO or default.xex',command=lambda:start(reuse=updating));button.pack(side='left') def poll(): nonlocal running,success while not messages.empty():