Merge default.xex refresh and verified character preparation caching

crates/skate-game/src/setup.rs

@@ -10,11 +10,18 @@ fn installed(base: &Path) -> Result<Option<(PathBuf, serde_json::Value)>, String
     let marker: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?;
     if marker["version"].as_u64() != Some(1) { return Err("Unsupported installation version".into()); }
     let relative = Path::new(marker["directory"].as_str().ok_or("Invalid installation path")?);
-    if relative.is_absolute() || relative.components().any(|c| !matches!(c, std::path::Component::Normal(_))) {
+    let parts: Vec<_> = relative.components().collect();
+    if parts.len() != 2 || parts[0].as_os_str() != "installations"
+        || !parts[1].as_os_str().to_str().is_some_and(|s| s.len() == 32
+            && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) {
         return Err("Invalid installation path".into());
     }
     let assets = base.join(relative).join("assets");
-    if !assets.join("private/game.json").is_file() { return Ok(None); }
+    if !assets.is_dir() { return Ok(None); }
+    if !assets.canonicalize().map_err(|e| e.to_string())?
+        .starts_with(base.canonicalize().map_err(|e| e.to_string())?) {
+        return Err("Installation escapes its package data directory".into());
+    }
     Ok(Some((assets, marker)))
 }
 
@@ -37,6 +44,9 @@ pub(crate) fn asset_root() -> Result<PathBuf, String> {
     let existing = installed(&base)?;
     if let Some((assets, marker)) = &existing {
         if expected.as_ref().is_none_or(|versions| marker.get("pipelines") == Some(versions))
+            && assets.join("private/game.json").is_file()
+            && marker.get("outputs").is_none_or(|groups| groups.as_object().is_some_and(|groups|
+                groups.values().all(|files| receipt_present(assets.parent().unwrap(), files))))
             && customiser_current(assets, expected_customiser.as_deref()) {
             return Ok(assets.clone());
         }
@@ -72,5 +82,35 @@ fn customiser_current(assets: &Path, expected: Option<&str>) -> bool {
                 && v["set"].as_str().is_some_and(|s| s.len() == 32 && s.bytes().all(|b| b.is_ascii_hexdigit())))
             && ["library-v3.json", "extra-menu.json", "native-lighting.json", "native-roster/complete.json"].iter()
                 .all(|name| crate::customiser_parts::asset_directory(assets).join(name).is_file())
+            && ["catalog", "library", "menu", "lighting", "roster"].iter().all(|stage| {
+                let directory = crate::customiser_parts::asset_directory(assets);
+                std::fs::read(directory.join(format!("{stage}-complete.json"))).ok()
+                    .and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())
+                    .is_some_and(|v| receipt_present(&directory, &v["files"]))
+            })
     })
 }
+
+// Cheap launch-time completeness check. Setup verifies SHA-256 before reuse;
+// hashing every map on every game launch would read gigabytes unnecessarily.
+fn receipt_present(root: &Path, files: &serde_json::Value) -> bool {
+    files.as_object().is_some_and(|files| !files.is_empty() && files.iter().all(|(name, entry)| {
+        let relative = Path::new(name);
+        !relative.is_absolute()
+            && relative.components().all(|c| matches!(c, std::path::Component::Normal(_)))
+            && std::fs::metadata(root.join(relative)).ok()
+                .is_some_and(|m| m.is_file() && Some(m.len()) == entry["size"].as_u64())
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn receipts_reject_missing_files_empty_lists_and_traversal() {
+        let root = std::env::temp_dir();
+        assert!(!receipt_present(&root, &serde_json::json!({})));
+        assert!(!receipt_present(&root, &serde_json::json!({"../missing": {"size": 0}})));
+        assert!(!receipt_present(&root, &serde_json::json!({"nonexistent-skate-setup-test": {"size": 0}})));
+    }
+}

docs/installation.md

@@ -5,8 +5,8 @@
 runtime statically; the setup helper bundles Python, NumPy, Pillow and Tcl/Tk.
 It also bundles a small Rust RefPack decoder. Users need no compiler.
 
-On first launch, setup accepts a local Xbox 360 Skate 3 ISO, `default.xex`,
-or an extracted game folder. Selecting `default.xex` uses the surrounding
+On first launch and after an extractor update, setup asks for a local Xbox 360
+Skate 3 `default.xex` or ISO. Selecting `default.xex` uses the surrounding
 folder, which must still contain the full game data. Extracted copies skip
 ISO extraction and need no tool download. For ISO input, setup downloads
 the hash-pinned extract-xiso utility from XboxDev. Asset conversion runs in
@@ -36,7 +36,7 @@ The conversion pipeline:
    This validates loading, not gameplay. Conversion intermediates are removed
    after their output succeeds. Failed workspaces retain logs for diagnosis.
 
-Data lives under `%LOCALAPPDATA%/Skate3RustEngine`. `installation.json` points
+Data lives in `data` beside this portable copy's executable. `installation.json` points
 to the completed installation. Conversion tools are cached separately, so
 retrying does not download them again. `setup-error.log` reports setup errors;
 each installation also has `setup.log` with converter output.
@@ -103,19 +103,40 @@ core VLT/animation decoding invalidate dependent groups too. Shared map/material
 parsers can invalidate both maps and environment assets. New converter modules
 must be assigned to their consuming groups in `asset_pipeline/versions.py`.
 
-Refresh reuses the Xbox source path saved for this copy, asking for its location
-if it moved. ISO sources must be unpacked again when a refresh needs disc files;
-unchanged asset groups are copied, not reconverted. The chosen Xbox executable
+Refresh uses the same `Select ISO or default.xex` file picker as initial setup;
+the player never needs to select an intermediate extraction folder. ISO sources
+must be unpacked again when a refresh needs disc files; unchanged asset groups
+are reused without conversion. The chosen Xbox executable
 must match the original edition. Source paths stay in local installation records
 and are never included in published release metadata.
 
-Updates create a new generation within this copy's data directory, copy unchanged
-assets/maps/settings, and run only affected conversion stages. No hardlinks are
-used, so rebuilding an output cannot modify the previous generation. Runtime
-asset checks must pass before the installation record is atomically replaced.
-Cancellation/failure leaves the previous record and data intact. Previous data
-is retained for recovery; staging needs additional disk space. A copied stale
-record without fingerprints triggers a complete refresh once.
+Updates run only affected exports. Core/HUD/character/environment/map recipes
+have separate dependency identities; setup UI and transaction changes do not
+invalidate every map. Explicit, exact old/new fingerprint equivalences cover
+verified unchanged exports when migrating the earlier broad fingerprint scheme.
+Unrecognized versions still refresh normally.
+
+Core refreshes create a new installation before publication. Unchanged prepared
+maps and immutable character generations use hardlinks (copy fallback on filesystems without support); mutable user files and
+outputs being rebuilt have independent storage. Customiser-only refreshes keep
+the current core installation and publish a new character generation. Custom
+models, profiles, settings, selections and mods are retained. Every required
+stage, including the complete character library and pro roster, must succeed
+before the installation record changes. Failures preserve the previous record.
+A kernel lock prevents concurrent setup and releases automatically on process
+exit, including a crash; the harmless `setup.lock` file can remain on disk.
+
+Character catalog, library, menu, lighting and roster stages have separate
+source/dependency identities and SHA-256 output receipts. Completed stages can
+be reused across generations or resumed after an interruption; partial stages
+are rebuilt before any existence-based converter sees them. Setup verifies
+checksums before reuse. Startup checks receipt paths and file sizes, avoiding a
+full map checksum scan on every launch. Same-size corruption is detected during
+setup validation, rather than ordinary startup. Old generations remain for
+recovery; this still requires space for genuinely changed output.
+
+See [character preparation measurements](setup-preparation-performance.md) for
+benchmark conditions, stage timings and output equivalence checks.
 
 Installations from releases that used the old global asset store need one setup
 in the new per-copy layout. There is deliberately no automatic global migration.

docs/setup-preparation-performance.md

@@ -0,0 +1,113 @@
+# Character setup preparation measurements
+
+Measured 2026-09-10 on Windows 11 Pro, Ryzen 7 9800X3D (8 cores / 16 threads),
+32 GB RAM, Crucial P3 4 TB NVMe system drive. The owned Xbox source and all test
+outputs were on that drive. No maps were extracted, copied or converted for
+these measurements, and no game was launched.
+
+## Release-dependency measurement
+
+Python 3.13.9, NumPy 2.2.6 and Pillow 11.3.0, matching `requirements-setup.txt`.
+Both versions used the same freshly compiled native RefPack library from this
+checkout. The baseline Python sources were taken from commit `c8219a5`.
+
+| Stage | Before | After |
+| --- | ---: | ---: |
+| Catalog | 0.98 s | 1.18 s |
+| Clothing/body/material library | 426.43 s | 97.27 s |
+| Authored clothing lighting | 5.83 s | 2.04 s |
+| Menu | Not timed separately | 0.02 s |
+| All 41 pro/special characters | Not timed separately | 140.59 s |
+
+The new complete staged customiser run took **245.62 s**, including source
+hashing, all five stages, output receipts and publication. There is no measured
+old full-customiser total, so the library comparison must not be advertised as
+a full-setup speed multiplier. The new stage timings include receipt creation;
+the old timings surround the corresponding converter function.
+
+Each cold-output run started with no generated character library, decoded
+textures or models. Only the two required, already prepared stock inputs were
+reused: OnBoard.abin and the female fallback recipe. This is **not** a cold OS
+cache test: filesystem caches were not flushed, and this was a development PC
+with other checks running during parts of the measurements. These are single
+observations, not controlled repeated trials or guarantees for other hardware.
+
+A subsequent generation reused matching, checksummed stages in **19.93 s** total:
+catalog 0.30 s, library 17.29 s, menu 0.01 s, lighting 0.08 s, roster 0.64 s.
+This included checksum verification, generation-relative JSON reference updates
+and hardlinking immutable output. It demonstrates validated reuse across a
+setup-code change with unchanged export identities; it is not a fresh extraction.
+
+## Profile and diagnosis
+
+Before editing converters, the full library and smaller model/texture samples
+were profiled. The source-only profile used Python 3.13.9, NumPy 2.4.3 and Pillow
+12.1.1 with the Python RefPack fallback. Full-library wall time was 805.53 s
+before and 288.83 s after. Profiling overhead and overlapping development work
+mean these figures should not be compared directly with the release-dependency
+numbers above.
+
+The old profile spent 424.37 s in 2,920 texture decodes and 171.97 s in 480 GLB
+conversions, including 83.33 s repeatedly constructing the animation source.
+It parsed RX2 geometry 6,240 times. The new profile spent 66.04 s decoding the
+same textures. The source fallback's archive decompression then dominated;
+release packaging already supplies a native decoder, so a freshly built native
+library was used for the second measurement.
+
+The character-only path loaded `rx2_parser.py` by filename without making its
+sibling `rx2_fast.py` importable. It silently selected scalar pixel loops;
+other setup stages could incidentally make the fast decoder available. Loading
+the matching sibling explicitly removes that import-order dependency. A
+12-texture diagnostic sample went from 0.595 s to 0.158 s.
+
+The library now retains one read-only animation source and reference skeleton
+per invocation, and passes each item's already parsed mesh and morphs into its
+GLB conversion. Item-local data is discarded each iteration, bounding memory.
+Lighting resolves authored shader parameters once per model instead of once per
+material variant; each variant still receives its own texture and conflict check.
+No extra concurrency was added: eliminating duplicated work and ensuring the
+existing vectorized decoder is used produced the measured improvement without
+additional worker memory or contention.
+
+## Correctness and recovery evidence
+
+- Source-profile comparison: all 4,189 GLB/PNG outputs were byte-identical,
+  and library metadata was equal: 480 models, 2,427 materials, 170 tattoos.
+- Release-dependency comparison: all 4,218 GLB/PNG outputs, including specular
+  masks, were byte-identical. Base library, final lit library and native lighting
+  metadata matched after normalizing the generation-relative path prefix.
+- The complete new pipeline published all 41 pro/special characters successfully.
+  Native roster output was not separately compared against an old full roster.
+- Owned-output checks verified geometry bounds, finite buffers, skin weights,
+  all 22 live morph targets, secondary UVs, hair opacity and referenced textures.
+- Synthetic recovery tests cover default.xex validation, failed finalization,
+  preservation of maps/user profiles/mods, crash-released locking, incomplete
+  stages, same-size corruption, dependency invalidation, and a crash between
+  publishing a generation and deleting its pending record.
+- `cargo check --locked -p skate-game --bin skate3rust --no-default-features`
+  passed; no game was automatically launched. Gameplay remains user-tested.
+
+## Reproducing stage measurements
+
+With setup dependencies installed, run the character helper against an explicitly
+chosen prepared asset root containing its stock inputs:
+
+```text
+python -m tools.asset_pipeline.customiser_setup --game PATH_TO_DEFAULT_XEX --assets PATH_TO_ASSETS
+```
+
+Stage timings and reuse flags are written inside the selected private character
+generation as `timings.json`. To measure fresh output, use a separate owned test
+asset root with the required stock inputs, rather than deleting working user
+assets. To profile, run the same module through Python's `cProfile`. Keep derived
+game content and machine-specific benchmark paths out of commits.
+
+`pipeline-equivalence.json` contains exact old/new pairs for the unchanged core,
+HUD, default-character, environment and map exports at `c8219a5`, covering LF and
+CRLF package source conventions. The export recipe split preserves the original
+operations. Character converter reuse and decoder dispatch were checked by the
+byte comparisons above; map exporters were not modified. New fingerprints
+normalize text line endings. Each equivalence is bound to its specific new
+fingerprint, so later exporter edits cannot inherit an old bypass. Unknown
+historical fingerprints still require conversion; no general stale-version
+exception exists.

tools/asset_pipeline/asset_exports.py

@@ -0,0 +1,77 @@
+"""Owned-disc exports, separate from setup transaction orchestration."""
+import json
+import shutil
+from pathlib import Path
+from . import install as engine
+from .vlt import convert as convert_vlt
+from .physics_skeleton import convert as convert_skeleton
+
+def core(game_root, stage, work, report, log, converted=None):
+    private=stage/"assets/private"
+    stock=private/"stock"
+    report('Extracting animation banks, graphs and gameplay inputs')
+    engine.extract(game_root/'data/big/miscload.big',stock)
+    engine.extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg')
+    # Some disc banks are loose files rather than members of miscload.
+    loose=game_root/'data/anim'
+    if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True)
+    report('Converting physics and difficulty settings')
+    database=work/'database'
+    engine.extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in {
+        'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'})
+    names=(engine.TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines()
+    converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names)
+    (stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8')
+    skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin')
+    (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8')
+    return converted
+
+
+def hud(game_root, stage, work, report, log, converted=None):
+    private=stage/"assets/private"
+    stock=private/"stock"
+    for folder in (private/'hud',private/'session-marker'):
+        if folder.is_dir():engine.remove_intermediate(folder,stage)
+    report('Preparing original scoring and session-marker HUD assets')
+    engine.run(engine.task(engine.TOOLS/'prepare_runtime_huds.py', '--game', game_root,
+             '--assets', stage/'assets', '--work', work/'hud'), log, report)
+
+
+def character(game_root, stage, work, report, log, converted=None):
+    private=stage/"assets/private"
+    stock=private/"stock"
+    report('Preparing the skater model and textures')
+    manifest=json.loads((engine.TOOLS/'default_skater_retail_manifest.json').read_text())
+    needed=set()
+    for c in manifest['components']:
+        needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower())
+        needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values())
+    engine.extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed)
+    character=work/'character'
+    engine.run(engine.task(engine.TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character,
+             '--private-root',private/'default_skater','--utt-root',engine.TOOLS/'vendor/utt'),log,report)
+    report('Building the skater model and rig')
+    from .character_glb import convert as write_character
+    write_character(character/'selected/models',private,manifest)
+    from .character_lighting import convert as write_character_lighting
+    write_character_lighting(character/'selected/models',private,converted)
+    game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000',
+                   'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph',
+                   'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'}
+    (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
+
+
+def environment(game_root, stage, work, report, log, converted=None):
+    private=stage/"assets/private"
+    stock=private/"stock"
+    report('Preparing retail sky domes')
+    from .sky import convert as write_skies
+    write_skies(game_root,stage/'assets',converted)
+    from .render_parameters import convert as write_render_parameters
+    write_render_parameters(stage/'assets',converted)
+    report('Extracting original travel destinations and location names')
+    from .teleports import convert as write_teleports
+    write_teleports(game_root,stage/'assets',converted)
+    report('Preparing global foliage backdrops')
+    from .backdrop import convert as write_backdrops
+    write_backdrops(game_root,stage/'assets',converted)

tools/asset_pipeline/character_glb.py

@@ -32,11 +32,12 @@ class Glb:
         path.write_bytes(struct.pack('<III',0x46546c67,2,28+len(text)+len(self.data))+
                         struct.pack('<I4s',len(text),b'JSON')+text+struct.pack('<I4s',len(self.data),b'BIN\0')+self.data)
 
-def convert(models,private,recipe,*,stock=None,output=None,materials=None,live_morphs=False,geometry_only=False,reference_models=None):
-    source=AnimSource((stock or private/'stock')/'data/anim/OnBoard.abin')
-    skeleton=SkeletonSet(str(models))
+def convert(models,private,recipe,*,stock=None,output=None,materials=None,live_morphs=False,geometry_only=False,reference_models=None,
+            source=None,reference_skeleton=None,parsed_models=None,decoded_morphs=None):
+    source=source or AnimSource((stock or private/'stock')/'data/anim/OnBoard.abin')
+    skeleton=SkeletonSet(str(models),parsed_models)
     if reference_models:
-        reference_skeleton=SkeletonSet(str(reference_models))
+        reference_skeleton=reference_skeleton or SkeletonSet(str(reference_models))
         skeleton.bind={**reference_skeleton.bind,**skeleton.bind}
     if skeleton.errors:raise ValueError(str(skeleton.errors))
     mapping=skeleton.index_map(ABIN.BONE_NAMES,source.parents)
@@ -74,10 +75,11 @@ def convert(models,private,recipe,*,stock=None,output=None,materials=None,live_m
     if {m['folder'] for m in meshes}!=set(components):raise ValueError('Incomplete character mesh set')
     for mesh in meshes:
         slot=mesh['folder'];component=components[slot];path=models/slot/mesh['name']
-        parsed=RX2.parse_rx2(str(path));raw=[m for m in parsed['meshes'] if m.get('positions') and m.get('indices')]
+        parsed=parsed_models[str(path)] if parsed_models is not None else RX2.parse_rx2(str(path))
+        raw=[m for m in parsed['meshes'] if m.get('positions') and m.get('indices')]
         if len(raw)!=1:raise ValueError('Ambiguous character mesh '+slot)
         raw=raw[0];positions=np.asarray(mesh['pos'],dtype=np.float64)
-        morphs=decode_dense_morphs(path,parsed,len(positions),RX2)
+        morphs=decoded_morphs[str(path)] if decoded_morphs is not None else decode_dense_morphs(path,parsed,len(positions),RX2)
         if [m['name'] for m in morphs]!=recipe['morph_assembly']['expected_targets'][slot]:raise ValueError('Unexpected morph set '+slot)
         if 'normals' in raw:
             normals=np.asarray(raw['normals'],dtype=np.float64)

tools/asset_pipeline/customisation_library.py

@@ -50,7 +50,7 @@ def variant_label(slot, model_name, native_name):
 
 def prepare(config):
     from tools.extract_default_skater import import_rx2_parser,decode_texture,parse_fallback_recipe
-    from tools.asset_pipeline.retail_character import RX2,decode_dense_morphs
+    from tools.asset_pipeline.retail_character import RX2,decode_dense_morphs,AnimSource,SkeletonSet
     from tools.asset_pipeline.character_glb import convert
     assets=Path(config['assets']);cache=Path(config.get('directory',assets/'private/customisation'));out=cache/'library';out.mkdir(exist_ok=True)
     catalog=json.loads((cache/'catalog.json').read_text());archive=BigArchive(Path(config['game_root'])/'data/content/createacharacter.big')
@@ -84,6 +84,10 @@ def prepare(config):
         dest.parent.mkdir(parents=True,exist_ok=True)
         if not dest.exists():shutil.copyfile(src,dest)
     defaults={x['material_id']:x for x in recipe_base['components']}
+    # Read-only shared rig inputs live only for this preparation invocation.
+    # Per-item meshes and morphs are discarded after conversion, bounding RAM.
+    animation=AnimSource(assets/'private/stock/data/anim/OnBoard.abin')
+    reference_skeleton=SkeletonSet(str(reference))
     for part in catalog['components']:
         for model in part['models']:
             if part['slot']=='Misc':continue  # Non-geometric stamp catalogue, indexed below.
@@ -97,7 +101,11 @@ def prepare(config):
                     morphs=decode_dense_morphs(mesh_file,parsed,len(mesh['positions']),RX2)
                     recipe=copy.deepcopy(recipe_base);recipe['components']=[dict(slot=slot,textures={},tint=[1.,1.,1.])]
                     recipe['morph_assembly']['expected_targets']={slot:[m['name'] for m in morphs]}
-                    convert(folder.parent,assets/'private',recipe,output=path,geometry_only=True,live_morphs=True,reference_models=reference)
+                    temporary=path.with_suffix('.tmp')
+                    convert(folder.parent,assets/'private',recipe,output=temporary,geometry_only=True,live_morphs=True,reference_models=reference,
+                            source=animation,reference_skeleton=reference_skeleton,
+                            parsed_models={str(mesh_file):parsed},decoded_morphs={str(mesh_file):morphs})
+                    temporary.replace(path)
                 groups=[[v['id'] for v in group] for group in lod['material_instances']]
                 model_data[key]=dict(slot=slot,name=friendly(model['name']),flags={k[4:]:v for k,v in model['flags'].items() if v},
                     materials=groups[0],groups=groups,scene=path.relative_to(assets).as_posix())

tools/asset_pipeline/customiser_cache.py

@@ -0,0 +1,116 @@
+"""Versioned, checksummed character stages, scoped to one portable install."""
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import time
+from .setup_state import atomic_json, receipt, valid_receipt
+
+SHARED = ('owned_game/**/*.py', 'vendor/utt/**/*.py', 'vendor/utt/**/*.json',
+          'asset_pipeline/fast_refpack.py', 'asset_pipeline/refpack_native.rs',
+          'requirements-setup.txt')
+GEOMETRY = ('asset_pipeline/character_glb.py', 'asset_pipeline/retail_character.py',
+            'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py',
+            'extract_default_skater.py', 'default_skater_retail_manifest.json')
+SOURCES = {
+    'catalog': ('asset_pipeline/customisation_catalog.py', 'asset_pipeline/customisation_native.py',
+                'asset_pipeline/vlt.py', 'asset_pipeline/names.txt'),
+    'library': GEOMETRY + ('asset_pipeline/customisation_library.py', 'asset_pipeline/customisation_worker.py'),
+    'menu': ('asset_pipeline/customisation_profiles.py',),
+    'lighting': ('asset_pipeline/customiser_lighting.py', 'asset_pipeline/environment.py',
+                 'asset_pipeline/native_roster.py', 'extract_default_skater.py'),
+    'roster': GEOMETRY + ('asset_pipeline/native_roster.py', 'mixamo_to_skate/*.py'),
+}
+OUTPUTS = {
+    'catalog': ('catalog.json', 'native.json', 'database', 'source/data/content/recipe'),
+    'library': ('library', 'decoded', 'source/data/content/createacharacter', 'library-base.json'),
+    'menu': ('extra-menu.json',),
+    'lighting': ('specular', 'native-lighting.json', 'library-v3.json'),
+    'roster': ('native-roster',),
+}
+REQUIRED = {
+    'catalog': ('catalog.json', 'native.json', 'database/collections.json'),
+    'library': ('library-base.json',),
+    'menu': ('extra-menu.json',),
+    'lighting': ('library-v3.json', 'native-lighting.json'),
+    'roster': ('native-roster/complete.json',),
+}
+
+
+def versions(tools=None):
+    tools = tools or Path(__file__).resolve().parents[1]
+    result = {}
+    for stage, patterns in SOURCES.items():
+        digest = hashlib.sha256()
+        for path in sorted({p for pat in SHARED+patterns for p in tools.glob(pat)
+                            if p.is_file() and not p.name.startswith('test_')}):
+            digest.update(path.relative_to(tools).as_posix().encode()+b'\0')
+            digest.update(hashlib.sha256(path.read_bytes().replace(b'\r\n', b'\n')).digest())
+        if stage != 'catalog':digest.update(result['catalog'].encode())
+        if stage == 'lighting':digest.update(result['library'].encode())
+        result[stage] = digest.hexdigest()
+    return result
+
+
+def read(path):
+    try:
+        data = json.loads(path.read_text())
+        return data if isinstance(data, dict) else {}
+    except (OSError, ValueError):return {}
+
+
+def stage_valid(directory, name, saved):
+    files = saved.get('files')
+    return (isinstance(files, dict) and set(REQUIRED[name]).issubset(files)
+            and valid_receipt(directory, files))
+
+
+def stage(directory, previous, name, version, source, action, assets, report):
+    start = time.perf_counter()
+    marker = directory/(name+'-complete.json')
+    saved = read(marker)
+    if (saved.get('version') == version and saved.get('source') == source
+            and stage_valid(directory, name, saved)):
+        report(f'Character {name}: resumed complete stage ({time.perf_counter()-start:.2f}s)')
+        return
+    # Incomplete stage output must never reach an exists()-based converter.
+    for relative in OUTPUTS[name]:
+        path = directory/relative
+        if path.is_dir():shutil.rmtree(path)
+        else:path.unlink(missing_ok=True)
+    cached = read(previous/(name+'-complete.json')) if previous else {}
+    reused = (previous and cached.get('version') == version and cached.get('source') == source
+              and stage_valid(previous, name, cached))
+    if reused:
+        old_prefix = previous.relative_to(assets).as_posix()+'/'
+        new_prefix = directory.relative_to(assets).as_posix()+'/'
+        for relative in cached['files']:
+            src = previous/relative; dst = directory/relative
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            if src.suffix == '.json':
+                # Runtime references are relative to assets, including generation.
+                dst.write_text(src.read_text().replace(old_prefix,new_prefix))
+            else:
+                try:dst.hardlink_to(src)
+                except OSError:shutil.copy2(src,dst)
+    else:
+        action()
+    if any(not (directory/path).is_file() for path in REQUIRED[name]):
+        raise RuntimeError('Incomplete character stage: '+name)
+    paths = []
+    for relative in OUTPUTS[name]:
+        path = directory/relative
+        paths.extend(path.rglob('*') if path.is_dir() else [path])
+    files = receipt(directory, paths)
+    if not files:raise RuntimeError('Character stage produced no output: '+name)
+    atomic_json(marker, dict(version=version, source=source, files=files))
+    elapsed = time.perf_counter()-start
+    timing = read(directory/'timings.json')
+    timing[name] = {'seconds': round(elapsed,3), 'reused': bool(reused)}
+    atomic_json(directory/'timings.json', timing)
+    report(f'Character {name}: {"reused" if reused else "prepared"} in {elapsed:.2f}s')
+
+
+def complete(directory):
+    return all(stage_valid(directory, name, read(directory/(name+'-complete.json')))
+               for name in SOURCES)

tools/asset_pipeline/customiser_lighting.py

@@ -60,12 +60,12 @@ def prepare(game, assets, directory, library):
                 continue
             lod = next(l for l in model['lods'] if l['index'] == 0)
             raw = cac.read(cac_entries[lod['path'].lower()])
+            authored = material_data(raw, collections, mdl_parser)
             for mid in library['models'][model['id']]['materials']:
                 if mid not in library['materials']:
                     continue
                 textures = {t['channel']: t['id'] for t in catalog['materials'][mid]['textures']}
-                data = material_data(raw, collections, mdl_parser,
-                                     mask(cac, cac_entries, 'createacharacter', textures.get('specular')))
+                data = {**authored, 'specular': mask(cac, cac_entries, 'createacharacter', textures.get('specular'))}
                 material = library['materials'][mid]
                 if 'lighting' in material and material['lighting'] != data:
                     raise ValueError('Shared material has conflicting authored shader parameters: '+mid)

tools/asset_pipeline/customiser_setup.py

@@ -9,11 +9,12 @@ import re
 
 def fingerprint(tools=None):
     tools = tools or Path(__file__).resolve().parents[1]
-    paths = {p for pattern in ('asset_pipeline/customis*.py', 'asset_pipeline/native_roster.py',
+    paths = {p for pattern in ('asset_pipeline/customis*.py', 'asset_pipeline/setup_state.py', 'asset_pipeline/native_roster.py',
                               'asset_pipeline/character_glb.py', 'asset_pipeline/retail_character.py',
                               'asset_pipeline/vlt.py', 'asset_pipeline/environment.py',
                               'asset_pipeline/names.txt', 'extract_default_skater.py',
-                              'owned_game/**/*.py', 'vendor/utt/**/*.py',
+                              'owned_game/**/*.py', 'vendor/utt/**/*.py', 'vendor/utt/**/*.json',
+                              'asset_pipeline/fast_refpack.py', 'asset_pipeline/refpack_native.rs',
                               'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py',
                               'mixamo_to_skate/*.py', 'requirements-setup.txt',
                               'default_skater_retail_manifest.json') for p in tools.glob(pattern)
@@ -21,13 +22,14 @@ def fingerprint(tools=None):
     digest = hashlib.sha256()
     for path in sorted(paths):
         digest.update(path.relative_to(tools).as_posix().encode()+b'\0')
-        digest.update(hashlib.sha256(path.read_bytes()).digest())
+        digest.update(hashlib.sha256(path.read_bytes().replace(b'\r\n', b'\n')).digest())
     return digest.hexdigest()
 
 
 def source_fingerprint(game):
     digest = hashlib.sha256()
-    for name in ('data/content/createacharacter.big', 'data/content/marquee.big', 'data/big/db.big'):
+    for name in ('data/content/createacharacter.big', 'data/content/marquee.big', 'data/big/db.big',
+                 'data/big/miscload.big', 'default.xex'):
         with (game/name).open('rb') as source:
             digest.update(hashlib.file_digest(source, 'sha256').digest())
     return digest.hexdigest()
@@ -39,7 +41,10 @@ def prepare(game, assets, report=print):
     from tools.asset_pipeline.customisation_profiles import generate
     from tools.asset_pipeline.customiser_lighting import prepare as lighting
     from tools.asset_pipeline.native_roster import prepare as native_roster
+    from . import customiser_cache as cache
     base = assets/'private/customisation'
+    source = source_fingerprint(game)
+    old_directory = None
     marker = base/'current.json'
     version = fingerprint()
     if marker.is_file():
@@ -48,11 +53,14 @@ def prepare(game, assets, report=print):
         except (OSError, ValueError):
             saved = {}
         if (isinstance(saved, dict) and isinstance(saved.get('set'), str)
-                and re.fullmatch('[0-9a-f]{32}', saved['set']) and saved.get('fingerprint') == version
+                and re.fullmatch('[0-9a-f]{32}', saved['set']) and saved.get('fingerprint') == version and saved.get('source') == source
                 and all((base/'sets'/saved['set']/name).is_file() for name in
-                        ('library-v3.json', 'extra-menu.json', 'native-lighting.json', 'native-roster/complete.json'))):
+                        ('library-v3.json', 'extra-menu.json', 'native-lighting.json', 'native-roster/complete.json'))
+                and cache.complete(base/'sets'/saved['set'])):
             return
-    source = source_fingerprint(game)
+    saved = cache.read(marker)
+    if isinstance(saved.get('set'), str) and re.fullmatch('[0-9a-f]{32}', saved['set']):
+        old_directory = base/'sets'/saved['set']
     pending = base/'pending.json'
     try:
         previous = json.loads(pending.read_text())
@@ -60,33 +68,44 @@ def prepare(game, assets, report=print):
         previous = {}
     reuse = (isinstance(previous, dict) and isinstance(previous.get('set'), str)
              and previous.get('fingerprint') == version and previous.get('source') == source
+             and previous.get('set') != saved.get('set')
              and re.fullmatch('[0-9a-f]{32}', previous['set']))
     identity = previous['set'] if reuse else uuid.uuid4().hex
     directory = base/'sets'/identity
     directory.mkdir(parents=True, exist_ok=True)
-    pending.write_text(json.dumps(dict(set=identity, fingerprint=version, source=source)))
+    cache.atomic_json(pending, dict(set=identity, fingerprint=version, source=source))
     # The generation is invisible until every model/material has been prepared.
     # Old generations and user profiles remain usable after any failure.
     report('Preparing character customiser clothing, bodies and textures')
-    catalog(game, directory)
-    data = library(dict(assets=str(assets), game_root=str(game), directory=str(directory), library_index='library-v3.json'))
-    if data['errors'] or not data['models']:
-        raise RuntimeError('Character library preparation failed: '+str(data['errors'])[:1000])
-    for profile in data['defaults'].values():
-        for selection in profile['selections'].values():
-            if selection['asset_id'] not in data['models'] or selection['material_id'] not in data['materials']:
-                raise RuntimeError('Incomplete default character outfit')
-    (directory/'extra-menu.json').write_text(json.dumps(generate(json.loads((directory/'native.json').read_text()))))
+    stage_versions = cache.versions()
+    def run_stage(name, action):
+        cache.stage(directory, old_directory, name, stage_versions[name], source, action, assets, report)
+    run_stage('catalog', lambda: catalog(game, directory))
+    def build_library():
+        data = library(dict(assets=str(assets), game_root=str(game), directory=str(directory), library_index='library-base.json'))
+        if data['errors'] or not data['models']:
+            raise RuntimeError('Character library preparation failed: '+str(data['errors'])[:1000])
+        for profile in data['defaults'].values():
+            for selection in profile['selections'].values():
+                if selection['asset_id'] not in data['models'] or selection['material_id'] not in data['materials']:
+                    raise RuntimeError('Incomplete default character outfit')
+    run_stage('library', build_library)
+    run_stage('menu', lambda: (directory/'extra-menu.json').write_text(json.dumps(generate(json.loads((directory/'native.json').read_text())))))
     report('Preparing authored clothing lighting for custom and pro skaters')
-    lighting(game, assets, directory, data)
+    run_stage('lighting', lambda: lighting(game, assets, directory, json.loads((directory/'library-base.json').read_text())))
     report('Preparing all owned pro and special character models')
-    roster = native_roster(game, assets, directory/'native-roster',
-                          directory/'database/collections.json', directory/'roster-work')
-    if not roster or any(item['status'] != 'ready' for item in roster):
-        raise RuntimeError('Native character roster preparation did not complete')
-    (directory/'native-roster/complete.json').write_text(json.dumps({'characters': len(roster)}))
+    def build_roster():
+        # Work textures were not published or receipted; discard failed work.
+        import shutil
+        if (directory/'roster-work').exists():shutil.rmtree(directory/'roster-work')
+        roster = native_roster(game, assets, directory/'native-roster',
+                              directory/'database/collections.json', directory/'roster-work')
+        if not roster or any(item['status'] != 'ready' for item in roster):
+            raise RuntimeError('Native character roster preparation did not complete')
+        (directory/'native-roster/complete.json').write_text(json.dumps({'characters': len(roster)}))
+    run_stage('roster', build_roster)
     temporary = marker.with_suffix('.tmp')
-    temporary.write_text(json.dumps(dict(version=1, set=identity, fingerprint=version)))
+    temporary.write_text(json.dumps(dict(version=1, set=identity, fingerprint=version, source=source)))
     temporary.replace(marker)
     pending.unlink(missing_ok=True)
 
@@ -96,17 +115,20 @@ def install(iso, base, game_exe, report, refresh=False):
     # Extract an ISO once and share that source across core and character jobs.
     # Existing groups/fingerprints stay unchanged; a character-only update never
     # reconverts maps or overwrites settings and imported character libraries.
+    from .setup_state import source_directory, setup_lock
+    base.parent.mkdir(parents=True, exist_ok=True)
     selected = iso.resolve()
-    with tempfile.TemporaryDirectory(prefix='character-source-', dir=base.parent) as temp:
+    with setup_lock(base), tempfile.TemporaryDirectory(prefix='character-source-', dir=base.parent) as temp:
         if selected.suffix.lower() == '.iso':
+            if not selected.is_file():raise RuntimeError('Select an existing Skate 3 Xbox 360 ISO')
             source = Path(temp)/'disc'
             extractor = core.dependency(base/'tools', 'extract-xiso', core.XISO_URL, core.XISO_SHA, report)
             with (Path(temp)/'extract.log').open('w') as log:
                 core.run([extractor, '-x', selected, '-d', source], log, report)
         else:
-            source = selected if selected.is_dir() else selected.parent
-        stage = core.install(iso, base, game_exe, report, game_root=source, refresh=refresh)
-        prepare(source, stage/'assets', report)
+            source = source_directory(selected)
+        stage = core._install(iso, base, game_exe, report, game_root=source, refresh=refresh,
+                             finalize=lambda stage: prepare(source, stage/'assets', report))
         return stage
 
 
@@ -120,4 +142,5 @@ if __name__ == '__main__':
     if args.fingerprint:
         print(fingerprint())
     else:
-        prepare(args.game, args.assets)
+        from tools.asset_pipeline.setup_state import source_directory
+        prepare(source_directory(args.game), args.assets)

tools/asset_pipeline/group_receipts.py

@@ -0,0 +1,67 @@
+"""Validate prepared groups without treating user files as extractor output."""
+import json
+from .setup_state import receipt, valid_receipt
+
+ROOTS = {
+    'core': ('assets/private/stock/data/anim', 'assets/private/stock/data/state',
+             'assets/private/stock/data/config', 'assets/private/stock/data/cacrecipes',
+             'assets/private/stock/skater-collections.json', 'assets/private/stock/physics-skeletons.json'),
+    'hud': ('assets/private/hud', 'assets/private/session-marker'),
+    'character': ('assets/private/game.json', 'assets/private/skater.glb', 'assets/private/default_skater',
+                  'assets/private/native-character', 'assets/private/character-lighting.json'),
+    'environment': ('assets/private/native-skies', 'assets/private/native-backdrops',
+                    'assets/private/render-parameters.json', 'assets/private/exposure.json',
+                    'assets/private/exposure-profiles.json', 'assets/private/teleports.json'),
+    'maps': ('assets/private/native-props', 'maps.json'),
+}
+
+
+def files(root, group):
+    result = []
+    for name in ROOTS[group]:
+        path = root/name
+        result.extend(path.rglob('*') if path.is_dir() else [path])
+    if group == 'core':
+        stock = root/'assets/private/stock'
+        result.extend(p for p in stock.rglob('*')
+                      if not p.is_relative_to(stock/'data/content/createacharacter'))
+    if group == 'maps':
+        for item in json.loads((root/'maps.json').read_text()):
+            path = (root/item['path']).resolve()
+            if not path.is_relative_to((root/'maps').resolve()):
+                raise ValueError('Invalid prepared map path')
+            result.append(path)
+    return result
+
+
+def record(root, group):
+    return receipt(root, files(root, group))
+
+
+def damaged(root, marker, exclude=()):
+    saved = marker.get('outputs', {})
+    result = set()
+    for group, roots in ROOTS.items():
+        if group in exclude:continue
+        if group in saved:
+            if not valid_receipt(root, saved[group]):result.add(group)
+        else:
+            # Legacy packages have no receipts: require their declared roots
+            # and verify the map checksums they already recorded. The runtime
+            # validator also checks legacy core inputs before publication.
+            if any(not (root/name).exists() for name in roots):result.add(group)
+            if group == 'hud' and group not in result:
+                from tools.install_prepared_hud import runtime_files
+                try:
+                    runtime_files(root/'assets/private/hud')
+                    runtime_files(root/'assets/private/session-marker', marker=True)
+                except (OSError, ValueError, KeyError, TypeError):result.add(group)
+            if group == 'maps' and group not in result:
+                try:
+                    maps = json.loads((root/'maps.json').read_text())
+                    checks = {m['path']: {'size': (root/m['path']).stat().st_size,
+                                         'sha256': m['sha256']} for m in maps}
+                    if not valid_receipt(root, checks):result.add(group)
+                except (OSError, ValueError, KeyError, TypeError):result.add(group)
+    if 'core' in result:result.update(ROOTS)
+    return result

tools/asset_pipeline/install.py

@@ -3,8 +3,6 @@ from pathlib import Path
 import hashlib,json,os,shutil,subprocess,sys,time,urllib.request,uuid,zipfile
 from concurrent.futures import ThreadPoolExecutor,as_completed
 from tools.owned_game.big import BigArchive
-from .vlt import convert as convert_vlt
-from .physics_skeleton import convert as convert_skeleton
 
 TOOLS=Path(__file__).resolve().parents[1]
 
@@ -152,14 +150,20 @@ def convert_map(archive,work,maps,stage,game_exe,log,report):
     return entry
 
 
-def install(iso,base,game_exe,report,game_root=None,refresh=False):
+def install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None):
+    from .setup_state import setup_lock
+    with setup_lock(base):
+        return _install(iso,base,game_exe,report,game_root,refresh,finalize)
+
+
+def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None):
     from .versions import fingerprints, changed_groups, installed, GROUPS
+    from .group_receipts import damaged, record
+    from .setup_state import atomic_json
+    from . import asset_exports as exports
     target_versions=fingerprints()
     previous=installed(base) if refresh else None
     groups=changed_groups(previous[1].get('pipelines',{}),target_versions) if previous else set(GROUPS)
-    if not groups:
-        report('Game assets are already current')
-        return previous[0]
     source=str((iso if iso is not None else game_root).resolve())
     if game_root is None and iso is not None:
         selected=iso.resolve()
@@ -169,133 +173,124 @@ def install(iso,base,game_exe,report,game_root=None,refresh=False):
                 raise RuntimeError('Select default.xex inside your extracted Skate 3 game folder')
             game_root=selected.parent
     base=base.resolve();base.mkdir(parents=True,exist_ok=True)
-    lock=base/'setup.lock'
-    try:fd=os.open(lock,os.O_CREAT|os.O_EXCL|os.O_WRONLY)
-    except FileExistsError:raise RuntimeError('Setup is already running, or a previous setup was interrupted. Close it before retrying; an abandoned setup.lock can be removed from '+str(base))
-    os.close(fd)
-    try:
-        install_id=uuid.uuid4().hex
-        stage=base/'installations'/install_id
-        stage.mkdir(parents=True)
-        if previous:
-            report('Preparing an asset update; keeping the previous installation until it succeeds')
-            for name in ('assets','maps','settings'):
-                if (previous[0]/name).is_dir():shutil.copytree(previous[0]/name,stage/name)
-            shutil.copy2(previous[0]/'maps.json',stage/'maps.json')
-        private=stage/'assets/private';private.mkdir(parents=True,exist_ok=True)
-        maps=stage/'maps';maps.mkdir(exist_ok=True)
-        work=stage/'conversion';work.mkdir()
-        with (stage/'setup.log').open('w',encoding='utf-8') as log:
-            if game_root is None:
-                iso=iso.resolve()
-                if not iso.is_file() or iso.suffix.lower()!='.iso':raise RuntimeError('Select an Xbox 360 Skate 3 ISO')
-                extractor=dependency(base/'tools','extract-xiso',XISO_URL,XISO_SHA,report)
-                game_root=work/'disc'
-                report('Extracting your ISO')
-                run([extractor,'-x',iso,'-d',game_root],log,report)
-            else:game_root=game_root.resolve()
-            for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big',
-                             'data/content/createacharacter.big','data/content/worldDIST_University.big']:
-                if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required)
-            source_hash=digest(game_root/'default.xex')
-            if previous and previous[1].get('source_hash',source_hash)!=source_hash:
-                raise RuntimeError('Select the same Xbox game edition used to set up this copy')
-            stock=private/'stock'
-            if 'core' in groups:
-                report('Extracting animation banks, graphs and gameplay inputs')
-                extract(game_root/'data/big/miscload.big',stock)
-                extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg')
-                # Some disc banks are loose files rather than members of miscload.
-                loose=game_root/'data/anim'
-                if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True)
-                report('Converting physics and difficulty settings')
-                database=work/'database'
-                extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in {
-                    'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'})
-                names=(TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines()
-                converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names)
-                (stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8')
-                skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin')
-                (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8')
-            else:
-                converted=json.loads((stock/'skater-collections.json').read_text(encoding='utf-8'))
-            if 'hud' in groups:
-                for folder in (private/'hud',private/'session-marker'):
-                    if folder.is_dir():remove_intermediate(folder,stage)
-                report('Preparing original scoring and session-marker HUD assets')
-                run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root,
-                         '--assets', stage/'assets', '--work', work/'hud'), log, report)
-            if 'character' in groups:
-                report('Preparing the skater model and textures')
-                manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text())
-                needed=set()
-                for c in manifest['components']:
-                    needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower())
-                    needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values())
-                extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed)
-                character=work/'character'
-                run(task(TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character,
-                         '--private-root',private/'default_skater','--utt-root',TOOLS/'vendor/utt'),log,report)
-                report('Building the skater model and rig')
-                from .character_glb import convert as write_character
-                write_character(character/'selected/models',private,manifest)
-                from .character_lighting import convert as write_character_lighting
-                write_character_lighting(character/'selected/models',private,converted)
-                game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000',
-                               'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph',
-                               'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'}
-                (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
-            if 'environment' in groups:
-                report('Preparing retail sky domes')
-                from .sky import convert as write_skies
-                write_skies(game_root,stage/'assets',converted)
-                from .render_parameters import convert as write_render_parameters
-                write_render_parameters(stage/'assets',converted)
-                report('Extracting original travel destinations and location names')
-                from .teleports import convert as write_teleports
-                write_teleports(game_root,stage/'assets',converted)
-                report('Preparing global foliage backdrops')
-                from .backdrop import convert as write_backdrops
-                write_backdrops(game_root,stage/'assets',converted)
-            if 'maps' in groups:
-                report('Preparing authored movable-object models')
-                from .dynamic_props import prepare_catalog
-                prepare_catalog(game_root,work/'dmo')
-            report('Validating skater, input and animation data')
-            run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
-            if 'maps' in groups:
-                archives=list((game_root/'data/content').glob('worldDIST_*.big'))
-                archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower()))
-                workers=map_workers()
-                report(f'Converting {len(archives)} maps with {workers} workers')
-                def map_job(archive):
-                    result=work/(archive.stem+'.json')
-                    with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log:
-                        run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage,
-                                 '--game-exe',game_exe,'--result',result),map_log,report)
-                    return json.loads(result.read_text(encoding='utf-8'))
-                completed={}
-                with ThreadPoolExecutor(max_workers=workers) as pool:
-                    # Start the expensive districts together so one does not
-                    # remain queued behind a string of small parks.
-                    futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)}
-                    for future in as_completed(futures):
-                        archive=futures[future];completed[archive.name]=future.result()
-                        report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}")
-                catalog=[completed[a.name] for a in archives]
-                if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted')
-            report('Validating installed runtime inputs')
-            run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
-            settings=stage/'settings';settings.mkdir(exist_ok=True)
-            if not previous:
-                (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8')
-            if 'maps' in groups:
-                (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8')
-            remove_intermediate(work,stage)
-            # Publish only after every conversion and the runtime's own load succeeds.
-            marker=base/'installation.json.new'
-            marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id,'source':source,'source_hash':source_hash,'pipelines':target_versions}),encoding='utf-8')
-            marker.replace(base/'installation.json')
-            report('Setup complete')
-            return stage
-    finally:lock.unlink(missing_ok=True)
+    from .setup_state import source_directory
+    if game_root is not None:
+        game_root=source_directory(game_root)
+        if previous and previous[1].get('source_hash') not in (None,digest(game_root/'default.xex')):
+            raise RuntimeError('Select the same Xbox game edition used to set up this copy')
+    if previous:groups.update(damaged(*previous, exclude=groups))
+    def outputs(stage):
+        saved=previous[1].get('outputs',{}) if previous else {}
+        return {g:saved[g] if g not in groups and g in saved else record(stage,g) for g in GROUPS}
+    if not groups:
+        with (base/'refresh-validation.log').open('w',encoding='utf-8') as log:
+            run([game_exe,'--assets',previous[0]/'assets','--test-world','--check-assets'],log,report)
+        if finalize:finalize(previous[0])
+        atomic_json(base/'installation.json', {**previous[1], 'pipelines':target_versions,
+                    'outputs':outputs(previous[0]), 'source':source})
+        report('Game assets are current')
+        return previous[0]
+    install_id=uuid.uuid4().hex
+    stage=base/'installations'/install_id
+    stage.mkdir(parents=True)
+    if previous:
+        report('Preparing an asset update; keeping the previous installation until it succeeds')
+        immutable_maps = ({(previous[0]/item['path']).resolve() for item in
+                           json.loads((previous[0]/'maps.json').read_text())}
+                          if 'maps' not in groups else set())
+        from .customiser_cache import SOURCES as character_stages
+        sets=previous[0]/'assets/private/customisation/sets'
+        immutable_sets=[p.resolve() for p in sets.glob('*') if p.is_dir()
+                        and all((p/(name+'-complete.json')).is_file() for name in character_stages)]
+        for entry in previous[0].iterdir():
+            if entry.name in {'conversion','setup.log'} or entry.name.endswith('-conversion.log'):continue
+            # Unchanged maps and immutable character generations share storage.
+            # Mutable user data and rebuilt outputs get independent files.
+            def copy_map(src,dst):
+                if Path(src).resolve() in immutable_maps:
+                    try:os.link(src,dst)
+                    except OSError:shutil.copy2(src,dst)
+                else:shutil.copy2(src,dst)
+                return dst
+            def copy_asset(src,dst):
+                if any(Path(src).resolve().is_relative_to(root) for root in immutable_sets):
+                    try:os.link(src,dst)
+                    except OSError:shutil.copy2(src,dst)
+                else:shutil.copy2(src,dst)
+                return dst
+            if entry.is_dir():
+                shutil.copytree(entry,stage/entry.name,
+                    copy_function=copy_map if entry.name=='maps' and 'maps' not in groups
+                    else copy_asset if entry.name=='assets' else shutil.copy2)
+            else:shutil.copy2(entry,stage/entry.name)
+    private=stage/'assets/private';private.mkdir(parents=True,exist_ok=True)
+    maps=stage/'maps';maps.mkdir(exist_ok=True)
+    work=stage/'conversion';work.mkdir()
+    with (stage/'setup.log').open('w',encoding='utf-8') as log:
+        if game_root is None:
+            iso=iso.resolve()
+            if not iso.is_file() or iso.suffix.lower()!='.iso':raise RuntimeError('Select an Xbox 360 Skate 3 ISO')
+            extractor=dependency(base/'tools','extract-xiso',XISO_URL,XISO_SHA,report)
+            game_root=work/'disc'
+            report('Extracting your ISO')
+            run([extractor,'-x',iso,'-d',game_root],log,report)
+        else:game_root=game_root.resolve()
+        for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big',
+                         'data/content/createacharacter.big','data/content/worldDIST_University.big']:
+            if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required)
+        source_hash=digest(game_root/'default.xex')
+        if previous and previous[1].get('source_hash',source_hash)!=source_hash:
+            raise RuntimeError('Select the same Xbox game edition used to set up this copy')
+        stock=private/'stock'
+        if 'core' in groups:
+            converted=exports.core(game_root,stage,work,report,log)
+        else:
+            converted=json.loads((stock/'skater-collections.json').read_text(encoding='utf-8'))
+        if 'hud' in groups:
+            exports.hud(game_root,stage,work,report,log)
+        if 'character' in groups:
+            exports.character(game_root,stage,work,report,log,converted)
+        if 'environment' in groups:
+            exports.environment(game_root,stage,work,report,log,converted)
+        if 'maps' in groups:
+            report('Preparing authored movable-object models')
+            from .dynamic_props import prepare_catalog
+            prepare_catalog(game_root,work/'dmo')
+        report('Validating skater, input and animation data')
+        run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
+        if 'maps' in groups:
+            archives=list((game_root/'data/content').glob('worldDIST_*.big'))
+            archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower()))
+            workers=map_workers()
+            report(f'Converting {len(archives)} maps with {workers} workers')
+            def map_job(archive):
+                result=work/(archive.stem+'.json')
+                with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log:
+                    run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage,
+                             '--game-exe',game_exe,'--result',result),map_log,report)
+                return json.loads(result.read_text(encoding='utf-8'))
+            completed={}
+            with ThreadPoolExecutor(max_workers=workers) as pool:
+                # Start the expensive districts together so one does not
+                # remain queued behind a string of small parks.
+                futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)}
+                for future in as_completed(futures):
+                    archive=futures[future];completed[archive.name]=future.result()
+                    report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}")
+            catalog=[completed[a.name] for a in archives]
+            if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted')
+        report('Validating installed runtime inputs')
+        run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
+        settings=stage/'settings';settings.mkdir(exist_ok=True)
+        if not previous:
+            (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8')
+        if 'maps' in groups:
+            (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8')
+        remove_intermediate(work,stage)
+        if finalize:finalize(stage)
+        # Publish only after every conversion, including the customiser, succeeds.
+        marker=base/'installation.json.new'
+        marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id,'source':source,'source_hash':source_hash,'pipelines':target_versions,'outputs':outputs(stage)}),encoding='utf-8')
+        marker.replace(base/'installation.json')
+        report('Setup complete')
+        return stage

tools/asset_pipeline/pipeline-equivalence.json

@@ -0,0 +1,52 @@
+{
+  "core": [
+    [
+      "590dc6f7aab2f2ce8784f89f3c46e77e532dc84baaf052f18bc70593df2021e2",
+      "4ced35322fac72787fdc812d9f8833ce6849522602cd9d15e2976872d9f2b96b"
+    ],
+    [
+      "79ec94813288e499a39b4f79db97c6b285f8878bd827914d217968236b1336a1",
+      "4ced35322fac72787fdc812d9f8833ce6849522602cd9d15e2976872d9f2b96b"
+    ]
+  ],
+  "hud": [
+    [
+      "a59a88bb5549668abf8c023bf3f9e706be7e0282ea94ca94f34a50fc66e61ca1",
+      "f1c596920b43dcf1018d616750877799e9a3fa5f8f2f5f84284e52c141a6d060"
+    ],
+    [
+      "b2a391109e8ab62477a589bce8097d0f3607ce169fb328eeebc7ee7ae3127fc0",
+      "f1c596920b43dcf1018d616750877799e9a3fa5f8f2f5f84284e52c141a6d060"
+    ]
+  ],
+  "character": [
+    [
+      "ae6808fb8819b6d549de5cf1e94032aa3f52103885c5a40d959c3f6c45663f60",
+      "7d89c09e4ae5457c3d61140048de72bba901146a897cf8f843b9010148a6af1a"
+    ],
+    [
+      "faeff08d897667ad0ebf1731e4a748875b65d628c4f724864bab589a1dbd59e5",
+      "7d89c09e4ae5457c3d61140048de72bba901146a897cf8f843b9010148a6af1a"
+    ]
+  ],
+  "environment": [
+    [
+      "0d38b652d3c23e7a799427bea91c6326a565d519e3b4b91ef110d7396cd6d52d",
+      "19aaa1e8235dc6f4887479e69eb8397cfee41908207dc6c5f011cfc109f73845"
+    ],
+    [
+      "ed0a83a303d249089ce8c0f893d8249b2c4633a4f6174c3895599088cadabf0d",
+      "19aaa1e8235dc6f4887479e69eb8397cfee41908207dc6c5f011cfc109f73845"
+    ]
+  ],
+  "maps": [
+    [
+      "33a989085013f5ec221401ee41cd936674df3e7dedb25696c7195de2e37e6ccd",
+      "3047b6aad327e1c0bab286c84fe6390ced1cf1c50bd70ab4fad5e6aac3e42705"
+    ],
+    [
+      "a3d38f780d95881f5369757e1db6a95635331c93053121e8b6277c41db5125e9",
+      "3047b6aad327e1c0bab286c84fe6390ced1cf1c50bd70ab4fad5e6aac3e42705"
+    ]
+  ]
+}

tools/asset_pipeline/retail_character.py

@@ -98,7 +98,7 @@ class AnimSource:
 class SkeletonSet:
     """Blender-safe copy of SkateAnimViewer's RX2-to-ABIN mapper."""
 
-    def __init__(self, folder):
+    def __init__(self, folder, parsed_models=None):
         self.models = []
         self.bind = {}
         self.ibp = {}
@@ -108,7 +108,7 @@ class SkeletonSet:
         )
         for path in files:
             try:
-                parsed = RX2.parse_rx2(path)
+                parsed = parsed_models[path] if parsed_models is not None else RX2.parse_rx2(path)
             except Exception as error:
                 self.errors.append(f"{Path(path).name}: {error}")
                 continue

tools/asset_pipeline/setup_state.py

@@ -0,0 +1,80 @@
+"""Portable setup locking, source validation and immutable output receipts."""
+from contextlib import contextmanager
+import hashlib
+import json
+import os
+from pathlib import Path
+
+
+def source_directory(selected):
+    selected = Path(selected).resolve()
+    if selected.is_file() and selected.name.lower() == 'default.xex':
+        selected = selected.parent
+    elif not selected.is_dir():
+        raise RuntimeError('Select your Skate 3 default.xex or Xbox 360 ISO')
+    for name in ('default.xex', 'data/big/miscload.big', 'data/big/miscboot.big',
+                 'data/big/db.big', 'data/content/createacharacter.big',
+                 'data/content/marquee.big', 'data/content/worldDIST_University.big'):
+        if not (selected/name).is_file():
+            raise RuntimeError('Keep the Skate 3 game content beside default.xex; missing '+name)
+    return selected
+
+
+@contextmanager
+def setup_lock(base):
+    base.mkdir(parents=True, exist_ok=True)
+    # Kernel locks are released on process death. Keep the lock file in place:
+    # unlinking it allows two processes to lock different file identities.
+    with (base/'setup.lock').open('a+b') as stream:
+        stream.seek(0, 2)
+        if not stream.tell():
+            stream.write(b'0'); stream.flush()
+        stream.seek(0)
+        try:
+            if os.name == 'nt':
+                import msvcrt
+                msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+            else:
+                import fcntl
+                fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as error:
+            raise RuntimeError('Setup is already running for this copy') from error
+        try:
+            yield
+        finally:
+            stream.seek(0)
+            if os.name == 'nt':
+                msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def sha(path):
+    with path.open('rb') as stream:
+        return hashlib.file_digest(stream, 'sha256').hexdigest()
+
+
+def receipt(root, paths):
+    return {p.relative_to(root).as_posix(): {'size': p.stat().st_size, 'sha256': sha(p)}
+            for p in sorted(set(paths)) if p.is_file()}
+
+
+def valid_receipt(root, files):
+    if not isinstance(files, dict) or not files:
+        return False
+    try:
+        for name, item in files.items():
+            path = (root/name).resolve()
+            if not path.is_relative_to(root.resolve()) or not path.is_file():
+                return False
+            if path.stat().st_size != item['size'] or sha(path) != item['sha256']:
+                return False
+        return True
+    except (OSError, KeyError, TypeError, ValueError):
+        return False
+
+
+def atomic_json(path, value):
+    temporary = path.with_suffix(path.suffix+'.tmp')
+    temporary.write_text(json.dumps(value, separators=(',', ':')), encoding='utf-8')
+    temporary.replace(path)

tools/asset_pipeline/test_customiser_setup.py

@@ -32,6 +32,21 @@ class CharacterSetup(unittest.TestCase):
                 with self.assertRaises(RuntimeError): s.prepare(Path(temp)/'source', assets, lambda _: None)
             self.assertEqual(json.loads((base/'current.json').read_text()), old)
 
+    def test_crash_after_publication_cannot_resume_inside_the_live_generation(self):
+        with tempfile.TemporaryDirectory() as temp:
+            assets=Path(temp)/'assets';base=assets/'private/customisation'
+            live=base/'sets'/('a'*32);live.mkdir(parents=True)
+            (live/'catalog.json').write_text('working output')
+            old={'set':'a'*32,'fingerprint':'v','source':'owned-disc'}
+            (base/'current.json').write_text(json.dumps(old))
+            (base/'pending.json').write_text(json.dumps(old))
+            with patch.object(s,'fingerprint',return_value='v'), \
+                 patch('tools.asset_pipeline.customisation_catalog.prepare',side_effect=RuntimeError('failed')):
+                with self.assertRaises(RuntimeError):s.prepare(Path(temp)/'source',assets,lambda _:None)
+            self.assertEqual((live/'catalog.json').read_text(),'working output')
+            self.assertEqual(json.loads((base/'current.json').read_text()),old)
+            self.assertNotEqual(json.loads((base/'pending.json').read_text())['set'],old['set'])
+
     def test_fresh_generation_contains_customiser_profiles_lighting_and_native_roster(self):
         with tempfile.TemporaryDirectory() as temp:
             assets = Path(temp)/'assets'
@@ -39,11 +54,15 @@ class CharacterSetup(unittest.TestCase):
                         defaults={'male': {'selections': {'Body': {'asset_id': 'body', 'material_id': 'cloth'}}}})
             def catalog(game, out):
                 (out/'native.json').write_text('{}')
+                (out/'catalog.json').write_text('{}')
+                (out/'database').mkdir(exist_ok=True)
+                (out/'database/collections.json').write_text('{}')
             def library(config):
-                (Path(config['directory'])/'library-v3.json').write_text(json.dumps(data))
+                (Path(config['directory'])/config['library_index']).write_text(json.dumps(data))
                 return data
             def lighting(game, assets, directory, data):
                 (directory/'native-lighting.json').write_text('{"pro":{}}')
+                (directory/'library-v3.json').write_text(json.dumps(data))
             def roster(game, assets, library, collections, work):
                 library.mkdir(exist_ok=True)
                 return roster_results.pop(0)
@@ -69,7 +88,11 @@ class CharacterSetup(unittest.TestCase):
         with tempfile.TemporaryDirectory() as temp:
             root = Path(temp); source = root/'source'; source.mkdir()
             base = root/'data'; installed = base/'installations/old'
-            with patch('tools.asset_pipeline.install.install', return_value=installed) as install, \
+            def core_install(*args, **kwargs):
+                kwargs['finalize'](installed)
+                return installed
+            with patch('tools.asset_pipeline.setup_state.source_directory', return_value=source), \
+                 patch('tools.asset_pipeline.install._install', side_effect=core_install) as install, \
                  patch.object(s, 'prepare') as prepare:
                 s.install(source, base, Path('unused.exe'), lambda _: None, refresh=True)
             self.assertTrue(install.call_args.kwargs['refresh'])

tools/asset_pipeline/test_setup_recovery.py

@@ -0,0 +1,92 @@
+"""Publication, interruption and cache-integrity regressions using tiny files."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+from . import test_versions
+from .install import install
+from . import versions, customiser_cache as cache
+from .setup_state import source_directory, setup_lock, receipt, valid_receipt
+
+
+class SetupRecovery(unittest.TestCase):
+    def test_xex_refresh_failure_never_publishes_core_or_edits_user_data(self):
+        with tempfile.TemporaryDirectory() as temp:
+            base, old, source, current, marker = test_versions.AssetVersions().fixture(Path(temp))
+            (old/'mods').mkdir(); (old/'mods/user.lua').write_text('user mod')
+            (old/'maps/user.skate').write_text('user map')
+            (old/'assets/imported-profile.json').write_text('user profile')
+            def fail(stage):
+                self.assertEqual(json.loads((base/'installation.json').read_text()), marker)
+                self.assertTrue(os.path.samefile(old/'maps/University.skate', stage/'maps/University.skate'))
+                self.assertFalse(os.path.samefile(old/'maps/user.skate', stage/'maps/user.skate'))
+                self.assertEqual((stage/'mods/user.lua').read_text(), 'user mod')
+                (stage/'assets/imported-profile.json').write_text('staged change')
+                raise RuntimeError('customiser interrupted')
+            with patch.object(versions,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run'):
+                with self.assertRaisesRegex(RuntimeError,'customiser interrupted'):
+                    install(source/'default.xex',base,Path('unused.exe'),lambda _:None,refresh=True,finalize=fail)
+            self.assertEqual(json.loads((base/'installation.json').read_text()),marker)
+            self.assertEqual((old/'assets/imported-profile.json').read_text(),'user profile')
+
+    def test_current_core_still_validates_selected_source_and_runs_finalizer(self):
+        with tempfile.TemporaryDirectory() as temp:
+            base, old, source, current, marker = test_versions.AssetVersions().fixture(Path(temp))
+            marker['pipelines']=current
+            (base/'installation.json').write_text(json.dumps(marker))
+            with patch.object(versions,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run'), patch('builtins.print'):
+                with self.assertRaisesRegex(RuntimeError,'character failed'):
+                    install(source/'default.xex',base,Path('unused.exe'),lambda _:None,refresh=True,
+                            finalize=lambda _: (_ for _ in ()).throw(RuntimeError('character failed')))
+            self.assertEqual(json.loads((base/'installation.json').read_text()),marker)
+            with self.assertRaisesRegex(RuntimeError,'default.xex'):
+                source_directory(source/'other.xex')
+            (source/'data/content/createacharacter.big').unlink()
+            with self.assertRaisesRegex(RuntimeError,'createacharacter.big'):
+                source_directory(source/'default.xex')
+
+    def test_process_death_releases_lock_without_manual_file_deletion(self):
+        with tempfile.TemporaryDirectory() as temp:
+            base=Path(temp)
+            script="from pathlib import Path; from tools.asset_pipeline.setup_state import setup_lock; import time; " \
+                   "ctx=setup_lock(Path(__import__('sys').argv[1])); ctx.__enter__(); print('locked',flush=True); time.sleep(60)"
+            child=subprocess.Popen([getattr(sys, '_base_executable', sys.executable),'-c',script,str(base)],stdout=subprocess.PIPE,text=True)
+            try:
+                self.assertEqual(child.stdout.readline().strip(),'locked')
+                with self.assertRaisesRegex(RuntimeError,'already running'):
+                    with setup_lock(base):pass
+            finally:
+                child.kill();child.wait();child.stdout.close()
+            with setup_lock(base):pass
+
+    def test_stage_reuses_only_matching_verified_outputs_and_repairs_partial_files(self):
+        with tempfile.TemporaryDirectory() as temp:
+            assets=Path(temp);old=assets/'old';new=assets/'new'
+            old.mkdir();new.mkdir()
+            def build(directory, text):
+                (directory/'extra-menu.json').write_text(json.dumps({'path':directory.relative_to(assets).as_posix()+'/item', 'text':text}))
+            cache.stage(old,None,'menu','v1','disc',lambda:build(old,'good'),assets,lambda _:None)
+            with patch.object(cache,'shutil', wraps=cache.shutil):
+                cache.stage(new,old,'menu','v1','disc',lambda:self.fail('valid cache must be reused'),assets,lambda _:None)
+            self.assertEqual(json.loads((new/'extra-menu.json').read_text())['path'],'new/item')
+            (new/'extra-menu.json').write_text('interrupted')
+            cache.stage(new,None,'menu','v1','disc',lambda:build(new,'repaired'),assets,lambda _:None)
+            self.assertEqual(json.loads((new/'extra-menu.json').read_text())['text'],'repaired')
+            cache.stage(new,old,'menu','v2','disc',lambda:build(new,'new extractor'),assets,lambda _:None)
+            self.assertEqual(json.loads((new/'extra-menu.json').read_text())['text'],'new extractor')
+            cache.stage(new,old,'menu','v1','different disc',lambda:build(new,'new source'),assets,lambda _:None)
+            self.assertEqual(json.loads((new/'extra-menu.json').read_text())['text'],'new source')
+
+    def test_same_size_corruption_and_escaping_receipts_are_not_current(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root=Path(temp);file=root/'asset';file.write_bytes(b'good')
+            files=receipt(root,[file]);file.write_bytes(b'evil')
+            self.assertFalse(valid_receipt(root,files))
+            self.assertFalse(valid_receipt(root,{'../asset':files['asset']}))
+
+
+if __name__=='__main__':unittest.main()

tools/asset_pipeline/test_versions.py

@@ -9,6 +9,34 @@ from tools.asset_pipeline.install import install
 
 
 class AssetVersions(unittest.TestCase):
+    def test_orchestration_does_not_invalidate_exporters_and_line_endings_are_stable(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root=Path(temp); (root/'asset_pipeline').mkdir()
+            install=root/'asset_pipeline/install.py'
+            install.write_text('def extract(): return 1\ndef convert_map(): return 2\ndef install(): return 3\n')
+            before=v.fingerprints(root)
+            install.write_text(install.read_text().replace('return 3','return 4'))
+            self.assertEqual(v.fingerprints(root),before)
+            install.write_text(install.read_text().replace('return 2','return 5'))
+            self.assertEqual(v.changed_groups(before,v.fingerprints(root)),{'maps'})
+            exporter=root/'asset_pipeline/asset_exports.py'
+            exporter.write_text('def core(): return 1\ndef hud(): return 2\n')
+            before=v.fingerprints(root)
+            exporter.write_text(exporter.read_text().replace('return 2','return 3'))
+            self.assertEqual(v.changed_groups(before,v.fingerprints(root)),{'hud'})
+            resource=root/'asset_pipeline/names.txt';resource.write_bytes(b'one\ntwo\n')
+            before=v.fingerprints(root);resource.write_bytes(b'one\r\ntwo\r\n')
+            self.assertEqual(v.fingerprints(root),before)
+
+    def test_equivalences_only_accept_exact_old_and_new_pair(self):
+        current=v.fingerprints()
+        migrations=json.loads(Path(v.__file__).with_name('pipeline-equivalence.json').read_text())
+        # Known HEAD export migration retains all current core/map content.
+        old={g:migrations[g][0][0] for g in v.GROUPS}
+        self.assertEqual(v.changed_groups(old,current),set())
+        changed={**current,'maps':'future-exporter'}
+        self.assertEqual(v.changed_groups(old,changed),{'maps'})
+
     def test_hud_change_does_not_reconvert_maps(self):
         with tempfile.TemporaryDirectory() as temp:
             root=Path(temp)
@@ -34,13 +62,18 @@ class AssetVersions(unittest.TestCase):
         (private/'hud').mkdir();(private/'hud/old.txt').write_text('old')
         (old/'maps').mkdir();(old/'maps/University.skate').write_bytes(b'unchanged map')
         (old/'settings').mkdir();(old/'settings/default-map.json').write_text('"custom.skate"')
-        (old/'maps.json').write_text('[{"name":"University"}]')
+        (old/'maps.json').write_text('[{"name":"University","path":"maps/University.skate"}]')
         current={group:'new' for group in v.GROUPS}
         marker={'version':1,'directory':'installations/'+'a'*32,'pipelines':{**current,'hud':'old'}}
         (base/'installation.json').write_text(json.dumps(marker))
+        from tools.asset_pipeline.group_receipts import record
+        for group in v.GROUPS:
+            # Minimal synthetic receipt; tests stub converters, never retail input.
+            marker.setdefault('outputs',{})[group] = record(old,group) or record(old,'character')
+        (base/'installation.json').write_text(json.dumps(marker))
         source=root/'disc'
         for name in ('default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big',
-                     'data/content/createacharacter.big','data/content/worldDIST_University.big'):
+                     'data/content/createacharacter.big','data/content/marquee.big','data/content/worldDIST_University.big'):
             p=source/name;p.parent.mkdir(parents=True,exist_ok=True);p.touch()
         return base,old,source,current,marker
 
@@ -66,7 +99,8 @@ class AssetVersions(unittest.TestCase):
             with patch.object(v,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run',side_effect=RuntimeError('failed')):
                 with self.assertRaises(RuntimeError):install(source,base,Path('unused.exe'),lambda _:None,refresh=True)
             self.assertEqual(v.installed(base)[1],marker)
-            self.assertFalse((base/'setup.lock').exists())
+            from tools.asset_pipeline.setup_state import setup_lock
+            with setup_lock(base):pass  # Released even on failure; file persists.
 
 
 if __name__=='__main__':unittest.main()

tools/asset_pipeline/versions.py

@@ -1,20 +1,22 @@
 """Content identities for independently refreshable owned-disc asset groups."""
 import argparse
+import ast
 import hashlib
 import json
 from pathlib import Path
 import re
 
 GROUPS = ('core', 'hud', 'character', 'environment', 'maps')
-COMMON = ('owned_game/**/*.py', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/install.py',
-          'asset_pipeline/versions.py', 'asset_pipeline/refpack_native.rs')
+COMMON = ('owned_game/**/*.py', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/refpack_native.rs',
+          'requirements-setup.txt')
 PARSERS = ('vendor/utt/**/*.py', 'vendor/university/**/*.py', 'vendor/utt/**/*.json', 'vendor/university/**/*.json')
 SOURCES = {
     'core': ('asset_pipeline/vlt.py', 'asset_pipeline/names.txt', 'asset_pipeline/physics_skeleton.py'),
     'hud': ('prepare_hud.py', 'prepare_runtime_huds.py', 'install_prepared_hud.py',
             'extract_session_marker.py', 'vendor/skate3_ui/**/*.py', 'vendor/skate3_ui/**/*.json'),
     'character': PARSERS + ('extract_default_skater.py', 'default_skater_retail_manifest.json',
-                  'asset_pipeline/character*.py'),
+                  'asset_pipeline/character*.py', 'asset_pipeline/retail_character.py',
+                  'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py'),
     'environment': PARSERS + ('asset_pipeline/sky.py', 'asset_pipeline/backdrop.py',
                   'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py',
                   'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py',
@@ -32,15 +34,28 @@ def fingerprints(tools=None):
         paths = {p for pattern in COMMON + patterns for p in tools.glob(pattern)
                  if p.is_file() and not p.name.startswith('test_') and '__pycache__' not in p.parts}
         digest = hashlib.sha256()
+        # Version the actual per-group recipe and shared extraction primitive,
+        # without invalidating maps when a dialog/lock/publication rule changes.
+        for filename, names in (('asset_pipeline/asset_exports.py', {group}),
+                                ('asset_pipeline/install.py', {'extract'} | ({'convert_map'} if group == 'maps' else set()))):
+            path = tools/filename
+            if path.is_file():
+                nodes = ast.parse(path.read_text()).body
+                for node in nodes:
+                    if isinstance(node, (ast.Import, ast.ImportFrom)) or getattr(node, 'name', None) in names:
+                        digest.update(ast.dump(node, include_attributes=False).encode())
         for path in sorted(paths, key=lambda p: p.relative_to(tools).as_posix()):
             digest.update(path.relative_to(tools).as_posix().encode() + b'\0')
-            digest.update(hashlib.sha256(path.read_bytes()).digest())
+            digest.update(hashlib.sha256(path.read_bytes().replace(b'\r\n', b'\n')).digest())
         result[group] = digest.hexdigest()
     return result
 
 
 def changed_groups(previous, current):
-    changed = {name for name in GROUPS if previous.get(name) != current[name]}
+    migrations = Path(__file__).with_name('pipeline-equivalence.json')
+    equivalent = json.loads(migrations.read_text()) if migrations.is_file() else {}
+    changed = {name for name in GROUPS if previous.get(name) != current[name]
+               and [previous.get(name), current[name]] not in equivalent.get(name, [])}
     # Other exports consume the core VLT/skeleton/animation data.
     if 'core' in changed:
         changed.update(GROUPS)
@@ -58,7 +73,7 @@ def installed(base):
     root = (base/directory).resolve()
     if not root.is_relative_to(base.resolve()):
         raise ValueError('Installation escapes its package data directory')
-    if not (root/'assets/private/game.json').is_file():
+    if not root.is_dir():
         return None
     return root, value
 

tools/extract_default_skater.py

@@ -216,6 +216,17 @@ def import_rx2_parser(utt_root: Path):
         raise RuntimeError(f"Could not load RX2 decoder: {module_path}")
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
+    # Loading by filename does not put the sibling directory on sys.path.
+    # In particular, character-only setup used the slow scalar fallback while
+    # map setup happened to make rx2_fast importable. Load the matching bundled
+    # implementation explicitly, independent of import order or global paths.
+    fast_spec = importlib.util.spec_from_file_location("skate3_local_rx2_fast", utt_root / "rx2_fast.py")
+    fast = importlib.util.module_from_spec(fast_spec)
+    fast_spec.loader.exec_module(fast)
+    for name in ('decode_dxt1', 'decode_dxt1_normal', 'decode_dxt3', 'decode_dxt5',
+                 'decode_ati2', '_untile360', '_decode_raw_a8r8g8b8',
+                 '_decode_raw_b5g6r5', '_decode_raw_a8'):
+        setattr(module, name, getattr(fast, name))
     return module
 
 

tools/setup.py

@@ -41,20 +41,18 @@ def main():
     if icon.is_file():window.iconbitmap(str(icon))
     frame=ttk.Frame(window,padding=24);frame.pack(fill='both',expand=True)
     ttk.Label(frame,text='Update game assets' if updating else 'Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16))
-    ttk.Label(frame,text=('Changed asset groups: '+(', '.join(sorted(changed)) or 'character customiser')+'.\nOnly changed groups will be prepared again.\nYour previous character data remains until preparation succeeds.\nUse Update game assets to reuse your selected Xbox source,\nor choose its new location if it has moved.') if updating else 'Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, customiser, animations and disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.',
+    ttk.Label(frame,text=('Asset version changes: '+(', '.join(sorted(changed)) or 'checking prepared content')+'.\nOnly changed or incomplete groups will be prepared again.\nYour previous character data remains until preparation succeeds.\nSelect your Skate 3 default.xex (or ISO) to continue.\nKeep the game data beside default.xex.') if updating else 'Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, customiser, animations and disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.',
               font=('Segoe UI',11),justify='left').pack(anchor='w')
     status=tk.StringVar(value='Choose your game to begin.')
     ttk.Label(frame,textvariable=status,wraplength=600).pack(anchor='w',pady=(18,8))
     progress=ttk.Progressbar(frame,mode='indeterminate');progress.pack(fill='x')
     messages=queue.Queue();running=False;success=False
-    def start(folder=False,reuse=False):
+    def start():
         nonlocal running
-        saved=previous[1].get('source') if previous else None
-        iso=saved if reuse and saved and Path(saved).exists() else (filedialog.askdirectory(parent=window,title='Select the Skate 3 folder containing default.xex') if folder else
-             filedialog.askopenfilename(parent=window,title='Select your Skate 3 Xbox 360 game',
-                 filetypes=[('Xbox 360 game','*.iso *.xex'),('Xbox 360 ISO','*.iso'),('Xbox executable','*.xex')]))
+        iso=filedialog.askopenfilename(parent=window,title='Select your Skate 3 default.xex or Xbox 360 ISO',
+            filetypes=[('Skate 3 game','default.xex *.iso'),('Skate 3 executable','default.xex'),('Xbox 360 ISO','*.iso')])
         if not iso:return
-        button.config(state='disabled');folder_button.config(state='disabled');running=True;progress.start()
+        button.config(state='disabled');running=True;progress.start()
         def work():
             try:
                 install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating)
@@ -69,8 +67,7 @@ def main():
             messagebox.showinfo('Setup running','Wait for the current conversion to finish. Your source game files are not modified.',parent=window)
         else:window.destroy()
     buttons=ttk.Frame(frame);buttons.pack(anchor='e',pady=18)
-    folder_button=ttk.Button(buttons,text='Select extracted folder',command=lambda:start(True));folder_button.pack(side='left',padx=(0,8))
-    button=ttk.Button(buttons,text='Update game assets' if updating else 'Select ISO or default.xex',command=lambda:start(reuse=updating));button.pack(side='left')
+    button=ttk.Button(buttons,text='Select ISO or default.xex',command=start);button.pack(side='left')
     def poll():
         nonlocal running,success
         while not messages.empty():
@@ -78,7 +75,7 @@ def main():
             if kind=='done':
                 running=False;success=True;progress.stop();window.destroy();return
             if kind=='error':
-                running=False;progress.stop();button.config(state='normal');folder_button.config(state='normal')
+                running=False;progress.stop();button.config(state='normal')
                 messagebox.showerror('Setup could not finish',text+'\n\nDetails: '+str(args.base/'setup-error.log'),parent=window)
         window.after(100,poll)
     window.protocol('WM_DELETE_WINDOW',close)