From 4eb0acf7862aebd03298737dab19297617912c45 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 00:18:07 +0100 Subject: [PATCH 1/5] Install prepared original HUD caches and report resolved asset paths --- crates/skate-game/src/scoring_hud.rs | 4 +- crates/skate-game/src/session_marker/hud.rs | 3 +- docs/hud-installation.md | 47 +++++++++++ tools/extract_session_marker.py | 31 ++++--- tools/install_prepared_hud.py | 90 +++++++++++++++++++++ tools/test_install_prepared_hud.py | 74 +++++++++++++++++ 6 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 docs/hud-installation.md create mode 100644 tools/install_prepared_hud.py create mode 100644 tools/test_install_prepared_hud.py diff --git a/crates/skate-game/src/scoring_hud.rs b/crates/skate-game/src/scoring_hud.rs index 1b80d30e..4c4c79cb 100644 --- a/crates/skate-game/src/scoring_hud.rs +++ b/crates/skate-game/src/scoring_hud.rs @@ -211,9 +211,9 @@ fn setup( }, )); commands.insert_resource(hud); - info!("Original scoring HUD loaded"); + info!("Original scoring HUD loaded from {}", root.display()); } - Err(error) => error!("Original scoring HUD could not load: {error}"), + Err(error) => error!("Original scoring HUD could not load from {}: {error}. See docs/hud-installation.md", root.display()), } } // Rasterize at output pixel resolution; retain the original 1280x720 APT diff --git a/crates/skate-game/src/session_marker/hud.rs b/crates/skate-game/src/session_marker/hud.rs index 6d1d9d74..bc37bbb7 100644 --- a/crates/skate-game/src/session_marker/hud.rs +++ b/crates/skate-game/src/session_marker/hud.rs @@ -77,7 +77,7 @@ fn load( Ok(h) => h, Err(e) => { warn!( - "Original session-marker HUD unavailable at {}: {e}", + "Original session-marker HUD unavailable at {}: {e}. See docs/hud-installation.md", folder.display() ); return; @@ -123,6 +123,7 @@ fn load( error!("Invalid marker HUD geometry"); return; } + info!("Original session-marker HUD loaded from {}", folder.display()); commands .spawn((MarkerHudRoot, Transform::default(), Visibility::Hidden)) .with_children(|parent| { diff --git a/docs/hud-installation.md b/docs/hud-installation.md new file mode 100644 index 00000000..31fe4d64 --- /dev/null +++ b/docs/hud-installation.md @@ -0,0 +1,47 @@ +# Original HUD installation + +The runtime reads both caches from the selected `--assets` directory: + +| HUD | Default manifest | Optional development override | +| --- | --- | --- | +| Trick scoring | `private/hud/runtime/trickdisplay.json` | `SKATE_SCORING_HUD_ROOT` | +| Session marker | `private/session-marker/hud.json` | `SKATE3_SESSION_MARKER_OVERLAY` | + +Merging HUD code does not install ignored, extracted game assets. Earlier feature +launchers used private caches through environment overrides; the combined build +with only `--assets` could not find either cache. Setup now prepares both with +the existing extractors. The marker extractor defaults to the vendored UI toolkit; +the external `--ui-toolkit` argument remains supported. + +For an existing installation, reuse prepared caches without reconverting maps: + +```powershell +python tools/install_prepared_hud.py --assets --scoring-cache --marker-cache +``` + +This validates both manifests and referenced texture sizes before copying any +files, checks available marker texture hashes, and verifies byte-identical copies. +It refuses differing installed files unless `--replace` is supplied. It installs +only the runtime files, preserving authored geometry, fonts, images and actions. +No copyrighted assets are committed or bundled with the source. + +If no prepared caches exist, prepare only the two HUDs from the owned game folder: + +```powershell +python tools/prepare_hud.py --game --output /private/hud --collections /private/stock/skater-collections.json +python tools/extract_session_marker.py --game --output /private/session-marker +``` + +Startup logs report the loaded paths or a load failure. Check the executable and +asset paths in the launcher as well: `PLAY.bat` delegates to `Launch.ps1`, which +selects `bin/skate3rust.exe`, not a feature build elsewhere in `bin`. + +Session-marker prompts appear while holding LB during gameplay; they are not an +always-visible panel. Scoring follows the original trickdisplay movie. Its camera +renders layer 31 offscreen before the presentation UI; the marker camera renders +layer 29 at order 1 after presentation. The existing scoring startup dependency +on `PresentationSetup` must be retained. Neither HUD uses the world mesh layers. + +The installation repair and release compilation do not establish GPU visibility +or gameplay parity. Manual checks remain: resume gameplay, hold LB for the marker +panel, and perform and land a trick for the scoring display. diff --git a/tools/extract_session_marker.py b/tools/extract_session_marker.py index 1ecbddaf..4fc77d0e 100644 --- a/tools/extract_session_marker.py +++ b/tools/extract_session_marker.py @@ -9,6 +9,7 @@ import hashlib import json from pathlib import Path import sys +import importlib # Presentation calibration against the supplied retail HUD captures. These are # host compositor adjustments, not recovered APT constants. Keep the original @@ -17,10 +18,11 @@ PANEL_OPACITY = 0.75 SHADOW_COVERAGE_GAMMA = 2.2 -def compile_hud(cache_root: Path, output: Path) -> None: - from skate3_ui_extract.scene_graph import AssetCache, SceneFlattener, transform_point - from skate3_ui_extract.bitmap_font import measure_bitmap_text - from skate3_ui_extract.timeline import playback_frame +def compile_hud(cache_root: Path, output: Path, toolkit="vendor.skate3_ui") -> None: + scene = importlib.import_module(toolkit + ".scene_graph") + AssetCache, SceneFlattener, transform_point = scene.AssetCache, scene.SceneFlattener, scene.transform_point + measure_bitmap_text = importlib.import_module(toolkit + ".bitmap_font").measure_bitmap_text + playback_frame = importlib.import_module(toolkit + ".timeline").playback_frame cache = AssetCache(cache_root) bundle = "data/fe/source/screens/hud2/hudphonelist" labels = json.loads((cache_root / "metadata/languages/english_global.json").read_text()) @@ -148,21 +150,26 @@ def compile_hud(cache_root: Path, output: Path) -> None: +def prepare(game: Path, output: Path, toolkit="vendor.skate3_ui"): + extract_project = importlib.import_module(toolkit + ".project").extract_project + cache = output / "source-cache" + extract_project(game, cache, include_dynamic=True, prefixes=( + "data/fe/source/screens/hud2/hudphonelist", "data/fe/source/controls/button_item2", + "data/fe/source/images/buttons/xbox360"), update=True) + compile_hud(cache, output, toolkit) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--game", type=Path, required=True) - parser.add_argument("--ui-toolkit", type=Path, required=True) + parser.add_argument("--ui-toolkit", type=Path, help="Optional external toolkit; defaults to the vendored extractor") parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.resolve().is_relative_to(args.game.resolve()): parser.error("Output must be outside the owned game source") - sys.path.insert(0, str(args.ui_toolkit.resolve())) - from skate3_ui_extract.project import extract_project - cache = args.output / "source-cache" - extract_project(args.game, cache, include_dynamic=True, prefixes=( - "data/fe/source/screens/hud2/hudphonelist", "data/fe/source/controls/button_item2", - "data/fe/source/images/buttons/xbox360"), update=True) - compile_hud(cache, args.output) + if args.ui_toolkit: + sys.path.insert(0, str(args.ui_toolkit.resolve())) + prepare(args.game, args.output, "skate3_ui_extract" if args.ui_toolkit else "vendor.skate3_ui") print(f"Private original session-marker HUD: {args.output / 'hud.json'}") diff --git a/tools/install_prepared_hud.py b/tools/install_prepared_hud.py new file mode 100644 index 00000000..bba8ba50 --- /dev/null +++ b/tools/install_prepared_hud.py @@ -0,0 +1,90 @@ +"""Install already extracted original HUD caches into an existing asset root. + +Copies only runtime manifests and referenced RGBA files, byte for byte. No game, +disc extraction, map conversion or renderer is started. Both caches are checked +before writing; existing differing files require an explicit --replace. +""" +from pathlib import Path +import argparse +import hashlib +import json +import shutil + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def contained(root, relative): + path = (root / relative).resolve() + if not path.is_relative_to(root.resolve()) or path == root.resolve(): + raise ValueError(f"HUD path escapes cache: {relative}") + return path + + +def runtime_files(root, marker=False): + manifest = "hud.json" if marker else "runtime/trickdisplay.json" + data = json.loads((root / manifest).read_text(encoding="utf-8")) + if data.get("version") != 1: + raise ValueError(f"Unsupported HUD version: {root}") + textures = [] + if marker: + if data.get("canvas") != [1280, 720] or not data.get("meshes"): + raise ValueError("Invalid session-marker HUD") + textures = [(t["file"], t["width"], t["height"], t.get("output_sha256")) + for t in data["textures"]] + else: + if data.get("format") != "skate3-scoring-hud" or data.get("unresolved_fonts"): + raise ValueError("Invalid scoring HUD or unresolved original fonts") + for shape in data["shapes"].values(): + for primitive in shape: + t = primitive["texture"] + textures.append((t["rgba"], t["width"], t["height"], None)) + for font in data["fonts"].values(): + size = font["definition"]["textures"][0] + textures.append((font["texture"], size["width"], size["height"], None)) + files = {manifest: digest(root / manifest)} + for relative, width, height, expected in textures: + path = contained(root, relative) + if width <= 0 or height <= 0 or path.stat().st_size != width * height * 4: + raise ValueError(f"Invalid RGBA size: {path}") + actual = digest(path) + if expected and expected != actual: + raise ValueError(f"Original HUD texture hash mismatch: {path}") + files[relative] = actual + return files + + +def install(assets, scoring_cache, marker_cache, replace=False): + plan = [] + for source, name, marker in [(scoring_cache, "hud", False), + (marker_cache, "session-marker", True)]: + destination = assets / "private" / name + for relative, sha256 in runtime_files(source, marker).items(): + target = contained(destination, relative) + if target.exists() and digest(target) != sha256 and not replace: + raise ValueError(f"Different installed HUD file (use --replace): {target}") + plan.append((contained(source, relative), target, sha256)) + for source, target, sha256 in plan: + if target.exists() and digest(target) == sha256: + continue + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + if digest(target) != sha256: + raise ValueError(f"Installed HUD hash mismatch: {target}") + return {str(target.relative_to(assets.resolve())): sha for _, target, sha in plan} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assets", type=Path, required=True) + parser.add_argument("--scoring-cache", type=Path, required=True) + parser.add_argument("--marker-cache", type=Path, required=True) + parser.add_argument("--replace", action="store_true") + args = parser.parse_args() + files = install(args.assets, args.scoring_cache, args.marker_cache, args.replace) + print(json.dumps({"assets": str(args.assets.resolve()), "files": files}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/test_install_prepared_hud.py b/tools/test_install_prepared_hud.py new file mode 100644 index 00000000..37ad47b5 --- /dev/null +++ b/tools/test_install_prepared_hud.py @@ -0,0 +1,74 @@ +import hashlib +import json +from pathlib import Path +import tempfile +import unittest + +from install_prepared_hud import install + + +class HudInstallationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + self.assets, self.score, self.marker = [root / p for p in ("assets", "score", "marker")] + (self.score / "runtime").mkdir(parents=True) + self.marker.mkdir() + self.pixels = bytes([12, 34, 56, 78]) + for cache in (self.score, self.marker): + (cache / "atlas.rgba").write_bytes(self.pixels) + self.score_json = {"format": "skate3-scoring-hud", "version": 1, + "shapes": {"1": [{"texture": {"rgba": "atlas.rgba", "width": 1, "height": 1}}]}, + "fonts": {"native": {"texture": "atlas.rgba", "definition": { + "textures": [{"width": 1, "height": 1}]}}}} + (self.score / "runtime/trickdisplay.json").write_text(json.dumps(self.score_json)) + self.marker_json = {"version": 1, "canvas": [1280, 720], "meshes": [{}], + "textures": [{"file": "atlas.rgba", "width": 1, "height": 1, + "output_sha256": hashlib.sha256(self.pixels).hexdigest()}]} + self.write_marker() + + def write_marker(self): + (self.marker / "hud.json").write_text(json.dumps(self.marker_json)) + + def run_install(self, **kwargs): + return install(self.assets, self.score, self.marker, **kwargs) + + def test_preserves_bytes_and_repeat_install_is_a_noop(self): + files = self.run_install() + before = {p: (self.assets / p).stat().st_mtime_ns for p in files} + self.assertEqual(files, self.run_install()) + self.assertEqual(before, {p: (self.assets / p).stat().st_mtime_ns for p in files}) + self.assertEqual((self.assets / "private/hud/atlas.rgba").read_bytes(), self.pixels) + self.assertEqual((self.assets / "private/session-marker/hud.json").read_bytes(), + (self.marker / "hud.json").read_bytes()) + + def test_invalid_second_cache_does_not_partially_install_first(self): + (self.marker / "atlas.rgba").write_bytes(b"bad") + with self.assertRaisesRegex(ValueError, "RGBA size"): + self.run_install() + self.assertFalse(self.assets.exists()) + + def test_hash_and_path_failures(self): + (self.marker / "atlas.rgba").write_bytes(b"fake") + with self.assertRaisesRegex(ValueError, "hash mismatch"): + self.run_install() + self.marker_json["textures"][0]["file"] = "../outside.rgba" + self.write_marker() + with self.assertRaisesRegex(ValueError, "escapes cache"): + self.run_install() + self.assertFalse(self.assets.exists()) + + def test_differing_existing_file_requires_replace(self): + self.run_install() + target = self.assets / "private/hud/atlas.rgba" + target.write_bytes(b"keep") + with self.assertRaisesRegex(ValueError, "Different installed"): + self.run_install() + self.assertEqual(target.read_bytes(), b"keep") + self.run_install(replace=True) + self.assertEqual(target.read_bytes(), self.pixels) + + +if __name__ == "__main__": + unittest.main() From a05c8c06b382065f516b2cee465df868a752a855 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 00:18:07 +0100 Subject: [PATCH 2/5] Extract complete owned game content and prepare HUDs and customisation during setup --- README.md | 7 +- docs/setup-content.md | 63 ++++++++++++ tools/asset_pipeline/install.py | 17 +++- tools/asset_pipeline/original_content.py | 97 +++++++++++++++++++ tools/asset_pipeline/runtime_content.py | 87 +++++++++++++++++ tools/asset_pipeline/test_original_content.py | 75 ++++++++++++++ tools/asset_pipeline/test_runtime_content.py | 32 ++++++ tools/owned_game/big.py | 41 +++++++- tools/prepare_customisation.py | 28 ++++++ tools/setup.py | 2 +- 10 files changed, 444 insertions(+), 5 deletions(-) create mode 100644 docs/setup-content.md create mode 100644 tools/asset_pipeline/original_content.py create mode 100644 tools/asset_pipeline/runtime_content.py create mode 100644 tools/asset_pipeline/test_original_content.py create mode 100644 tools/asset_pipeline/test_runtime_content.py create mode 100644 tools/prepare_customisation.py diff --git a/README.md b/README.md index aaa67fa2..45b8e352 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,12 @@ Editable examples live in `sdk/examples/`; `tools/package_mod.py` validates and Extract the Windows release ZIP and run `skate3rust.exe`. Select your Skate 3 Xbox 360 ISO, or select `default.xex` in an extracted game folder. Keep its -`data` folder alongside it. Setup prepares the skater, animations and all disc maps, then -starts University. No Blender, Python or Rust installation is needed. +`data` folder alongside it. Setup extracts all original content, then prepares the +skater, clothing, animations, original HUDs, environment assets and all disc maps +before starting University. No Blender, Python or Rust installation is needed. ISO extraction needs internet access. The first conversion can take a while. +Allow space for both the original source cache and converted runtime assets. +See [setup content coverage](docs/setup-content.md) and [HUD cache repair](docs/hud-installation.md). Use an XInput controller to play. Escape opens graphics, difficulty and map settings. Maps can be switched without restarting the game. diff --git a/docs/setup-content.md b/docs/setup-content.md new file mode 100644 index 00000000..a12c80dd --- /dev/null +++ b/docs/setup-content.md @@ -0,0 +1,63 @@ +# Setup content ownership + +Setup owns extraction and preparation. Features must read installed asset paths; +private development launchers and environment overrides are not installation steps. + +The selected owned game tree is extracted into `assets/private/original` before +runtime conversion. All EB BIG v3 and BIG4/BIGF banks are decoded, including nested +audio banks, and loose files are copied. Each bank has its own namespace so two +archives with the same member name cannot overwrite each other. The private +`original/manifest.json` records source archive hashes, member names, output sizes +and decoded hashes. Repeating this extraction verifies and reuses matching files. +Unsafe paths, ambiguous names and unsupported archive formats fail explicitly. +The installation retains this source cache; temporary map conversion files are +still removed after successful preparation. + +| Runtime content | Setup owner | +| --- | --- | +| Stock animation banks, action/motion/camera graphs, gestures, input and recipes | Stock BIG and loose animation extraction | +| Physics, difficulty, scoring and other VLT tuning; skeleton metadata | VLT and physics-skeleton converters | +| Skater mesh, textures and authored character lighting | Character and lighting converters | +| Resident character choices, original textures, tattoos and native presets | `prepare_customisation.py` using existing CAC converters | +| Original trickdisplay actions, shapes and bitmap fonts | `prepare_hud.py` | +| Original session-marker geometry, labels and Xbox button images | `extract_session_marker.py` using the vendored toolkit | +| Retail skies, render/exposure settings, travel destinations and foliage | Existing environment converters | +| Authored movable-object models and placement | DMO catalog and per-map conversion | +| Every disc district map, textures, collision and retained extensions | Existing map jobs | + +The final `runtime_content` check verifies manifests and referenced files, +including HUD RGBA lengths, character scene/texture dependencies, lighting, +per-map skies and movable objects. It writes `private/content-manifest.json`. +Only after this check and the existing runtime asset checks pass does setup +publish `installation.json` and report success. CAC conversion errors now stop +setup rather than publishing an incomplete library. + +Extraction is not a claim of full engine support for every original format. +Unported audio/frontend data remains in the indexed source cache. The ocean PCA +table is executable-resident: its existing converter accepts a hash-verified +mapped TU3 image, which ordinary disc setup does not produce. The owned XEX is +preserved, but that optional table still needs the mapped-image converter; the +renderer retains its documented fallback when absent. User-authored animations +and third-party mods are not disc assets and are not generated by extraction. + +Existing installations can add the original HUDs using the targeted +[cache repair](hud-installation.md), or prepare only missing feature caches with +the respective tools. Updating source code does not automatically rerun setup +or replace an already installed package. + +## Offline validation + +The 2026-09-09 setup integration was checked without running the game, recomp, +Steam or gameplay tests. Archive tests cover BIG4 RefPack payloads, nested banks, +duplicate names, unsafe paths, reuse and repair. The owned source inventory reads +51 top-level banks and identifies 28 nested EB banks. HUD installation tests cover +missing/corrupt files and byte preservation; the vendored marker compiler produces +the same 15 meshes and seven texture payloads as the prepared native overlay. +The data-only original movie audit passes 1,800 frames. The existing owned CAC +audit validates 480 models, 2,427 materials and 170 tattoos. +The dependency validator also passes against 3,435 prepared runtime files assembled +from the existing caches with a University map. The packaged setup worker passes +data-only archive extraction and HUD installation checks; no setup GUI is launched. + +Full installation and GPU/gameplay behavior still require manual validation; +prepared maps and dependencies were reused rather than rebuilding the installation. diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index 62c55ce1..569a815a 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -163,8 +163,12 @@ def install(iso,base,game_exe,report,game_root=None): run([extractor,'-x',iso,'-d',game_root],log,report) else:game_root=game_root.resolve() for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big', + 'data/big/fedata.big','data/big/fetexture.big','data/big/fedynamic.big', 'data/content/createacharacter.big','data/content/worldDIST_University.big']: if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required) + report('Extracting and indexing all original game content') + from .original_content import extract as extract_original + extract_original(game_root, private/'original', report) report('Extracting animation banks, graphs and gameplay inputs') stock=private/'stock' extract(game_root/'data/big/miscload.big',stock) @@ -181,6 +185,11 @@ def install(iso,base,game_exe,report,game_root=None): (stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8') skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin') (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8') + report('Preparing original scoring and session-marker HUD assets') + run(task(TOOLS/'prepare_hud.py', '--game', game_root, '--output', private/'hud', + '--collections', stock/'skater-collections.json'), log, report) + run(task(TOOLS/'extract_session_marker.py', '--game', game_root, + '--output', private/'session-marker'), log, report) report('Preparing the skater model and textures') manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text()) needed=set() @@ -196,6 +205,9 @@ def install(iso,base,game_exe,report,game_root=None): write_character(character/'selected/models',private,manifest) from .character_lighting import convert as write_character_lighting write_character_lighting(character/'selected/models',private,converted) + report('Preparing original character customisation models, textures and menus') + run(task(TOOLS/'prepare_customisation.py', '--game', game_root, + '--assets', stage/'assets'), log, report) game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000', 'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph', 'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'} @@ -210,7 +222,7 @@ def install(iso,base,game_exe,report,game_root=None): write_teleports(game_root,stage/'assets',converted) report('Preparing global foliage backdrops') from .backdrop import convert as write_backdrops - write_backdrops(game_root,stage/'assets',converted) + backdrop_count = write_backdrops(game_root,stage/'assets',converted) report('Preparing authored movable-object models') from .dynamic_props import prepare_catalog prepare_catalog(game_root,work/'dmo') @@ -241,6 +253,9 @@ def install(iso,base,game_exe,report,game_root=None): settings=stage/'settings';settings.mkdir() (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8') (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8') + report('Checking original content and every prepared runtime asset group') + from .runtime_content import validate as validate_content + validate_content(stage, catalog, expected_backdrops=backdrop_count) remove_intermediate(work,stage) # Publish only after every conversion and the runtime's own load succeeds. marker=base/'installation.json.new' diff --git a/tools/asset_pipeline/original_content.py b/tools/asset_pipeline/original_content.py new file mode 100644 index 00000000..5bcaa12b --- /dev/null +++ b/tools/asset_pipeline/original_content.py @@ -0,0 +1,97 @@ +"""Lossless, indexed extraction of the selected owned game tree. + +Keep each bank in its own namespace: duplicated names in different banks must +not overwrite one another. Container decoding is separate from runtime format +conversion; unported audio/frontend/etc. payloads remain original private data. +""" +from pathlib import Path +import hashlib +import json +import shutil + +from tools.owned_game.big import BigArchive + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def destination(root, relative): + target = (root / BigArchive.safe_relative(relative)).resolve() + if not target.is_relative_to(root.resolve()): + raise ValueError(f"Original content path escaped output: {relative}") + return target + + +def extract(game_root, output, report=print): + game_root, output = game_root.resolve(strict=True), output.resolve() + if not game_root.is_dir(): + raise ValueError("Owned game source must be a directory") + if output.is_relative_to(game_root) or game_root.is_relative_to(output): + raise ValueError("Original content output must be separate from the owned source") + output.mkdir(parents=True, exist_ok=True) + index = output / "manifest.json" + prior = json.loads(index.read_text(encoding="utf-8")) if index.is_file() else {} + records = [] + old = {r["output"]: r for r in prior.get("files", [])} + claimed = set() + + def claim(relative): + key = relative.casefold() + if key in claimed: + raise ValueError(f"Ambiguous original content path: {relative}") + claimed.add(key) + return destination(output, relative) + + def bank(path, relative, lineage, depth=0): + if depth > 16: + raise ValueError("Excessive nested original archives") + archive = BigArchive(path) + identity = digest(path) + report(f"Extracting original bank: {lineage} ({len(archive.entries)} entries)") + for entry in archive.entries: + name = BigArchive.safe_relative(entry.path).as_posix() + member = relative + "/" + name + target = claim(member) + previous = old.get(member, {}) + if not (previous.get("archive_sha256") == identity and target.is_file() + and target.stat().st_size == entry.unpacked_size + and digest(target) == previous.get("sha256")): + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(archive.read(entry)) + records.append(dict(output=member, archive=lineage, entry=entry.path, + archive_sha256=identity, size=entry.unpacked_size, sha256=digest(target))) + if name.lower().endswith(".big"): + bank(target, member + ".contents", lineage + "!/" + name, depth + 1) + + for source in sorted(game_root.rglob("*")): + if not source.is_file(): + continue + relative = source.relative_to(game_root).as_posix() + if source.suffix.lower() == ".big": + bank(source, "banks/" + relative, relative) + else: + name = "loose/" + relative + target = claim(name) + sha256 = digest(source) + if not target.is_file() or digest(target) != sha256: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + if digest(target) != sha256: + raise ValueError(f"Original file copy failed: {relative}") + records.append(dict(output=name, source=relative, size=source.stat().st_size, sha256=sha256)) + manifest = dict(version=1, files=records) + temporary = index.with_suffix(".json.new") + temporary.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + temporary.replace(index) + return manifest + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--game", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + extract(args.game, args.output) diff --git a/tools/asset_pipeline/runtime_content.py b/tools/asset_pipeline/runtime_content.py new file mode 100644 index 00000000..aa271f8d --- /dev/null +++ b/tools/asset_pipeline/runtime_content.py @@ -0,0 +1,87 @@ +"""Verify installed runtime dependencies before publishing setup completion.""" +import hashlib +import json +from pathlib import Path + +from tools.install_prepared_hud import runtime_files +from .original_content import destination + + +def validate(stage, maps, expected_backdrops=None): + assets = stage / "assets" + files = {} + + def require(relative): + path = destination(assets, relative) + if relative in files: + return path + if not path.is_file() or not path.stat().st_size: + raise ValueError(f"Setup is missing runtime content: {relative}") + with path.open("rb") as stream: + sha = hashlib.file_digest(stream, "sha256").hexdigest() + files[relative] = sha + return path + + def read(relative): + return json.loads(require(relative).read_text(encoding="utf-8")) + + manifest = read("private/game.json") + for key in ("character_scene", "action_graph", "motion_graph"): + require(manifest[key]) + for name in ("OnBoard", "OffBoard"): + require(f"private/stock/data/anim/{name}.abin") + for name in ("skater-collections.json", "physics-skeletons.json", "data/config/input.cfg", + "data/script/camera/Default_cameragraph.stategraph", "data/camera/1.shk", "data/camera/2.shk"): + require("private/stock/" + name) + for folder, marker in (("hud", False), ("session-marker", True)): + for relative in runtime_files(assets / "private" / folder, marker): + require("private/" + folder + "/" + relative) + library = read("private/customisation/library-v3.json") + if library.get("version") != 3 or library.get("errors") or not library.get("models"): + raise ValueError("Setup character library is incomplete") + read("private/customisation/extra-menu.json") + for model in library["models"].values(): + require(model["scene"]) + for material in library["materials"].values(): + for key in ("diffuse", "normal", "rough", "opacity"): + if material.get(key): + require(material[key]) + for tattoo in library["tattoos"].values(): + require(tattoo["texture"]) + lighting = read("private/character-lighting.json") + for material in lighting["materials"].values(): + if material.get("specular"): + require(material["specular"]) + for name in ("render-parameters", "exposure", "exposure-profiles", "teleports"): + read(f"private/{name}.json") + backdrops = list((assets / "private/native-backdrops").glob("*.skate")) + if expected_backdrops is not None and len(backdrops) != expected_backdrops: + raise ValueError("Setup foliage backdrop count does not match conversion") + for backdrop in backdrops: + require(backdrop.relative_to(assets).as_posix()) + if not maps: + raise ValueError("Setup has no converted maps") + for row in maps: + path = destination(stage, row["path"]) + if not path.is_file() or not path.stat().st_size: + raise ValueError(f"Setup map is missing: {row['path']}") + name = row["name"] + sky = read(f"private/native-skies/{name}.json") + for suffix, width, height in (("rgba", sky["width"], sky["height"]), + ("sun.rgba", sky["sun_width"], sky["sun_height"])): + if require(f"private/native-skies/{name}.{suffix}").stat().st_size != width * height * 4: + raise ValueError(f"Invalid sky texture: {name}.{suffix}") + props = read(f"private/native-props/{name}.json") + if props.get("instances"): + require(f"private/native-props/{name}.skate") + original = read("private/original/manifest.json") + if original.get("version") != 1 or not original.get("files"): + raise ValueError("Setup has no indexed original content") + result = dict(version=1, runtime_files=files, maps=maps, + original_files=len(original["files"]), + preserved_source_only=["Unported audio and frontend payloads", + "Executable-resident tables without a disc decoder (ocean PCA)"]) + temporary = assets / "private/content-manifest.json.new" + temporary.write_text(json.dumps(result, indent=2), encoding="utf-8") + temporary.replace(assets / "private/content-manifest.json") + return result diff --git a/tools/asset_pipeline/test_original_content.py b/tools/asset_pipeline/test_original_content.py new file mode 100644 index 00000000..6faeb33c --- /dev/null +++ b/tools/asset_pipeline/test_original_content.py @@ -0,0 +1,75 @@ +from pathlib import Path +import json +import struct +import tempfile +import unittest + +from tools.asset_pipeline.original_content import extract +from tools.owned_game.big import BigArchive + + +def big4(entries): + end = 16 + sum(9 + len(name.encode()) for name, _ in entries) + size = end + sum(len(data) for _, data in entries) + directory, payload, offset = bytearray(), bytearray(), end + for name, data in entries: + directory += struct.pack('>II', offset, len(data)) + name.encode() + b'\0' + payload += data + offset += len(data) + return b'BIG4' + struct.pack('II', len(entries), end) + directory + payload + + +class OriginalContentTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.game, self.out = self.root/'game', self.root/'output' + self.game.mkdir() + + def test_nested_banks_duplicate_names_loose_files_and_repair(self): + (self.game/'a.big').write_bytes(big4([('same', b'a'), ('nested.big', big4([('child', b'c')]))])) + (self.game/'b.big').write_bytes(big4([('same', b'b')])) + (self.game/'loose.bin').write_bytes(b'original') + first = extract(self.game, self.out, lambda _: None) + self.assertEqual(len(first['files']), 5) + self.assertEqual((self.out/'banks/a.big/same').read_bytes(), b'a') + self.assertEqual((self.out/'banks/b.big/same').read_bytes(), b'b') + self.assertEqual((self.out/'banks/a.big/nested.big.contents/child').read_bytes(), b'c') + target = self.out/'banks/a.big/same' + stamp = target.stat().st_mtime_ns + self.assertEqual(extract(self.game, self.out, lambda _: None), first) + self.assertEqual(stamp, target.stat().st_mtime_ns) + target.write_bytes(b'x') + extract(self.game, self.out, lambda _: None) + self.assertEqual(target.read_bytes(), b'a') + + def test_big4_refpack_and_invalid_offsets(self): + # Header, literal of three bytes, end marker. + path = self.game/'compressed.big' + path.write_bytes(big4([('data', b'\x10\xfb\x00\x00\x03\xffabc')])) + archive = BigArchive(path) + self.assertEqual(archive.read(archive.entries[0]), b'abc') + raw = bytearray(path.read_bytes()) + raw[16:20] = struct.pack('>I', len(raw) + 1) + path.write_bytes(raw) + with self.assertRaisesRegex(ValueError, 'bounds'): + BigArchive(path) + + def test_unsafe_member_and_source_overlap_fail(self): + (self.game/'bad.big').write_bytes(big4([('../escape', b'bad')])) + with self.assertRaisesRegex(ValueError, 'unsafe'): + extract(self.game, self.out, lambda _: None) + self.assertFalse((self.out/'manifest.json').exists()) + with self.assertRaisesRegex(ValueError, 'separate'): + extract(self.game, self.game/'output', lambda _: None) + + def test_case_collisions_do_not_publish_completed_index(self): + (self.game/'bad.big').write_bytes(big4([('Name', b'a'), ('name', b'b')])) + with self.assertRaisesRegex(ValueError, 'Ambiguous'): + extract(self.game, self.out, lambda _: None) + self.assertFalse((self.out/'manifest.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/asset_pipeline/test_runtime_content.py b/tools/asset_pipeline/test_runtime_content.py new file mode 100644 index 00000000..419274ed --- /dev/null +++ b/tools/asset_pipeline/test_runtime_content.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from tools.asset_pipeline.runtime_content import validate + + +class RuntimeContentTests(unittest.TestCase): + def test_missing_hud_cannot_publish_complete_content(self): + with tempfile.TemporaryDirectory() as directory: + stage = Path(directory) + assets = stage/'assets' + def write(name, value=b'fixture'): + path = assets/name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + write('private/game.json', json.dumps(dict(character_scene='private/skater.glb', + action_graph='private/action', motion_graph='private/motion')).encode()) + for name in ('private/skater.glb', 'private/action', 'private/motion', + 'private/stock/data/anim/OnBoard.abin', 'private/stock/data/anim/OffBoard.abin'): + write(name) + for name in ('skater-collections.json', 'physics-skeletons.json', 'data/config/input.cfg', + 'data/script/camera/Default_cameragraph.stategraph', 'data/camera/1.shk', 'data/camera/2.shk'): + write('private/stock/'+name) + with self.assertRaises(FileNotFoundError): + validate(stage, []) + self.assertFalse((assets/'private/content-manifest.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/owned_game/big.py b/tools/owned_game/big.py index 9af625fa..70a733f1 100644 --- a/tools/owned_game/big.py +++ b/tools/owned_game/big.py @@ -74,13 +74,17 @@ def _decompress_chunkref(data: bytes, expected_size: int) -> bytes: class BigArchive: - """Streaming reader for the EB BIG v3 archives used by Skate 3.""" + """Streaming reader for Skate 3's EB BIG v3 and legacy BIG4/BIGF banks.""" def __init__(self, path: Path): self.path = Path(path).resolve() self.file_size = self.path.stat().st_size with self.path.open("rb") as stream: header = stream.read(48) + if header[:4] in (b"BIG4", b"BIGF"): + self.metadata = b"" + self.entries = self._parse_big4(stream, header) + return if len(header) != 48: raise FormatError(f"{self.path}: truncated BIG header") header_reader = Reader(header, str(self.path)) @@ -93,6 +97,41 @@ class BigArchive: self.metadata = stream.read(metadata_size) self.entries = self._parse() + def _parse_big4(self, stream, header): + """Older EA containers, including the disc's music overlays bank.""" + if len(header) < 16: + raise FormatError("truncated BIG4 header") + count = int.from_bytes(header[8:12], "big") + end = int.from_bytes(header[12:16], "big") + if count > 1_000_000 or end < 16 or end > self.file_size: + raise FormatError("invalid BIG4 directory bounds") + stream.seek(16) + directory = stream.read(end - 16) + cursor = 0 + entries = [] + for index in range(count): + if cursor + 8 > len(directory): + raise FormatError("truncated BIG4 entry") + offset = int.from_bytes(directory[cursor:cursor+4], "big") + size = int.from_bytes(directory[cursor+4:cursor+8], "big") + name_end = directory.find(b"\0", cursor + 8) + if name_end < 0 or offset < end or offset + size > self.file_size: + raise FormatError("invalid BIG4 entry bounds") + name = directory[cursor+8:name_end].decode("utf-8") + self.safe_relative(name) + stream.seek(offset) + prefix = stream.read(min(size, 6)) + compressed = len(prefix) >= 2 and prefix[0] in (0x10, 0x90) and prefix[1] == 0xfb + unpacked = size + if compressed: + length = 6 if prefix[0] == 0x90 else 5 + if len(prefix) < length: + raise FormatError("truncated BIG4 RefPack header") + unpacked = int.from_bytes(prefix[2:length], "big") + entries.append(BigEntry(index, name, offset, size, unpacked, int(compressed))) + cursor = name_end + 1 + return entries + def _parse(self) -> list[BigEntry]: reader = Reader(self.metadata, str(self.path)) if reader.u16be(0) != 0x4542: diff --git a/tools/prepare_customisation.py b/tools/prepare_customisation.py new file mode 100644 index 00000000..fb844d27 --- /dev/null +++ b/tools/prepare_customisation.py @@ -0,0 +1,28 @@ +"""Prepare original resident character choices during setup, without a game.""" +import argparse +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tools.asset_pipeline.customisation_catalog import prepare as catalog +from tools.asset_pipeline.customisation_library import prepare as library +from tools.asset_pipeline.customisation_profiles import generate + + +def prepare(game, assets): + cache = assets / "private/customisation" + catalog(game, cache) + result = library({"game_root": str(game), "assets": str(assets), "library_index": "library-v3.json"}) + if result["errors"]: + raise ValueError(f"Original character library is incomplete: {result['errors']}") + native = json.loads((cache / "native.json").read_text(encoding="utf-8")) + (cache / "extra-menu.json").write_text(json.dumps(generate(native), indent=2), encoding="utf-8") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--game", type=Path, required=True) + parser.add_argument("--assets", type=Path, required=True) + args = parser.parse_args() + prepare(args.game, args.assets) diff --git a/tools/setup.py b/tools/setup.py index 913a200e..feca9009 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -29,7 +29,7 @@ def main(): if icon.is_file():window.iconbitmap(str(icon)) frame=ttk.Frame(window,padding=24);frame.pack(fill='both',expand=True) ttk.Label(frame,text='Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16)) - ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.', + ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup extracts all original content and prepares the maps,\nskater, clothing, animations, HUDs and environment assets.\nNo other apps need installing.\n\nAllow free disk space for original and converted content.\nISO extraction needs internet access.', font=('Segoe UI',11),justify='left').pack(anchor='w') status=tk.StringVar(value='Choose your game to begin.') ttk.Label(frame,textvariable=status,wraplength=600).pack(anchor='w',pady=(18,8)) From cfafc213c05fef3f13836c7627411b2f077dec51 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 01:06:22 +0100 Subject: [PATCH 3/5] Resolve scoring and HUD font records by native VLT identity --- crates/skate-data/src/scoring.rs | 40 ++++++++++++++++++++++++++++---- tools/prepare_hud.py | 5 ++-- tools/test_hud_font_identity.py | 37 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 tools/test_hud_font_identity.py diff --git a/crates/skate-data/src/scoring.rs b/crates/skate-data/src/scoring.rs index e4520b96..5a3b7795 100644 --- a/crates/skate-data/src/scoring.rs +++ b/crates/skate-data/src/scoring.rs @@ -10,6 +10,16 @@ use skate_core::{ const SCORABLE: &str = "Hash_6918469984A8C596"; const TUNING: &str = "Hash_349215E2E817703C"; +fn scorable_keys(data: &Collections) -> std::collections::BTreeSet { + // Setup's name dictionary may resolve the class as `scoring_trick`. + // Presence checks must use the same native identities as Collections::field. + data.entries() + .iter() + .filter(|row| crate::attrib_hash::numeric_name(&row.class_name) == SCORABLE) + .map(|row| crate::attrib_hash::numeric_name(&row.key)) + .collect() +} + #[derive(Clone, Debug)] pub struct Definition { pub metadata: Scorable, @@ -53,14 +63,11 @@ impl ScoringData { } pub fn load(data: &Collections) -> Result { let mut definitions = Vec::new(); + let available = scorable_keys(data); for (id, &(identifier, class, score_type)) in IDENTIFIERS.iter().enumerate() { // The executable enum includes unused entries without VLT records. // Only actual records are eligible for recognition. - if !data.entries().iter().any(|r| { - r.class_name == SCORABLE - && (r.key == identifier - || r.key == crate::attrib_hash::numeric_name(identifier)) - }) { + if !available.contains(&crate::attrib_hash::numeric_name(identifier)) { continue; } let words = |name| data.words::<1>(SCORABLE, identifier, name).map(|v| v[0]); @@ -132,6 +139,29 @@ impl ScoringData { } } +#[cfg(test)] +mod identity_tests { + use super::*; + + #[test] + fn scoring_presence_accepts_readable_and_numeric_vault_names() { + let key = IDENTIFIERS[0].0; + let numeric_key = crate::attrib_hash::numeric_name(key); + for class in ["scoring_trick", SCORABLE, "0x6918469984a8c596"] { + for name in [key, numeric_key.as_str()] { + let data: Collections = serde_json::from_value(serde_json::json!({ + "version": 1, + "collections": [{"class": class, "key": name, "parent": "", + "source": "fixture", "sha256": "", "fields": {}}, + {"class": "unrelated", "key": "other", "parent": "", + "source": "fixture", "sha256": "", "fields": {}}] + })).unwrap(); + assert_eq!(scorable_keys(&data), [numeric_key.clone()].into()); + } + } + } +} + #[derive(Clone, Debug)] pub struct CollectorTuning { scalars: std::collections::BTreeMap, diff --git a/tools/prepare_hud.py b/tools/prepare_hud.py index f81b6a08..a1bbccee 100644 --- a/tools/prepare_hud.py +++ b/tools/prepare_hud.py @@ -24,11 +24,12 @@ def font_mapping(collections: Path, cache: AssetCache) -> dict: data = json.loads(collections.read_text(encoding='utf-8')) result = {} for row in data['collections']: - if row['class'] != 'Hash_FECFBCAF356518C4': + class_id = int(row['class'][5:], 16) if row['class'].startswith('Hash_') else hash64(row['class']) + if class_id != 0xFECFBCAF356518C4: continue def field(name): return row['fields'].get(name, row['fields'].get(f'Hash_{hash64(name):016X}')) - apt_name = row['fields']['Hash_340832CCFD9FDEB4']['data'] + apt_name = field('AptName')['data'] filename = field('FileName')['data'] matches = [r for r in cache.manifest['bundles'] if 'font' in r and Path(r['name']).name == filename] diff --git a/tools/test_hud_font_identity.py b/tools/test_hud_font_identity.py new file mode 100644 index 00000000..5d98dd84 --- /dev/null +++ b/tools/test_hud_font_identity.py @@ -0,0 +1,37 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from prepare_hud import font_mapping +from asset_pipeline.vlt import hash64 + + +class Cache: + manifest = {'bundles': [{'name': 'fonts/original', 'font': {'family': 'Original family'}}]} + + def add_font_alias(self, name, target, provenance): + self.alias = (name, target) + + +class FontIdentityTests(unittest.TestCase): + def test_readable_and_numeric_exports_resolve_the_same_original_font(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'collections.json' + for named_class in (True, False): + for named_fields in (True, False): + fields = {'AptName': {'data': 'Original APT name'}, 'FileName': {'data': 'original'}} + fields.update({key: {'data': '3F800000'} for key in ('ScaleX', 'ScaleY', 'OffsetX', 'OffsetY')}) + if not named_fields: + fields = {f'Hash_{hash64(key):016X}': value for key, value in fields.items()} + path.write_text(json.dumps({'collections': [{ + 'class': 'fe_fonts' if named_class else 'Hash_FECFBCAF356518C4', + 'key': 'original', 'sha256': '', 'fields': fields}]})) + cache = Cache() + result = font_mapping(path, cache) + self.assertEqual(cache.alias, ('Original APT name', 'Original family')) + self.assertEqual(result['Original APT name']['native_layout']['ScaleX'], 1.0) + + +if __name__ == '__main__': + unittest.main() From 3427261467f33d667845ed5e8d9e941b524e6f36 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 01:23:31 +0100 Subject: [PATCH 4/5] Keep original installer and add only automatic native HUD export --- README.md | 8 +- docs/hud-installation.md | 7 ++ docs/setup-content.md | 63 ------------ tools/asset_pipeline/install.py | 18 +--- tools/asset_pipeline/original_content.py | 97 ------------------- tools/asset_pipeline/runtime_content.py | 87 ----------------- tools/asset_pipeline/test_original_content.py | 75 -------------- tools/asset_pipeline/test_runtime_content.py | 32 ------ tools/owned_game/big.py | 41 +------- tools/prepare_customisation.py | 28 ------ tools/prepare_hud.py | 3 +- tools/prepare_runtime_huds.py | 31 ++++++ tools/setup.py | 2 +- 13 files changed, 47 insertions(+), 445 deletions(-) delete mode 100644 docs/setup-content.md delete mode 100644 tools/asset_pipeline/original_content.py delete mode 100644 tools/asset_pipeline/runtime_content.py delete mode 100644 tools/asset_pipeline/test_original_content.py delete mode 100644 tools/asset_pipeline/test_runtime_content.py delete mode 100644 tools/prepare_customisation.py create mode 100644 tools/prepare_runtime_huds.py diff --git a/README.md b/README.md index 45b8e352..6f675c89 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,10 @@ Editable examples live in `sdk/examples/`; `tools/package_mod.py` validates and Extract the Windows release ZIP and run `skate3rust.exe`. Select your Skate 3 Xbox 360 ISO, or select `default.xex` in an extracted game folder. Keep its -`data` folder alongside it. Setup extracts all original content, then prepares the -skater, clothing, animations, original HUDs, environment assets and all disc maps -before starting University. No Blender, Python or Rust installation is needed. +`data` folder alongside it. Setup prepares the skater, animations and all disc maps, then +starts University. The original scoring and session-marker HUD assets are also +exported automatically during setup. No Blender, Python or Rust installation is needed. ISO extraction needs internet access. The first conversion can take a while. -Allow space for both the original source cache and converted runtime assets. -See [setup content coverage](docs/setup-content.md) and [HUD cache repair](docs/hud-installation.md). Use an XInput controller to play. Escape opens graphics, difficulty and map settings. Maps can be switched without restarting the game. diff --git a/docs/hud-installation.md b/docs/hud-installation.md index 31fe4d64..520522a0 100644 --- a/docs/hud-installation.md +++ b/docs/hud-installation.md @@ -13,6 +13,13 @@ with only `--assets` could not find either cache. Setup now prepares both with the existing extractors. The marker extractor defaults to the vendored UI toolkit; the external `--ui-toolkit` argument remains supported. +The original installer calls `prepare_runtime_huds.py` after VLT conversion. +It exports only the scoring movie and marker subtree needed by the runtime, +validates both, and installs their manifests and referenced RGBA payloads. +Intermediate extraction files remain in the existing conversion workspace and +are removed by the installer's normal cleanup. No full-disc or audio extraction +stage is added, and the original map/skater/environment setup flow is retained. + For an existing installation, reuse prepared caches without reconverting maps: ```powershell diff --git a/docs/setup-content.md b/docs/setup-content.md deleted file mode 100644 index a12c80dd..00000000 --- a/docs/setup-content.md +++ /dev/null @@ -1,63 +0,0 @@ -# Setup content ownership - -Setup owns extraction and preparation. Features must read installed asset paths; -private development launchers and environment overrides are not installation steps. - -The selected owned game tree is extracted into `assets/private/original` before -runtime conversion. All EB BIG v3 and BIG4/BIGF banks are decoded, including nested -audio banks, and loose files are copied. Each bank has its own namespace so two -archives with the same member name cannot overwrite each other. The private -`original/manifest.json` records source archive hashes, member names, output sizes -and decoded hashes. Repeating this extraction verifies and reuses matching files. -Unsafe paths, ambiguous names and unsupported archive formats fail explicitly. -The installation retains this source cache; temporary map conversion files are -still removed after successful preparation. - -| Runtime content | Setup owner | -| --- | --- | -| Stock animation banks, action/motion/camera graphs, gestures, input and recipes | Stock BIG and loose animation extraction | -| Physics, difficulty, scoring and other VLT tuning; skeleton metadata | VLT and physics-skeleton converters | -| Skater mesh, textures and authored character lighting | Character and lighting converters | -| Resident character choices, original textures, tattoos and native presets | `prepare_customisation.py` using existing CAC converters | -| Original trickdisplay actions, shapes and bitmap fonts | `prepare_hud.py` | -| Original session-marker geometry, labels and Xbox button images | `extract_session_marker.py` using the vendored toolkit | -| Retail skies, render/exposure settings, travel destinations and foliage | Existing environment converters | -| Authored movable-object models and placement | DMO catalog and per-map conversion | -| Every disc district map, textures, collision and retained extensions | Existing map jobs | - -The final `runtime_content` check verifies manifests and referenced files, -including HUD RGBA lengths, character scene/texture dependencies, lighting, -per-map skies and movable objects. It writes `private/content-manifest.json`. -Only after this check and the existing runtime asset checks pass does setup -publish `installation.json` and report success. CAC conversion errors now stop -setup rather than publishing an incomplete library. - -Extraction is not a claim of full engine support for every original format. -Unported audio/frontend data remains in the indexed source cache. The ocean PCA -table is executable-resident: its existing converter accepts a hash-verified -mapped TU3 image, which ordinary disc setup does not produce. The owned XEX is -preserved, but that optional table still needs the mapped-image converter; the -renderer retains its documented fallback when absent. User-authored animations -and third-party mods are not disc assets and are not generated by extraction. - -Existing installations can add the original HUDs using the targeted -[cache repair](hud-installation.md), or prepare only missing feature caches with -the respective tools. Updating source code does not automatically rerun setup -or replace an already installed package. - -## Offline validation - -The 2026-09-09 setup integration was checked without running the game, recomp, -Steam or gameplay tests. Archive tests cover BIG4 RefPack payloads, nested banks, -duplicate names, unsafe paths, reuse and repair. The owned source inventory reads -51 top-level banks and identifies 28 nested EB banks. HUD installation tests cover -missing/corrupt files and byte preservation; the vendored marker compiler produces -the same 15 meshes and seven texture payloads as the prepared native overlay. -The data-only original movie audit passes 1,800 frames. The existing owned CAC -audit validates 480 models, 2,427 materials and 170 tattoos. -The dependency validator also passes against 3,435 prepared runtime files assembled -from the existing caches with a University map. The packaged setup worker passes -data-only archive extraction and HUD installation checks; no setup GUI is launched. - -Full installation and GPU/gameplay behavior still require manual validation; -prepared maps and dependencies were reused rather than rebuilding the installation. diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index 569a815a..7f50edee 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -163,12 +163,8 @@ def install(iso,base,game_exe,report,game_root=None): run([extractor,'-x',iso,'-d',game_root],log,report) else:game_root=game_root.resolve() for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big', - 'data/big/fedata.big','data/big/fetexture.big','data/big/fedynamic.big', 'data/content/createacharacter.big','data/content/worldDIST_University.big']: if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required) - report('Extracting and indexing all original game content') - from .original_content import extract as extract_original - extract_original(game_root, private/'original', report) report('Extracting animation banks, graphs and gameplay inputs') stock=private/'stock' extract(game_root/'data/big/miscload.big',stock) @@ -186,10 +182,8 @@ def install(iso,base,game_exe,report,game_root=None): skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin') (stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8') report('Preparing original scoring and session-marker HUD assets') - run(task(TOOLS/'prepare_hud.py', '--game', game_root, '--output', private/'hud', - '--collections', stock/'skater-collections.json'), log, report) - run(task(TOOLS/'extract_session_marker.py', '--game', game_root, - '--output', private/'session-marker'), log, report) + run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root, + '--assets', stage/'assets', '--work', work/'hud'), log, report) report('Preparing the skater model and textures') manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text()) needed=set() @@ -205,9 +199,6 @@ def install(iso,base,game_exe,report,game_root=None): write_character(character/'selected/models',private,manifest) from .character_lighting import convert as write_character_lighting write_character_lighting(character/'selected/models',private,converted) - report('Preparing original character customisation models, textures and menus') - run(task(TOOLS/'prepare_customisation.py', '--game', game_root, - '--assets', stage/'assets'), log, report) game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000', 'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph', 'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'} @@ -222,7 +213,7 @@ def install(iso,base,game_exe,report,game_root=None): write_teleports(game_root,stage/'assets',converted) report('Preparing global foliage backdrops') from .backdrop import convert as write_backdrops - backdrop_count = write_backdrops(game_root,stage/'assets',converted) + write_backdrops(game_root,stage/'assets',converted) report('Preparing authored movable-object models') from .dynamic_props import prepare_catalog prepare_catalog(game_root,work/'dmo') @@ -253,9 +244,6 @@ def install(iso,base,game_exe,report,game_root=None): settings=stage/'settings';settings.mkdir() (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8') (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8') - report('Checking original content and every prepared runtime asset group') - from .runtime_content import validate as validate_content - validate_content(stage, catalog, expected_backdrops=backdrop_count) remove_intermediate(work,stage) # Publish only after every conversion and the runtime's own load succeeds. marker=base/'installation.json.new' diff --git a/tools/asset_pipeline/original_content.py b/tools/asset_pipeline/original_content.py deleted file mode 100644 index 5bcaa12b..00000000 --- a/tools/asset_pipeline/original_content.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Lossless, indexed extraction of the selected owned game tree. - -Keep each bank in its own namespace: duplicated names in different banks must -not overwrite one another. Container decoding is separate from runtime format -conversion; unported audio/frontend/etc. payloads remain original private data. -""" -from pathlib import Path -import hashlib -import json -import shutil - -from tools.owned_game.big import BigArchive - - -def digest(path): - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def destination(root, relative): - target = (root / BigArchive.safe_relative(relative)).resolve() - if not target.is_relative_to(root.resolve()): - raise ValueError(f"Original content path escaped output: {relative}") - return target - - -def extract(game_root, output, report=print): - game_root, output = game_root.resolve(strict=True), output.resolve() - if not game_root.is_dir(): - raise ValueError("Owned game source must be a directory") - if output.is_relative_to(game_root) or game_root.is_relative_to(output): - raise ValueError("Original content output must be separate from the owned source") - output.mkdir(parents=True, exist_ok=True) - index = output / "manifest.json" - prior = json.loads(index.read_text(encoding="utf-8")) if index.is_file() else {} - records = [] - old = {r["output"]: r for r in prior.get("files", [])} - claimed = set() - - def claim(relative): - key = relative.casefold() - if key in claimed: - raise ValueError(f"Ambiguous original content path: {relative}") - claimed.add(key) - return destination(output, relative) - - def bank(path, relative, lineage, depth=0): - if depth > 16: - raise ValueError("Excessive nested original archives") - archive = BigArchive(path) - identity = digest(path) - report(f"Extracting original bank: {lineage} ({len(archive.entries)} entries)") - for entry in archive.entries: - name = BigArchive.safe_relative(entry.path).as_posix() - member = relative + "/" + name - target = claim(member) - previous = old.get(member, {}) - if not (previous.get("archive_sha256") == identity and target.is_file() - and target.stat().st_size == entry.unpacked_size - and digest(target) == previous.get("sha256")): - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(archive.read(entry)) - records.append(dict(output=member, archive=lineage, entry=entry.path, - archive_sha256=identity, size=entry.unpacked_size, sha256=digest(target))) - if name.lower().endswith(".big"): - bank(target, member + ".contents", lineage + "!/" + name, depth + 1) - - for source in sorted(game_root.rglob("*")): - if not source.is_file(): - continue - relative = source.relative_to(game_root).as_posix() - if source.suffix.lower() == ".big": - bank(source, "banks/" + relative, relative) - else: - name = "loose/" + relative - target = claim(name) - sha256 = digest(source) - if not target.is_file() or digest(target) != sha256: - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, target) - if digest(target) != sha256: - raise ValueError(f"Original file copy failed: {relative}") - records.append(dict(output=name, source=relative, size=source.stat().st_size, sha256=sha256)) - manifest = dict(version=1, files=records) - temporary = index.with_suffix(".json.new") - temporary.write_text(json.dumps(manifest, indent=2), encoding="utf-8") - temporary.replace(index) - return manifest - - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--game", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - extract(args.game, args.output) diff --git a/tools/asset_pipeline/runtime_content.py b/tools/asset_pipeline/runtime_content.py deleted file mode 100644 index aa271f8d..00000000 --- a/tools/asset_pipeline/runtime_content.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Verify installed runtime dependencies before publishing setup completion.""" -import hashlib -import json -from pathlib import Path - -from tools.install_prepared_hud import runtime_files -from .original_content import destination - - -def validate(stage, maps, expected_backdrops=None): - assets = stage / "assets" - files = {} - - def require(relative): - path = destination(assets, relative) - if relative in files: - return path - if not path.is_file() or not path.stat().st_size: - raise ValueError(f"Setup is missing runtime content: {relative}") - with path.open("rb") as stream: - sha = hashlib.file_digest(stream, "sha256").hexdigest() - files[relative] = sha - return path - - def read(relative): - return json.loads(require(relative).read_text(encoding="utf-8")) - - manifest = read("private/game.json") - for key in ("character_scene", "action_graph", "motion_graph"): - require(manifest[key]) - for name in ("OnBoard", "OffBoard"): - require(f"private/stock/data/anim/{name}.abin") - for name in ("skater-collections.json", "physics-skeletons.json", "data/config/input.cfg", - "data/script/camera/Default_cameragraph.stategraph", "data/camera/1.shk", "data/camera/2.shk"): - require("private/stock/" + name) - for folder, marker in (("hud", False), ("session-marker", True)): - for relative in runtime_files(assets / "private" / folder, marker): - require("private/" + folder + "/" + relative) - library = read("private/customisation/library-v3.json") - if library.get("version") != 3 or library.get("errors") or not library.get("models"): - raise ValueError("Setup character library is incomplete") - read("private/customisation/extra-menu.json") - for model in library["models"].values(): - require(model["scene"]) - for material in library["materials"].values(): - for key in ("diffuse", "normal", "rough", "opacity"): - if material.get(key): - require(material[key]) - for tattoo in library["tattoos"].values(): - require(tattoo["texture"]) - lighting = read("private/character-lighting.json") - for material in lighting["materials"].values(): - if material.get("specular"): - require(material["specular"]) - for name in ("render-parameters", "exposure", "exposure-profiles", "teleports"): - read(f"private/{name}.json") - backdrops = list((assets / "private/native-backdrops").glob("*.skate")) - if expected_backdrops is not None and len(backdrops) != expected_backdrops: - raise ValueError("Setup foliage backdrop count does not match conversion") - for backdrop in backdrops: - require(backdrop.relative_to(assets).as_posix()) - if not maps: - raise ValueError("Setup has no converted maps") - for row in maps: - path = destination(stage, row["path"]) - if not path.is_file() or not path.stat().st_size: - raise ValueError(f"Setup map is missing: {row['path']}") - name = row["name"] - sky = read(f"private/native-skies/{name}.json") - for suffix, width, height in (("rgba", sky["width"], sky["height"]), - ("sun.rgba", sky["sun_width"], sky["sun_height"])): - if require(f"private/native-skies/{name}.{suffix}").stat().st_size != width * height * 4: - raise ValueError(f"Invalid sky texture: {name}.{suffix}") - props = read(f"private/native-props/{name}.json") - if props.get("instances"): - require(f"private/native-props/{name}.skate") - original = read("private/original/manifest.json") - if original.get("version") != 1 or not original.get("files"): - raise ValueError("Setup has no indexed original content") - result = dict(version=1, runtime_files=files, maps=maps, - original_files=len(original["files"]), - preserved_source_only=["Unported audio and frontend payloads", - "Executable-resident tables without a disc decoder (ocean PCA)"]) - temporary = assets / "private/content-manifest.json.new" - temporary.write_text(json.dumps(result, indent=2), encoding="utf-8") - temporary.replace(assets / "private/content-manifest.json") - return result diff --git a/tools/asset_pipeline/test_original_content.py b/tools/asset_pipeline/test_original_content.py deleted file mode 100644 index 6faeb33c..00000000 --- a/tools/asset_pipeline/test_original_content.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path -import json -import struct -import tempfile -import unittest - -from tools.asset_pipeline.original_content import extract -from tools.owned_game.big import BigArchive - - -def big4(entries): - end = 16 + sum(9 + len(name.encode()) for name, _ in entries) - size = end + sum(len(data) for _, data in entries) - directory, payload, offset = bytearray(), bytearray(), end - for name, data in entries: - directory += struct.pack('>II', offset, len(data)) + name.encode() + b'\0' - payload += data - offset += len(data) - return b'BIG4' + struct.pack('II', len(entries), end) + directory + payload - - -class OriginalContentTests(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.game, self.out = self.root/'game', self.root/'output' - self.game.mkdir() - - def test_nested_banks_duplicate_names_loose_files_and_repair(self): - (self.game/'a.big').write_bytes(big4([('same', b'a'), ('nested.big', big4([('child', b'c')]))])) - (self.game/'b.big').write_bytes(big4([('same', b'b')])) - (self.game/'loose.bin').write_bytes(b'original') - first = extract(self.game, self.out, lambda _: None) - self.assertEqual(len(first['files']), 5) - self.assertEqual((self.out/'banks/a.big/same').read_bytes(), b'a') - self.assertEqual((self.out/'banks/b.big/same').read_bytes(), b'b') - self.assertEqual((self.out/'banks/a.big/nested.big.contents/child').read_bytes(), b'c') - target = self.out/'banks/a.big/same' - stamp = target.stat().st_mtime_ns - self.assertEqual(extract(self.game, self.out, lambda _: None), first) - self.assertEqual(stamp, target.stat().st_mtime_ns) - target.write_bytes(b'x') - extract(self.game, self.out, lambda _: None) - self.assertEqual(target.read_bytes(), b'a') - - def test_big4_refpack_and_invalid_offsets(self): - # Header, literal of three bytes, end marker. - path = self.game/'compressed.big' - path.write_bytes(big4([('data', b'\x10\xfb\x00\x00\x03\xffabc')])) - archive = BigArchive(path) - self.assertEqual(archive.read(archive.entries[0]), b'abc') - raw = bytearray(path.read_bytes()) - raw[16:20] = struct.pack('>I', len(raw) + 1) - path.write_bytes(raw) - with self.assertRaisesRegex(ValueError, 'bounds'): - BigArchive(path) - - def test_unsafe_member_and_source_overlap_fail(self): - (self.game/'bad.big').write_bytes(big4([('../escape', b'bad')])) - with self.assertRaisesRegex(ValueError, 'unsafe'): - extract(self.game, self.out, lambda _: None) - self.assertFalse((self.out/'manifest.json').exists()) - with self.assertRaisesRegex(ValueError, 'separate'): - extract(self.game, self.game/'output', lambda _: None) - - def test_case_collisions_do_not_publish_completed_index(self): - (self.game/'bad.big').write_bytes(big4([('Name', b'a'), ('name', b'b')])) - with self.assertRaisesRegex(ValueError, 'Ambiguous'): - extract(self.game, self.out, lambda _: None) - self.assertFalse((self.out/'manifest.json').exists()) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/asset_pipeline/test_runtime_content.py b/tools/asset_pipeline/test_runtime_content.py deleted file mode 100644 index 419274ed..00000000 --- a/tools/asset_pipeline/test_runtime_content.py +++ /dev/null @@ -1,32 +0,0 @@ -import json -from pathlib import Path -import tempfile -import unittest - -from tools.asset_pipeline.runtime_content import validate - - -class RuntimeContentTests(unittest.TestCase): - def test_missing_hud_cannot_publish_complete_content(self): - with tempfile.TemporaryDirectory() as directory: - stage = Path(directory) - assets = stage/'assets' - def write(name, value=b'fixture'): - path = assets/name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(value) - write('private/game.json', json.dumps(dict(character_scene='private/skater.glb', - action_graph='private/action', motion_graph='private/motion')).encode()) - for name in ('private/skater.glb', 'private/action', 'private/motion', - 'private/stock/data/anim/OnBoard.abin', 'private/stock/data/anim/OffBoard.abin'): - write(name) - for name in ('skater-collections.json', 'physics-skeletons.json', 'data/config/input.cfg', - 'data/script/camera/Default_cameragraph.stategraph', 'data/camera/1.shk', 'data/camera/2.shk'): - write('private/stock/'+name) - with self.assertRaises(FileNotFoundError): - validate(stage, []) - self.assertFalse((assets/'private/content-manifest.json').exists()) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/owned_game/big.py b/tools/owned_game/big.py index 70a733f1..9af625fa 100644 --- a/tools/owned_game/big.py +++ b/tools/owned_game/big.py @@ -74,17 +74,13 @@ def _decompress_chunkref(data: bytes, expected_size: int) -> bytes: class BigArchive: - """Streaming reader for Skate 3's EB BIG v3 and legacy BIG4/BIGF banks.""" + """Streaming reader for the EB BIG v3 archives used by Skate 3.""" def __init__(self, path: Path): self.path = Path(path).resolve() self.file_size = self.path.stat().st_size with self.path.open("rb") as stream: header = stream.read(48) - if header[:4] in (b"BIG4", b"BIGF"): - self.metadata = b"" - self.entries = self._parse_big4(stream, header) - return if len(header) != 48: raise FormatError(f"{self.path}: truncated BIG header") header_reader = Reader(header, str(self.path)) @@ -97,41 +93,6 @@ class BigArchive: self.metadata = stream.read(metadata_size) self.entries = self._parse() - def _parse_big4(self, stream, header): - """Older EA containers, including the disc's music overlays bank.""" - if len(header) < 16: - raise FormatError("truncated BIG4 header") - count = int.from_bytes(header[8:12], "big") - end = int.from_bytes(header[12:16], "big") - if count > 1_000_000 or end < 16 or end > self.file_size: - raise FormatError("invalid BIG4 directory bounds") - stream.seek(16) - directory = stream.read(end - 16) - cursor = 0 - entries = [] - for index in range(count): - if cursor + 8 > len(directory): - raise FormatError("truncated BIG4 entry") - offset = int.from_bytes(directory[cursor:cursor+4], "big") - size = int.from_bytes(directory[cursor+4:cursor+8], "big") - name_end = directory.find(b"\0", cursor + 8) - if name_end < 0 or offset < end or offset + size > self.file_size: - raise FormatError("invalid BIG4 entry bounds") - name = directory[cursor+8:name_end].decode("utf-8") - self.safe_relative(name) - stream.seek(offset) - prefix = stream.read(min(size, 6)) - compressed = len(prefix) >= 2 and prefix[0] in (0x10, 0x90) and prefix[1] == 0xfb - unpacked = size - if compressed: - length = 6 if prefix[0] == 0x90 else 5 - if len(prefix) < length: - raise FormatError("truncated BIG4 RefPack header") - unpacked = int.from_bytes(prefix[2:length], "big") - entries.append(BigEntry(index, name, offset, size, unpacked, int(compressed))) - cursor = name_end + 1 - return entries - def _parse(self) -> list[BigEntry]: reader = Reader(self.metadata, str(self.path)) if reader.u16be(0) != 0x4542: diff --git a/tools/prepare_customisation.py b/tools/prepare_customisation.py deleted file mode 100644 index fb844d27..00000000 --- a/tools/prepare_customisation.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Prepare original resident character choices during setup, without a game.""" -import argparse -import json -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from tools.asset_pipeline.customisation_catalog import prepare as catalog -from tools.asset_pipeline.customisation_library import prepare as library -from tools.asset_pipeline.customisation_profiles import generate - - -def prepare(game, assets): - cache = assets / "private/customisation" - catalog(game, cache) - result = library({"game_root": str(game), "assets": str(assets), "library_index": "library-v3.json"}) - if result["errors"]: - raise ValueError(f"Original character library is incomplete: {result['errors']}") - native = json.loads((cache / "native.json").read_text(encoding="utf-8")) - (cache / "extra-menu.json").write_text(json.dumps(generate(native), indent=2), encoding="utf-8") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--game", type=Path, required=True) - parser.add_argument("--assets", type=Path, required=True) - args = parser.parse_args() - prepare(args.game, args.assets) diff --git a/tools/prepare_hud.py b/tools/prepare_hud.py index a1bbccee..bae03220 100644 --- a/tools/prepare_hud.py +++ b/tools/prepare_hud.py @@ -49,8 +49,7 @@ def font_mapping(collections: Path, cache: AssetCache) -> dict: def prepare(game: Path, output: Path, collections: Path): - extract_project(game, output, prefixes=("data/fe/source/screens/hud2/", - "data/fe/source/controls/"), update=True) + extract_project(game, output, prefixes=("data/fe/source/screens/hud2/trickdisplay2",), update=True) cache = AssetCache(output) mappings = font_mapping(collections,cache) name = "data/fe/source/screens/hud2/trickdisplay2" diff --git a/tools/prepare_runtime_huds.py b/tools/prepare_runtime_huds.py new file mode 100644 index 00000000..63e5d7a0 --- /dev/null +++ b/tools/prepare_runtime_huds.py @@ -0,0 +1,31 @@ +"""Export the two HUDs used by the engine, as one automatic setup stage. + +Only runtime manifests and referenced RGBA payloads enter the installation. +Extraction intermediates stay in setup's conversion workspace and are removed +by its normal successful-install cleanup. No audio or full-disc extraction. +""" +import argparse +from pathlib import Path + +from prepare_hud import prepare as prepare_scoring +from extract_session_marker import prepare as prepare_marker +from install_prepared_hud import install + + +def prepare(game, assets, work): + scoring = work / 'scoring' + marker = work / 'session-marker' + prepare_scoring(game, scoring, assets / 'private/stock/skater-collections.json') + prepare_marker(game, marker) + files = install(assets, scoring, marker) + print(f'Original runtime HUDs ready: {len(files)} verified files', flush=True) + return files + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--game', type=Path, required=True) + parser.add_argument('--assets', type=Path, required=True) + parser.add_argument('--work', type=Path, required=True) + args = parser.parse_args() + prepare(args.game, args.assets, args.work) diff --git a/tools/setup.py b/tools/setup.py index feca9009..913a200e 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -29,7 +29,7 @@ def main(): if icon.is_file():window.iconbitmap(str(icon)) frame=ttk.Frame(window,padding=24);frame.pack(fill='both',expand=True) ttk.Label(frame,text='Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16)) - ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup extracts all original content and prepares the maps,\nskater, clothing, animations, HUDs and environment assets.\nNo other apps need installing.\n\nAllow free disk space for original and converted content.\nISO extraction needs internet access.', + ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.', font=('Segoe UI',11),justify='left').pack(anchor='w') status=tk.StringVar(value='Choose your game to begin.') ttk.Label(frame,textvariable=status,wraplength=600).pack(anchor='w',pady=(18,8)) From 6aa47622850f5a7d1888d553d2491f8354a66de3 Mon Sep 17 00:00:00 2001 From: chazmm Date: Wed, 9 Sep 2026 01:52:25 +0100 Subject: [PATCH 5/5] Speed up owned-disc setup while preserving runtime asset content --- docs/installer-performance.md | 48 ++++++++ tools/asset_pipeline/dynamic_props.py | 32 +++++- tools/asset_pipeline/install.py | 29 ++++- tools/asset_pipeline/map_writer.py | 106 ++++++++++++------ tools/asset_pipeline/test_dynamic_props.py | 32 +++++- tools/asset_pipeline/test_map_writer.py | 63 +++++++++++ tools/asset_pipeline/test_streams.py | 51 +++++++++ .../tools/prepare_hawaiian_dream.py | 21 ++-- .../tools/skate3_streams.py | 14 ++- 9 files changed, 341 insertions(+), 55 deletions(-) create mode 100644 docs/installer-performance.md create mode 100644 tools/asset_pipeline/test_map_writer.py create mode 100644 tools/asset_pipeline/test_streams.py diff --git a/docs/installer-performance.md b/docs/installer-performance.md new file mode 100644 index 00000000..251ed0af --- /dev/null +++ b/docs/installer-performance.md @@ -0,0 +1,48 @@ +# Installer performance and fidelity + +Setup keeps the same asset conversion formats, lossless compression settings, +map validation, and publication checks. The optimizations remove repeated work: + +- SFIL padding checks use a view of the input instead of copying the remaining + stream for each asset. District cells share the same ATOC lookup table. +- Map preparation creates its output directories once. Direct installation + avoids temporary model/texture RX2 copies which the writer never read and + setup previously deleted. Standalone extraction still writes these sources + by default; simulation and irradiance sources remain available in both modes. +- Spawn selection consumes collision meshes during their existing decode pass, + retaining triangle order, floating-point operations, and tie behavior. +- Model serialization reads each mesh's arrays once. Prop transforms only read + arrays belonging to the selected mesh. +- The installation builds and validates its shared prop catalog once, then + passes a JSON snapshot to map workers. It lives inside the current conversion + workspace and is removed after successful installation. +- Two bounded texture-packing threads overlap lossless zlib compression, with + the original texture order and compression level. Map-worker limits retain + the existing CPU and available-memory checks. + +Map entries in `maps.json` now include `phase_seconds` for extraction, +preparation, collision packaging, map writing, prop export, validation, and +hashing/cleanup. These are diagnostic timings, not runtime asset inputs. + +## Verification + +A Windows x64 comparison on 2026-09-09 used packaged installers, the same +extracted owned disc, all ten maps, three map workers, and the same runtime +executable. A full installation took 117.8 seconds before and 91.5 seconds after +the changes. A subsequent baseline run took 103.0 seconds, putting the observed +reduction at 11–22%, depending on the baseline run. These are local warm-cache +measurements; ISO unpacking and downloading the ISO tool were outside them. + +The comparison used the same output path for both runs and checked every file +under `assets`, `maps`, and `settings`: no missing or extra files; 2,231 files +were byte-identical, including all ten maps. Five prop packages had identical +binary payloads and decoded metadata, with differences only in JSON object key +order. Prop texture metadata is now sorted to make its order deterministic. +Timing records and logs were excluded from asset equality checks. + +Both complete installations passed every existing runtime asset check. The +asset-pipeline suite passed 48 tests, with one unrelated owned-customisation +fixture skipped because it is opt-in. New fixtures cover serial/parallel texture +byte equality, cubemap orientation, compressible and incompressible textures, +worker failures, spawn ties, catalog matrix precision, selective mesh reads, +and malformed or incomplete SFIL input. diff --git a/tools/asset_pipeline/dynamic_props.py b/tools/asset_pipeline/dynamic_props.py index f919dd71..f9ad1d38 100644 --- a/tools/asset_pipeline/dynamic_props.py +++ b/tools/asset_pipeline/dynamic_props.py @@ -97,7 +97,8 @@ def template_meshes(raw): def transform_mesh(arrays, index, transform): - values = {key: np.array(value, copy=True) for key, value in arrays.items() if key.endswith('_'+str(index))} + # NpzFile.items() reads every array before the filter runs. + values = {key: np.array(arrays[key], copy=True) for key in arrays if key.endswith('_'+str(index))} key = f'vertices_{index}' values[key] = (values[key] @ transform[:3, :3] + transform[3, :3]).astype('f4') normal_matrix = np.linalg.inv(transform[:3, :3]).T @@ -148,9 +149,31 @@ def catalog(cache_roots): return templates, textures -def export(manifest_path, cache_roots, output): - district = json.loads(manifest_path.read_text()) +def save_catalog(cache_roots, output): + """Build once per installation; JSON keeps this local cache non-executable.""" templates, textures = catalog(cache_roots) + serialised = {key: dict(value, matrix=value['matrix'].tolist(), + model_matrix=value['model_matrix'].tolist(), npz=str(value['npz'].resolve())) + for key, value in templates.items()} + temporary = output.with_suffix('.new') + temporary.write_text(json.dumps(dict(version=1, templates=serialised, textures=textures)), encoding='utf-8') + temporary.replace(output) + + +def load_catalog(path): + data = json.loads(path.read_text(encoding='utf-8')) + if data['version'] != 1: + raise ValueError('Unsupported DMO catalog version') + for value in data['templates'].values(): + value['matrix'] = np.array(value['matrix'], dtype=np.float64) + value['model_matrix'] = np.array(value['model_matrix'], dtype=np.float64) + value['npz'] = Path(value['npz']) + return data['templates'], data['textures'] + + +def export(manifest_path, cache_roots, output, *, catalog_path=None): + district = json.loads(manifest_path.read_text()) + templates, textures = catalog(cache_roots) if catalog_path is None else load_catalog(catalog_path) placements = {} for source in district['simulation_assets']: raw = (manifest_path.parent/source['rx2']).read_bytes() @@ -180,7 +203,7 @@ def export(manifest_path, cache_roots, output): report['instances'].append(dict(item, model_asset=template['asset_id'], meshes=len(meshes))) if models: manifest = dict(map_name=district['map_name'], district_name=district['district_name'], - models=models, textures={key:textures[key] for key in used}, + models=models, textures={key:textures[key] for key in sorted(used)}, normal_texture_policy=dict(excluded_texture_ids=[]), grind_splines=[]) (root/'manifest.json').write_text(json.dumps(manifest)) # Publish only a complete package; preserve the previous one on failure. @@ -209,6 +232,7 @@ def prepare_catalog(game_root, work): prepare(stream_directory=stream, output_root=root, utt_root=vendor/'utt', district_name=stream.name, map_name=stream.name, raw_texture_cache=True) roots.append(root) + save_catalog(roots, work/'catalog.json') return roots diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index 7f50edee..b318ba0b 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -1,6 +1,6 @@ """Local owned-disc installation. No game content is downloaded or packaged.""" from pathlib import Path -import hashlib,json,os,shutil,subprocess,sys,urllib.request,uuid,zipfile +import hashlib,json,os,shutil,subprocess,sys,time,urllib.request,uuid,zipfile from concurrent.futures import ThreadPoolExecutor,as_completed from tools.owned_game.big import BigArchive from .vlt import convert as convert_vlt @@ -98,38 +98,57 @@ def extract(archive,destination,entries=None): return data def convert_map(archive,work,maps,stage,game_exe,log,report): + timings={};started=time.perf_counter() + def finished(phase): + nonlocal started + now=time.perf_counter();timings[phase]=round(now-started,3);started=now + report(f'{archive.stem}: {phase} {timings[phase]:.3f}s') map_tools=TOOLS/'vendor/university/tools/vanilla_map_extraction/tools' sys.path.insert(0,str(map_tools)) from prepare_hawaiian_dream import prepare from prepare_university import EXCLUDED_NORMAL_TEXTURE_IDS from build_retail_collision_archive import build_archive - from .map_writer import write as write_map + from .map_writer import write as write_map, SpawnSelector district=archive.stem.removeprefix('world') label=district.removeprefix('DIST_') district_work=work/district extract(archive,district_work/'raw') + finished('extract') stream=district_work/'raw/data/content/world/stream'/district if not stream.is_dir():raise RuntimeError('Missing district stream '+str(stream)) + spawn=SpawnSelector(district) manifest_path=prepare(stream_directory=stream,output_root=district_work/'intermediate', utt_root=TOOLS/'vendor/utt',district_name=district,map_name=label, package_name='Skate 3 owned disc',cache_format='skate3-rust-map-v1', # Smaller parks keep their textures in Pres rather than a Tex stream. texture_stream_names=('Tex',) if any(stream.glob('cTex_*.xsf')) else (), - excluded_normal_texture_ids=EXCLUDED_NORMAL_TEXTURE_IDS,raw_texture_cache=True) + excluded_normal_texture_ids=EXCLUDED_NORMAL_TEXTURE_IDS,raw_texture_cache=True, + collision_consumer=spawn.consider, + # Model/texture RX2 copies are unused by the direct writer and were + # deleted after conversion. Keep simulation and irradiance sources. + write_render_sources=False) + finished('prepare') collision=district_work/'collision.rwcmset' build_archive(manifest_path,collision) + finished('collision_archive') final=maps/(label+'.skate') - write_map(manifest_path,final,collision,report) + write_map(manifest_path,final,collision,report,prepared_spawn=spawn.result(label)) + finished('write_map') from .dynamic_props import export as write_props caches=list((work/'dmo/cache').glob('DMO_*')) if not caches: raise RuntimeError('Missing prepared DMO catalog') - placed, unresolved=write_props(manifest_path,caches,stage/'assets/private/native-props'/(label+'.skate')) + placed, unresolved=write_props(manifest_path,caches,stage/'assets/private/native-props'/(label+'.skate'), + catalog_path=work/'dmo/catalog.json') + finished('props') report(f'{label}: placed {placed} authored DMO instances, {unresolved} unresolved templates') report('Checking converted map: '+label) run([game_exe,'--assets',stage/'assets','--map',final,'--check-assets'],log,report) + finished('validate') entry={'name':label,'path':'maps/'+final.name,'sha256':digest(final)} remove_intermediate(district_work,work) + finished('hash_and_cleanup') + entry['phase_seconds']=timings return entry diff --git a/tools/asset_pipeline/map_writer.py b/tools/asset_pipeline/map_writer.py index a727e1c6..6365f5f9 100644 --- a/tools/asset_pipeline/map_writer.py +++ b/tools/asset_pipeline/map_writer.py @@ -1,6 +1,8 @@ """Direct retail cache -> SKATE14 writer. All geometry remains in Y-up metres.""" from pathlib import Path import io,json,struct,sys,zlib +from collections import deque +from concurrent.futures import ThreadPoolExecutor import numpy as np from PIL import Image from tools.asset_pipeline.retail_material import _retail_shader_family,_retail_render_flags @@ -14,27 +16,49 @@ def stored(f,data): method=1 if len(packed)>=len(data):method=0;packed=data u(f,method,len(packed));f.write(packed) + +def packed_texture(root,name,entry): + if 'rgba' in entry: + width,height=entry['width'],entry['height'] + pixels=np.frombuffer((root/entry['rgba']).read_bytes(),dtype=np.uint8).reshape(height,width,4) + rgba=(pixels if entry.get('cube_faces')==6 else pixels[::-1]).tobytes() + else: + with Image.open(root/entry['png']) as source: + image=source.convert('RGBA') + if entry.get('cube_faces')!=6:image=image.transpose(Image.Transpose.FLIP_TOP_BOTTOM) + width,height=image.size;rgba=image.tobytes() + result=io.BytesIO();string(result,name);u(result,width,height,1);stored(result,rgba) + return result.getvalue() + +def write_textures(output,root,textures): + # zlib releases the GIL. Bound outstanding work to two textures rather + # than retaining an entire district's decoded/compressed images. + names=iter(sorted(textures)) + with ThreadPoolExecutor(max_workers=2) as pool: + pending=deque() + for _ in range(2): + name=next(names,None) + if name is not None:pending.append(pool.submit(packed_texture,root,name,textures[name])) + while pending: + output.write(pending.popleft().result()) + name=next(names,None) + if name is not None:pending.append(pool.submit(packed_texture,root,name,textures[name])) def normalise(a):return a/np.maximum(np.linalg.norm(a,axis=1,keepdims=True),1e-20) -def spawn_point(manifest,root): - from retail_collision_mesh import decode_rx2_clustered_meshes - university=manifest['district_name']=='DIST_University' - best=None - for entry in manifest['simulation_assets']: - if not entry.get('collision_meshes'):continue - # Bound the exact same search before decoding distant collision - # meshes. A triangle's centroid cannot be closer than its AABB. - bounds=[mesh['bounds'] for mesh in entry['collision_meshes']] - if university: - if not any(b['minimum'][0]<=330<=b['maximum'][0] and - b['minimum'][2]<=-710<=b['maximum'][2] for b in bounds):continue - elif best is not None: - def nearest_square(b): - return sum(max(b['minimum'][j],-b['maximum'][j],0.)**2 for j in (0,2)) - if min(map(nearest_square,bounds))>best[0]+.01:continue - for mesh in decode_rx2_clustered_meshes((root/entry['rx2']).read_bytes()): +class SpawnSelector: + """Keep the original triangle search order while consuming decoded meshes.""" + def __init__(self, district_name): + self.university=district_name=='DIST_University' + self.best=None + + def consider(self, meshes): + university=self.university;best=self.best + for mesh in meshes: if not mesh.triangles:continue if university and not (mesh.bounds_min[0]<=330<=mesh.bounds_max[0] and mesh.bounds_min[2]<=-710<=mesh.bounds_max[2]):continue + if not university and best is not None: + nearest=sum(max(mesh.bounds_min[j],-mesh.bounds_max[j],0.)**2 for j in (0,2)) + if nearest>best[0]+.01:continue triangles=np.asarray([(tri.a,tri.b,tri.c) for tri in mesh.triangles],dtype=np.float64) a,b,c=triangles[:,0],triangles[:,1],triangles[:,2] cross=np.cross(b-a,c-a);length=np.linalg.norm(cross,axis=1) @@ -56,10 +80,30 @@ def spawn_point(manifest,root): points[:,1]+=1. index=np.argmin(scores);score=scores[index] if np.isfinite(score) and (best is None or scoreselector.best[0]+.01:continue + selector.consider(decode_rx2_clustered_meshes((root/entry['rx2']).read_bytes())) + return selector.result(manifest['map_name']) + +def write(manifest_path,output,collision,report=lambda _:None, *, render_only=False, prepared_spawn=None): root=manifest_path.parent;m=json.loads(manifest_path.read_text());textures=m['textures'] ids={name:i+1 for i,name in enumerate(sorted(textures))} excluded=set(m['normal_texture_policy']['excluded_texture_ids']) @@ -68,9 +112,13 @@ def write(manifest_path,output,collision,report=lambda _:None, *, render_only=Fa ('mat','=len(pos):raise ValueError('Map face is outside vertex array') shader=mesh.get('shader_name') or '';roles=mesh.get('retail_texture_ids',{}) @@ -115,7 +163,7 @@ def write(manifest_path,output,collision,report=lambda _:None, *, render_only=Fa record['frame'][:,:3]=np.rint(np.clip(binormal,-1,1)*127).astype('i1');record['frame'][:,3]=np.rint(sign*127).astype('i1') if not np.isfinite(pos).all():raise ValueError('Non-finite map geometry') vertices.write(record.tobytes());indices.write((faces+nv).astype('5I', payload, 0, 0, 1, 1, 32, 160) diff --git a/tools/asset_pipeline/test_map_writer.py b/tools/asset_pipeline/test_map_writer.py new file mode 100644 index 00000000..bb2589ce --- /dev/null +++ b/tools/asset_pipeline/test_map_writer.py @@ -0,0 +1,63 @@ +import io +import struct +import tempfile +import unittest +import zlib +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +from PIL import Image +from .map_writer import SpawnSelector, write_textures + + +class MapWriterTests(unittest.TestCase): + def test_parallel_textures_match_serial_format_exactly(self): + with tempfile.TemporaryDirectory() as work: + root=Path(work);textures={};expected=io.BytesIO() + rng=np.random.default_rng(23) + for name, cube, png in [('z-cube', True, False), ('a-rgba', False, False), ('m-png', False, True), ('b-flat', False, False)]: + pixels=rng.integers(0,256,(24 if cube else 4,4,4),dtype=np.uint8) + if name=='b-flat':pixels[:]=0 + path=root/(name+('.png' if png else '.rgba')) + if png:Image.fromarray(pixels).save(path) + else:path.write_bytes(pixels.tobytes()) + textures[name]=dict(width=4,height=len(pixels),cube_faces=6 if cube else 1, + **{'png' if png else 'rgba':path.name}) + for name,entry in sorted(textures.items()): + if 'png' in entry: + with Image.open(root/entry['png']) as image:pixels=np.array(image.convert('RGBA')) + else:pixels=np.frombuffer((root/entry['rgba']).read_bytes(),dtype=np.uint8).reshape(entry['height'],4,4) + raw=(pixels if entry['cube_faces']==6 else pixels[::-1]).tobytes() + packed=zlib.compress(raw,1);method=1 + if len(packed)>=len(raw):packed=raw;method=0 + encoded=name.encode() + expected.write(struct.pack('I',raw,16,128) + struct.pack_into('>Q3I',raw,128,123,32,128,256) + raw[256:288]=bytes(range(32)) + record=AssetRecord(123,0,32,128,128,0,4,b'') + return bytes(raw)+tail,record + + def test_raw_assets_padding_and_shared_index(self): + class NoSuffixCopy(bytes): + def __getitem__(self,key): + if isinstance(key,slice) and key.start and key.stop is None: + raise AssertionError('Copied the entire remaining stream') + return super().__getitem__(key) + raw,record=self.fixture() + with patch.object(Path,'read_bytes',return_value=NoSuffixCopy(raw)): + normal=read_sfil('fixture.xsf',[record]) + shared=read_sfil('fixture.xsf',[record],record_index={123:record}) + self.assertEqual(normal,shared) + self.assertEqual(normal[0].data,bytes(range(32))) + self.assertEqual(normal[0].source_offset,128) + + def test_truncated_nonzero_tail_is_still_rejected(self): + raw,record=self.fixture(b'\0\1') + with patch.object(Path,'read_bytes',return_value=raw): + with self.assertRaisesRegex(StreamFormatError,'truncated'): + read_sfil('fixture.xsf',[record]) + + def test_missing_and_unknown_records_are_still_rejected(self): + raw,record=self.fixture() + missing=AssetRecord(456,0,32,128,128,0,4,b'') + with patch.object(Path,'read_bytes',return_value=raw): + with self.assertRaisesRegex(StreamFormatError,'missing'): + read_sfil('fixture.xsf',[record,missing]) + self.assertEqual(len(read_sfil('fixture.xsf',[record,missing],require_all_records=False)),1) + with self.assertRaisesRegex(StreamFormatError,'absent'): + read_sfil('fixture.xsf',[missing]) + + +if __name__=='__main__':unittest.main() diff --git a/tools/vendor/university/tools/vanilla_map_extraction/tools/prepare_hawaiian_dream.py b/tools/vendor/university/tools/vanilla_map_extraction/tools/prepare_hawaiian_dream.py index bc820902..4401d359 100644 --- a/tools/vendor/university/tools/vanilla_map_extraction/tools/prepare_hawaiian_dream.py +++ b/tools/vendor/university/tools/vanilla_map_extraction/tools/prepare_hawaiian_dream.py @@ -90,10 +90,10 @@ def _default_stream_directory(workspace: Path) -> Path: ) -def _write_rx2(output_root: Path, category: str, asset: StreamAsset) -> Path: +def _write_rx2(output_root: Path, category: str, asset: StreamAsset, *, write: bool = True) -> Path: target = output_root / "rx2" / category / f"{asset.record.asset_id:016X}.rx2" - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(asset.data) + if write: + target.write_bytes(asset.data) return target @@ -407,12 +407,19 @@ def prepare( grind_coordinate_mode: str = "world_space", excluded_static_model_asset_ids: tuple[str, ...] = (), raw_texture_cache: bool = False, + collision_consumer=None, + write_render_sources: bool = True, ) -> Path: sys.path.insert(0, str(utt_root)) import rx2_parser from mdl_parser import parse_rx2 as parse_model output_root.mkdir(parents=True, exist_ok=True) + # Create each output directory once, rather than thousands of failing + # mkdir calls against directories already created for previous assets. + for directory in ("rx2/textures", "rx2/models", "rx2/presentation_other", + "rx2/simulation", "textures/by_asset", "models"): + (output_root / directory).mkdir(parents=True, exist_ok=True) presentation_assets = load_district_stream( stream_directory, "Pres", @@ -476,7 +483,7 @@ def prepare( asset_id = asset.record.asset_id source_file = asset.source_path.name if asset.record.asset_type == ASSET_TYPE_TEXTURE: - rx2_path = _write_rx2(output_root, "textures", asset) + rx2_path = _write_rx2(output_root, "textures", asset, write=write_render_sources) base_texture_id = _texture_id(asset.data, asset_id) parsed = rx2_parser.parse_rx2(asset.data) if not parsed.textures: @@ -507,7 +514,6 @@ def prepare( / "by_asset" / f"{asset_id:016X}_{texture_index}{'.rgba' if raw_texture_cache else '.png'}" ) - png_path.parent.mkdir(parents=True, exist_ok=True) if raw_texture_cache:png_path.write_bytes(texture.rgba) else:texture.save_png(png_path) new_entry = { @@ -556,14 +562,13 @@ def prepare( known_digests.add(texture_digest) textures[texture_id] = new_entry elif asset.record.asset_type == ASSET_TYPE_MODEL: - rx2_path = _write_rx2(output_root, "models", asset) + rx2_path = _write_rx2(output_root, "models", asset, write=write_render_sources) parsed = parse_model( asset.data, source_path=rx2_path, strict=False, ) npz_path = output_root / "models" / f"{asset_id:016X}.npz" - npz_path.parent.mkdir(parents=True, exist_ok=True) arrays: dict[str, numpy.ndarray] = {} mesh_entries: list[dict[str, object]] = [] source_material_groups = _group_material_parameters(parsed.materials) @@ -791,6 +796,8 @@ def prepare( rx2_path = _write_rx2(output_root, "simulation", asset) asset_grinds = decode_grind_splines(asset.data) asset_collision_meshes = decode_rx2_clustered_meshes(asset.data) + if collision_consumer is not None: + collision_consumer(asset_collision_meshes) grind_spline_asset_count += bool(asset_grinds) collision_mesh_asset_count += bool(asset_collision_meshes) collision_mesh_count += len(asset_collision_meshes) diff --git a/tools/vendor/university/tools/vanilla_map_extraction/tools/skate3_streams.py b/tools/vendor/university/tools/vanilla_map_extraction/tools/skate3_streams.py index f352f87f..dd8007c6 100644 --- a/tools/vendor/university/tools/vanilla_map_extraction/tools/skate3_streams.py +++ b/tools/vendor/university/tools/vanilla_map_extraction/tools/skate3_streams.py @@ -196,13 +196,15 @@ def read_sfil( records: list[AssetRecord], *, require_all_records: bool = True, + record_index: dict[int, AssetRecord] | None = None, ) -> list[StreamAsset]: source_path = Path(path) source = source_path.read_bytes() if len(source) < SFIL_HEADER_SIZE or source[:4] != b"SFIL": raise StreamFormatError(f"{source_path} is not an SFIL stream") - by_id = {record.asset_id: record for record in records} + by_id = record_index if record_index is not None else {record.asset_id: record for record in records} + source_view = memoryview(source) first_asset_offset = struct.unpack_from(">I", source, 16)[0] if first_asset_offset < SFIL_HEADER_SIZE or first_asset_offset > len(source): raise StreamFormatError( @@ -214,10 +216,12 @@ def read_sfil( seen_ids: set[int] = set() while cursor < len(source): - if not any(source[cursor:]): + # bytes[cursor:] copied the entire remaining stream for every asset. + # A view preserves the same padding check without quadratic copying. + if not any(source_view[cursor:]): break if cursor + SFIL_SECTION_DATA_OFFSET > len(source): - if any(source[cursor:]): + if any(source_view[cursor:]): raise StreamFormatError(f"truncated SFIL asset at 0x{cursor:X}") break @@ -288,7 +292,7 @@ def read_sfil( ) cursor += asset_stride - missing = set(by_id) - seen_ids + missing = set(by_id) - seen_ids if require_all_records else set() if require_all_records and missing: missing_text = ", ".join(f"0x{asset_id:016X}" for asset_id in sorted(missing)) raise StreamFormatError(f"ATOC assets missing from SFIL: {missing_text}") @@ -323,11 +327,13 @@ def load_district_stream( ) assets_by_id: dict[int, StreamAsset] = {} + record_index = {record.asset_id: record for record in records} for stream_file in stream_files: for asset in read_sfil( stream_file, records, require_all_records=False, + record_index=record_index, ): asset_id = asset.record.asset_id previous = assets_by_id.get(asset_id)