From 616be77a283890ee443996ed6213ce9f5e69f968 Mon Sep 17 00:00:00 2001 From: chazmm Date: Tue, 8 Sep 2026 16:07:30 +0100 Subject: [PATCH] Restore global tree-wall backdrops omitted from district streams --- crates/skate-data/examples/inspect_skate.rs | 13 +- crates/skate-data/src/skate_map.rs | 17 ++- crates/skate-data/tests/skate_map.rs | 17 +++ crates/skate-game/src/retail_backdrop.rs | 36 +++++ crates/skate-game/src/retail_render.rs | 4 + crates/skate-game/src/world.rs | 1 + docs/retail-renderer.md | 41 ++++++ tools/asset_pipeline/backdrop.py | 138 ++++++++++++++++++++ tools/asset_pipeline/install.py | 3 + tools/asset_pipeline/map_writer.py | 10 +- tools/asset_pipeline/test_backdrop.py | 44 +++++++ 11 files changed, 318 insertions(+), 6 deletions(-) create mode 100644 crates/skate-game/src/retail_backdrop.rs create mode 100644 tools/asset_pipeline/backdrop.py create mode 100644 tools/asset_pipeline/test_backdrop.py diff --git a/crates/skate-data/examples/inspect_skate.rs b/crates/skate-data/examples/inspect_skate.rs index 56cd0792..3d8cfa1f 100644 --- a/crates/skate-data/examples/inspect_skate.rs +++ b/crates/skate-data/examples/inspect_skate.rs @@ -1,7 +1,18 @@ fn main() { let mut failed = false; + let mut render_only = false; for path in std::env::args_os().skip(1) { - match skate_data::skate_map::SkateMap::load(std::path::Path::new(&path)) { + if path == "--render-only" { + render_only = true; + continue; + } + let result = if render_only { + std::fs::read(&path).map_err(|e| e.to_string()) + .and_then(|bytes| skate_data::skate_map::SkateMap::parse_render_only(&bytes)) + } else { + skate_data::skate_map::SkateMap::load(std::path::Path::new(&path)) + }; + match result { Ok(m) => { println!( "{} v{}: vertices={} triangles={} collision={} textures={} rails={} doors={} lights={} routes={} spawn={:?} heading={}", diff --git a/crates/skate-data/src/skate_map.rs b/crates/skate-data/src/skate_map.rs index 6c229971..52d05733 100644 --- a/crates/skate-data/src/skate_map.rs +++ b/crates/skate-data/src/skate_map.rs @@ -343,6 +343,21 @@ impl SkateMap { Self::parse(&data).map_err(|e| format!("{}: {e}", path.display())) } pub fn parse(data: &[u8]) -> Result { + Self::parse_inner(data, true) + } + /// Decode a supplemental presentation package without requiring collision. + /// Playable map loading continues to use `parse`/`load`. + pub fn parse_render_only(data: &[u8]) -> Result { + let map = Self::parse_inner(data, false)?; + if !map.geometry.collision.is_empty() || !map.rails.is_empty() + || !map.doors.is_empty() || !map.lights.is_empty() || !map.routes.is_empty() + || map.extensions.iter().any(|e| e.tag != *b"WMET") + { + return Err("SKATE render-only package contains non-presentation data".into()); + } + Ok(map) + } + fn parse_inner(data: &[u8], require_collision: bool) -> Result { let mut r = Reader { bytes: data, at: 0 }; let magic = r.take(8)?; if &magic[..5] != b"SKATE" @@ -637,7 +652,7 @@ impl SkateMap { }); } } - if geometry.collision.is_empty() && !extensions.iter().any(|e| e.tag == *b"RWCM" && e.schema == 1 && !e.payload.is_empty()) { + if require_collision && geometry.collision.is_empty() && !extensions.iter().any(|e| e.tag == *b"RWCM" && e.schema == 1 && !e.payload.is_empty()) { return Err("SKATE requires triangle collision or an embedded RWCM archive".into()); } if r.at != data.len() { diff --git a/crates/skate-data/tests/skate_map.rs b/crates/skate-data/tests/skate_map.rs index d48ca217..013e49b7 100644 --- a/crates/skate-data/tests/skate_map.rs +++ b/crates/skate-data/tests/skate_map.rs @@ -1,5 +1,22 @@ use skate_data::skate_map::SkateMap; +#[test] +fn presentation_loading_does_not_relax_playable_map_validation() { + let mut data = fixture(1); + let counts = 8 + 4 + 4 + "Fixture".len() + (4 + 12) * 4; + data[counts + 16..counts + 24].fill(0); // No collision or rails. + let needle: Vec = [0u32, 1, 2].into_iter().flat_map(u32::to_le_bytes).collect(); + let indices = data.windows(12).position(|w| w == needle).unwrap(); + data.truncate(indices + 12); + assert!(SkateMap::parse(&data).unwrap_err().contains("requires triangle collision")); + let render = SkateMap::parse_render_only(&data).unwrap(); + assert_eq!(render.geometry.indices, [0, 1, 2]); + assert!(render.geometry.collision.is_empty()); + data[indices..indices+4].copy_from_slice(&99u32.to_le_bytes()); + assert!(SkateMap::parse_render_only(&data).unwrap_err().contains("indices")); + assert!(SkateMap::parse_render_only(&fixture(1)).unwrap_err().contains("non-presentation")); +} + fn u(b: &mut Vec, value: u32) { b.extend(value.to_le_bytes()); } diff --git a/crates/skate-game/src/retail_backdrop.rs b/crates/skate-game/src/retail_backdrop.rs new file mode 100644 index 00000000..598930a3 --- /dev/null +++ b/crates/skate-game/src/retail_backdrop.rs @@ -0,0 +1,36 @@ +//! Global authored foliage that lives outside the district stream. +use bevy::prelude::*; +use skate_data::skate_map::SkateMap; +use std::path::Path; + +pub(crate) fn spawn_backdrop( + name: &str, + asset_root: &Path, + commands: &mut Commands, + meshes: &mut Assets, + materials: &mut Assets, + retail_materials: &mut Assets, + images: &mut Assets, +) { + let path = asset_root.join("private/native-backdrops").join(format!("{name}.skate")); + if !path.is_file() { + return; + } + let map = match std::fs::read(&path).map_err(|e| e.to_string()) + .and_then(|data| SkateMap::parse_render_only(&data)) { + Ok(map) => map, + Err(error) => { + error!("SKATE_BACKDROP: {}: {error}", path.display()); + return; + } + }; + // This package contributes presentation only; never route it into physics. + if map.name != name || !map.geometry.collision.is_empty() + || !map.lights.is_empty() || !map.doors.is_empty() || !map.rails.is_empty() + { + error!("SKATE_BACKDROP: invalid render-only package {}", path.display()); + return; + } + info!("SKATE_BACKDROP: {name} authored foliage triangles={}", map.geometry.indices.len() / 3); + crate::skate_world::spawn(&map, commands, meshes, materials, retail_materials, images); +} diff --git a/crates/skate-game/src/retail_render.rs b/crates/skate-game/src/retail_render.rs index a78a8263..f6e9b922 100644 --- a/crates/skate-game/src/retail_render.rs +++ b/crates/skate-game/src/retail_render.rs @@ -40,6 +40,10 @@ pub(crate) struct RetailScene(pub bool); mod sky; pub(crate) use sky::{spawn_sky, RetailSkyMaterial}; +#[path = "retail_backdrop.rs"] +mod backdrop; +pub(crate) use backdrop::spawn_backdrop; + #[derive(Component, ExtractComponent, Clone, Copy, ShaderType, Default)] pub(crate) struct RetailTone { pub enabled: Vec4, diff --git a/crates/skate-game/src/world.rs b/crates/skate-game/src/world.rs index f14639dd..3681dbc6 100644 --- a/crates/skate-game/src/world.rs +++ b/crates/skate-game/src/world.rs @@ -39,6 +39,7 @@ fn spawn( // Physics has already consumed the package. Render assets own their // uploaded data; keeping another full city package wastes gigabytes. crate::skate_world::spawn(&map, &mut commands, &mut meshes, &mut materials, &mut retail_materials, &mut images); + crate::retail_render::spawn_backdrop(&map.name, &config.asset_root, &mut commands, &mut meshes, &mut materials, &mut retail_materials, &mut images); crate::retail_render::spawn_sky(&map.name, &config.asset_root, &mut commands, &mut meshes, &mut images, &mut sky_materials, &mut retail_materials); return; } diff --git a/docs/retail-renderer.md b/docs/retail-renderer.md index ea47ad62..3a1c8255 100644 --- a/docs/retail-renderer.md +++ b/docs/retail-renderer.md @@ -143,3 +143,44 @@ reported viewpoint: solid foliage appearing in the missing area implicates the normal shading/alpha path; an empty area requires checking placement, camera or source selection. Unset the variable to restore normal rendering. This diagnostic has been prepared without launching the game. + +### Missing global tree-wall backdrop resolved (2026-09-08) + +The user-run opaque-foliage diagnostic left the same empty background. The +missing surfaces are in `world/models/DIST_Water_University.rx2`, not in +`DIST_University` or its proxy stream. Despite its name, this global model has +both an ocean plane and a separate tree material: mesh 1 contains 1,572 vertices +and 786 tree-wall triangles. World fields `951898F6C0FA6856` and +`CA5A157A65E75934` select this model and its texture resource. The global +Industrial model likewise contains 716 foliage triangles. + +`tools.asset_pipeline.backdrop` follows those world references and exports only +the tree-family presentation meshes into `private/native-backdrops/.skate`. +Mesh-to-material and material-to-texture selection use binary GUIDs: texture +name suffixes differ from the global texture resource GUIDs. Original positions, +indices, UVs, masked alpha, two-sided rendering and baked lightmaps are retained. +No water surfaces or synthetic forest placements are added. + +The renderer loads this supplemental package after the district and before sky +setup, through the existing retail mesh/material path. `parse_render_only` is +an explicit data-reader entry point for presentation packages; ordinary playable +map loading still requires collision. Presentation loading rejects collision, +rails, doors, lights, routes and extensions other than WMET. This does not alter +the playable map's collision or gameplay state. Future installation runs prepare +the supplement automatically; existing assets can be updated with: + +```powershell +python -m tools.asset_pipeline.backdrop --game-root --assets +``` + +Static verification: the University supplement contains exactly the source's +786 triangles and matching vertex positions/indices/UVs. Zero of these triangles +match the 86 existing district TreeWall meshes after unordered-vertex 1 mm +quantization. Both University and Industrial supplements pass the runtime data +reader's offline inspector. Synthetic GUID tests and map-reader regression tests +pass. The executable was compiled but not launched; visual confirmation remains +for the user. The normal launcher clears SKATE_DEBUG_FOLIAGE. + +For integration with staged map switching, prepare this supplement alongside the +district, before applying sky lighting. Track its entities and assets in the +same map lifetime. The current adapter is in `retail_backdrop.rs`. diff --git a/tools/asset_pipeline/backdrop.py b/tools/asset_pipeline/backdrop.py new file mode 100644 index 00000000..ff6836c0 --- /dev/null +++ b/tools/asset_pipeline/backdrop.py @@ -0,0 +1,138 @@ +"""Retain foliage in authored global world models outside district streams.""" +import json +import hashlib +import struct +import sys +import tempfile +from pathlib import Path + +from .environment import Collections, field, key_hash +from .map_writer import write +from .sky import _texture +from tools.owned_game.big import BigArchive + + +def texture_groups(raw, table, channels): + """Read binary channel GUIDs; display-name suffixes are not resource IDs.""" + groups = [] + for section in table.entries: + if section.type_id != 0x00eb0005: + continue + o = section.f0 + h = struct.unpack_from('>8I', raw, o) + if not h[1] or (h[4] - h[3]) != h[1] * 32: + raise ValueError('Unexpected global model material layout') + for j in range(h[1]): + v = struct.unpack_from('>8I', raw, o+h[3]+j*32) + at = o+v[0] + kind = raw[at:raw.index(b'\0', at)].decode('ascii') + if kind == 'Name': + groups.append({}) + if kind in channels: + groups[-1][kind] = (v[4] << 32) | v[5] + return groups + + +def convert(game_root, assets, converted): + vendor = Path(__file__).resolve().parents[1] / 'vendor' + sys.path.insert(0, str(vendor / 'utt')) + sys.path.insert(0, str(vendor / 'university/tools/vanilla_map_extraction/tools')) + import numpy as np + import mdl_parser + import rx2_parser + import prepare_hawaiian_dream as prep + from retail_lightmap_uv import decode_lightmap_uvs + from retail_texture_decode import B5G6R5_FORMAT_ID, decode_b5g6r5 + + archive = BigArchive(game_root / 'data/big/miscload.big') + entries = {e.path: e for e in archive.entries} + collections = Collections(converted) + output = assets / 'private/native-backdrops' + output.mkdir(parents=True, exist_ok=True) + available = {p.stem.removeprefix('world') for p in (game_root/'data/content').glob('worldDIST_*.big')} + count = 0 + for (cls, _), row in collections.rows.items(): + if cls != key_hash('world'): + continue + own_stream = next((v['data'] for k, v in row['fields'].items() + if key_hash(k) == key_hash('WorldStream')), None) + if own_stream not in available: + continue + fields, _ = collections.resolve('world', row['key']) + # These global presentation resources are separate from WorldStream. + if key_hash('Hash_951898F6C0FA6856') not in fields: + continue + model_name = field(fields, 'Hash_951898F6C0FA6856') + if not model_name: + continue + texture_name = field(fields, 'Hash_CA5A157A65E75934') + raw = archive.read(entries['data/content/' + model_name + '.rx2']) + model = mdl_parser.parse_rx2(raw) + groups, bindings = prep._bind_material_groups_by_guid( + raw, prep._group_material_parameters(model.materials), len(model.meshes), + allow_import_order_fallback=False) + selected = [(i, prep._material_metadata(groups, i)) for i in range(len(model.meshes))] + selected = [(i, m) for i, m in selected if m['shader_name'] in ('tree.default', 'animated.tree')] + if not selected: + continue + textures = rx2_parser.parse_rx2(archive.read(entries['data/content/' + texture_name + '.rx2'])) + table = rx2_parser.RX2File(raw) + table.parse() + channel_groups = texture_groups(raw, table, prep.RETAIL_TEXTURE_CHANNELS) + with tempfile.TemporaryDirectory(prefix='skate-backdrop-') as temporary: + root = Path(temporary) + arrays, meshes, exported_textures = {}, [], {} + for i, material in selected: + mesh = model.meshes[i] + binding = bindings[i] + meshes.append(dict(material, index=i, name=groups[i]['Name'][0], + retail_material_guid=f"0x{binding['material_guid']:016X}", + retail_material_handle=f"0x{binding['material_handle']:08X}", + retail_material_group_index=binding['group_index'], source_offsets=mesh.source_offsets)) + arrays[f'vertices_{i}'] = mesh.vertices + arrays[f'faces_{i}'] = mesh.faces + arrays[f'uvs_{i}'] = mesh.uvs + arrays[f'normals_{i}'] = mesh.normals + lm = decode_lightmap_uvs(raw, vertex_buffer_offset=mesh.source_offsets['vertex_buffer'], + vertex_count=mesh.vertex_count, vertex_stride=mesh.vertex_stride, attributes=mesh.attributes) + if lm is not None: + arrays[f'lightmap_uvs_{i}'] = lm.values + for role, name in material['retail_texture_ids'].items(): + if name in exported_textures: + continue + texture = _texture(textures, channel_groups[binding['group_index']][role]) + rgba = texture.rgba + if texture.fmt_id == B5G6R5_FORMAT_ID: + rgba = decode_b5g6r5(textures.data[texture.data_offset:texture.data_offset+texture.buffer_size], + texture.width, texture.height) + path = name + '.rgba' + (root/path).write_bytes(rgba) + exported_textures[name] = dict(width=texture.width, height=texture.height, rgba=path) + np.savez(root/'model.npz', **arrays) + name = own_stream.removeprefix('DIST_') + manifest = dict(map_name=name, district_name=own_stream, models=[dict( + asset_id='0x'+hashlib.sha256(raw).hexdigest()[:16], npz='model.npz', meshes=meshes)], + textures=exported_textures, normal_texture_policy=dict(excluded_texture_ids=[]), grind_splines=[], + source_model=model_name, source_textures=texture_name, source_sha256=hashlib.sha256(raw).hexdigest()) + path = root/'manifest.json' + path.write_text(json.dumps(manifest)) + write(path, output/(name+'.skate'), None, render_only=True) + count += 1 + return count + + +if __name__ == '__main__': + import argparse + from .vlt import convert as convert_vlt + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--game-root', type=Path, required=True) + parser.add_argument('--assets', type=Path, required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix='skate-backdrop-db-') as work: + database = BigArchive(args.game_root/'data/big/db.big') + needed = {'skaterschema.bin', 'skaterschema.vlt', 'skatercollections.bin', 'skatercollections.vlt'} + database.extract_entries([e for e in database.entries if Path(e.path).name.lower() in needed], Path(work)) + stem = Path(work)/'data/db' + names = (Path(__file__).parent/'names.txt').read_text(encoding='utf-8').splitlines() + converted = convert_vlt(stem/'skaterschema', stem/'skatercollections', names) + print('Prepared', convert(args.game_root, args.assets, converted), 'authored foliage backdrops') diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index 4ad5cd10..070bbfe0 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -195,6 +195,9 @@ def install(iso,base,game_exe,report,game_root=None): report('Preparing retail sky domes') from .sky import convert as write_skies write_skies(game_root,stage/'assets',converted) + report('Preparing global foliage backdrops') + from .backdrop import convert as write_backdrops + write_backdrops(game_root,stage/'assets',converted) report('Validating skater, input and animation data') run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) archives=list((game_root/'data/content').glob('worldDIST_*.big')) diff --git a/tools/asset_pipeline/map_writer.py b/tools/asset_pipeline/map_writer.py index a17f776c..22843321 100644 --- a/tools/asset_pipeline/map_writer.py +++ b/tools/asset_pipeline/map_writer.py @@ -59,7 +59,7 @@ def spawn_point(manifest,root): if best is None:raise ValueError('No supported spawn surface in '+manifest['map_name']) return best[1] -def write(manifest_path,output,collision,report=lambda _:None): +def write(manifest_path,output,collision,report=lambda _:None, *, render_only=False): root=manifest_path.parent;m=json.loads(manifest_path.read_text());textures=m['textures'] ids={name:i+1 for i,name in enumerate(sorted(textures))} excluded=set(m['normal_texture_policy']['excluded_texture_ids']) @@ -115,7 +115,7 @@ def write(manifest_path,output,collision,report=lambda _:None): record['frame'][:,:3]=np.rint(np.clip(binormal,-1,1)*127).astype('i1');record['frame'][:,3]=np.rint(sign*127).astype('i1') if not np.isfinite(pos).all():raise ValueError('Non-finite map geometry') vertices.write(record.tobytes());indices.write((faces+nv).astype('8I', raw, 0, 0, len(parameters), 0, 32, + 32 + len(parameters)*32, 0, 0, 0) + at = 256 + for i, (kind, guid) in enumerate(parameters): + word = kind.encode() + b'\0' + raw[at:at+len(word)] = word + struct.pack_into('>8I', raw, 32+i*32, at, 0, 0, 0, + guid >> 32, guid & 0xffffffff, 0, 0) + at += len(word) + table = SimpleNamespace(entries=[SimpleNamespace(type_id=0xeb0005, f0=0)]) + self.assertEqual(texture_groups(bytes(raw), table, ('diffuse', 'normal', 'lightmap')), + [{'normal': 19}, {'diffuse': 0x123456789ABCDEF0, 'lightmap': 23}]) + + def test_texture_guid_uses_handle_not_list_position(self): + raw = bytearray(64) + struct.pack_into('>2I', raw, 0, 1, 8) + struct.pack_into('>6I', raw, 8, 0, 0, 0x12345678, 0x9abcdef0, 0, 27) + wanted = SimpleNamespace(index=27) + textures = SimpleNamespace(data=bytes(raw), + entries=[SimpleNamespace(type_id=0xeb000b, f0=0)], + textures=[SimpleNamespace(index=1), wanted]) + self.assertIs(_texture(textures, 0x123456789ABCDEF0), wanted) + with self.assertRaises(ValueError): + _texture(textures, 123) + + +if __name__ == '__main__': + unittest.main()