Restore global tree-wall backdrops omitted from district streams

crates/skate-data/examples/inspect_skate.rs

@@ -1,7 +1,18 @@
 fn main() {
     let mut failed = false;
+    let mut render_only = false;
     for path in std::env::args_os().skip(1) {
-        match skate_data::skate_map::SkateMap::load(std::path::Path::new(&path)) {
+        if path == "--render-only" {
+            render_only = true;
+            continue;
+        }
+        let result = if render_only {
+            std::fs::read(&path).map_err(|e| e.to_string())
+                .and_then(|bytes| skate_data::skate_map::SkateMap::parse_render_only(&bytes))
+        } else {
+            skate_data::skate_map::SkateMap::load(std::path::Path::new(&path))
+        };
+        match result {
             Ok(m) => {
                 println!(
                     "{} v{}: vertices={} triangles={} collision={} textures={} rails={} doors={} lights={} routes={} spawn={:?} heading={}",

crates/skate-data/src/skate_map.rs

@@ -343,6 +343,21 @@ impl SkateMap {
         Self::parse(&data).map_err(|e| format!("{}: {e}", path.display()))
     }
     pub fn parse(data: &[u8]) -> Result<Self, String> {
+        Self::parse_inner(data, true)
+    }
+    /// Decode a supplemental presentation package without requiring collision.
+    /// Playable map loading continues to use `parse`/`load`.
+    pub fn parse_render_only(data: &[u8]) -> Result<Self, String> {
+        let map = Self::parse_inner(data, false)?;
+        if !map.geometry.collision.is_empty() || !map.rails.is_empty()
+            || !map.doors.is_empty() || !map.lights.is_empty() || !map.routes.is_empty()
+            || map.extensions.iter().any(|e| e.tag != *b"WMET")
+        {
+            return Err("SKATE render-only package contains non-presentation data".into());
+        }
+        Ok(map)
+    }
+    fn parse_inner(data: &[u8], require_collision: bool) -> Result<Self, String> {
         let mut r = Reader { bytes: data, at: 0 };
         let magic = r.take(8)?;
         if &magic[..5] != b"SKATE"
@@ -637,7 +652,7 @@ impl SkateMap {
                 });
             }
         }
-        if geometry.collision.is_empty() && !extensions.iter().any(|e| e.tag == *b"RWCM" && e.schema == 1 && !e.payload.is_empty()) {
+        if require_collision && geometry.collision.is_empty() && !extensions.iter().any(|e| e.tag == *b"RWCM" && e.schema == 1 && !e.payload.is_empty()) {
             return Err("SKATE requires triangle collision or an embedded RWCM archive".into());
         }
         if r.at != data.len() {

crates/skate-data/tests/skate_map.rs

@@ -1,5 +1,22 @@
 use skate_data::skate_map::SkateMap;
 
+#[test]
+fn presentation_loading_does_not_relax_playable_map_validation() {
+    let mut data = fixture(1);
+    let counts = 8 + 4 + 4 + "Fixture".len() + (4 + 12) * 4;
+    data[counts + 16..counts + 24].fill(0); // No collision or rails.
+    let needle: Vec<u8> = [0u32, 1, 2].into_iter().flat_map(u32::to_le_bytes).collect();
+    let indices = data.windows(12).position(|w| w == needle).unwrap();
+    data.truncate(indices + 12);
+    assert!(SkateMap::parse(&data).unwrap_err().contains("requires triangle collision"));
+    let render = SkateMap::parse_render_only(&data).unwrap();
+    assert_eq!(render.geometry.indices, [0, 1, 2]);
+    assert!(render.geometry.collision.is_empty());
+    data[indices..indices+4].copy_from_slice(&99u32.to_le_bytes());
+    assert!(SkateMap::parse_render_only(&data).unwrap_err().contains("indices"));
+    assert!(SkateMap::parse_render_only(&fixture(1)).unwrap_err().contains("non-presentation"));
+}
+
 fn u(b: &mut Vec<u8>, value: u32) {
     b.extend(value.to_le_bytes());
 }

crates/skate-game/src/retail_backdrop.rs

@@ -0,0 +1,36 @@
+//! Global authored foliage that lives outside the district stream.
+use bevy::prelude::*;
+use skate_data::skate_map::SkateMap;
+use std::path::Path;
+
+pub(crate) fn spawn_backdrop(
+    name: &str,
+    asset_root: &Path,
+    commands: &mut Commands,
+    meshes: &mut Assets<Mesh>,
+    materials: &mut Assets<StandardMaterial>,
+    retail_materials: &mut Assets<super::RetailWorldMaterial>,
+    images: &mut Assets<Image>,
+) {
+    let path = asset_root.join("private/native-backdrops").join(format!("{name}.skate"));
+    if !path.is_file() {
+        return;
+    }
+    let map = match std::fs::read(&path).map_err(|e| e.to_string())
+        .and_then(|data| SkateMap::parse_render_only(&data)) {
+        Ok(map) => map,
+        Err(error) => {
+            error!("SKATE_BACKDROP: {}: {error}", path.display());
+            return;
+        }
+    };
+    // This package contributes presentation only; never route it into physics.
+    if map.name != name || !map.geometry.collision.is_empty()
+        || !map.lights.is_empty() || !map.doors.is_empty() || !map.rails.is_empty()
+    {
+        error!("SKATE_BACKDROP: invalid render-only package {}", path.display());
+        return;
+    }
+    info!("SKATE_BACKDROP: {name} authored foliage triangles={}", map.geometry.indices.len() / 3);
+    crate::skate_world::spawn(&map, commands, meshes, materials, retail_materials, images);
+}

crates/skate-game/src/retail_render.rs

@@ -40,6 +40,10 @@ pub(crate) struct RetailScene(pub bool);
 mod sky;
 pub(crate) use sky::{spawn_sky, RetailSkyMaterial};
 
+#[path = "retail_backdrop.rs"]
+mod backdrop;
+pub(crate) use backdrop::spawn_backdrop;
+
 #[derive(Component, ExtractComponent, Clone, Copy, ShaderType, Default)]
 pub(crate) struct RetailTone {
     pub enabled: Vec4,

crates/skate-game/src/world.rs

@@ -39,6 +39,7 @@ fn spawn(
         // Physics has already consumed the package. Render assets own their
         // uploaded data; keeping another full city package wastes gigabytes.
         crate::skate_world::spawn(&map, &mut commands, &mut meshes, &mut materials, &mut retail_materials, &mut images);
+        crate::retail_render::spawn_backdrop(&map.name, &config.asset_root, &mut commands, &mut meshes, &mut materials, &mut retail_materials, &mut images);
         crate::retail_render::spawn_sky(&map.name, &config.asset_root, &mut commands, &mut meshes, &mut images, &mut sky_materials, &mut retail_materials);
         return;
     }

docs/retail-renderer.md

@@ -143,3 +143,44 @@ reported viewpoint: solid foliage appearing in the missing area implicates the
 normal shading/alpha path; an empty area requires checking placement, camera or
 source selection. Unset the variable to restore normal rendering. This diagnostic
 has been prepared without launching the game.
+
+### Missing global tree-wall backdrop resolved (2026-09-08)
+
+The user-run opaque-foliage diagnostic left the same empty background. The
+missing surfaces are in `world/models/DIST_Water_University.rx2`, not in
+`DIST_University` or its proxy stream. Despite its name, this global model has
+both an ocean plane and a separate tree material: mesh 1 contains 1,572 vertices
+and 786 tree-wall triangles. World fields `951898F6C0FA6856` and
+`CA5A157A65E75934` select this model and its texture resource. The global
+Industrial model likewise contains 716 foliage triangles.
+
+`tools.asset_pipeline.backdrop` follows those world references and exports only
+the tree-family presentation meshes into `private/native-backdrops/<map>.skate`.
+Mesh-to-material and material-to-texture selection use binary GUIDs: texture
+name suffixes differ from the global texture resource GUIDs. Original positions,
+indices, UVs, masked alpha, two-sided rendering and baked lightmaps are retained.
+No water surfaces or synthetic forest placements are added.
+
+The renderer loads this supplemental package after the district and before sky
+setup, through the existing retail mesh/material path. `parse_render_only` is
+an explicit data-reader entry point for presentation packages; ordinary playable
+map loading still requires collision. Presentation loading rejects collision,
+rails, doors, lights, routes and extensions other than WMET. This does not alter
+the playable map's collision or gameplay state. Future installation runs prepare
+the supplement automatically; existing assets can be updated with:
+
+```powershell
+python -m tools.asset_pipeline.backdrop --game-root <owned-game> --assets <assets-root>
+```
+
+Static verification: the University supplement contains exactly the source's
+786 triangles and matching vertex positions/indices/UVs. Zero of these triangles
+match the 86 existing district TreeWall meshes after unordered-vertex 1 mm
+quantization. Both University and Industrial supplements pass the runtime data
+reader's offline inspector. Synthetic GUID tests and map-reader regression tests
+pass. The executable was compiled but not launched; visual confirmation remains
+for the user. The normal launcher clears SKATE_DEBUG_FOLIAGE.
+
+For integration with staged map switching, prepare this supplement alongside the
+district, before applying sky lighting. Track its entities and assets in the
+same map lifetime. The current adapter is in `retail_backdrop.rs`.

tools/asset_pipeline/backdrop.py

@@ -0,0 +1,138 @@
+"""Retain foliage in authored global world models outside district streams."""
+import json
+import hashlib
+import struct
+import sys
+import tempfile
+from pathlib import Path
+
+from .environment import Collections, field, key_hash
+from .map_writer import write
+from .sky import _texture
+from tools.owned_game.big import BigArchive
+
+
+def texture_groups(raw, table, channels):
+    """Read binary channel GUIDs; display-name suffixes are not resource IDs."""
+    groups = []
+    for section in table.entries:
+        if section.type_id != 0x00eb0005:
+            continue
+        o = section.f0
+        h = struct.unpack_from('>8I', raw, o)
+        if not h[1] or (h[4] - h[3]) != h[1] * 32:
+            raise ValueError('Unexpected global model material layout')
+        for j in range(h[1]):
+            v = struct.unpack_from('>8I', raw, o+h[3]+j*32)
+            at = o+v[0]
+            kind = raw[at:raw.index(b'\0', at)].decode('ascii')
+            if kind == 'Name':
+                groups.append({})
+            if kind in channels:
+                groups[-1][kind] = (v[4] << 32) | v[5]
+    return groups
+
+
+def convert(game_root, assets, converted):
+    vendor = Path(__file__).resolve().parents[1] / 'vendor'
+    sys.path.insert(0, str(vendor / 'utt'))
+    sys.path.insert(0, str(vendor / 'university/tools/vanilla_map_extraction/tools'))
+    import numpy as np
+    import mdl_parser
+    import rx2_parser
+    import prepare_hawaiian_dream as prep
+    from retail_lightmap_uv import decode_lightmap_uvs
+    from retail_texture_decode import B5G6R5_FORMAT_ID, decode_b5g6r5
+
+    archive = BigArchive(game_root / 'data/big/miscload.big')
+    entries = {e.path: e for e in archive.entries}
+    collections = Collections(converted)
+    output = assets / 'private/native-backdrops'
+    output.mkdir(parents=True, exist_ok=True)
+    available = {p.stem.removeprefix('world') for p in (game_root/'data/content').glob('worldDIST_*.big')}
+    count = 0
+    for (cls, _), row in collections.rows.items():
+        if cls != key_hash('world'):
+            continue
+        own_stream = next((v['data'] for k, v in row['fields'].items()
+                           if key_hash(k) == key_hash('WorldStream')), None)
+        if own_stream not in available:
+            continue
+        fields, _ = collections.resolve('world', row['key'])
+        # These global presentation resources are separate from WorldStream.
+        if key_hash('Hash_951898F6C0FA6856') not in fields:
+            continue
+        model_name = field(fields, 'Hash_951898F6C0FA6856')
+        if not model_name:
+            continue
+        texture_name = field(fields, 'Hash_CA5A157A65E75934')
+        raw = archive.read(entries['data/content/' + model_name + '.rx2'])
+        model = mdl_parser.parse_rx2(raw)
+        groups, bindings = prep._bind_material_groups_by_guid(
+            raw, prep._group_material_parameters(model.materials), len(model.meshes),
+            allow_import_order_fallback=False)
+        selected = [(i, prep._material_metadata(groups, i)) for i in range(len(model.meshes))]
+        selected = [(i, m) for i, m in selected if m['shader_name'] in ('tree.default', 'animated.tree')]
+        if not selected:
+            continue
+        textures = rx2_parser.parse_rx2(archive.read(entries['data/content/' + texture_name + '.rx2']))
+        table = rx2_parser.RX2File(raw)
+        table.parse()
+        channel_groups = texture_groups(raw, table, prep.RETAIL_TEXTURE_CHANNELS)
+        with tempfile.TemporaryDirectory(prefix='skate-backdrop-') as temporary:
+            root = Path(temporary)
+            arrays, meshes, exported_textures = {}, [], {}
+            for i, material in selected:
+                mesh = model.meshes[i]
+                binding = bindings[i]
+                meshes.append(dict(material, index=i, name=groups[i]['Name'][0],
+                    retail_material_guid=f"0x{binding['material_guid']:016X}",
+                    retail_material_handle=f"0x{binding['material_handle']:08X}",
+                    retail_material_group_index=binding['group_index'], source_offsets=mesh.source_offsets))
+                arrays[f'vertices_{i}'] = mesh.vertices
+                arrays[f'faces_{i}'] = mesh.faces
+                arrays[f'uvs_{i}'] = mesh.uvs
+                arrays[f'normals_{i}'] = mesh.normals
+                lm = decode_lightmap_uvs(raw, vertex_buffer_offset=mesh.source_offsets['vertex_buffer'],
+                    vertex_count=mesh.vertex_count, vertex_stride=mesh.vertex_stride, attributes=mesh.attributes)
+                if lm is not None:
+                    arrays[f'lightmap_uvs_{i}'] = lm.values
+                for role, name in material['retail_texture_ids'].items():
+                    if name in exported_textures:
+                        continue
+                    texture = _texture(textures, channel_groups[binding['group_index']][role])
+                    rgba = texture.rgba
+                    if texture.fmt_id == B5G6R5_FORMAT_ID:
+                        rgba = decode_b5g6r5(textures.data[texture.data_offset:texture.data_offset+texture.buffer_size],
+                                            texture.width, texture.height)
+                    path = name + '.rgba'
+                    (root/path).write_bytes(rgba)
+                    exported_textures[name] = dict(width=texture.width, height=texture.height, rgba=path)
+            np.savez(root/'model.npz', **arrays)
+            name = own_stream.removeprefix('DIST_')
+            manifest = dict(map_name=name, district_name=own_stream, models=[dict(
+                asset_id='0x'+hashlib.sha256(raw).hexdigest()[:16], npz='model.npz', meshes=meshes)],
+                textures=exported_textures, normal_texture_policy=dict(excluded_texture_ids=[]), grind_splines=[],
+                source_model=model_name, source_textures=texture_name, source_sha256=hashlib.sha256(raw).hexdigest())
+            path = root/'manifest.json'
+            path.write_text(json.dumps(manifest))
+            write(path, output/(name+'.skate'), None, render_only=True)
+            count += 1
+    return count
+
+
+if __name__ == '__main__':
+    import argparse
+    from .vlt import convert as convert_vlt
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--game-root', type=Path, required=True)
+    parser.add_argument('--assets', type=Path, required=True)
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory(prefix='skate-backdrop-db-') as work:
+        database = BigArchive(args.game_root/'data/big/db.big')
+        needed = {'skaterschema.bin', 'skaterschema.vlt', 'skatercollections.bin', 'skatercollections.vlt'}
+        database.extract_entries([e for e in database.entries if Path(e.path).name.lower() in needed], Path(work))
+        stem = Path(work)/'data/db'
+        names = (Path(__file__).parent/'names.txt').read_text(encoding='utf-8').splitlines()
+        converted = convert_vlt(stem/'skaterschema', stem/'skatercollections', names)
+        print('Prepared', convert(args.game_root, args.assets, converted), 'authored foliage backdrops')

tools/asset_pipeline/install.py

@@ -195,6 +195,9 @@ def install(iso,base,game_exe,report,game_root=None):
             report('Preparing retail sky domes')
             from .sky import convert as write_skies
             write_skies(game_root,stage/'assets',converted)
+            report('Preparing global foliage backdrops')
+            from .backdrop import convert as write_backdrops
+            write_backdrops(game_root,stage/'assets',converted)
             report('Validating skater, input and animation data')
             run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
             archives=list((game_root/'data/content').glob('worldDIST_*.big'))

tools/asset_pipeline/map_writer.py

@@ -59,7 +59,7 @@ def spawn_point(manifest,root):
     if best is None:raise ValueError('No supported spawn surface in '+manifest['map_name'])
     return best[1]
 
-def write(manifest_path,output,collision,report=lambda _:None):
+def write(manifest_path,output,collision,report=lambda _:None, *, render_only=False):
     root=manifest_path.parent;m=json.loads(manifest_path.read_text());textures=m['textures']
     ids={name:i+1 for i,name in enumerate(sorted(textures))}
     excluded=set(m['normal_texture_policy']['excluded_texture_ids'])
@@ -115,7 +115,7 @@ def write(manifest_path,output,collision,report=lambda _:None):
                     record['frame'][:,:3]=np.rint(np.clip(binormal,-1,1)*127).astype('i1');record['frame'][:,3]=np.rint(sign*127).astype('i1')
                 if not np.isfinite(pos).all():raise ValueError('Non-finite map geometry')
                 vertices.write(record.tobytes());indices.write((faces+nv).astype('<u4').tobytes());nv+=len(pos);ni+=faces.size;nm+=1
-    report('Selecting starting position: '+m['map_name']);spawn=spawn_point(m,root)
+    report('Selecting starting position: '+m['map_name']);spawn=(0.,0.,0.) if render_only else spawn_point(m,root)
     # Match the supplied exporter's environment defaults. Native sky shaders
     # remain a separate runtime feature; no geometry is synthesized here.
     environment=[.10,.36,.75,.64,.82,1.,.18,.24,.30,0.,11.,0.,18.,0.,
@@ -145,8 +145,10 @@ def write(manifest_path,output,collision,report=lambda _:None):
                 raw=bytes.fromhex(segment)
                 if len(raw)!=120:raise ValueError('Invalid native spline segment')
                 f.write(np.frombuffer(raw,dtype='>u4').astype('<u4').tobytes())
-        u(f,2)
-        for tag,data in [(b'RWCM',collision.read_bytes()),(b'WMET',json.dumps(m,separators=(',',':')).encode())]:
+        extensions=[(b'WMET',json.dumps(m,separators=(',',':')).encode())]
+        if not render_only:extensions.insert(0,(b'RWCM',collision.read_bytes()))
+        u(f,len(extensions))
+        for tag,data in extensions:
             f.write(tag);u(f,1,len(data));stored(f,data)
     report('Map written: '+m['map_name'])
 

tools/asset_pipeline/test_backdrop.py

@@ -0,0 +1,44 @@
+"""Global resource GUID selection without owned-game fixtures."""
+import struct
+import unittest
+from types import SimpleNamespace
+
+from .backdrop import texture_groups
+from .sky import _texture
+
+
+class BackdropTests(unittest.TestCase):
+    def test_missing_channels_do_not_shift_later_materials(self):
+        # Ocean has no diffuse here; foliage follows it. Name text and GUIDs
+        # deliberately differ, as they do in the global University resource.
+        parameters = [('Name', 0), ('normal', 19), ('Name', 0),
+                      ('diffuse', 0x123456789ABCDEF0), ('lightmap', 23)]
+        raw = bytearray(512)
+        struct.pack_into('>8I', raw, 0, 0, len(parameters), 0, 32,
+                         32 + len(parameters)*32, 0, 0, 0)
+        at = 256
+        for i, (kind, guid) in enumerate(parameters):
+            word = kind.encode() + b'\0'
+            raw[at:at+len(word)] = word
+            struct.pack_into('>8I', raw, 32+i*32, at, 0, 0, 0,
+                             guid >> 32, guid & 0xffffffff, 0, 0)
+            at += len(word)
+        table = SimpleNamespace(entries=[SimpleNamespace(type_id=0xeb0005, f0=0)])
+        self.assertEqual(texture_groups(bytes(raw), table, ('diffuse', 'normal', 'lightmap')),
+                         [{'normal': 19}, {'diffuse': 0x123456789ABCDEF0, 'lightmap': 23}])
+
+    def test_texture_guid_uses_handle_not_list_position(self):
+        raw = bytearray(64)
+        struct.pack_into('>2I', raw, 0, 1, 8)
+        struct.pack_into('>6I', raw, 8, 0, 0, 0x12345678, 0x9abcdef0, 0, 27)
+        wanted = SimpleNamespace(index=27)
+        textures = SimpleNamespace(data=bytes(raw),
+            entries=[SimpleNamespace(type_id=0xeb000b, f0=0)],
+            textures=[SimpleNamespace(index=1), wanted])
+        self.assertIs(_texture(textures, 0x123456789ABCDEF0), wanted)
+        with self.assertRaises(ValueError):
+            _texture(textures, 123)
+
+
+if __name__ == '__main__':
+    unittest.main()