Merge retail scenery, lighting and travel with current map lifecycle

tools/asset_pipeline/backdrop.py

@@ -0,0 +1,138 @@
+"""Retain foliage in authored global world models outside district streams."""
+import json
+import hashlib
+import struct
+import sys
+import tempfile
+from pathlib import Path
+
+from .environment import Collections, field, key_hash
+from .map_writer import write
+from .sky import _texture
+from tools.owned_game.big import BigArchive
+
+
+def texture_groups(raw, table, channels):
+    """Read binary channel GUIDs; display-name suffixes are not resource IDs."""
+    groups = []
+    for section in table.entries:
+        if section.type_id != 0x00eb0005:
+            continue
+        o = section.f0
+        h = struct.unpack_from('>8I', raw, o)
+        if not h[1] or (h[4] - h[3]) != h[1] * 32:
+            raise ValueError('Unexpected global model material layout')
+        for j in range(h[1]):
+            v = struct.unpack_from('>8I', raw, o+h[3]+j*32)
+            at = o+v[0]
+            kind = raw[at:raw.index(b'\0', at)].decode('ascii')
+            if kind == 'Name':
+                groups.append({})
+            if kind in channels:
+                groups[-1][kind] = (v[4] << 32) | v[5]
+    return groups
+
+
+def convert(game_root, assets, converted):
+    vendor = Path(__file__).resolve().parents[1] / 'vendor'
+    sys.path.insert(0, str(vendor / 'utt'))
+    sys.path.insert(0, str(vendor / 'university/tools/vanilla_map_extraction/tools'))
+    import numpy as np
+    import mdl_parser
+    import rx2_parser
+    import prepare_hawaiian_dream as prep
+    from retail_lightmap_uv import decode_lightmap_uvs
+    from retail_texture_decode import B5G6R5_FORMAT_ID, decode_b5g6r5
+
+    archive = BigArchive(game_root / 'data/big/miscload.big')
+    entries = {e.path: e for e in archive.entries}
+    collections = Collections(converted)
+    output = assets / 'private/native-backdrops'
+    output.mkdir(parents=True, exist_ok=True)
+    available = {p.stem.removeprefix('world') for p in (game_root/'data/content').glob('worldDIST_*.big')}
+    count = 0
+    for (cls, _), row in collections.rows.items():
+        if cls != key_hash('world'):
+            continue
+        own_stream = next((v['data'] for k, v in row['fields'].items()
+                           if key_hash(k) == key_hash('WorldStream')), None)
+        if own_stream not in available:
+            continue
+        fields, _ = collections.resolve('world', row['key'])
+        # These global presentation resources are separate from WorldStream.
+        if key_hash('Hash_951898F6C0FA6856') not in fields:
+            continue
+        model_name = field(fields, 'Hash_951898F6C0FA6856')
+        if not model_name:
+            continue
+        texture_name = field(fields, 'Hash_CA5A157A65E75934')
+        raw = archive.read(entries['data/content/' + model_name + '.rx2'])
+        model = mdl_parser.parse_rx2(raw)
+        groups, bindings = prep._bind_material_groups_by_guid(
+            raw, prep._group_material_parameters(model.materials), len(model.meshes),
+            allow_import_order_fallback=False)
+        selected = [(i, prep._material_metadata(groups, i)) for i in range(len(model.meshes))]
+        selected = [(i, m) for i, m in selected if m['shader_name'] in ('tree.default', 'animated.tree', 'ocean.reflection', 'environment.reflective_simple')]
+        if not selected:
+            continue
+        textures = rx2_parser.parse_rx2(archive.read(entries['data/content/' + texture_name + '.rx2']))
+        table = rx2_parser.RX2File(raw)
+        table.parse()
+        channel_groups = texture_groups(raw, table, prep.RETAIL_TEXTURE_CHANNELS)
+        with tempfile.TemporaryDirectory(prefix='skate-backdrop-') as temporary:
+            root = Path(temporary)
+            arrays, meshes, exported_textures = {}, [], {}
+            for i, material in selected:
+                mesh = model.meshes[i]
+                binding = bindings[i]
+                meshes.append(dict(material, index=i, name=groups[i]['Name'][0],
+                    retail_material_guid=f"0x{binding['material_guid']:016X}",
+                    retail_material_handle=f"0x{binding['material_handle']:08X}",
+                    retail_material_group_index=binding['group_index'], source_offsets=mesh.source_offsets))
+                arrays[f'vertices_{i}'] = mesh.vertices
+                arrays[f'faces_{i}'] = mesh.faces
+                arrays[f'uvs_{i}'] = mesh.uvs
+                arrays[f'normals_{i}'] = mesh.normals
+                lm = decode_lightmap_uvs(raw, vertex_buffer_offset=mesh.source_offsets['vertex_buffer'],
+                    vertex_count=mesh.vertex_count, vertex_stride=mesh.vertex_stride, attributes=mesh.attributes)
+                if lm is not None:
+                    arrays[f'lightmap_uvs_{i}'] = lm.values
+                for role, name in material['retail_texture_ids'].items():
+                    if name in exported_textures:
+                        continue
+                    texture = _texture(textures, channel_groups[binding['group_index']][role])
+                    rgba = texture.rgba
+                    if texture.fmt_id == B5G6R5_FORMAT_ID:
+                        rgba = decode_b5g6r5(textures.data[texture.data_offset:texture.data_offset+texture.buffer_size],
+                                            texture.width, texture.height)
+                    path = name + '.rgba'
+                    (root/path).write_bytes(rgba)
+                    exported_textures[name] = dict(width=texture.width, height=texture.height, rgba=path)
+            np.savez(root/'model.npz', **arrays)
+            name = own_stream.removeprefix('DIST_')
+            manifest = dict(map_name=name, district_name=own_stream, models=[dict(
+                asset_id='0x'+hashlib.sha256(raw).hexdigest()[:16], npz='model.npz', meshes=meshes)],
+                textures=exported_textures, normal_texture_policy=dict(excluded_texture_ids=[]), grind_splines=[],
+                source_model=model_name, source_textures=texture_name, source_sha256=hashlib.sha256(raw).hexdigest())
+            path = root/'manifest.json'
+            path.write_text(json.dumps(manifest))
+            write(path, output/(name+'.skate'), None, render_only=True)
+            count += 1
+    return count
+
+
+if __name__ == '__main__':
+    import argparse
+    from .vlt import convert as convert_vlt
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--game-root', type=Path, required=True)
+    parser.add_argument('--assets', type=Path, required=True)
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory(prefix='skate-backdrop-db-') as work:
+        database = BigArchive(args.game_root/'data/big/db.big')
+        needed = {'skaterschema.bin', 'skaterschema.vlt', 'skatercollections.bin', 'skatercollections.vlt'}
+        database.extract_entries([e for e in database.entries if Path(e.path).name.lower() in needed], Path(work))
+        stem = Path(work)/'data/db'
+        names = (Path(__file__).parent/'names.txt').read_text(encoding='utf-8').splitlines()
+        converted = convert_vlt(stem/'skaterschema', stem/'skatercollections', names)
+        print('Prepared', convert(args.game_root, args.assets, converted), 'authored foliage backdrops')

tools/asset_pipeline/character_lighting.py

@@ -0,0 +1,55 @@
+"""Export authored character shader parameters and default SH coefficients."""
+import json
+import struct
+import sys
+from PIL import Image
+from pathlib import Path
+from .environment import Collections, key_hash
+
+
+def convert(models, private, converted):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'vendor/utt'))
+    import mdl_parser
+    collections = Collections(converted)
+    result = {'materials': {}}
+    recipe = json.loads((Path(__file__).resolve().parents[1] / 'default_skater_retail_manifest.json').read_text())
+    components = {c['slot']: c for c in recipe['components']}
+    for slot in sorted(models.iterdir()):
+        if not slot.is_dir():
+            continue
+        files = list(slot.glob('*.rx2'))
+        if len(files) != 1:
+            raise ValueError('Ambiguous character lighting model: '+slot.name)
+        model = mdl_parser.parse_rx2(files[0].read_bytes())
+        shaders = {p.value for p in model.materials if p.kind == 'AttribulatorMaterialName'}
+        if len(shaders) != 1:
+            raise ValueError('Ambiguous character shader: '+slot.name)
+        shader = shaders.pop()
+        family, key = shader.split('.', 1)
+        fields, _ = collections.resolve('material_'+family, key)
+        rows = fields[key_hash('m_params')]['array']['items']
+        result['materials']['Retail_'+slot.name] = dict(shader=shader,
+            params=[struct.unpack('>4f', bytes.fromhex(row)) for row in rows])
+        specular = components[slot.name]['textures'].get('specular')
+        if specular:
+            # The native skin shader consumes R, not luminance or roughness.
+            source = private / 'default_skater/textures/decoded' / (specular+'.png')
+            destination = private / 'native-character' / (slot.name+'_specular.png')
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            with Image.open(source) as image:
+                image.convert('RGB').getchannel('R').save(destination)
+            result['materials']['Retail_'+slot.name]['specular'] = 'private/native-character/'+destination.name
+    fields, _ = collections.resolve('Hash_6A2B5946514B8490', 'freeskate')
+    result['default_sh'] = [struct.unpack('>3f', bytes.fromhex(row))
+        for row in fields[key_hash('Hash_F45DFCAB6EC195CD')]['array']['items']]
+    (private/'character-lighting.json').write_text(json.dumps(result))
+
+
+if __name__ == '__main__':
+    import argparse
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--models', type=Path, required=True)
+    p.add_argument('--private', type=Path, required=True)
+    p.add_argument('--collections', type=Path, required=True)
+    a = p.parse_args()
+    convert(a.models, a.private, json.loads(a.collections.read_text()))

tools/asset_pipeline/dynamic_props.py

@@ -0,0 +1,221 @@
+"""Export authored district DMO locators and their referenced model geometry.
+
+This preserves initial placement, not DMO simulation. Native 0x825876D0 reads
+the locator matrix and template ID at +112; RX2 EB001D records are 128 bytes.
+No name matching, random placement, ground snapping or collision synthesis.
+"""
+import argparse
+import copy
+import hashlib
+import json
+import struct
+import sys
+import tempfile
+from pathlib import Path
+
+import numpy as np
+from .map_writer import write
+from tools.owned_game.big import BigArchive
+
+
+def sections(raw):
+    if raw[:7] != b'\x89RW4xb2':
+        raise ValueError('Expected Xbox RX2')
+    count = struct.unpack_from('>I', raw, 32)[0]
+    table = struct.unpack_from('>I', raw, 48)[0]
+    if table + count*24 > len(raw):
+        raise ValueError('Truncated RX2 section table')
+    return [struct.unpack_from('>6I', raw, table+i*24) for i in range(count)]
+
+
+def matrix(raw, at):
+    value = np.array(struct.unpack_from('>16f', raw, at)).reshape(4, 4)
+    if not np.isfinite(value).all() or not np.allclose(value[:, 3], [0, 0, 0, 1], atol=1e-5):
+        raise ValueError('Invalid DMO affine matrix')
+    if abs(np.linalg.det(value[:3, :3])) < 1e-8:
+        raise ValueError('Singular DMO matrix')
+    return value
+
+
+def records(raw, kind, stride):
+    for offset, _, size, _, _, type_id in sections(raw):
+        if type_id != kind:
+            continue
+        if offset+size > len(raw) or size < 32:
+            raise ValueError('Invalid DMO section extent')
+        count = struct.unpack_from('>I', raw, offset+4)[0]
+        start, strings = struct.unpack_from('>2I', raw, offset+12)
+        if start != 32 or strings != start+count*stride or strings > size:
+            raise ValueError('Unexpected DMO record layout')
+        for index in range(count):
+            yield offset, offset+start+index*stride, size
+
+
+def locators(raw):
+    result = []
+    for base, at, size in records(raw, 0xEB001D, 128):
+        name_offset = struct.unpack_from('>I', raw, at+124)[0]
+        if not 32 <= name_offset < size:
+            raise ValueError('Invalid DMO locator name offset')
+        end = raw.index(b'\0', base+name_offset, base+size)
+        result.append(dict(instance_id=f'{struct.unpack_from(">Q", raw, at+96)[0]:016X}',
+            locator_id=f'{struct.unpack_from(">Q", raw, at+104)[0]:016X}',
+            template_id=f'{struct.unpack_from(">Q", raw, at+112)[0]:016X}',
+            matrix=matrix(raw, at).tolist(), name=raw[base+name_offset:end].decode('utf-8'),
+            source_offset=at, bounds=np.array(struct.unpack_from('>8f', raw, at+64)).reshape(2, 4)[:, :3].tolist()))
+    return result
+
+
+def template_meshes(raw):
+    """Resolve EB000D tInstance -> EB0001 model -> EB0023 mesh -> declaration."""
+    table = sections(raw)
+    result = {}
+    for _, at, _ in records(raw, 0xEB000D, 160):
+        key = f'{struct.unpack_from(">Q", raw, at+104)[0]:016X}'
+        index = struct.unpack_from('>I', raw, at+128)[0]
+        if index >= len(table) or table[index][5] != 0xEB0001:
+            raise ValueError('Unresolved DMO model reference')
+        base, _, size, _, _, _ = table[index]
+        start = struct.unpack_from('>I', raw, base+36)[0]
+        count = struct.unpack_from('>H', raw, base+48)[0]
+        if base+size > len(raw) or start+count*8 > size:
+            raise ValueError('Invalid DMO model mesh table')
+        mesh_info = []
+        for j in range(count):
+            mesh_index = struct.unpack_from('>I', raw, base+start+j*8)[0]
+            if mesh_index >= len(table) or table[mesh_index][5] != 0xEB0023:
+                raise ValueError('Unresolved DMO mesh reference')
+            descriptor = struct.unpack_from('>I', raw, table[mesh_index][0]+40)[0]
+            if descriptor >= len(table) or table[descriptor][5] != 0x200E9:
+                raise ValueError('Unresolved DMO vertex declaration')
+            mesh_info.append(table[descriptor][0])
+        if key in result:
+            raise ValueError('Duplicate DMO template ID')
+        result[key] = dict(mesh_info=mesh_info, matrix=matrix(raw, at),
+                          model_matrix=matrix(raw, base+struct.unpack_from('>I', raw, base+32)[0]))
+    return result
+
+
+def transform_mesh(arrays, index, transform):
+    values = {key: np.array(value, copy=True) for key, value in arrays.items() if key.endswith('_'+str(index))}
+    key = f'vertices_{index}'
+    values[key] = (values[key] @ transform[:3, :3] + transform[3, :3]).astype('f4')
+    normal_matrix = np.linalg.inv(transform[:3, :3]).T
+    for prefix in ('normals', 'retail_normals'):
+        key = f'{prefix}_{index}'
+        if key in values:
+            n = values[key] @ normal_matrix
+            values[key] = (n/np.maximum(np.linalg.norm(n, axis=1, keepdims=True), 1e-20)).astype('f4')
+    if np.linalg.det(transform[:3, :3]) < 0:
+        values[f'faces_{index}'] = values[f'faces_{index}'][:, [0, 2, 1]]
+    return values
+
+
+def catalog(cache_roots):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'vendor/utt'))
+    import rx2_parser
+    from .backdrop import texture_groups
+    templates, textures = {}, {}
+    for root in cache_roots:
+        m = json.loads((root/'manifest.json').read_text())
+        for key, value in m['textures'].items():
+            entry = dict(value)
+            for path_key in ('rgba', 'png'):
+                if path_key in entry:
+                    entry[path_key] = str((root/entry[path_key]).resolve())
+            if key in textures and Path(textures[key]['rgba']).read_bytes() != Path(entry['rgba']).read_bytes():
+                raise ValueError('Conflicting DMO texture '+key)
+            textures[key] = entry
+        for model in m['models']:
+            raw = (root/model['rx2']).read_bytes()
+            table = rx2_parser.RX2File(raw)
+            table.parse()
+            roles = set(role for mesh in model['meshes'] for role in mesh['retail_texture_ids'])
+            bindings = texture_groups(raw, table, roles)
+            for key, definition in template_meshes(raw).items():
+                meshes = [copy.deepcopy(mesh) for mesh in model['meshes'] if mesh['source_offsets']['mesh_info'] in definition['mesh_info']]
+                for mesh in meshes:
+                    # DMO resources have a high-bit namespace that display names
+                    # omit. Bind the binary channel GUID; never strip that bit.
+                    group = bindings[mesh['retail_material_group_index']]
+                    mesh['retail_texture_ids'] = {role:f'0x{group[role]:016x}' for role in mesh['retail_texture_ids']}
+                    mesh['texture_id'] = mesh['retail_texture_ids'].get('diffuse', mesh['retail_texture_ids'].get('transparent'))
+                if len(meshes) != len(definition['mesh_info']):
+                    raise ValueError('DMO model geometry omitted '+key)
+                if key in templates:
+                    raise ValueError('Conflicting DMO template '+key)
+                templates[key] = dict(definition, meshes=meshes, npz=root/model['npz'], asset_id=model['asset_id'])
+    return templates, textures
+
+
+def export(manifest_path, cache_roots, output):
+    district = json.loads(manifest_path.read_text())
+    templates, textures = catalog(cache_roots)
+    placements = {}
+    for source in district['simulation_assets']:
+        raw = (manifest_path.parent/source['rx2']).read_bytes()
+        for item in locators(raw):
+            item['source_asset'] = source['asset_id']
+            item['source_sha256'] = hashlib.sha256(raw).hexdigest()
+            key = item['instance_id']
+            if key in placements and placements[key] != item:
+                raise ValueError('Conflicting DMO locator '+key)
+            placements[key] = item
+    report = dict(map=district['map_name'], instances=[], unresolved=[], simulation='initial placement only')
+    with tempfile.TemporaryDirectory(prefix='skate-dmo-') as work:
+        root = Path(work); models = []; used = set()
+        for number, item in enumerate(placements.values()):
+            template = templates.get(item['template_id'])
+            if template is None:
+                report['unresolved'].append(item)
+                continue
+            transform = template['model_matrix'] @ template['matrix'] @ np.array(item['matrix'])
+            arrays = {}; meshes = copy.deepcopy(template['meshes'])
+            with np.load(template['npz'], allow_pickle=False) as original:
+                for mesh in meshes:
+                    arrays.update(transform_mesh(original, mesh['index'], transform))
+                    used.update(mesh['retail_texture_ids'].values())
+            path = f'{number}.npz'; np.savez(root/path, **arrays)
+            models.append(dict(asset_id=item['instance_id'], meshes=meshes, npz=path))
+            report['instances'].append(dict(item, model_asset=template['asset_id'], meshes=len(meshes)))
+        if models:
+            manifest = dict(map_name=district['map_name'], district_name=district['district_name'],
+                models=models, textures={key:textures[key] for key in used},
+                normal_texture_policy=dict(excluded_texture_ids=[]), grind_splines=[])
+            (root/'manifest.json').write_text(json.dumps(manifest))
+            # Publish only a complete package; preserve the previous one on failure.
+            temporary = output.with_suffix('.skate.new')
+            try:
+                write(root/'manifest.json', temporary, None, render_only=True)
+                temporary.replace(output)
+            finally:
+                temporary.unlink(missing_ok=True)
+        else:
+            output.unlink(missing_ok=True)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.with_suffix('.json').write_text(json.dumps(report, indent=2))
+    return len(report['instances']), len(report['unresolved'])
+
+
+def prepare_catalog(game_root, work):
+    vendor = Path(__file__).resolve().parents[1]/'vendor'
+    sys.path.insert(0, str(vendor/'university/tools/vanilla_map_extraction/tools'))
+    from prepare_hawaiian_dream import prepare
+    archive = BigArchive(game_root/'data/content/worlddmo.big')
+    archive.extract_entries(archive.entries, work/'raw')
+    roots = []
+    for stream in (work/'raw/data/content/world/dmo').iterdir():
+        root = work/'cache'/stream.name
+        prepare(stream_directory=stream, output_root=root, utt_root=vendor/'utt',
+                district_name=stream.name, map_name=stream.name, raw_texture_cache=True)
+        roots.append(root)
+    return roots
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--manifest', type=Path, required=True)
+    p.add_argument('--cache', type=Path, nargs='+', required=True)
+    p.add_argument('--output', type=Path, required=True)
+    a = p.parse_args()
+    print('DMO placed/unresolved:', export(a.manifest, a.cache, a.output))

tools/asset_pipeline/environment.py

@@ -0,0 +1,99 @@
+"""Resolve authored world/environment references without inventing frame rows."""
+import math
+import struct
+from .vlt import hash64
+
+# Names recovered from schema/debug strings; unnamed fields keep their hashes.
+NAMES = ('world', 'render_locations', 'material_sky', 'material_fog',
+         'sun_position', 'm_params', 'fog_far', 'fog_near', 'fog_colour',
+         'fog_power', 'fog_max')
+
+
+def key_hash(name):
+    return int(name[5:], 16) if name.startswith('Hash_') else hash64(name)
+
+
+class Collections:
+    def __init__(self, data):
+        self.rows = {(key_hash(r['class']), key_hash(r['key'])): r
+                     for r in data['collections']}
+
+    def resolve(self, cls, key):
+        cls = key_hash(cls) if isinstance(cls, str) else cls
+        key = key_hash(key) if isinstance(key, str) else key
+        fields, chain, seen = {}, [], set()
+        while key:
+            if key in seen:
+                raise ValueError('Cyclic environment inheritance')
+            seen.add(key)
+            row = self.rows[cls, key]
+            chain.append(row['key'])
+            for name, value in row['fields'].items():
+                fields.setdefault(key_hash(name), value)
+            key = key_hash(row['parent'])
+        return fields, chain
+
+
+def field(fields, name):
+    return fields[key_hash(name)]['data']
+
+
+def floats(raw, count):
+    values = struct.unpack('>' + 'f' * count, bytes.fromhex(raw))
+    if not all(math.isfinite(v) for v in values):
+        raise ValueError('Non-finite environment value')
+    return list(values)
+
+
+def world_environment(collections, world_key):
+    world, world_chain = collections.resolve('world', world_key)
+    location_key = struct.unpack('>Q', bytes.fromhex(
+        field(world, 'Hash_7E23D10785C43717'))[:8])[0]
+    location, location_chain = collections.resolve('render_locations', location_key)
+    sun = floats(field(location, 'sun_position'), 3)
+    length = math.sqrt(sum(v*v for v in sun))
+    if length < 1e-6:
+        raise ValueError('Invalid authored sun direction')
+    fog_cls, fog_key, _ = struct.unpack('>3Q', bytes.fromhex(field(location, 'material_fog')))
+    fog, fog_chain = collections.resolve(fog_cls, fog_key)
+    return {
+        'world_chain': world_chain, 'location_chain': location_chain,
+        'sun_direction': [v/length for v in sun],
+        'anchor_height': floats(field(location, 'Hash_2E9AAECD1C29F81F'), 1)[0],
+        'sky_model': field(world, 'Hash_C7A0A84F018E87BA'),
+        'sky_textures': field(world, 'Hash_2A0BF629355AFB15'),
+        # Keep the authored values alongside the derived native shader rows.
+        'fog': {'chain': fog_chain, **{n: floats(field(fog, n), 4 if n == 'fog_colour' else 1)
+                for n in ('fog_near', 'fog_far', 'fog_colour', 'fog_power', 'fog_max')}},
+    }
+
+
+def sky_parameters(collections, shader):
+    effect, key = shader.split('.', 1)
+    if effect != 'sky':
+        raise ValueError('Not a sky material: ' + shader)
+    fields, chain = collections.resolve('material_sky', key)
+    params = fields[hash64('m_params')].get('array')
+    if not params or not params['items']:
+        raise ValueError('Sky parameters require complete VLT array payloads')
+    row = floats(params['items'][0], 4)
+    if row[0] <= 0 or row[1] < 0:
+        raise ValueError('Invalid authored sky parameters')
+    return {'material_chain': chain, 'sun_scale': row[0], 'multiplier': row[1]}
+
+
+def fog_parameters(fog):
+    """TU3 0x828012D0: schema offsets 20/28/16/24 -> ramp and colour.
+
+    0x82F825F0 initializes the negative-one vector used for colour alpha.
+    Colour RGB is already linear here; do not square it again.
+    """
+    near, far, power, maximum = (fog[n][0] for n in
+        ('fog_near', 'fog_far', 'fog_power', 'fog_max'))
+    colour = fog['fog_colour']
+    if (len(colour) != 4 or not all(math.isfinite(v) for v in
+            [near, far, power, maximum, *colour]) or far <= near or near < 0
+            or power <= 0 or not 0 <= maximum <= 1):
+        raise ValueError('Invalid authored fog range/colour')
+    return {'ramp': [1 / (far - near), -near / (far - near), power, 0],
+            'colour': [v * maximum for v in colour[:3]] + [-maximum]}

tools/asset_pipeline/install.py

@@ -120,6 +120,12 @@ def convert_map(archive,work,maps,stage,game_exe,log,report):
     build_archive(manifest_path,collision)
     final=maps/(label+'.skate')
     write_map(manifest_path,final,collision,report)
+    from .dynamic_props import export as write_props
+    caches=list((work/'dmo/cache').glob('DMO_*'))
+    if not caches:
+        raise RuntimeError('Missing prepared DMO catalog')
+    placed, unresolved=write_props(manifest_path,caches,stage/'assets/private/native-props'/(label+'.skate'))
+    report(f'{label}: placed {placed} authored DMO instances, {unresolved} unresolved templates')
     report('Checking converted map: '+label)
     run([game_exe,'--assets',stage/'assets','--map',final,'--check-assets'],log,report)
     entry={'name':label,'path':'maps/'+final.name,'sha256':digest(final)}
@@ -188,13 +194,26 @@ def install(iso,base,game_exe,report,game_root=None):
             report('Building the skater model and rig')
             from .character_glb import convert as write_character
             write_character(character/'selected/models',private,manifest)
+            from .character_lighting import convert as write_character_lighting
+            write_character_lighting(character/'selected/models',private,converted)
             game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000',
                            'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph',
                            'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'}
             (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
             report('Preparing retail sky domes')
             from .sky import convert as write_skies
-            write_skies(game_root,stage/'assets')
+            write_skies(game_root,stage/'assets',converted)
+            from .render_parameters import convert as write_render_parameters
+            write_render_parameters(stage/'assets',converted)
+            report('Extracting original travel destinations and location names')
+            from .teleports import convert as write_teleports
+            write_teleports(game_root,stage/'assets',converted)
+            report('Preparing global foliage backdrops')
+            from .backdrop import convert as write_backdrops
+            write_backdrops(game_root,stage/'assets',converted)
+            report('Preparing authored movable-object models')
+            from .dynamic_props import prepare_catalog
+            prepare_catalog(game_root,work/'dmo')
             report('Validating skater, input and animation data')
             run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
             archives=list((game_root/'data/content').glob('worldDIST_*.big'))

tools/asset_pipeline/irradiance.py

@@ -0,0 +1,51 @@
+"""Export retail RWOBJECTTYPE_IRRADIANCEDATA (0xEB0024) without resampling."""
+import json
+import math
+import struct
+from pathlib import Path
+
+
+def decode(raw):
+    if raw[:7] != b'\x89RW4xb2':
+        raise ValueError('Not an Xbox RX2 resource')
+    count, table = struct.unpack_from('>I', raw, 32)[0], struct.unpack_from('>I', raw, 48)[0]
+    groups = []
+    for i in range(count):
+        offset, _, size, _, _, kind = struct.unpack_from('>6I', raw, table + i*24)
+        if kind != 0xEB0024:
+            continue
+        n, start = struct.unpack_from('>2I', raw, offset)
+        if start < 8 or start+n*160 > size or offset+size > len(raw):
+            raise ValueError('Invalid irradiance record extent')
+        records = []
+        for j in range(n):
+            values = struct.unpack_from('>39fI', raw, offset+start+j*160)
+            if not all(math.isfinite(x) for x in values[:39]):
+                raise ValueError('Non-finite irradiance sample')
+            records.append(struct.pack('<39fI', *values))
+        if records:
+            groups.append(records)
+    return groups
+
+
+def write(manifest_path, output):
+    manifest = json.loads(manifest_path.read_text())
+    groups = []
+    for asset in manifest['other_presentation_assets']:
+        groups.extend(decode((manifest_path.parent / asset['rx2']).read_bytes()))
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open('wb') as f:
+        f.write(b'SHP1' + struct.pack('<I', len(groups)))
+        for records in groups:
+            f.write(struct.pack('<I', len(records)))
+            f.write(b''.join(records))
+    return sum(map(len, groups))
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--manifest', type=Path, required=True)
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    print('Exported', write(args.manifest, args.output), 'irradiance samples')

tools/asset_pipeline/map_writer.py

@@ -59,7 +59,7 @@ def spawn_point(manifest,root):
     if best is None:raise ValueError('No supported spawn surface in '+manifest['map_name'])
     return best[1]
 
-def write(manifest_path,output,collision,report=lambda _:None):
+def write(manifest_path,output,collision,report=lambda _:None, *, render_only=False):
     root=manifest_path.parent;m=json.loads(manifest_path.read_text());textures=m['textures']
     ids={name:i+1 for i,name in enumerate(sorted(textures))}
     excluded=set(m['normal_texture_policy']['excluded_texture_ids'])
@@ -115,7 +115,7 @@ def write(manifest_path,output,collision,report=lambda _:None):
                     record['frame'][:,:3]=np.rint(np.clip(binormal,-1,1)*127).astype('i1');record['frame'][:,3]=np.rint(sign*127).astype('i1')
                 if not np.isfinite(pos).all():raise ValueError('Non-finite map geometry')
                 vertices.write(record.tobytes());indices.write((faces+nv).astype('<u4').tobytes());nv+=len(pos);ni+=faces.size;nm+=1
-    report('Selecting starting position: '+m['map_name']);spawn=spawn_point(m,root)
+    report('Selecting starting position: '+m['map_name']);spawn=(0.,0.,0.) if render_only else spawn_point(m,root)
     # Match the supplied exporter's environment defaults. Native sky shaders
     # remain a separate runtime feature; no geometry is synthesized here.
     environment=[.10,.36,.75,.64,.82,1.,.18,.24,.30,0.,11.,0.,18.,0.,
@@ -145,10 +145,15 @@ def write(manifest_path,output,collision,report=lambda _:None):
                 raw=bytes.fromhex(segment)
                 if len(raw)!=120:raise ValueError('Invalid native spline segment')
                 f.write(np.frombuffer(raw,dtype='>u4').astype('<u4').tobytes())
-        u(f,2)
-        for tag,data in [(b'RWCM',collision.read_bytes()),(b'WMET',json.dumps(m,separators=(',',':')).encode())]:
+        extensions=[(b'WMET',json.dumps(m,separators=(',',':')).encode())]
+        if not render_only:extensions.insert(0,(b'RWCM',collision.read_bytes()))
+        u(f,len(extensions))
+        for tag,data in extensions:
             f.write(tag);u(f,1,len(data));stored(f,data)
     report('Map written: '+m['map_name'])
+    if not render_only:
+        from tools.asset_pipeline.irradiance import write as write_irradiance
+        write_irradiance(manifest_path, output.with_suffix('.irradiance'))
 
 if __name__=='__main__':
     import argparse

tools/asset_pipeline/ocean_pca.py

@@ -0,0 +1,41 @@
+"""Extract TU3's built-in PCA ocean animation from an owned mapped image.
+
+Addresses are corroborated by cPCAWaterAnimationData::Init (827905B0) and
+its update routine (82790858). No game content is embedded in this tool.
+"""
+import hashlib
+import json
+import math
+import struct
+from pathlib import Path
+
+
+def extract(image, output, base=0x82000000):
+    data = Path(image).read_bytes()
+    identity = hashlib.sha256(data).hexdigest()
+    if identity != 'f4aa113eb541bfba03dbc108cf5ab43f58c965b20fa3b82f9c40938a0ad841c4':
+        raise ValueError('Unsupported mapped executable build for PCA table extraction')
+    frames = []
+    for frame in range(30):
+        mean = struct.unpack_from('>3f', data, 0x830118D8-base+frame*12)
+        weights = struct.unpack_from('>24f', data, 0x83011A40-base+frame*96)
+        if not all(math.isfinite(v) for v in (*mean, *weights)):
+            raise ValueError('Non-finite ocean PCA table')
+        # Shader model order: mean X/Z/Y and weight pairs R/B/G.
+        rows = [[mean[0]/255., mean[2]/255., mean[1]/255., 0.]]
+        for start in (0, 4, 16, 20, 8, 12):
+            rows.append([v/255. for v in weights[start:start+4]])
+        frames.append(rows)
+    output = Path(output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(dict(source_sha256=identity, hz=30., frames=frames)), encoding='utf-8')
+    return len(frames)
+
+
+if __name__ == '__main__':
+    import argparse
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--image', type=Path, required=True)
+    p.add_argument('--output', type=Path, required=True)
+    a = p.parse_args()
+    print('Extracted', extract(a.image, a.output), 'ocean PCA frames')

tools/asset_pipeline/refresh_textures.py

@@ -0,0 +1,105 @@
+"""Refresh SKATE14 texture payloads from a newly decoded RX2 manifest cache.
+
+Material, geometry, collision and metadata bytes remain unchanged. Publication
+is atomic and never overwrites the input map. No renderer is launched.
+"""
+import hashlib
+import json
+import os
+import struct
+import tempfile
+import zlib
+from pathlib import Path
+import numpy as np
+
+
+def refresh(manifest, source, output):
+    manifest, source, output = map(Path, (manifest, source, output))
+    if source.resolve() == output.resolve():
+        raise ValueError('Refresh requires a separate output map')
+    data = json.loads(manifest.read_text())
+    output.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with source.open('rb') as src:
+            def read(n):
+                value = src.read(n)
+                if len(value) != n:
+                    raise ValueError('Truncated SKATE14 package')
+                return value
+            def uint():
+                return struct.unpack('<I', read(4))[0]
+            def text():
+                return read(uint()).decode('utf-8')
+            if read(8) != b'SKATE14\0' or uint() != 0x12345678:
+                raise ValueError('Texture refresh requires SKATE14')
+            text()
+            read(49 * 4)
+            counts = [uint() for _ in range(9)]
+            for _ in range(counts[0]):
+                text()
+                read(4 + 7*4 + 8 + 4 + 16 + 4 + 16)
+                if uint():
+                    read(16)
+                    text()
+                    read(8)
+                    for _ in range(uint()):
+                        text()
+                        read(16)
+                    for _ in range(uint()):
+                        text()
+                        for _ in range(uint()):
+                            text()
+                    text()
+            prefix_end = src.tell()
+            src.seek(0)
+            prefix = read(prefix_end)
+            changed = []
+            with tempfile.NamedTemporaryFile(dir=output.parent, suffix='.skate.tmp', delete=False) as dst:
+                temporary = Path(dst.name)
+                dst.write(prefix)
+                for _ in range(counts[1]):
+                    start = src.tell()
+                    name = text()
+                    width, height, _space = uint(), uint(), uint()
+                    data_start = src.tell()
+                    method, size = uint(), uint()
+                    packed = read(size)
+                    if method not in (0, 1):
+                        raise ValueError('Unsupported texture compression')
+                    old = zlib.decompress(packed) if method else packed
+                    end = src.tell()
+                    entry = data['textures'][name]
+                    if (width, height) != (entry['width'], entry['height']):
+                        raise ValueError('Texture dimensions changed: ' + name)
+                    cache = (manifest.parent / entry['rgba']).read_bytes()
+                    if len(cache) != width*height*4 or len(old) != len(cache):
+                        raise ValueError('Texture byte count mismatch: ' + name)
+                    array = np.frombuffer(cache, np.uint8).reshape(height, width, 4)
+                    new = (array if entry.get('cube_faces') == 6 else array[::-1]).tobytes()
+                    src.seek(start)
+                    dst.write(read(data_start-start))
+                    src.seek(end)
+                    if new != old:
+                        replacement = zlib.compress(new, 1)
+                        dst.write(struct.pack('<II', 1, len(replacement)))
+                        dst.write(replacement)
+                        changed.append(dict(name=name, width=width, height=height, format=entry.get('format'),
+                            changed_pixels=int(np.any(np.frombuffer(old, np.uint8).reshape(-1, 4) !=
+                                np.frombuffer(new, np.uint8).reshape(-1, 4), axis=1).sum())))
+                    else:
+                        dst.write(struct.pack('<II', method, size))
+                        dst.write(packed)
+                old_tail_offset, new_tail_offset = src.tell(), dst.tell()
+                tail_hash = hashlib.sha256()
+                while chunk := src.read(1024*1024):
+                    tail_hash.update(chunk)
+                    dst.write(chunk)
+            os.replace(temporary, output)
+            temporary = None
+            return dict(changed=changed, count=len(changed), textures=counts[1],
+                prefix_sha256=hashlib.sha256(prefix).hexdigest(), tail_sha256=tail_hash.hexdigest(),
+                old_tail_offset=old_tail_offset, new_tail_offset=new_tail_offset)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)

tools/asset_pipeline/render_parameters.py

@@ -0,0 +1,49 @@
+"""Export complete authored parameter rows for additional presentation shaders."""
+import json
+import struct
+from .environment import Collections, key_hash
+
+
+def exposure_parameters(collections, location):
+    fields, chain = collections.resolve('render_locations', location)
+    key = int(fields[key_hash('Hash_C345507C4B9B6F62')]['data'][:16], 16)
+    rendering, rendering_chain = collections.resolve('rendering', key)
+    values = {n: struct.unpack('>f', bytes.fromhex(rendering[key_hash('auto_exposure_'+n)]['data']))[0]
+              for n in ('target_luminance', 'min', 'max', 'damping')}
+    return dict(values, location_chain=chain, rendering_chain=rendering_chain)
+
+
+def convert(assets, converted):
+    collections = Collections(converted)
+    result = {}
+    aliases = ('default', 'reflection', 'backlituvscroll', 'transparent',
+               'flowing', 'flowingalpha', 'alpha', 'skatepark', 'videoscreen')
+    for (cls, key), row in collections.rows.items():
+        for family in ('water', 'ocean', 'incandescent'):
+            if cls != key_hash('material_' + family):
+                continue
+            fields, _ = collections.resolve(cls, key)
+            raw = fields.get(key_hash('m_params'), {}).get('array', {}).get('items', [])
+            if not raw:
+                continue
+            name = next((n for n in aliases if key_hash(n) == key), row['key'])
+            result[family + '.' + name] = [struct.unpack('>4f', bytes.fromhex(r)) for r in raw]
+    path = assets / 'private/render-parameters.json'
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(result), encoding='utf-8')
+    fields, _ = collections.resolve('Hash_1FFDC8E3ACA07C1F', 'auto_exposure')
+    exposure = {n: struct.unpack('>f', bytes.fromhex(fields[key_hash('auto_exposure_'+n)]['data']))[0]
+                for n in ('target_luminance', 'min', 'max', 'damping')}
+    (assets / 'private/exposure.json').write_text(json.dumps(exposure), encoding='utf-8')
+    profiles = {}
+    for (cls, key), row in collections.rows.items():
+        if cls != key_hash('world'):
+            continue
+        own_stream = next((v['data'] for k, v in row['fields'].items() if key_hash(k) == key_hash('WorldStream')), None)
+        if not own_stream or not own_stream.startswith('DIST_'):
+            continue
+        world, _ = collections.resolve(cls, key)
+        location = int(world[key_hash('Hash_7E23D10785C43717')]['data'][:16], 16)
+        profiles[own_stream.removeprefix('DIST_').casefold()] = exposure_parameters(collections, location)
+    (assets / 'private/exposure-profiles.json').write_text(json.dumps(profiles, indent=2), encoding='utf-8')
+    return len(result)

tools/asset_pipeline/retail_material.py

@@ -31,6 +31,8 @@ def _retail_shader_family(shader_name: str) -> int:
         return 12
     if shader.startswith("water.flowing"):
         return 30
+    if shader in ("water.default", "water.alpha", "water.skatepark"):
+        return 33
     if shader.startswith("ocean.default"):
         return 31
     if shader.startswith("ocean.reflection"):

tools/asset_pipeline/sky.py

@@ -1,43 +1,112 @@
-"""Extract retail sky geometry and textures through RX2 GUID bindings."""
-import json,struct,sys
+"""Extract authored retail skies through world references and RX2 GUID bindings."""
+import json
+import struct
+import sys
 from pathlib import Path
 from tools.owned_game.big import BigArchive
+from .environment import Collections, NAMES, field, key_hash, sky_parameters, world_environment, fog_parameters
 
-def convert(game_root, assets):
-    sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'vendor/utt'))
-    import mdl_parser,rx2_parser
-    archive=BigArchive(game_root/'data/big/miscload.big')
-    entries={e.path:e for e in archive.entries}
-    output=assets/'private/native-skies';output.mkdir(parents=True,exist_ok=True)
-    for name,suffix in [('University',''),('DownTown','_downtown'),('Industrial','_industrial')]:
-        prefix='data/content/world/models/DIST_skybox'+suffix
-        raw=archive.read(entries[prefix+'.rx2'])
-        model=mdl_parser.parse_rx2(raw)
-        table=rx2_parser.RX2File(raw);table.parse()
-        diffuse=None
-        for section in table.entries:
-            if section.type_id!=0x00eb0005:continue
-            o=section.f0;h=struct.unpack_from('>8I',raw,o)
-            if (h[4]-h[3])//h[1]!=32:raise ValueError('Unexpected sky material layout')
-            for i in range(h[1]):
-                v=struct.unpack_from('>8I',raw,o+h[3]+i*32)
-                at=o+v[0];kind=raw[at:raw.index(b'\0',at)]
-                if kind==b'diffuse':diffuse=(v[4]<<32)|v[5]
-        textures=rx2_parser.parse_rx2(archive.read(entries[prefix+'_Textures.rx2']))
-        handle=None
-        for section in textures.entries:
-            if section.type_id!=0x00eb000b:continue
-            o=section.f0;n,start=struct.unpack_from('>2I',textures.data,o)
-            for i in range(n):
-                _,_,hi,lo,kind,index=struct.unpack_from('>6I',textures.data,o+start+i*24)
-                if (hi<<32)|lo==diffuse:handle=index
-        texture=next((t for t in textures.textures if t.index==handle),None)
-        if texture is None:raise ValueError('Sky diffuse GUID did not resolve: '+name)
-        if len(model.meshes)!=1 or model.meshes[0].uvs is None:raise ValueError('Unexpected sky mesh layout')
-        mesh=model.meshes[0]
-        (output/(name+'.rgba')).write_bytes(texture.rgba)
+
+def _material(raw, table):
+    channels = {}
+    for section in table.entries:
+        if section.type_id != 0x00eb0005:
+            continue
+        o = section.f0
+        h = struct.unpack_from('>8I', raw, o)
+        if not h[1] or (h[4]-h[3])//h[1] != 32:
+            raise ValueError('Unexpected sky material layout')
+        for i in range(h[1]):
+            v = struct.unpack_from('>8I', raw, o+h[3]+i*32)
+            at = o+v[0]
+            kind = raw[at:raw.index(b'\0', at)].decode('ascii')
+            if kind in ('diffuse', 'specular'):
+                channels[kind] = (v[4]<<32)|v[5]
+    return channels
+
+
+def _texture(textures, guid):
+    handle = None
+    for section in textures.entries:
+        if section.type_id != 0x00eb000b:
+            continue
+        o = section.f0
+        n, start = struct.unpack_from('>2I', textures.data, o)
+        for i in range(n):
+            _, _, hi, lo, _, index = struct.unpack_from('>6I', textures.data, o+start+i*24)
+            if (hi<<32)|lo == guid:
+                handle = index
+    texture = next((t for t in textures.textures if t.index == handle), None)
+    if texture is None:
+        raise ValueError(f'Sky texture GUID did not resolve: {guid:016X}')
+    return texture
+
+
+def convert(game_root, assets, converted):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'vendor/utt'))
+    import mdl_parser
+    import rx2_parser
+    collections = Collections(converted)
+    archive = BigArchive(game_root/'data/big/miscload.big')
+    entries = {e.path: e for e in archive.entries}
+    output = assets/'private/native-skies'
+    output.mkdir(parents=True, exist_ok=True)
+    available = {p.stem.removeprefix('world') for p in (game_root/'data/content').glob('worldDIST_*.big')}
+    count = 0
+    for (cls, _), row in collections.rows.items():
+        if cls != key_hash('world'):
+            continue
+        own_stream = next((v['data'] for k, v in row['fields'].items()
+                           if key_hash(k) == key_hash('WorldStream')), None)
+        if own_stream not in available:
+            continue
+        name = own_stream.removeprefix('DIST_')
+        env = world_environment(collections, row['key'])
+        raw = archive.read(entries['data/content/'+env['sky_model']+'.rx2'])
+        model = mdl_parser.parse_rx2(raw)
+        table = rx2_parser.RX2File(raw)
+        table.parse()
+        channels = _material(raw, table)
+        textures = rx2_parser.parse_rx2(archive.read(entries['data/content/'+env['sky_textures']+'.rx2']))
+        diffuse = _texture(textures, channels['diffuse'])
+        sun = _texture(textures, channels['specular'])
+        shader = next(p.value for p in model.materials if p.kind == 'AttribulatorMaterialName')
+        env.update(sky_parameters(collections, shader))
+        env['fog_frame'] = fog_parameters(env['fog'])
+        if len(model.meshes) != 1 or model.meshes[0].uvs is None:
+            raise ValueError('Unexpected sky mesh layout')
+        mesh = model.meshes[0]
+        if len(sun.rgba) != sun.width*sun.height*4 or sun.height != 16:
+            raise ValueError('Unexpected sky sun-gradient dimensions')
+        (output/(name+'.rgba')).write_bytes(diffuse.rgba)
+        (output/(name+'.sun.rgba')).write_bytes(sun.rgba)
         (output/(name+'.json')).write_text(json.dumps({
-            'width':texture.width,'height':texture.height,
-            'positions':mesh.vertices.tolist(),'uvs':mesh.uvs.tolist(),
-            'indices':mesh.faces.reshape(-1).tolist(),
-        },separators=(',',':')),encoding='utf-8')
+            'width': diffuse.width, 'height': diffuse.height,
+            'positions': mesh.vertices.tolist(), 'uvs': mesh.uvs.tolist(),
+            'indices': mesh.faces.reshape(-1).tolist(),
+            'environment': env, 'sun_width': sun.width, 'sun_height': sun.height,
+            'source_sha256': row['sha256'],
+        }, separators=(',', ':')), encoding='utf-8')
+        count += 1
+    if not count:
+        raise ValueError('No authored world skies resolved')
+    return count
+
+
+if __name__ == '__main__':
+    import argparse
+    import tempfile
+    from .vlt import convert as convert_vlt
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--game-root', type=Path, required=True)
+    parser.add_argument('--assets', type=Path, required=True)
+    args = parser.parse_args()
+    # A targeted update reads only the small database pair and sky resources.
+    with tempfile.TemporaryDirectory(prefix='skate-sky-') as work:
+        database = BigArchive(args.game_root/'data/big/db.big')
+        needed = {'skaterschema.bin', 'skaterschema.vlt', 'skatercollections.bin', 'skatercollections.vlt'}
+        database.extract_entries([e for e in database.entries if Path(e.path).name.lower() in needed], Path(work))
+        stem = Path(work)/'data/db'
+        names = (Path(__file__).parent/'names.txt').read_text(encoding='utf-8').splitlines()
+        converted = convert_vlt(stem/'skaterschema', stem/'skatercollections', [*names, *NAMES])
+        print('Prepared', convert(args.game_root, args.assets, converted), 'authored skies')

tools/asset_pipeline/teleports.py

@@ -0,0 +1,120 @@
+"""Join retail FE destinations to localized names and RX2 location matrices.
+
+Owned content stays in private sidecars. No guessed coordinates or renamed spots.
+LocationManager 828A3D50 loads data/content/global_locators; EB0009 records
+contain a 64-byte matrix and a section-relative name pointer at +104.
+"""
+import hashlib
+import json
+import struct
+import tempfile
+from pathlib import Path
+
+from tools.owned_game.big import BigArchive
+from .dynamic_props import sections, matrix
+from .environment import Collections, key_hash
+from .vlt import vault
+
+
+def cstring(raw, offset, limit=None):
+    limit = len(raw) if limit is None else limit
+    if not 0 <= offset < limit <= len(raw):
+        raise ValueError('Invalid location string offset')
+    return raw[offset:raw.index(b'\0', offset, limit)]
+
+
+def language_table(raw):
+    if len(raw) < 36 or struct.unpack_from('<I', raw)[0] != 0x39000:
+        raise ValueError('Unsupported retail language table')
+    count, start, strings = struct.unpack_from('<3I', raw, 8)
+    start += 8
+    strings += 8
+    if start + count * 8 != strings or strings > len(raw):
+        raise ValueError('Invalid retail language index')
+    result = {}
+    for index in range(count):
+        key, offset = struct.unpack_from('<2I', raw, start + index * 8)
+        if key in result:
+            raise ValueError('Duplicate language key')
+        result[key] = cstring(raw, strings + offset)
+    return result
+
+
+def location_records(raw):
+    result = []
+    for base, _, size, _, _, kind in sections(raw):
+        if kind != 0xEB0009:
+            continue
+        count, _, _, start, end = struct.unpack_from('>5I', raw, base)
+        if base + size > len(raw) or start != 32 or end != start + count * 128 or end > size:
+            raise ValueError('Invalid location record table')
+        for index in range(count):
+            at = base + start + index * 128
+            pointer = struct.unpack_from('>I', raw, at + 104)[0]
+            if not end <= pointer < size:
+                raise ValueError('Location name overlaps records')
+            result.append({'locator': cstring(raw, base + pointer, base + size).decode('utf-8'),
+                           'matrix': matrix(raw, at).tolist(), 'source_offset': at})
+    return result
+
+
+def convert(game_root, assets, converted):
+    game_root, assets = Path(game_root), Path(assets)
+    archive = BigArchive(game_root / 'data/big/miscboot.big')
+    tables = {}
+    for language in ('english', 'labels'):
+        entry = next(e for e in archive.entries if f'/{language}/' in e.path and '_Global_' in e.path)
+        tables[language] = language_table(archive.read(entry))
+    labels = {value.strip().decode('ascii'): key for key, value in tables['labels'].items()}
+    database = BigArchive(game_root / 'data/big/db.big')
+    with tempfile.TemporaryDirectory(prefix='skate-teleports-') as tmp:
+        root = Path(tmp)
+        entries = [e for e in database.entries if e.path in
+                   ('data/db/skatercollections.bin', 'data/db/skatercollections.vlt')]
+        database.extract_entries(entries, root)
+        _, binary, _ = vault(root / 'data/db/skatercollections')
+    locators = {}
+    for path in sorted((game_root / 'data/content/global_locators').rglob('*.rx2')):
+        raw = path.read_bytes()
+        stream = path.parent.name
+        for record in location_records(raw):
+            record.update(map=stream.removeprefix('DIST_'), source=str(path.relative_to(game_root)),
+                          source_sha256=hashlib.sha256(raw).hexdigest())
+            locators.setdefault((stream.casefold(), record['locator']), []).append(record)
+    collections = Collections(converted)
+    destinations = []
+    # Own location artwork distinguishes selectable FE destinations from base
+    # templates, startup checkpoints and park-layout aliases.
+    for (cls, key), row in collections.rows.items():
+        if cls != key_hash('fe_locations') or not any(key_hash(k) == key_hash('Hash_786A2D24F475342B') for k in row['fields']):
+            continue
+        fields, _ = collections.resolve(cls, key)
+        label = fields[key_hash('Hash_174B301910A902C9')]['data']
+        if label not in labels:
+            continue  # Template placeholders have no shipped localized name.
+        text = tables['english'][labels[label]]
+        # 0xAB is the retail font's skate wordmark glyph. The portable UI has
+        # no such font glyph; spell out the same brand instead of showing «.
+        name = text.replace(b'\xab', b'skate.').decode('ascii').strip()
+        locator = cstring(binary, int(fields[key_hash('location')]['data'], 16)).decode('utf-8')
+        world_key = int(fields[key_hash('World')]['data'][:16], 16)
+        world, _ = collections.resolve('world', world_key)
+        stream = world[key_hash('WorldStream')]['data']
+        if not (game_root / 'data/content' / ('world' + stream + '.big')).exists():
+            continue
+        matches = locators.get((stream.casefold(), locator), [])
+        unique = {json.dumps(m['matrix']) for m in matches}
+        if len(unique) > 1:
+            raise ValueError('Ambiguous authored teleport: ' + locator)
+        item = dict(id=row['key'], name=name, map=stream.removeprefix('DIST_'),
+                    label_key=label, locator=locator, matrix=None)
+        if matches:
+            item.update(matches[0])
+        else:
+            item['unavailable_reason'] = 'Authored destination locator was not found in the extracted content'
+        destinations.append(item)
+    destinations.sort(key=lambda d: (d['map'].casefold(), d['name'].casefold()))
+    output = assets / 'private/teleports.json'
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps({'version': 1, 'destinations': destinations}, indent=2), encoding='utf-8')
+    return destinations

tools/asset_pipeline/test_backdrop.py

@@ -0,0 +1,44 @@
+"""Global resource GUID selection without owned-game fixtures."""
+import struct
+import unittest
+from types import SimpleNamespace
+
+from .backdrop import texture_groups
+from .sky import _texture
+
+
+class BackdropTests(unittest.TestCase):
+    def test_missing_channels_do_not_shift_later_materials(self):
+        # Ocean has no diffuse here; foliage follows it. Name text and GUIDs
+        # deliberately differ, as they do in the global University resource.
+        parameters = [('Name', 0), ('normal', 19), ('Name', 0),
+                      ('diffuse', 0x123456789ABCDEF0), ('lightmap', 23)]
+        raw = bytearray(512)
+        struct.pack_into('>8I', raw, 0, 0, len(parameters), 0, 32,
+                         32 + len(parameters)*32, 0, 0, 0)
+        at = 256
+        for i, (kind, guid) in enumerate(parameters):
+            word = kind.encode() + b'\0'
+            raw[at:at+len(word)] = word
+            struct.pack_into('>8I', raw, 32+i*32, at, 0, 0, 0,
+                             guid >> 32, guid & 0xffffffff, 0, 0)
+            at += len(word)
+        table = SimpleNamespace(entries=[SimpleNamespace(type_id=0xeb0005, f0=0)])
+        self.assertEqual(texture_groups(bytes(raw), table, ('diffuse', 'normal', 'lightmap')),
+                         [{'normal': 19}, {'diffuse': 0x123456789ABCDEF0, 'lightmap': 23}])
+
+    def test_texture_guid_uses_handle_not_list_position(self):
+        raw = bytearray(64)
+        struct.pack_into('>2I', raw, 0, 1, 8)
+        struct.pack_into('>6I', raw, 8, 0, 0, 0x12345678, 0x9abcdef0, 0, 27)
+        wanted = SimpleNamespace(index=27)
+        textures = SimpleNamespace(data=bytes(raw),
+            entries=[SimpleNamespace(type_id=0xeb000b, f0=0)],
+            textures=[SimpleNamespace(index=1), wanted])
+        self.assertIs(_texture(textures, 0x123456789ABCDEF0), wanted)
+        with self.assertRaises(ValueError):
+            _texture(textures, 123)
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/asset_pipeline/test_dynamic_props.py

@@ -0,0 +1,52 @@
+import struct
+import unittest
+import numpy as np
+
+from .dynamic_props import locators, transform_mesh, template_meshes
+
+
+def resource(kind, payload):
+    raw = bytearray(88+len(payload))
+    raw[:7] = b'\x89RW4xb2'
+    struct.pack_into('>I', raw, 32, 1)
+    struct.pack_into('>I', raw, 48, 64)
+    struct.pack_into('>6I', raw, 64, 88, 0, len(payload), 0, 0, kind)
+    raw[88:] = payload
+    return raw
+
+
+class DynamicPropsTests(unittest.TestCase):
+    def test_locator_id_and_row_matrix(self):
+        payload = bytearray(166)
+        struct.pack_into('>5I', payload, 0, 0, 1, 1, 32, 160)
+        m = np.eye(4); m[3, :3] = [10, 20, -30]
+        struct.pack_into('>16f', payload, 32, *m.ravel())
+        struct.pack_into('>3Q2I', payload, 128, 123, 456, 789, 0, 160)
+        payload[160:] = b'bench\0'
+        rows = locators(resource(0xEB001D, payload))
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]['template_id'], '0000000000000315')
+        self.assertEqual(rows[0]['matrix'][3], [10, 20, -30, 1])
+        self.assertEqual(rows[0]['name'], 'bench')
+        struct.pack_into('>I', payload, 16, 159)
+        with self.assertRaisesRegex(ValueError, 'layout'):
+            locators(resource(0xEB001D, payload))
+
+    def test_scaled_rotation_and_normal_transform(self):
+        m = np.array([[0, 0, -2, 0], [0, 3, 0, 0], [4, 0, 0, 0], [10, 20, 30, 1]], dtype=float)
+        arrays = {'vertices_0': np.array([[1, 2, 3]]), 'normals_0': np.array([[0, 0, 1]]), 'faces_0': np.array([[0, 0, 0]])}
+        transformed = transform_mesh(arrays, 0, m)
+        np.testing.assert_allclose(transformed['vertices_0'], [[22, 26, 28]])
+        np.testing.assert_allclose(transformed['normals_0'], [[1, 0, 0]])
+        np.testing.assert_array_equal(arrays['vertices_0'], [[1, 2, 3]])
+
+    def test_missing_template_reference_is_rejected(self):
+        payload = bytearray(192)
+        struct.pack_into('>5I', payload, 0, 0, 1, 1, 32, 192)
+        struct.pack_into('>I', payload, 160, 100)
+        with self.assertRaisesRegex(ValueError, 'model reference'):
+            template_meshes(resource(0xEB000D, payload))
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/asset_pipeline/test_environment.py

@@ -0,0 +1,86 @@
+"""Synthetic environment references, independent of owned game assets."""
+import struct
+import unittest
+from .environment import Collections, field, key_hash, sky_parameters, world_environment, fog_parameters
+from .render_parameters import exposure_parameters
+
+
+def raw(*values):
+    return struct.pack('>'+'f'*len(values), *values).hex()
+
+
+def row(cls, key, parent='', **fields):
+    return {'class': cls, 'key': key, 'parent': parent,
+            'fields': {k: {'type': 'fixture', 'data': v} for k, v in fields.items()}}
+
+
+class EnvironmentTests(unittest.TestCase):
+    def test_exposure_follows_location_reference_and_inheritance(self):
+        rows = [row('render_locations', 'park', Hash_C345507C4B9B6F62=struct.pack('>2Q', key_hash('park_exposure'), 0).hex()),
+                row('rendering', 'base', auto_exposure_target_luminance=raw(.25), auto_exposure_min=raw(.75), auto_exposure_max=raw(2.5), auto_exposure_damping=raw(.5)),
+                row('rendering', 'park_exposure', 'base', auto_exposure_max=raw(1.5))]
+        result = exposure_parameters(Collections({'collections': rows}), 'park')
+        self.assertEqual(result['max'], 1.5)
+        self.assertEqual(result['min'], .75)
+        self.assertEqual(result['rendering_chain'], ['park_exposure', 'base'])
+    def test_world_reference_and_inheritance_select_default_fog(self):
+        rows = [
+            row('world', 'base', Hash_C7A0A84F018E87BA='world/models/sky',
+                Hash_2A0BF629355AFB15='world/models/sky_Textures',
+                Hash_7E23D10785C43717=struct.pack('>2Q', key_hash('park'), 0).hex()),
+            row('world', 'test', 'base'),
+            row('render_locations', 'default', sun_position=raw(1, 0, 0),
+                Hash_2E9AAECD1C29F81F=raw(150),
+                material_fog=struct.pack('>3Q', key_hash('material_fog'), key_hash('default'), 0).hex()),
+            row('render_locations', 'park', 'default', sun_position=raw(0, 3, 4)),
+            row('material_fog', 'default', fog_near=raw(20), fog_far=raw(1000),
+                fog_colour=raw(.1, .2, .3, 1), fog_power=raw(2), fog_max=raw(.5)),
+            row('material_fog', 'fog_default', 'default', fog_far=raw(200)),
+        ]
+        env = world_environment(Collections({'collections': rows}), 'test')
+        self.assertEqual(env['sun_direction'], [0, .6, .8])
+        self.assertEqual(env['anchor_height'], 150)
+        self.assertEqual(env['location_chain'], ['park', 'default'])
+        self.assertEqual(env['fog']['fog_far'], [1000])
+        self.assertEqual(env['sky_model'], 'world/models/sky')
+
+    def test_native_fog_rows_and_distance_boundaries(self):
+        fog = dict(fog_near=[100.], fog_far=[2000.], fog_power=[2.],
+                   fog_max=[.5], fog_colour=[.1, .2, .3, 1.])
+        frame = fog_parameters(fog)
+        self.assertEqual(frame['colour'], [.05, .1, .15, -.5])
+        def opacity(distance):
+            x = max(0., min(1., distance * frame['ramp'][0] + frame['ramp'][1]))
+            return .5 * x ** frame['ramp'][2]
+        self.assertEqual(opacity(0.), 0.)
+        self.assertAlmostEqual(opacity(100.), 0.)
+        self.assertAlmostEqual(opacity(1050.), .125)
+        self.assertAlmostEqual(opacity(2000.), .5)
+        self.assertAlmostEqual(opacity(3000.), .5)
+        fog['fog_far'] = [100.]
+        with self.assertRaises(ValueError): fog_parameters(fog)
+
+    def test_unnamed_hashes_and_cycle_rejection(self):
+        cls = f'Hash_{key_hash("world"):016X}'
+        c = Collections({'collections': [row(cls, 'a', 'b', value='child'),
+                                         row(cls, 'b', value='parent')]})
+        values, _ = c.resolve('world', 'a')
+        self.assertEqual(field(values, 'value'), 'child')
+        c.rows[key_hash('world'), key_hash('b')]['parent'] = 'a'
+        with self.assertRaises(ValueError):
+            c.resolve('world', 'a')
+
+    def test_sky_requires_complete_array_and_shader_collection(self):
+        r = row('material_sky', 'default')
+        r['fields']['m_params'] = {'type': 'Attrib::Types::Vector4', 'data': 'header'}
+        c = Collections({'collections': [r]})
+        with self.assertRaises(ValueError):
+            sky_parameters(c, 'sky.default')
+        r['fields']['m_params']['array'] = {'items': [raw(.5, .25, 0, 0)]}
+        self.assertEqual(sky_parameters(c, 'sky.default')['multiplier'], .25)
+        with self.assertRaises(KeyError):
+            sky_parameters(c, 'sky.missing')
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/asset_pipeline/test_irradiance.py

@@ -0,0 +1,37 @@
+"""RX2 irradiance layout checks without owned-game fixtures."""
+import struct
+import unittest
+
+from .irradiance import decode
+
+
+def resource():
+    data = bytearray(272)
+    data[:7] = b'\x89RW4xb2'
+    struct.pack_into('>I', data, 32, 1)
+    struct.pack_into('>I', data, 48, 64)
+    struct.pack_into('>6I', data, 64, 96, 0, 176, 0, 0, 0xEB0024)
+    struct.pack_into('>2I', data, 96, 1, 16)
+    values = [i / 4 for i in range(39)]
+    struct.pack_into('>39fI', data, 112, *values, 0xDEDEDEDE)
+    return data, values
+
+
+class IrradianceTests(unittest.TestCase):
+    def test_preserves_coefficients_position_and_link_word(self):
+        data, values = resource()
+        groups = decode(data)
+        self.assertEqual(len(groups), 1)
+        self.assertEqual(struct.unpack('<39fI', groups[0][0]), (*values, 0xDEDEDEDE))
+
+    def test_rejects_records_outside_section(self):
+        data, _ = resource()
+        struct.pack_into('>I', data, 96, 2)
+        with self.assertRaises(ValueError):
+            decode(data)
+
+    def test_rejects_nonfinite_sh(self):
+        data, _ = resource()
+        struct.pack_into('>f', data, 112, float('nan'))
+        with self.assertRaises(ValueError):
+            decode(data)

tools/asset_pipeline/test_refresh_textures.py

@@ -0,0 +1,35 @@
+import json
+import struct
+import tempfile
+import unittest
+from pathlib import Path
+from .refresh_textures import refresh
+
+
+class RefreshTests(unittest.TestCase):
+    def test_payload_refresh_preserves_tail_and_failed_refresh_preserves_output(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            string = lambda x: struct.pack('<I', len(x)) + x
+            prefix = b'SKATE14\0' + struct.pack('<I', 0x12345678) + string(b'fixture') + bytes(49*4)
+            prefix += struct.pack('<9I', 0, 1, 0, 0, 0, 0, 0, 0, 0)
+            old = bytes(16)
+            header = string(b'texture') + struct.pack('<3I', 2, 2, 0)
+            tail = b'geometry/collision/metadata-preservation-fixture'
+            source = root/'input.skate'
+            source.write_bytes(prefix + header + struct.pack('<II', 0, 16) + old + tail)
+            cache = bytes(range(16))
+            (root/'new.rgba').write_bytes(cache)
+            manifest = root/'manifest.json'
+            manifest.write_text(json.dumps({'textures': {'texture': dict(width=2, height=2, rgba='new.rgba')}}))
+            output = root/'out.skate'
+            report = refresh(manifest, source, output)
+            self.assertEqual(report['count'], 1)
+            self.assertEqual(output.read_bytes()[:len(prefix)], prefix)
+            self.assertEqual(output.read_bytes()[report['new_tail_offset']:], tail)
+            saved = output.read_bytes()
+            (root/'new.rgba').write_bytes(b'bad')
+            with self.assertRaises(ValueError): refresh(manifest, source, output)
+            self.assertEqual(output.read_bytes(), saved)
+            with self.assertRaises(ValueError): refresh(manifest, source, source)
+            self.assertFalse(list(root.glob('*.tmp')))

tools/asset_pipeline/test_teleports.py

@@ -0,0 +1,36 @@
+import struct
+import unittest
+import numpy as np
+from .teleports import language_table, location_records
+from .test_dynamic_props import resource
+
+
+class TeleportTests(unittest.TestCase):
+    def test_language_join_uses_keys_not_storage_order(self):
+        # Offsets are relative to the chunk payload; strings need not follow
+        # the same physical order as the sorted hash index.
+        raw = bytearray(36 + 16 + 11)
+        struct.pack_into('<5I', raw, 0, 0x39000, len(raw)-8, 2, 28, 44)
+        struct.pack_into('<4I', raw, 36, 123, 6, 456, 0)
+        raw[52:] = b'First\0Last\0'
+        self.assertEqual(language_table(raw), {123: b'Last', 456: b'First'})
+        struct.pack_into('<I', raw, 40, 999)
+        with self.assertRaises(ValueError): language_table(raw)
+
+    def test_locator_preserves_heading_and_world_space_position(self):
+        payload = bytearray(165)
+        struct.pack_into('>5I', payload, 0, 1, 0, 0, 32, 160)
+        m = np.array([[0, 0, -1, 0], [0, 1, 0, 0], [1, 0, 0, 0], [350, 140, -725, 1]], dtype=float)
+        struct.pack_into('>16f', payload, 32, *m.ravel())
+        struct.pack_into('>I', payload, 136, 160)
+        payload[160:] = b'spot\0'
+        rows = location_records(resource(0xEB0009, payload))
+        self.assertEqual(rows[0]['locator'], 'spot')
+        np.testing.assert_array_equal(rows[0]['matrix'], m)
+        struct.pack_into('>I', payload, 136, 32)
+        with self.assertRaisesRegex(ValueError, 'overlaps'): location_records(resource(0xEB0009, payload))
+
+    def test_bad_record_count_is_rejected(self):
+        payload = bytearray(32)
+        struct.pack_into('>5I', payload, 0, 1, 0, 0, 32, 160)
+        with self.assertRaises(ValueError): location_records(resource(0xEB0009, payload))

tools/asset_pipeline/test_texture_decode.py

@@ -0,0 +1,46 @@
+"""Regression fixtures for padded Xbox tiles and BC colour palettes."""
+import importlib.util
+from pathlib import Path
+import struct
+import sys
+import unittest
+from unittest.mock import patch
+
+UTT = Path(__file__).resolve().parents[1] / 'vendor/utt'
+sys.path.insert(0, str(UTT))
+import rx2_fast
+
+# Exercise the standalone fallback as well as the NumPy implementation.
+spec = importlib.util.spec_from_file_location('rx2_scalar_test', UTT / 'rx2_parser.py')
+scalar = importlib.util.module_from_spec(spec)
+with patch.dict(sys.modules, {'rx2_fast': None}):
+    spec.loader.exec_module(scalar)
+
+
+class TextureDecodeTests(unittest.TestCase):
+    def test_padding_cannot_overwrite_visible_rows(self):
+        for width, pitch in ((w, p) for w in (4, 8, 16, 32) for p in (2, 4, 8, 16)):
+            with self.subTest(width=width, pitch=pitch):
+                units = max(32*32, 4096//pitch)
+                raw = bytearray(units*pitch)
+                for offset in range(units):
+                    x = scalar._x360_tiled_x(offset, width, pitch)
+                    y = scalar._x360_tiled_y(offset, width, pitch)
+                    if x < width and y < 32:
+                        raw[offset*pitch:offset*pitch+pitch] = bytes([1+y])*pitch
+                expected = b''.join(bytes([1+y])*(width*pitch) for y in range(32))
+                for decoder in (scalar, rx2_fast):
+                    self.assertEqual(decoder._untile360(raw, width, pitch)[:len(expected)], expected)
+
+    def test_bc1_keeps_transparent_palette(self):
+        colour = struct.pack('>HH', 0x001f, 0xf800) + b'\xff'*4
+        self.assertEqual(scalar._dxt1_block(colour, True), bytes(64))
+        self.assertEqual(rx2_fast._decode(colour, 4, 4, 1, False), bytes(64))
+
+    def test_bc2_bc3_use_four_colours_with_reversed_endpoints(self):
+        colour = struct.pack('>HH', 0x001f, 0xf800) + b'\xff'*4
+        for kind, alpha, block_decoder in ((3, b'\xaa'*8, scalar._dxt3_block),
+                                           (5, b'\x00\xaa'+bytes(6), scalar._dxt5_block)):
+            expected = bytes((170, 0, 85, 170))*16
+            self.assertEqual(block_decoder(alpha+colour), expected)
+            self.assertEqual(rx2_fast._decode(alpha+colour, 4, 4, kind, False), expected)

tools/vendor/university/tools/vanilla_map_extraction/tools/skate3_streams.py

@@ -315,6 +315,8 @@ def load_district_stream(
     root = Path(stream_directory)
     records = read_atoc(root / f"{district_name}_{stream_name}.xst")
     stream_files = sorted(root.glob(f"c{stream_name}_*.xsf"))
+    if not records and not stream_files:
+        return []
     if not stream_files:
         raise StreamFormatError(
             f"{root} contains no c{stream_name}_*.xsf stream files"

tools/vendor/utt/rx2_fast.py

@@ -17,7 +17,9 @@ def _layout(units,width,pitch):
     y=(((m//(aligned>>5))<<2)+((t>>(6+log))&1)+((b&2048)>>10))<<3
     y+=(((t&(((pitch<<6)-1)&~31))+((t&15)<<1))>>(3+log))&~1
     y+=(t&16)>>4
-    dest=y*width+x;valid=dest<units
+    # Padded tile columns are outside the logical image. Flattening them
+    # first aliases them onto later rows and overwrites valid blocks.
+    dest=y*width+x;valid=(x<width)&(dest<units)
     # Match scalar last-write ordering even for narrow padded surfaces.
     source=np.full(units,-1,dtype=np.int64)
     np.maximum.at(source,dest[valid],offset[valid])
@@ -39,7 +41,9 @@ def _blocks(data,w,h,size):
 def _colour(b,transparent):
     c=b[:,:4].astype(np.uint32);c0=c[:,0]*256+c[:,1];c1=c[:,2]*256+c[:,3]
     def rgb(v):return np.stack((((v>>11)&31)*255//31,((v>>5)&63)*255//63,(v&31)*255//31),axis=1)
-    a=rgb(c0);z=rgb(c1);four=c0>c1
+    # BC2/BC3 always use four colours. Only BC1 uses endpoint ordering
+    # to select the three-colour + transparent-black palette.
+    a=rgb(c0);z=rgb(c1);four=(c0>c1) | (not transparent)
     pal=np.zeros((len(b),4,4),dtype=np.uint8);pal[:,:,3]=255
     pal[:,0,:3]=a;pal[:,1,:3]=z
     pal[:,2,:3]=np.where(four[:,None],(2*a+z+1)//3,(a+z)//2)

tools/vendor/utt/rx2_parser.py

@@ -426,7 +426,7 @@ def _untile360(src, width_units, texel_pitch):
             y = _x360_tiled_y(j * width_units + i, width_units, texel_pitch)
             dst_idx = (y * width_units + x) * texel_pitch
             src_idx = (j * width_units + i) * texel_pitch
-            if dst_idx + texel_pitch <= len(dst) and src_idx + texel_pitch <= len(src):
+            if x < width_units and dst_idx + texel_pitch <= len(dst) and src_idx + texel_pitch <= len(src):
                 dst[dst_idx:dst_idx + texel_pitch] = src[src_idx:src_idx + texel_pitch]
     return bytes(dst)
 
@@ -473,7 +473,8 @@ def _dxt1_block(blk, transparent):
     c0, c1 = struct.unpack_from(">HH", blk, 0)
     r0, g0, b0 = _rgb565(c0)
     r1, g1, b1 = _rgb565(c1)
-    if c0 > c1:
+    # DXT3/DXT5 have independent alpha and always use four colour entries.
+    if c0 > c1 or not transparent:
         c2, c3 = _rgb565_interp((c0 >> 11) & 0x1F, (c1 >> 11) & 0x1F,
                                 (c0 >> 5) & 0x3F, (c1 >> 5) & 0x3F,
                                 c0 & 0x1F, c1 & 0x1F)