Preserve complete aligned VLT arrays alongside legacy field data

tools/asset_pipeline/test_vlt.py

@@ -0,0 +1,37 @@
+"""Synthetic fixtures for the binary array layout; contains no game data."""
+import struct
+import unittest
+from .vlt import array_items
+
+
+class ArrayLayoutTests(unittest.TestCase):
+    def test_vector4_header_alignment(self):
+        first = struct.pack('>4f', .75, .35, 0, 0)
+        second = struct.pack('>4f', 1, 1, 1, 1)
+        raw = struct.pack('>4H', 2, 2, 16, 0x8000) + bytes(8) + first + second
+        result = array_items(raw, 0, 16, 4)
+        self.assertEqual(result['items'], [first.hex().upper(), second.hex().upper()])
+        self.assertEqual(result['alignment'], 16)
+
+    def test_each_element_is_aligned_at_absolute_offset(self):
+        a, b = struct.pack('>3f', 2, 3, 4), struct.pack('>3f', 5, 6, 7)
+        raw = bytes(4) + struct.pack('>4H', 3, 2, 12, 0) + bytes(4)
+        raw += a + bytes(4) + b + bytes(4) + bytes(12)
+        result = array_items(raw, 4, 12, 4)
+        self.assertEqual(result['capacity'], 3)
+        self.assertEqual(result['items'], [a.hex().upper(), b.hex().upper()])
+
+    def test_empty_and_pointer_arrays(self):
+        self.assertEqual(array_items(struct.pack('>4H', 0, 0, 4, 0), 0, 4, 2)['items'], [])
+        raw = struct.pack('>4H2I', 2, 2, 4, 0, 0x1234, 0)
+        self.assertEqual(array_items(raw, 0, 4, 2)['items'], ['00001234', '00000000'])
+
+    def test_invalid_header_or_truncated_capacity_fails(self):
+        for raw in [struct.pack('>4H', 1, 2, 4, 0), struct.pack('>4H', 1, 1, 8, 0),
+                    struct.pack('>4HI', 2, 1, 4, 0, 123)]:
+            with self.assertRaises(ValueError):
+                array_items(raw, 0, 4, 2)
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/asset_pipeline/vlt.py

@@ -43,6 +43,28 @@ def take(data, at, size):
 def unpack(fmt,data,at):
     return struct.unpack('>'+fmt,take(data,at,struct.calcsize('>'+fmt)))
 
+def array_items(data, pos, size, alignment):
+    """ModernBase array: four u16 header lanes, then aligned elements.
+
+    Alignment in the schema is log2(bytes), not the array header's mask.
+    Retain each element separately: padding is not part of its value.
+    """
+    capacity, count, stride, _ = unpack('4H', data, pos)
+    if count > capacity or stride != size or not stride or alignment > 16:
+        raise ValueError(f'Invalid VLT array at {pos:#x}')
+    boundary = 1 << alignment
+    cursor = pos + 8
+    items = []
+    for index in range(capacity):
+        cursor = (cursor + boundary - 1) & -boundary
+        raw = take(data, cursor, stride)
+        if index < count:
+            items.append(raw.hex().upper())
+        cursor += stride
+    return {'capacity': capacity, 'element_size': stride,
+            'alignment': boundary, 'items': items}
+
+
 def chunks(data):
     at=0
     while at<len(data):
@@ -114,6 +136,10 @@ def convert(schema_stem, collections_stem, names):
             else:
                 raw=take(data,pos,length)
                 out[name(fkey)]={'type':t,'data':raw.hex().upper()}
+                if flags&1:
+                    # Keep the legacy data field unchanged for existing
+                    # gameplay consumers. New readers use the full array.
+                    out[name(fkey)]['array']=array_items(data,pos,n,alignment)
         for fk,f in fields.items():
             if f[4]&2 and layout: value(fk,cb,layout+f[1])
         entries=at+48+typeslen*8