Resolve VLT text array values before discarding the binary bank

tools/asset_pipeline/test_vlt.py

@@ -1,7 +1,7 @@
 """Synthetic fixtures for the binary array layout; contains no game data."""
 import struct
 import unittest
-from .vlt import array_items
+from .vlt import array_items, array_text_items
 
 
 class ArrayLayoutTests(unittest.TestCase):
@@ -32,6 +32,13 @@ class ArrayLayoutTests(unittest.TestCase):
             with self.assertRaises(ValueError):
                 array_items(raw, 0, 4, 2)
 
+    def test_text_array_resolves_relocated_pointers(self):
+        binary = bytes(8) + 'test\0caf\u00e9\0'.encode()
+        self.assertEqual(array_text_items(binary, ['00000008', '0000000D', '00000000']),
+                         ['test', 'caf\u00e9', ''])
+        with self.assertRaises(ValueError):
+            array_text_items(binary, ['0000FFFF'])
+
 
 if __name__ == '__main__':
     unittest.main()

tools/asset_pipeline/vlt.py

@@ -65,6 +65,20 @@ def array_items(data, pos, size, alignment):
             'alignment': boundary, 'items': items}
 
 
+def array_text_items(binary, items):
+    values = []
+    for raw in items:
+        ptr, = unpack('I', bytes.fromhex(raw), 0)
+        if not ptr:
+            values.append('')
+            continue
+        end = binary.find(b'\0', ptr)
+        if end < 0:
+            raise ValueError(f'Unterminated VLT array text at {ptr:#x}')
+        values.append(take(binary, ptr, end-ptr).decode('utf-8'))
+    return values
+
+
 def chunks(data):
     at=0
     while at<len(data):
@@ -140,6 +154,9 @@ def convert(schema_stem, collections_stem, names):
                     # Keep the legacy data field unchanged for existing
                     # gameplay consumers. New readers use the full array.
                     out[name(fkey)]['array']=array_items(data,pos,n,alignment)
+                    if t=='EA::Reflection::Text':
+                        array=out[name(fkey)]['array']
+                        array['text_items']=array_text_items(cb,array['items'])
         for fk,f in fields.items():
             if f[4]&2 and layout: value(fk,cb,layout+f[1])
         entries=at+48+typeslen*8