Read SKATE15 custom maps with lossless compressed storage transforms

crates/skate-data/src/skate_map.rs

@@ -1,8 +1,9 @@
-//! Little-endian SKATE01..14 reader. Layout follows the supplied
+//! Little-endian SKATE01..15 reader. Layout follows the supplied
 //! SK8R15/Source/tools/blender_owned_map/SKATE_FORMAT.md and
 //! owned/world/src/owned_map_package.cpp. No geometry or physics is inferred.
 use std::{io::Read, path::Path, time::Instant};
 mod texture_decode;
+mod storage_v15;
 
 #[derive(Debug, PartialEq)]
 pub struct SkateMap {
@@ -314,6 +315,7 @@ impl<'a> Reader<'a> {
 }
 /// Borrow compressed payloads while scanning; decompression does not mutate
 /// the package or depend on any other texture.
+#[derive(Clone, Copy)]
 struct StoredBlock<'a> { expected: usize, method: u32, bytes: &'a [u8] }
 impl StoredBlock<'_> {
     fn decode(&self) -> Result<Vec<u8>, String> {
@@ -342,6 +344,7 @@ impl StoredBlock<'_> {
                     .map_err(|e| format!("SKATE Zstandard: {e}"))?;
                 result
             }
+            3..=10 => return storage_v15::decode(method, bytes, expected),
             _ => return Err(format!("Unsupported SKATE storage method {method}")),
         };
         if decoded.len() != expected {
@@ -395,8 +398,8 @@ impl SkateMap {
             return Err("Invalid SKATE magic/version".into());
         }
         let version = (magic[5] - b'0') * 10 + magic[6] - b'0';
-        if !(1..=14).contains(&version) {
-            return Err("Unsupported SKATE version: reader supports 01 through 14; newer packages require an update".into());
+        if !(1..=15).contains(&version) {
+            return Err("Unsupported SKATE version: reader supports 01 through 15; newer packages require an update".into());
         }
         if r.u()? != 0x12345678 {
             return Err("Invalid SKATE endian marker".into());
@@ -428,6 +431,15 @@ impl SkateMap {
         for c in &mut counts[..n] {
             *c = r.u()?;
         }
+        let material_bytes = if version >= 15 {
+            let size = r.u()? as usize;
+            Some(r.stored(size)?)
+        } else { None };
+        let mut package_reader = None;
+        if let Some(bytes) = &material_bytes {
+            package_reader = Some(r);
+            r = Reader { bytes, at: 0 };
+        }
         r.check_count(counts[0], if version >= 2 { 80 } else { 48 })?;
         let mut materials = Vec::new();
         for _ in 0..counts[0] {
@@ -512,10 +524,14 @@ impl SkateMap {
             }
             materials.push(m);
         }
+        if let Some(package) = package_reader {
+            if r.at != r.bytes.len() { return Err("SKATE material block has trailing bytes".into()); }
+            r = package;
+        }
         let texture_started = Instant::now();
         r.check_count(counts[1], 20)?;
         let mut textures = Vec::new();
-        let mut texture_blocks = Vec::new();
+        let mut texture_blocks: Vec<StoredBlock<'_>> = Vec::new();
         for _ in 0..counts[1] {
             let name = r.string()?;
             let width = r.u()?;
@@ -534,6 +550,13 @@ impl SkateMap {
                 }
                 StoredBlock { expected, method: 0, bytes: r.take(bytes)? }
             };
+            let block = if version >= 15 && block.method == 11 {
+                let index = u32::from_le_bytes(block.bytes.try_into()
+                    .map_err(|_| "Invalid SKATE texture reference size")?) as usize;
+                let source = *texture_blocks.get(index).ok_or("Invalid SKATE forward texture reference")?;
+                if source.expected != expected { return Err("SKATE texture reference size mismatch".into()); }
+                source
+            } else { block };
             texture_blocks.push(block);
             textures.push(Texture {
                 name,

crates/skate-data/src/skate_map/storage_v15.rs

@@ -0,0 +1,152 @@
+//! Reversible SKATE15 storage transforms, matching the bundled exporter.
+use super::{Reader, StoredBlock};
+
+pub(super) fn decode(method: u32, bytes: &[u8], expected: usize) -> Result<Vec<u8>, String> {
+    let mut input = Reader { bytes, at: 0 };
+    let size = input.u()? as usize;
+    // Transformed records are at most a small multiple of their decoded size.
+    // Bound both lengths before allocating or decompressing untrusted data.
+    if size > expected.saturating_mul(3).saturating_add(16) || size > 2_147_483_648 {
+        return Err("SKATE transformed block exceeds size limit".into());
+    }
+    let data = StoredBlock {
+        expected: size, method: if method <= 6 { 2 } else { 1 },
+        bytes: &bytes[4..],
+    }.decode()?;
+    let mut source = Reader { bytes: &data, at: 0 };
+    let mut result = Vec::new();
+    match method {
+        3 | 7 => {
+            let width = source.u()? as usize;
+            let height = source.u()? as usize;
+            let row = width.checked_mul(4).ok_or("SKATE filtered width overflow")?;
+            if row.checked_mul(height) != Some(expected) || width > 16384 || height > 16384 {
+                return Err("SKATE filtered dimensions mismatch".into());
+            }
+            result.resize(expected, 0u8);
+            for y in 0..height {
+                let filter = source.take(1)?[0];
+                if filter > 4 { return Err("Invalid SKATE RGBA filter".into()); }
+                for (x, &value) in source.take(row)?.iter().enumerate() {
+                    let at = y * row + x;
+                    let left = if x >= 4 { result[at - 4] as i32 } else { 0 };
+                    let above = if y > 0 { result[at - row] as i32 } else { 0 };
+                    let upper_left = if x >= 4 && y > 0 { result[at - row - 4] as i32 } else { 0 };
+                    let predictor = match filter {
+                        1 => left, 2 => above, 3 => (left + above) / 2,
+                        4 => {
+                            let p = left + above - upper_left;
+                            let a = (p - left).abs();
+                            let b = (p - above).abs();
+                            let c = (p - upper_left).abs();
+                            if a <= b && a <= c { left } else if b <= c { above } else { upper_left }
+                        }
+                        _ => 0,
+                    };
+                    result[at] = value.wrapping_add(predictor as u8);
+                }
+            }
+        }
+        4 | 8 => {
+            if expected % 56 != 0 || data.len() != expected { return Err("Invalid SKATE vertex streams".into()); }
+            result.resize(expected, 0);
+            for (offset, length) in [(0,12), (12,12), (24,8), (32,8), (40,4), (44,8), (52,4)] {
+                for vertex in 0..expected / 56 {
+                    let at = vertex * 56 + offset;
+                    result[at..at + length].copy_from_slice(source.take(length)?);
+                }
+            }
+        }
+        5 | 9 => {
+            if expected % 4 != 0 { return Err("Invalid SKATE index block size".into()); }
+            let mut previous = 0i64;
+            for _ in 0..expected / 4 {
+                let mut zigzag = 0u64;
+                // A delta between u32 indices requires at most 33 bits.
+                for shift in (0..=28).step_by(7) {
+                    let byte = source.take(1)?[0];
+                    if shift == 28 && byte > 31 { return Err("Invalid SKATE index varint".into()); }
+                    zigzag |= u64::from(byte & 127) << shift;
+                    if byte & 128 == 0 { break; }
+                }
+                let current = previous + ((zigzag >> 1) as i64 ^ -((zigzag & 1) as i64));
+                let current = u32::try_from(current).map_err(|_| "SKATE index delta overflow")?;
+                result.extend(current.to_le_bytes());
+                previous = i64::from(current);
+            }
+        }
+        6 | 10 => {
+            if expected % 48 != 0 { return Err("Invalid SKATE collision block size".into()); }
+            let count = source.u()? as usize;
+            let vertices = source.take(count.checked_mul(12).ok_or("SKATE collision vertex overflow")?)?;
+            for _ in 0..expected / 48 {
+                for _ in 0..3 {
+                    let index = source.u()? as usize;
+                    if index >= count { return Err("Invalid SKATE collision vertex reference".into()); }
+                    result.extend_from_slice(&vertices[index * 12..index * 12 + 12]);
+                }
+                result.extend_from_slice(source.take(12)?);
+            }
+        }
+        _ => return Err("Invalid SKATE storage transform".into()),
+    }
+    if source.at != data.len() || result.len() != expected { return Err("SKATE transformed size mismatch or trailing data".into()); }
+    Ok(result)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+    fn unpack(method: u32, data: &[u8], expected: usize) -> Result<Vec<u8>, String> {
+        let mut bytes = (data.len() as u32).to_le_bytes().to_vec();
+        if method <= 6 { bytes.extend(zstd::stream::encode_all(data, 1).unwrap()); }
+        else {
+            let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast());
+            encoder.write_all(data).unwrap();
+            bytes.extend(encoder.finish().unwrap());
+        }
+        decode(method, &bytes, expected)
+    }
+    #[test]
+    fn both_codecs_restore_vertices_indices_collision_and_rgba_exactly() {
+        let vertices: Vec<u8> = (0..112).collect();
+        let mut soa = Vec::new();
+        for (offset, len) in [(0,12), (12,12), (24,8), (32,8), (40,4), (44,8), (52,4)] {
+            for v in 0..2 { soa.extend_from_slice(&vertices[v * 56 + offset..v * 56 + offset + len]); }
+        }
+        let mut collision = 3u32.to_le_bytes().to_vec();
+        collision.extend(0u8..36);
+        for index in [2u32, 0, 1] { collision.extend(index.to_le_bytes()); }
+        collision.extend(36u8..48);
+        let expected_collision: Vec<u8> = (24..36).chain(0..24).chain(36..48).collect();
+        let mut rgba = 2u32.to_le_bytes().to_vec();
+        rgba.extend(5u32.to_le_bytes());
+        for filter in 0..5u8 {
+            rgba.push(filter);
+            // Constant RGBA rows exercise all five PNG-style predictors.
+            rgba.extend(if filter == 0 { [10; 8] } else if filter == 1 { [10,10,10,10,0,0,0,0] }
+                else if filter == 3 { [5,5,5,5,0,0,0,0] } else { [0; 8] });
+        }
+        for shift in [0, 4] {
+            assert_eq!(unpack(3 + shift, &rgba, 40).unwrap(), vec![10; 40]);
+            assert_eq!(unpack(4 + shift, &soa, 112).unwrap(), vertices);
+            assert_eq!(unpack(5 + shift, &[0, 4, 1], 12).unwrap(), [0u32,2,1].into_iter().flat_map(u32::to_le_bytes).collect::<Vec<_>>());
+            assert_eq!(unpack(6 + shift, &collision, 48).unwrap(), expected_collision);
+        }
+    }
+    #[test]
+    fn malformed_transforms_fail_without_panics() {
+        for method in 3..=10 {
+            assert!(unpack(method, &[], 48).is_err());
+            assert!(decode(method, &[255; 4], 48).is_err());
+        }
+        assert!(unpack(9, &[1], 4).is_err()); // negative first index
+        assert!(unpack(9, &[128; 5], 4).is_err());
+        assert!(unpack(9, &[0, 0], 4).is_err()); // trailing varint
+        assert!(unpack(8, &[0; 56], 55).is_err());
+        let mut bad_collision = 0u32.to_le_bytes().to_vec();
+        bad_collision.extend([0; 24]);
+        assert!(unpack(10, &bad_collision, 48).is_err());
+    }
+}

crates/skate-data/tests/skate_map.rs

@@ -57,6 +57,7 @@ fn fixture(version: u8) -> Vec<u8> {
     if version >= 8 {
         u(&mut b, 1);
     }
+    let material_start = b.len();
     s(&mut b, "Surface");
     u(&mut b, 1);
     fs(&mut b, &[0.6, 0.1, 1., 0.5, 0.25, 0.8, 0.]);
@@ -78,6 +79,11 @@ fn fixture(version: u8) -> Vec<u8> {
     if version >= 12 {
         u(&mut b, 0);
     }
+    if version >= 15 {
+        let material = b.split_off(material_start);
+        u(&mut b, material.len() as u32);
+        stored(&mut b, &material, 2);
+    }
     s(&mut b, "Pixel");
     for n in [1, 1, 1] {
         u(&mut b, n);
@@ -172,7 +178,7 @@ fn stored(b: &mut Vec<u8>, data: &[u8], method: u8) {
 }
 #[test]
 fn reads_all_documented_versions_without_moving_geometry() {
-    for version in 1..=14 {
+    for version in 1..=15 {
         let map = SkateMap::parse(&fixture(version)).unwrap();
         assert_eq!(map.version, version);
         assert_eq!(map.spawn, [2., 0., 3.]);
@@ -194,6 +200,36 @@ fn reads_all_documented_versions_without_moving_geometry() {
         }
     }
 }
+
+#[test]
+fn v15_texture_references_preserve_order_and_reject_forward_or_wrong_size() {
+    let mut data = fixture(15);
+    let header_counts = 8 + 4 + 4 + "Fixture".len() + 49 * 4;
+    data[header_counts + 4..header_counts + 8].copy_from_slice(&3u32.to_le_bytes());
+    let mut needle = Vec::new();
+    s(&mut needle, "Pixel");
+    let start = data.windows(needle.len()).position(|w| w == needle).unwrap();
+    let metadata_end = start + needle.len() + 12;
+    let texture_end = metadata_end + 8 + 4; // fixture v15 uses raw RGBA
+    let mut references = Vec::new();
+    for source in [0, 1] {
+        references.extend_from_slice(&data[start..metadata_end]);
+        u(&mut references, 11);
+        u(&mut references, 4);
+        u(&mut references, source);
+    }
+    data.splice(texture_end..texture_end, references);
+    let map = SkateMap::parse(&data).unwrap();
+    assert_eq!(map.textures.len(), 3);
+    assert!(map.textures.iter().all(|t| t.rgba == [255, 128, 64, 255]));
+    let reference = texture_end + (metadata_end - start) + 8;
+    data[reference..reference + 4].copy_from_slice(&1u32.to_le_bytes());
+    assert!(SkateMap::parse(&data).unwrap_err().contains("forward texture reference"));
+    data[reference..reference + 4].copy_from_slice(&0u32.to_le_bytes());
+    let width = texture_end + needle.len();
+    data[width..width + 4].copy_from_slice(&2u32.to_le_bytes());
+    assert!(SkateMap::parse(&data).unwrap_err().contains("reference size mismatch"));
+}
 #[test]
 fn rejects_every_truncated_prefix_and_trailing_data() {
     let data = fixture(8);