diff --git a/crates/skate-data/examples/xex_unpack.rs b/crates/skate-data/examples/xex_unpack.rs new file mode 100644 index 00000000..b010078d --- /dev/null +++ b/crates/skate-data/examples/xex_unpack.rs @@ -0,0 +1,26 @@ +//! Unpacks a retail XEX2 to its mapped base image and locates the ocean PCA table. +//! Usage: cargo run -p skate-data --example xex_unpack -- [] +fn main() { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let xex = std::fs::read(&args[0]).expect("read XEX"); + let started = std::time::Instant::now(); + let image = skate_data::xex::XexImage::parse(&xex).unwrap_or_else(|e| panic!("{e}")); + println!( + "base 0x{:08X}, image {} bytes, {:.2} s", + image.base_address, + image.image.len(), + started.elapsed().as_secs_f64() + ); + match skate_data::ocean_pca::extract(&image) { + Ok(pca) => println!( + "ocean PCA table at 0x{:08X}, frame 0 mean {:?}, {:.2} s", + pca.means_address, + pca.frames[0][0], + started.elapsed().as_secs_f64() + ), + Err(e) => println!("ocean PCA: {e}"), + } + if let Some(out) = args.get(1) { + std::fs::write(out, &image.image).expect("write image"); + } +} diff --git a/crates/skate-data/src/lib.rs b/crates/skate-data/src/lib.rs index 7677f014..200f7e93 100644 --- a/crates/skate-data/src/lib.rs +++ b/crates/skate-data/src/lib.rs @@ -18,3 +18,5 @@ mod sha256; pub mod state_graph; pub mod skate_map; pub mod retail_collision; +pub mod xex; +pub mod ocean_pca; diff --git a/crates/skate-data/src/ocean_pca.rs b/crates/skate-data/src/ocean_pca.rs new file mode 100644 index 00000000..6d40b7da --- /dev/null +++ b/crates/skate-data/src/ocean_pca.rs @@ -0,0 +1,137 @@ +//! The built-in PCA ocean/water animation table (cPCAWaterAnimationData) from +//! an unpacked retail executable, written as `assets/private/ocean-pca.json`. +//! +//! TU3 keeps the 30 frame means at 0x830118D8 and the weights right after them +//! at 0x83011A40 (tools/asset_pipeline/ocean_pca.py). Other builds place the +//! table elsewhere, so it is located the way the code reaches it: a `lis` +//! followed by a D-form use of the same register forms each address, and +//! cPCAWaterAnimationData::Init forms both within a few instructions. +use crate::xex::XexImage; + +const FRAMES: usize = 30; +const MEANS: usize = FRAMES * 3; +const WEIGHTS: usize = FRAMES * 24; +const WEIGHTS_OFFSET: u32 = (MEANS * 4) as u32; // 0x168 +/// Instructions between `lis` and the instruction completing an address. +const ADDRESS_WINDOW: usize = 8; +/// Instructions between the two address loads. +const PAIR_WINDOW: usize = 32; + +pub struct OceanPca { + pub means_address: u32, + /// Per frame: mean row then six weight rows, in shader order, scaled /255. + pub frames: Vec<[[f32; 4]; 7]>, +} + +pub fn extract(image: &XexImage) -> Result { + let candidates = locate(image); + let means_address = match candidates.as_slice() { + [one] => *one, + [] => return Err("Ocean PCA table not found in the executable".into()), + many => { + return Err(format!( + "Ocean PCA table is ambiguous: {} candidates ({:08X?})", + many.len(), + many + )) + } + }; + let means = floats(image, means_address, MEANS).ok_or("Ocean PCA means out of range")?; + let weights = floats(image, means_address + WEIGHTS_OFFSET, WEIGHTS).ok_or("Ocean PCA weights out of range")?; + let frames = (0..FRAMES) + .map(|frame| { + let mean = &means[frame * 3..frame * 3 + 3]; + let weight = &weights[frame * 24..frame * 24 + 24]; + let mut rows = [[0.; 4]; 7]; + // Shader model order: mean X/Z/Y and weight pairs R/B/G. + rows[0] = [mean[0] / 255., mean[2] / 255., mean[1] / 255., 0.]; + for (row, start) in rows[1..].iter_mut().zip([0, 4, 16, 20, 8, 12]) { + *row = core::array::from_fn(|i| weight[start + i] / 255.); + } + rows + }) + .collect(); + Ok(OceanPca { means_address, frames }) +} + +impl OceanPca { + /// The JSON read by the game (`retail_render::read_pca`), matching + /// tools/asset_pipeline/ocean_pca.py. + pub fn to_json(&self, source_sha256: &str) -> String { + serde_json::json!({ + "source_sha256": source_sha256, + "means_address": format!("0x{:08X}", self.means_address), + "hz": 30.0, + "frames": self.frames, + }) + .to_string() + } +} + +fn floats(image: &XexImage, address: u32, count: usize) -> Option> { + let bytes = image.at(address, count * 4)?; + let values: Vec = bytes + .chunks_exact(4) + .map(|b| f32::from_be_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + values.iter().all(|v| v.is_finite()).then_some(values) +} + +/// A table holds 30 normal-like means (0..255 components) and nonzero weights. +fn plausible(image: &XexImage, address: u32) -> bool { + let (Some(means), Some(weights)) = ( + floats(image, address, MEANS), + floats(image, address + WEIGHTS_OFFSET, WEIGHTS), + ) else { + return false; + }; + means.iter().all(|v| (0.0..=255.0).contains(v)) + && means.iter().any(|v| *v != 0.) + && weights.iter().all(|v| v.abs() <= 255.) + && weights.iter().filter(|v| **v != 0.).count() > WEIGHTS / 2 +} + +fn locate(image: &XexImage) -> Vec { + let words: Vec = image + .image + .chunks_exact(4) + .map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + // (instruction index, address) for every lis + D-form completion. + let mut formed: Vec<(usize, u32)> = Vec::new(); + for (i, &w) in words.iter().enumerate() { + let (op, rd, ra) = (w >> 26, (w >> 21) & 31, (w >> 16) & 31); + if op != 15 || ra != 0 { + continue; + } + let high = (w & 0xffff) << 16; + for (j, &u) in words.iter().enumerate().skip(i + 1).take(ADDRESS_WINDOW) { + let (uop, urd, ura) = (u >> 26, (u >> 21) & 31, (u >> 16) & 31); + // addi and the integer/float loads and stores. + if ura == rd && matches!(uop, 14 | 32 | 34 | 36 | 40 | 44 | 48 | 50 | 52 | 54) { + let low = (u & 0xffff) as u16 as i16 as i32; + formed.push((j, high.wrapping_add(low as u32))); + } + if uop == 15 && urd == rd { + break; + } + } + } + let mut found: Vec = Vec::new(); + for (k, &(i, address)) in formed.iter().enumerate() { + let pair = formed[k.saturating_sub(64)..(k + 64).min(formed.len())] + .iter() + .any(|&(j, other)| other == address.wrapping_add(WEIGHTS_OFFSET) && i.abs_diff(j) <= PAIR_WINDOW); + if pair && !found.contains(&address) && plausible(image, address) { + found.push(address); + } + } + found +} + +/// Unpacks `xex`, extracts the table and returns the game's JSON. +pub fn json_from_xex(xex: &[u8]) -> Result { + let image = XexImage::parse(xex)?; + let pca = extract(&image)?; + Ok(pca.to_json(&crate::sha256::digest(&image.image))) +} diff --git a/crates/skate-data/src/xex/aes.rs b/crates/skate-data/src/xex/aes.rs new file mode 100644 index 00000000..3da05869 --- /dev/null +++ b/crates/skate-data/src/xex/aes.rs @@ -0,0 +1,162 @@ +//! AES-128 decryption (FIPS-197 inverse cipher), only what XEX2 needs: +//! ECB for the file key and CBC with a zero IV for the payload. + +const SBOX: [u8; 256] = [ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16, +]; + +fn xtime(x: u8) -> u8 { + (x << 1) ^ if x & 0x80 != 0 { 0x1b } else { 0 } +} + +fn mul(mut a: u8, mut b: u8) -> u8 { + let mut product = 0; + while b != 0 { + if b & 1 != 0 { + product ^= a; + } + a = xtime(a); + b >>= 1; + } + product +} + +pub(crate) struct Aes128 { + round_keys: [[u8; 16]; 11], + inverse_sbox: [u8; 256], +} + +impl Aes128 { + pub(crate) fn new(key: [u8; 16]) -> Self { + let mut words = [[0u8; 4]; 44]; + for (i, word) in words.iter_mut().take(4).enumerate() { + word.copy_from_slice(&key[i * 4..i * 4 + 4]); + } + let mut rcon = 1u8; + for i in 4..44 { + let mut t = words[i - 1]; + if i % 4 == 0 { + t = [SBOX[t[1] as usize] ^ rcon, SBOX[t[2] as usize], SBOX[t[3] as usize], SBOX[t[0] as usize]]; + rcon = xtime(rcon); + } + for j in 0..4 { + words[i][j] = words[i - 4][j] ^ t[j]; + } + } + let mut round_keys = [[0u8; 16]; 11]; + for (round, key) in round_keys.iter_mut().enumerate() { + for j in 0..4 { + key[j * 4..j * 4 + 4].copy_from_slice(&words[round * 4 + j]); + } + } + let mut inverse_sbox = [0u8; 256]; + for (i, &s) in SBOX.iter().enumerate() { + inverse_sbox[s as usize] = i as u8; + } + Self { round_keys, inverse_sbox } + } + + pub(crate) fn decrypt_block(&self, block: &mut [u8; 16]) { + let add = |block: &mut [u8; 16], key: &[u8; 16]| { + for (b, k) in block.iter_mut().zip(key) { + *b ^= k; + } + }; + add(block, &self.round_keys[10]); + for round in (0..10).rev() { + // Inverse ShiftRows (state is column-major: byte r + 4c). + let s = *block; + for c in 0..4 { + for r in 0..4 { + block[r + 4 * ((c + r) % 4)] = s[r + 4 * c]; + } + } + for b in block.iter_mut() { + *b = self.inverse_sbox[*b as usize]; + } + add(block, &self.round_keys[round]); + if round > 0 { + for c in 0..4 { + let col = [block[4 * c], block[4 * c + 1], block[4 * c + 2], block[4 * c + 3]]; + for r in 0..4 { + block[4 * c + r] = mul(col[r], 14) + ^ mul(col[(r + 1) % 4], 11) + ^ mul(col[(r + 2) % 4], 13) + ^ mul(col[(r + 3) % 4], 9); + } + } + } + } + } + + /// CBC with a zero IV over whole 16-byte blocks; a trailing partial + /// block is left untouched. + pub(crate) fn decrypt_cbc_zero_iv(&self, data: &mut [u8]) { + let mut previous = [0u8; 16]; + for chunk in data.chunks_exact_mut(16) { + let cipher: [u8; 16] = chunk.try_into().unwrap(); + let mut block = cipher; + self.decrypt_block(&mut block); + for (out, (plain, prev)) in chunk.iter_mut().zip(block.iter().zip(previous)) { + *out = plain ^ prev; + } + previous = cipher; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fips_197_appendix_c1_vector_decrypts() { + let key: [u8; 16] = core::array::from_fn(|i| i as u8); + let mut block = [ + 0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a, + ]; + Aes128::new(key).decrypt_block(&mut block); + let expected: [u8; 16] = core::array::from_fn(|i| (i as u8) * 0x11); + assert_eq!(block, expected); + } + + #[test] + fn cbc_chains_previous_ciphertext() { + // NIST SP 800-38A F.2.2 (CBC-AES128.Decrypt), first two blocks, but + // with a zero IV: the first plaintext is XORed with the real IV here. + let key = [ + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c, + ]; + let iv: [u8; 16] = core::array::from_fn(|i| i as u8); + let mut data = [ + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + ]; + Aes128::new(key).decrypt_cbc_zero_iv(&mut data); + for (b, v) in data[..16].iter_mut().zip(iv) { + *b ^= v; + } + assert_eq!( + data, + [ + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + ] + ); + } +} diff --git a/crates/skate-data/src/xex/lzx.rs b/crates/skate-data/src/xex/lzx.rs new file mode 100644 index 00000000..7e6e07ed --- /dev/null +++ b/crates/skate-data/src/xex/lzx.rs @@ -0,0 +1,354 @@ +//! LZX decompression as used by XEX2 "normal" compression: one continuous +//! stream, 32 KiB frames, no reset interval, no E8 translation. +//! Follows the classic (non-DELTA) LZX decoder of libmspack's lzxd.c. + +const FRAME: usize = 32768; +const MIN_MATCH: usize = 2; +const NUM_CHARS: usize = 256; +const PRETREE: usize = 20; +const LENGTH_TREE: usize = 249; +const ALIGNED_TREE: usize = 8; +const MAX_BITS: usize = 16; + +/// 16-bit little-endian words, consumed most-significant bit first. +struct Bits<'a> { + data: &'a [u8], + pos: usize, + buffer: u32, + left: u32, +} + +impl<'a> Bits<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0, buffer: 0, left: 0 } + } + fn byte(&mut self) -> u8 { + // Past the end the stream reads as zeros, like libmspack's padding. + let b = self.data.get(self.pos).copied().unwrap_or(0); + self.pos += 1; + b + } + fn ensure(&mut self, n: u32) { + while self.left < n { + let lo = u32::from(self.byte()); + let hi = u32::from(self.byte()); + self.buffer |= ((hi << 8) | lo) << (16 - self.left); + self.left += 16; + } + } + fn peek(&mut self, n: u32) -> u32 { + self.ensure(n); + self.buffer >> (32 - n) + } + fn remove(&mut self, n: u32) { + self.buffer = if n >= 32 { 0 } else { self.buffer << n }; + self.left -= n; + } + fn read(&mut self, n: u32) -> u32 { + if n == 0 { + return 0; + } + if n > 16 { + let hi = self.read(n - 16); + return (hi << 16) | self.read(16); + } + let v = self.peek(n); + self.remove(n); + v + } + /// libmspack's start-of-uncompressed-block realignment. + fn realign_to_bytes(&mut self) { + self.ensure(16); + if self.left > 16 { + self.pos -= 2; + } + self.left = 0; + self.buffer = 0; + } + /// End-of-frame realignment to a 16-bit boundary. + fn realign_frame(&mut self) { + if self.left > 0 { + self.ensure(16); + } + if self.left & 15 != 0 { + let n = self.left & 15; + self.remove(n); + } + } + fn le32(&mut self) -> u32 { + u32::from_le_bytes([self.byte(), self.byte(), self.byte(), self.byte()]) + } +} + +/// Canonical Huffman decoder built from code lengths (MSB-first codes). +struct Tree { + /// Symbols sorted by (length, symbol). + symbols: Vec, + /// Per length: number of codes. + counts: [u16; MAX_BITS + 1], + empty: bool, +} + +impl Tree { + fn new(lengths: &[u8]) -> Result { + let mut counts = [0u16; MAX_BITS + 1]; + for &l in lengths { + if usize::from(l) > MAX_BITS { + return Err("LZX code length exceeds 16".into()); + } + counts[usize::from(l)] += 1; + } + counts[0] = 0; + let mut symbols: Vec = (0..lengths.len() as u16).filter(|&s| lengths[s as usize] != 0).collect(); + symbols.sort_by_key(|&s| (lengths[s as usize], s)); + // Over-subscribed trees are invalid; incomplete ones are tolerated + // only when empty (libmspack accepts an all-zero tree). + let mut left: i32 = 1; + for len in 1..=MAX_BITS { + left = (left << 1) - i32::from(counts[len]); + if left < 0 { + return Err("LZX Huffman tree is over-subscribed".into()); + } + } + let empty = symbols.is_empty(); + if !empty && left != 0 { + return Err("LZX Huffman tree is incomplete".into()); + } + Ok(Self { symbols, counts, empty }) + } + + fn decode(&self, bits: &mut Bits<'_>) -> Result { + if self.empty { + return Err("LZX decode from an empty Huffman tree".into()); + } + let mut code: i32 = 0; + let mut first: i32 = 0; + let mut index: i32 = 0; + for len in 1..=MAX_BITS { + code |= bits.read(1) as i32; + let count = i32::from(self.counts[len]); + if code - first < count { + return Ok(usize::from(self.symbols[(index + code - first) as usize])); + } + index += count; + first = (first + count) << 1; + code <<= 1; + } + Err("LZX Huffman code not found".into()) + } +} + +fn read_lengths(bits: &mut Bits<'_>, lengths: &mut [u8], first: usize, last: usize) -> Result<(), String> { + let mut pre = [0u8; PRETREE]; + for l in &mut pre { + *l = bits.read(4) as u8; + } + let pretree = Tree::new(&pre)?; + let mut x = first; + while x < last { + let z = pretree.decode(bits)?; + match z { + 17 => { + let run = bits.read(4) as usize + 4; + fill(lengths, &mut x, last, run, 0)?; + } + 18 => { + let run = bits.read(5) as usize + 20; + fill(lengths, &mut x, last, run, 0)?; + } + 19 => { + let run = bits.read(1) as usize + 4; + let z = pretree.decode(bits)?; + let v = (i32::from(lengths[x]) - z as i32).rem_euclid(17) as u8; + fill(lengths, &mut x, last, run, v)?; + } + _ => { + lengths[x] = (i32::from(lengths[x]) - z as i32).rem_euclid(17) as u8; + x += 1; + } + } + } + Ok(()) +} + +fn fill(lengths: &mut [u8], x: &mut usize, last: usize, run: usize, value: u8) -> Result<(), String> { + if *x + run > last { + return Err("LZX length run overflows the tree".into()); + } + lengths[*x..*x + run].fill(value); + *x += run; + Ok(()) +} + +/// Decompresses `output_len` bytes with a window of `window_size` bytes. +pub(crate) fn decompress(input: &[u8], window_size: usize, output_len: usize) -> Result, String> { + let window_bits = window_size.trailing_zeros(); + if !window_size.is_power_of_two() || !(15..=21).contains(&window_bits) { + return Err(format!("Unsupported LZX window size {window_size}")); + } + let position_slots = match window_bits { + 20 => 42, + 21 => 50, + b => (b * 2) as usize, + }; + let mut extra_bits = [0u32; 51]; + let mut position_base = [0usize; 51]; + let mut j = 0; + let mut i = 0; + while i < 51 { + extra_bits[i] = j; + if i + 1 < 51 { + extra_bits[i + 1] = j; + } + if i != 0 && j < 17 { + j += 1; + } + i += 2; + } + let mut base = 0; + for i in 0..51 { + position_base[i] = base; + base += 1 << extra_bits[i]; + } + + let main_size = NUM_CHARS + position_slots * 8; + let mut main_lengths = vec![0u8; main_size]; + let mut length_lengths = vec![0u8; LENGTH_TREE]; + let mut bits = Bits::new(input); + let mut out = Vec::with_capacity(output_len); + let mut r = [1usize; 3]; + + // Header: E8 translation flag (+ 32-bit file size when set). + if bits.read(1) != 0 { + return Err("LZX E8 translation is not supported for XEX images".into()); + } + + let mut block_type = 0u32; + let mut block_remaining = 0usize; + let mut block_length = 0usize; + let mut main_tree: Option = None; + let mut length_tree: Option = None; + let mut aligned_tree: Option = None; + + while out.len() < output_len { + let frame_end = (out.len() + FRAME).min(output_len); + while out.len() < frame_end { + if block_remaining == 0 { + if block_type == 3 && block_length & 1 == 1 { + bits.byte(); + } + block_type = bits.read(3); + let hi = bits.read(16) as usize; + let lo = bits.read(8) as usize; + block_length = (hi << 8) | lo; + block_remaining = block_length; + match block_type { + 2 | 1 => { + if block_type == 2 { + let mut aligned = [0u8; ALIGNED_TREE]; + for l in &mut aligned { + *l = bits.read(3) as u8; + } + aligned_tree = Some(Tree::new(&aligned)?); + } + read_lengths(&mut bits, &mut main_lengths, 0, NUM_CHARS)?; + read_lengths(&mut bits, &mut main_lengths, NUM_CHARS, main_size)?; + main_tree = Some(Tree::new(&main_lengths)?); + read_lengths(&mut bits, &mut length_lengths, 0, LENGTH_TREE)?; + length_tree = Some(Tree::new(&length_lengths)?); + } + 3 => { + bits.realign_to_bytes(); + for slot in &mut r { + *slot = bits.le32() as usize; + } + } + _ => return Err(format!("Invalid LZX block type {block_type}")), + } + if block_remaining == 0 { + return Err("Empty LZX block".into()); + } + } + let this_run = block_remaining.min(frame_end - out.len()); + let target = out.len() + this_run; + if block_type == 3 { + for _ in 0..this_run { + let b = bits.byte(); + out.push(b); + } + block_remaining -= this_run; + continue; + } + let main = main_tree.as_ref().ok_or("LZX main tree missing")?; + let start = out.len(); + while out.len() < target { + let symbol = main.decode(&mut bits)?; + if symbol < NUM_CHARS { + out.push(symbol as u8); + continue; + } + let symbol = symbol - NUM_CHARS; + let mut length = symbol & 7; + if length == 7 { + length += length_tree.as_ref().ok_or("LZX length tree missing")?.decode(&mut bits)?; + } + length += MIN_MATCH; + let slot = symbol >> 3; + let offset = if slot > 2 { + let extra = extra_bits[slot]; + let offset = if block_type == 1 { + if slot == 3 { + 1 + } else { + position_base[slot] - 2 + bits.read(extra) as usize + } + } else { + let aligned = aligned_tree.as_ref().ok_or("LZX aligned tree missing")?; + let mut offset = position_base[slot] - 2; + if extra > 3 { + offset += (bits.read(extra - 3) as usize) << 3; + offset += aligned.decode(&mut bits)?; + } else if extra == 3 { + offset += aligned.decode(&mut bits)?; + } else if extra > 0 { + offset += bits.read(extra) as usize; + } else { + offset = 1; + } + offset + }; + r[2] = r[1]; + r[1] = r[0]; + r[0] = offset; + offset + } else if slot == 0 { + r[0] + } else if slot == 1 { + r.swap(0, 1); + r[0] + } else { + r.swap(0, 2); + r[0] + }; + if offset == 0 || offset > out.len() || offset > window_size { + return Err(format!("LZX match offset {offset} outside the window at {}", out.len())); + } + for _ in 0..length { + let b = out[out.len() - offset]; + out.push(b); + } + } + let decoded = out.len() - start; + if decoded > block_remaining { + return Err("LZX match overran its block".into()); + } + block_remaining -= decoded; + if out.len() > frame_end { + return Err("LZX match overran its frame".into()); + } + } + bits.realign_frame(); + } + out.truncate(output_len); + Ok(out) +} diff --git a/crates/skate-data/src/xex/mod.rs b/crates/skate-data/src/xex/mod.rs new file mode 100644 index 00000000..84b72497 --- /dev/null +++ b/crates/skate-data/src/xex/mod.rs @@ -0,0 +1,136 @@ +//! Xbox 360 XEX2 executables: recover the mapped base image (the bytes the +//! console loads at the image base) from a retail `default.xex`. +//! Layout follows the public XEX2 documentation used by Xenia and xextool. +//! Supports retail encryption and "normal" (LZX) or "basic" (raw) compression. +mod aes; +mod lzx; + +/// The published Xbox 360 retail XEX key (it decrypts the per-file key). +const RETAIL_KEY: [u8; 16] = [ + 0x20, 0xb1, 0x85, 0xa5, 0x9d, 0x28, 0xfd, 0xc3, 0x40, 0x58, 0x3f, 0xbb, 0x08, 0x96, 0xbf, 0x91, +]; + +const FILE_FORMAT_INFO: u32 = 0x0000_03ff; +const IMAGE_BASE_ADDRESS: u32 = 0x0001_0201; + +#[derive(Debug)] +pub struct XexImage { + /// Virtual address of `image[0]`. + pub base_address: u32, + pub image: Vec, +} + +fn be16(d: &[u8], at: usize) -> Result { + d.get(at..at + 2) + .map(|b| u16::from_be_bytes([b[0], b[1]])) + .ok_or_else(|| format!("Truncated XEX at {at}")) +} +fn be32(d: &[u8], at: usize) -> Result { + d.get(at..at + 4) + .map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + .ok_or_else(|| format!("Truncated XEX at {at}")) +} +fn span(d: &[u8], at: usize, n: usize) -> Result<&[u8], String> { + d.get(at..at.checked_add(n).ok_or("XEX offset overflow")?) + .ok_or_else(|| format!("Truncated XEX at {at} (need {n})")) +} + +impl XexImage { + pub fn parse(xex: &[u8]) -> Result { + if span(xex, 0, 4)? != b"XEX2" { + return Err("Not an XEX2 executable".into()); + } + let payload_offset = be32(xex, 8)? as usize; + let security = be32(xex, 16)? as usize; + let header_count = be32(xex, 20)? as usize; + let mut format = None; + let mut base_address = None; + for i in 0..header_count { + let key = be32(xex, 24 + i * 8)?; + let value = be32(xex, 28 + i * 8)?; + match key { + FILE_FORMAT_INFO => format = Some(value as usize), + IMAGE_BASE_ADDRESS => base_address = Some(value), + _ => {} + } + } + let format = format.ok_or("XEX has no file format header")?; + let encryption = be16(xex, format + 4)?; + let compression = be16(xex, format + 6)?; + // Security info: image size at +4, the encrypted file key at +0x150, + // the load address at +0x110. + let image_size = be32(xex, security + 4)? as usize; + let base_address = match base_address { + Some(address) => address, + None => be32(xex, security + 0x110)?, + }; + let mut payload = span(xex, payload_offset, xex.len().saturating_sub(payload_offset))?.to_vec(); + match encryption { + 0 => {} + 1 => { + let mut key: [u8; 16] = span(xex, security + 0x150, 16)?.try_into().unwrap(); + aes::Aes128::new(RETAIL_KEY).decrypt_block(&mut key); + aes::Aes128::new(key).decrypt_cbc_zero_iv(&mut payload); + } + other => return Err(format!("Unsupported XEX encryption type {other}")), + } + let image = match compression { + 1 => basic(xex, format, &payload, image_size)?, + 2 => normal(xex, format, &payload, image_size)?, + other => return Err(format!("Unsupported XEX compression type {other}")), + }; + if image.get(..2) != Some(b"MZ") { + return Err("Unpacked XEX image does not start with a PE header (wrong key?)".into()); + } + Ok(Self { base_address, image }) + } + + /// Bytes at a virtual address. + pub fn at(&self, address: u32, len: usize) -> Option<&[u8]> { + let offset = address.checked_sub(self.base_address)? as usize; + self.image.get(offset..offset.checked_add(len)?) + } +} + +/// Basic compression: (data size, zero size) pairs, zeros not stored. +fn basic(xex: &[u8], format: usize, payload: &[u8], image_size: usize) -> Result, String> { + let info_size = be32(xex, format)? as usize; + let blocks = info_size.saturating_sub(8) / 8; + let mut image = Vec::with_capacity(image_size); + let mut at = 0; + for i in 0..blocks { + let data = be32(xex, format + 8 + i * 8)? as usize; + let zeros = be32(xex, format + 12 + i * 8)? as usize; + image.extend_from_slice(span(payload, at, data)?); + at += data; + image.resize(image.len() + zeros, 0); + } + image.resize(image_size, 0); + Ok(image) +} + +/// Normal compression: a chain of blocks, each starting with the next +/// block's size and hash, then u16-length-prefixed LZX chunks. +fn normal(xex: &[u8], format: usize, payload: &[u8], image_size: usize) -> Result, String> { + let window_size = be32(xex, format + 8)? as usize; + let mut block_size = be32(xex, format + 12)? as usize; + let mut at = 0; + let mut stream = Vec::with_capacity(payload.len()); + while block_size != 0 { + let block = span(payload, at, block_size)?; + let next = be32(block, 0)? as usize; + let mut p = 24; + loop { + let chunk = be16(block, p)? as usize; + p += 2; + if chunk == 0 { + break; + } + stream.extend_from_slice(span(block, p, chunk)?); + p += chunk; + } + at += block_size; + block_size = next; + } + lzx::decompress(&stream, window_size, image_size) +} diff --git a/crates/skate-game/src/main.rs b/crates/skate-game/src/main.rs index 8782d122..cd02007e 100644 --- a/crates/skate-game/src/main.rs +++ b/crates/skate-game/src/main.rs @@ -60,6 +60,8 @@ fn main() -> bevy::app::AppExit { Ok(false) => {} } if let Some(code) = crash_report::entry() { std::process::exit(code); } + // Setup tool mode: `--extract-ocean-pca `. + if let Some(code) = extract_ocean_pca() { std::process::exit(code); } let _trace = match profiling::init() { Ok(guard) => guard, Err(error) => { eprintln!("{error}"); return bevy::app::AppExit::error(); } @@ -158,3 +160,30 @@ fn main() -> bevy::app::AppExit { #[cfg(test)] #[path = "tests/action_host.rs"] mod action_host_tests; + +/// Writes the retail water/ocean animation table for setup (see +/// skate_data::ocean_pca). Runs before any game initialization. +fn extract_ocean_pca() -> Option { + let mut args = std::env::args_os().skip(1); + if args.next()? != "--extract-ocean-pca" { + return None; + } + let (Some(xex), Some(out), None) = (args.next(), args.next(), args.next()) else { + eprintln!("Usage: skate3rust --extract-ocean-pca "); + return Some(2); + }; + let result = std::fs::read(&xex) + .map_err(|e| format!("{}: {e}", std::path::Path::new(&xex).display())) + .and_then(|bytes| skate_data::ocean_pca::json_from_xex(&bytes)) + .and_then(|json| std::fs::write(&out, json).map_err(|e| format!("{}: {e}", std::path::Path::new(&out).display()))); + match result { + Ok(()) => { + println!("OCEAN_PCA_READY"); + Some(0) + } + Err(error) => { + eprintln!("Ocean PCA extraction failed: {error}"); + Some(1) + } + } +} diff --git a/docs/hails-additions/09-water.md b/docs/hails-additions/09-water.md new file mode 100644 index 00000000..cad7c03e --- /dev/null +++ b/docs/hails-additions/09-water.md @@ -0,0 +1,201 @@ +# Water (investigation in progress) + +Branch: `gameplay/water`. Status: **water bail confirmed in play; water animation extraction added, awaiting in-play check.** + +## Problem + +Water does nothing in play: the skater rides over it or falls through it as if it +were ordinary ground or empty space. The retail game has water-specific behaviour +(the `IsInWater` motion-graph condition, the water ragdoll profile, and the +"special surface" path in the wipeout state). + +## Evidence so far + +### 1. Water is a retail collision surface type, and it is in the converted maps + +Retail collision units carry a 16-bit surface ID. The surface **type** is +`(surface >> 7) & 31`; the engine already treats type 12 as water: + +- `crates/skate-core/src/physics/board_ground.rs` — the board sets collision flag + bit 25 (`Body872`) and records the contact height (`Body864`, + `surface_twelve_height`) when any part touches type 12. +- `crates/skate-game/src/physics/wipeout_states/prediction.rs` — the wipeout + trajectory query tests `surface & 0xF80 == 0x600` (type 12). + +New tool: `crates/skate-data/examples/water_surfaces.rs` lists the surface types +in each map's embedded RWCM collision and details the type-12 triangles. + +``` +cargo run --locked --release -p skate-data --example water_surfaces -- data/installations//maps/*.skate +``` + +Results (installation `c82bd63f…`): + +| Map | Water triangles | Surface ID | Heights | Notes | +|---|---|---|---|---| +| DownTown | 355 (330 flat, all one-sided) | 1591 only | 0.4 m … 48.3 m, many levels | fountains/pools spread over 26 tiles | +| University | 322 (all flat, all one-sided) | 1591 only | 296 at 217.9 m, rest 67.9–71.0 m | reservoir plus smaller pools | +| Industrial, all parks | 0 | — | — | — | + +So water collision exists only in DownTown and University. Every collision +stream is already converted (cities use their `cSim_*_high` tiles; parks use +`cSim_Global`), and the volume filter from change 5 cannot drop water, because +water triangles carry a surface ID. + +Render materials agree: `water.alpha` / `water.flowingalpha` appear only in +DownTown and `water.default` / `water.flowing` only in University. Industrial's +harbour/sea is only the backdrop `ocean.reflection` mesh +(`assets/private/native-backdrops/Industrial.skate`), with no water collision +under it; the harbour bed is ordinary collision at about y = -3 (see change 4). + +### 2. The skater never learns it is in water (the missing link) + +The skater-side water signal comes from the collision output: + +- `CollisionOutputFields.flag_3481` → `Processed.flags_2488` bit 30 + (`0x4000_0000`), and +- `CollisionOutputFields.scalar_28` → `Processed.collision_scalar_2924` + (water height) + +(`crates/skate-core/src/player/input_phase/publication.rs:253-254`). The wipeout +state reads both (`wipeout_states/lifecycle.rs:81-82`, `update.rs:24-26`) to +enter the "special surface" path (ragdoll profile 10, `below_surface`, the +`IsInWater` motion-graph condition). + +**Nothing in the codebase ever writes `flag_3481` or `scalar_28`.** They stay +at their defaults (0), so the water path can never start, even on DownTown and +University where the data exists. + +The board already computes the matching values (`collision_flags` bit 25 and +`surface_twelve_height`), and the board manager (`offboard/board_manager/runtime.rs` +`surface()`) and the player state (`player_state/publication.rs`) already read bit 25. +The likely retail behaviour is that the physics output fill copies the board's +`Body872` bit 25 → `Collision+3481` and `Body864` → `Collision+28`. **This +is a hypothesis; the retail write site has not been found yet.** + +### 3. The skater's own contacts also classify water + +`SkeletonCollision` (82BD4A30, `crates/skate-core/src/physics/skeleton_body/collision_update.rs`) +sets `flags.material_12` and `material_12_height` when any body part touches +type 12. Its flags sit at native 4079/4080/4081 (materials 10/11/12); the +collision output's 3479/3480/3481 use the same layout 600 bytes lower, and +3479/3480 are already the material-10/11 results. This makes the skater +body the most likely retail source of Collision+3481/+28. + +### 4. Industrial's sea + +The user remembers that falling into Industrial's sea respawned the player in +retail, and suggests the sea may simply be out of reach in retail. Industrial's +harbour floor uses ordinary surfaces (concrete and so on) and its sea has no +collision of any kind on the disc (the proxy archives are visual only), so no +data-driven water detection is possible there. Not handled by this change. + +## Change + +`PlayerInputRuntime::publish_water` (`crates/skate-game/src/physics/player_input/mod.rs`), +called from `frame.rs` right after `publish_board` (after the packet reset and +the skeleton contact feedback for the frame): + +- skater body touching type 12 → `flag_3481 = 1`, `scalar_28 = material_12_height`; +- otherwise board touching type 12 (`collision_flags` bit 25) → `flag_3481 = 1`, + `scalar_28 = surface_twelve_height`; +- otherwise both stay at their reset values (0). + +Project choice: retail's writer is unconfirmed; the skater body wins over the +board because the wipeout water path simulates the body. Water triangles stay +solid one-sided collision, as before. + +### Water bail (added after the first in-play test) + +First in-play test (University fountain basin): the skater landed on the water +and rode on it; no bail. Expected: the published flag only feeds the wipeout +state, and nothing starts a bail on water. I searched every recovered bail check +(`skate-core/src/player/wipeout/`, the state selector, all motion-graph conditions +in the stock state XML) and found no water/type-12 trigger. The retail +trigger is in code that hasn't been recovered, and no decrypted executable is +available. + +`physics/wipeout.rs` `check_after_physics` now requests a bail (request slot 33, +`WATER_BAIL_REASON`, otherwise unused and not a runout reason) when the board +(`collision_flags` bit 25) or the skater body (`flags.material_12`) touches +water, unless the skater is already in WipeoutGround, Teleporting or Sleeping. +It then takes the ordinary path: state flag 65 → `PhysicsWantsWipeOut` → +WipeoutGround, where `Collision+3481` starts the water ragdoll and the +respawn timer. Logged as `WATER_BAIL tick=… state=…`. **Project choice**, not +recovered retail behaviour. + +Not changed: water stays solid. The water buoyancy code +(`wipeout_state::body::special_surface`, which pushes parts up when below +height + 0.1) suggests that in retail the body sinks slightly into water, so +water may not be solid for the body in retail. Left alone until play shows +whether that matters. + +### Water rendering: extract the ocean animation table from default.xex + +Water (family 33) and ocean (family 31) shaders need `assets/private/ocean-pca.json`, +which nothing in setup created (see open question 5), so 52 water/ocean materials +fell back to static shading (black for `ocean.default`, which has no diffuse). + +- `crates/skate-data/src/xex/`: XEX2 unpacker (retail AES-128 file-key and CBC + payload decryption, "normal" LZX or "basic" decompression) producing the + mapped base image. No new dependencies; AES is checked against FIPS-197 and + NIST SP 800-38A vectors. The owned disc `default.xex` (normal encryption, + LZX) unpacks to an 18,022,400-byte image at 0x82000000 in about 0.2 s + (sha256 `ce1e3ae5…`, not the TU3 image `ocean_pca.py` expects). +- `crates/skate-data/src/ocean_pca.rs`: locates cPCAWaterAnimationData's table + the way the code addresses it (`lis` + D-form pairs forming two addresses + 0x168 apart within 32 instructions, data plausibility check), so it works for + any build. The disc build has exactly one match: means at 0x82FC3978, weights + at 0x82FC3AE0 (TU3: 0x830118D8 / 0x83011A40); frame 0 mean is + (127.55, 251.83, 127.54), an "up" normal. JSON layout matches `ocean_pca.py`. +- `skate3rust --extract-ocean-pca ` (`main.rs`, runs + before any game initialization; prints `OCEAN_PCA_READY`). +- Setup: `ocean_pca.convert` runs it in the `environment` group + (`asset_exports.environment`, new `game_exe` argument; `versions.py` adds + `ocean_pca.py` to the group). Failure is optional content, like other + environment parts. Changing the environment recipe refreshes that group once. + +## Verification + +- Unit test `physics::player_input::tests::water_contact_prefers_the_skater_body_height`. +- `cargo test --locked -p skate-game --release --bin skate3rust -- water_contact`: passes. +- Release build staged into `bin\`; `--test-world --check-assets` → `SKATE_ASSETS_READY`. +- In play, first build: rode on top of the water, no bail (led to the water bail above). +- In play, second build: bails on University fountain basin (F2), University reservoir (F3) and DownTown fountain (F4). Log shows `WATER_BAIL ... state=KnownAir` on each drop. +- Asset-backed wipeout tests (`--ignored wipeout`): 4 pass; `marker_reply_restores_on_foot` + fails identically with these changes stashed (pre-existing). + +## Open questions + +1. Where does retail write `Collision+3481` / `+28`? The wiring above is the + best match to the evidence; it is not confirmed against retail. +2. Water triangles are solid one-sided surfaces in our collision world. Does the + retail board/body pass through type 12, or rest on it? (In play: does the + skater stand on the University reservoir?) +3. Industrial's sea: retail respawned the player (user's memory), possibly + out of reach in retail. No collision data exists for it; see section 4. +5. Water rendering (user: static texture on the fountains, black on the + University reservoir). Cause found: the log says "52 of 8546 world + materials use an unsupported shader family and render as family 1". Water + (family 33) and ocean (family 31) shaders require `assets/private/ocean-pca.json` + (`MaterialTuning::supported`, `read_pca`). Nothing in setup (here or on + upstream `60efdef`) creates that file; `tools/asset_pipeline/ocean_pca.py` + needs a decrypted, decompressed TU3 executable image (hard-coded sha256 and + table addresses 0x830118D8 / 0x83011A40). The disc `default.xex` is XEX2 with + normal encryption and LZX compression, and setup never unpacks it. Fallback + family 1 shows the static diffuse; `ocean.default` has no diffuse, hence black. + This affects upstream users equally. +4. What is surface type 13 (DownTown 88, University 17,790 triangles)? It may be + related (shallow water or a splash surface) or something unrelated, such as grass. + +## Files + +- `crates/skate-data/examples/water_surfaces.rs` (new, diagnostic only; also prints water/ocean render mesh heights). +- `crates/skate-game/src/physics/player_input/mod.rs` (`publish_water`, unit test). +- `crates/skate-game/src/physics/frame.rs` (call site). +- `crates/skate-game/src/physics/wipeout.rs` (water bail request). +- `mods/water-test-teleport/` (dev-only test mod, F2/F3/F4 teleports; not committed, not for upstream). +- `crates/skate-data/src/xex/{mod,aes,lzx}.rs`, `crates/skate-data/src/ocean_pca.rs`, `crates/skate-data/src/lib.rs`. +- `crates/skate-data/examples/xex_unpack.rs` (diagnostic: unpack + locate table). +- `crates/skate-game/src/main.rs` (`--extract-ocean-pca`). +- `tools/asset_pipeline/ocean_pca.py` (`convert`), `asset_exports.py`, `install.py`, `versions.py`. diff --git a/docs/hails-additions/README.md b/docs/hails-additions/README.md index 643563d2..a44ccffa 100644 --- a/docs/hails-additions/README.md +++ b/docs/hails-additions/README.md @@ -14,6 +14,7 @@ Each document stands alone. File paths are relative to the repository root. | 3 | [SDL3 gamepad input](03-sdl3-gamepad-input.md) | Game (Rust), build scripts, vendored `sdl3-sys` | Done, verified in play with an Xbox Elite Series 2 | | 4 | [Authored map spawns](04-map-spawns.md) | Asset pipeline (Python) | Done, all 10 maps confirmed in play | | 5 | [Invisible collision volumes](05-collision-volumes.md) | Game data loader (Rust) | Done, all four affected maps confirmed in play | +| 9 | [Water](09-water.md) | Game physics (Rust), diagnostics | Water flag wired up (DownTown, University); awaiting in-play test | ## Environment used for verification diff --git a/tools/asset_pipeline/asset_exports.py b/tools/asset_pipeline/asset_exports.py index bfdf66cd..2e61bcab 100644 --- a/tools/asset_pipeline/asset_exports.py +++ b/tools/asset_pipeline/asset_exports.py @@ -59,7 +59,7 @@ def character(game_root, stage, work, report, log, converted=None): (private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8') -def environment(game_root, stage, work, report, log, converted=None): +def environment(game_root, stage, work, report, log, converted=None, game_exe=None): private=stage/"assets/private" stock=private/"stock" report('Preparing retail sky domes') @@ -78,6 +78,11 @@ def environment(game_root, stage, work, report, log, converted=None): attempt('skies',lambda assets:write_skies(game_root,assets,converted)) from .render_parameters import convert as write_render_parameters attempt('parameters',lambda assets:write_render_parameters(assets,converted)) + if game_exe is not None: + # Water/ocean animation table, held only in the executable. + report('Extracting the original water animation') + from .ocean_pca import convert as write_ocean_pca + attempt('ocean',lambda assets:write_ocean_pca(game_exe,game_root/'default.xex',assets,log)) report('Extracting original travel destinations and location names') from .teleports import convert as write_teleports attempt('teleports',lambda assets:write_teleports(game_root,assets,converted)) diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index db36ac64..b497bb4d 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -433,7 +433,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None if 'character' in groups: exports.character(game_root,stage,work,report,log,converted) if 'environment' in groups: - exports.environment(game_root,stage,work,report,log,converted) + exports.environment(game_root,stage,work,report,log,converted,game_exe) if 'maps' in groups: report('Preparing authored movable-object models') from .dynamic_props import prepare_catalog diff --git a/tools/asset_pipeline/ocean_pca.py b/tools/asset_pipeline/ocean_pca.py index 7c435fe5..0adc48a4 100644 --- a/tools/asset_pipeline/ocean_pca.py +++ b/tools/asset_pipeline/ocean_pca.py @@ -1,5 +1,11 @@ -"""Extract TU3's built-in PCA ocean animation from an owned mapped image. +"""Extract the built-in PCA ocean/water animation (assets/private/ocean-pca.json). +`convert` is what setup uses: the game executable unpacks the owned +default.xex and locates the table in any build (skate_data::ocean_pca, +`skate3rust --extract-ocean-pca`). Without this file every water and ocean +material renders as a static fallback. + +`extract` is the original TU3-only path from an already mapped image. Addresses are corroborated by cPCAWaterAnimationData::Init (827905B0) and its update routine (82790858). No game content is embedded in this tool. """ @@ -32,6 +38,28 @@ def extract(image, output, base=0x82000000): return len(frames) +def convert(game_exe, xex, assets, log=None): + """Writes assets/private/ocean-pca.json from the owned default.xex.""" + import subprocess + from .install import spawn + output = Path(assets) / 'private/ocean-pca.json' + output.parent.mkdir(parents=True, exist_ok=True) + with spawn([str(game_exe), '--extract-ocean-pca', str(xex), str(output)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: + text = process.stdout.read().decode('utf-8', errors='replace') + code = process.wait() + if log is not None: + log.write(text) + log.flush() + if code or 'OCEAN_PCA_READY' not in text: + output.unlink(missing_ok=True) + raise RuntimeError('Ocean animation extraction failed: ' + (text.strip().splitlines() or ['no output'])[-1]) + frames = json.loads(output.read_text(encoding='utf-8'))['frames'] + if len(frames) != 30: + raise ValueError('Ocean animation table must have 30 frames') + return len(frames) + + if __name__ == '__main__': import argparse p = argparse.ArgumentParser(description=__doc__) diff --git a/tools/asset_pipeline/versions.py b/tools/asset_pipeline/versions.py index 921a554c..c18dc2d2 100644 --- a/tools/asset_pipeline/versions.py +++ b/tools/asset_pipeline/versions.py @@ -20,7 +20,8 @@ SOURCES = { 'environment': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/sky.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py', 'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py', - 'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py'), + 'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py', + 'asset_pipeline/ocean_pca.py'), 'maps': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py', 'asset_pipeline/environment.py', 'asset_pipeline/irradiance.py', 'asset_pipeline/retail_material.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/sky.py'),