mirror of
https://github.com/SK8-ENGINE/skate-3-rust-engine.git
synced 2026-10-26 19:47:44 +00:00
Water rendering: extract the ocean animation table from default.xex
Water and ocean shaders need assets/private/ocean-pca.json, which setup never created, so 52 water/ocean materials rendered static (or black). - skate_data::xex: XEX2 unpacker (retail AES file key + CBC payload, LZX or basic decompression), no new dependencies - skate_data::ocean_pca: locates cPCAWaterAnimationData's table in any build by its code addressing (disc build: 0x82FC3978; TU3: 0x830118D8) - skate3rust --extract-ocean-pca <default.xex> <out.json> - Setup environment group runs it (optional content); refreshes that group In-play check of the rendered water still pending. See docs/hails-additions/09-water.md.
This commit is contained in:
13 files changed
+1086
-4
No files matched your search
@@ -0,0 +1,26 @@
|
||||
//! Unpacks a retail XEX2 to its mapped base image and locates the ocean PCA table.
|
||||
//! Usage: cargo run -p skate-data --example xex_unpack -- <default.xex> [<image.bin>]
|
||||
fn main() {
|
||||
let args: Vec<_> = std::env::args_os().skip(1).collect();
|
||||
let xex = std::fs::read(&args[0]).expect("read XEX");
|
||||
let started = std::time::Instant::now();
|
||||
let image = skate_data::xex::XexImage::parse(&xex).unwrap_or_else(|e| panic!("{e}"));
|
||||
println!(
|
||||
"base 0x{:08X}, image {} bytes, {:.2} s",
|
||||
image.base_address,
|
||||
image.image.len(),
|
||||
started.elapsed().as_secs_f64()
|
||||
);
|
||||
match skate_data::ocean_pca::extract(&image) {
|
||||
Ok(pca) => println!(
|
||||
"ocean PCA table at 0x{:08X}, frame 0 mean {:?}, {:.2} s",
|
||||
pca.means_address,
|
||||
pca.frames[0][0],
|
||||
started.elapsed().as_secs_f64()
|
||||
),
|
||||
Err(e) => println!("ocean PCA: {e}"),
|
||||
}
|
||||
if let Some(out) = args.get(1) {
|
||||
std::fs::write(out, &image.image).expect("write image");
|
||||
}
|
||||
}
|
||||
@@ -18,3 +18,5 @@ mod sha256;
|
||||
pub mod state_graph;
|
||||
pub mod skate_map;
|
||||
pub mod retail_collision;
|
||||
pub mod xex;
|
||||
pub mod ocean_pca;
|
||||
@@ -0,0 +1,137 @@
|
||||
//! The built-in PCA ocean/water animation table (cPCAWaterAnimationData) from
|
||||
//! an unpacked retail executable, written as `assets/private/ocean-pca.json`.
|
||||
//!
|
||||
//! TU3 keeps the 30 frame means at 0x830118D8 and the weights right after them
|
||||
//! at 0x83011A40 (tools/asset_pipeline/ocean_pca.py). Other builds place the
|
||||
//! table elsewhere, so it is located the way the code reaches it: a `lis`
|
||||
//! followed by a D-form use of the same register forms each address, and
|
||||
//! cPCAWaterAnimationData::Init forms both within a few instructions.
|
||||
use crate::xex::XexImage;
|
||||
|
||||
const FRAMES: usize = 30;
|
||||
const MEANS: usize = FRAMES * 3;
|
||||
const WEIGHTS: usize = FRAMES * 24;
|
||||
const WEIGHTS_OFFSET: u32 = (MEANS * 4) as u32; // 0x168
|
||||
/// Instructions between `lis` and the instruction completing an address.
|
||||
const ADDRESS_WINDOW: usize = 8;
|
||||
/// Instructions between the two address loads.
|
||||
const PAIR_WINDOW: usize = 32;
|
||||
|
||||
pub struct OceanPca {
|
||||
pub means_address: u32,
|
||||
/// Per frame: mean row then six weight rows, in shader order, scaled /255.
|
||||
pub frames: Vec<[[f32; 4]; 7]>,
|
||||
}
|
||||
|
||||
pub fn extract(image: &XexImage) -> Result<OceanPca, String> {
|
||||
let candidates = locate(image);
|
||||
let means_address = match candidates.as_slice() {
|
||||
[one] => *one,
|
||||
[] => return Err("Ocean PCA table not found in the executable".into()),
|
||||
many => {
|
||||
return Err(format!(
|
||||
"Ocean PCA table is ambiguous: {} candidates ({:08X?})",
|
||||
many.len(),
|
||||
many
|
||||
))
|
||||
}
|
||||
};
|
||||
let means = floats(image, means_address, MEANS).ok_or("Ocean PCA means out of range")?;
|
||||
let weights = floats(image, means_address + WEIGHTS_OFFSET, WEIGHTS).ok_or("Ocean PCA weights out of range")?;
|
||||
let frames = (0..FRAMES)
|
||||
.map(|frame| {
|
||||
let mean = &means[frame * 3..frame * 3 + 3];
|
||||
let weight = &weights[frame * 24..frame * 24 + 24];
|
||||
let mut rows = [[0.; 4]; 7];
|
||||
// Shader model order: mean X/Z/Y and weight pairs R/B/G.
|
||||
rows[0] = [mean[0] / 255., mean[2] / 255., mean[1] / 255., 0.];
|
||||
for (row, start) in rows[1..].iter_mut().zip([0, 4, 16, 20, 8, 12]) {
|
||||
*row = core::array::from_fn(|i| weight[start + i] / 255.);
|
||||
}
|
||||
rows
|
||||
})
|
||||
.collect();
|
||||
Ok(OceanPca { means_address, frames })
|
||||
}
|
||||
|
||||
impl OceanPca {
|
||||
/// The JSON read by the game (`retail_render::read_pca`), matching
|
||||
/// tools/asset_pipeline/ocean_pca.py.
|
||||
pub fn to_json(&self, source_sha256: &str) -> String {
|
||||
serde_json::json!({
|
||||
"source_sha256": source_sha256,
|
||||
"means_address": format!("0x{:08X}", self.means_address),
|
||||
"hz": 30.0,
|
||||
"frames": self.frames,
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn floats(image: &XexImage, address: u32, count: usize) -> Option<Vec<f32>> {
|
||||
let bytes = image.at(address, count * 4)?;
|
||||
let values: Vec<f32> = bytes
|
||||
.chunks_exact(4)
|
||||
.map(|b| f32::from_be_bytes([b[0], b[1], b[2], b[3]]))
|
||||
.collect();
|
||||
values.iter().all(|v| v.is_finite()).then_some(values)
|
||||
}
|
||||
|
||||
/// A table holds 30 normal-like means (0..255 components) and nonzero weights.
|
||||
fn plausible(image: &XexImage, address: u32) -> bool {
|
||||
let (Some(means), Some(weights)) = (
|
||||
floats(image, address, MEANS),
|
||||
floats(image, address + WEIGHTS_OFFSET, WEIGHTS),
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
means.iter().all(|v| (0.0..=255.0).contains(v))
|
||||
&& means.iter().any(|v| *v != 0.)
|
||||
&& weights.iter().all(|v| v.abs() <= 255.)
|
||||
&& weights.iter().filter(|v| **v != 0.).count() > WEIGHTS / 2
|
||||
}
|
||||
|
||||
fn locate(image: &XexImage) -> Vec<u32> {
|
||||
let words: Vec<u32> = image
|
||||
.image
|
||||
.chunks_exact(4)
|
||||
.map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
|
||||
.collect();
|
||||
// (instruction index, address) for every lis + D-form completion.
|
||||
let mut formed: Vec<(usize, u32)> = Vec::new();
|
||||
for (i, &w) in words.iter().enumerate() {
|
||||
let (op, rd, ra) = (w >> 26, (w >> 21) & 31, (w >> 16) & 31);
|
||||
if op != 15 || ra != 0 {
|
||||
continue;
|
||||
}
|
||||
let high = (w & 0xffff) << 16;
|
||||
for (j, &u) in words.iter().enumerate().skip(i + 1).take(ADDRESS_WINDOW) {
|
||||
let (uop, urd, ura) = (u >> 26, (u >> 21) & 31, (u >> 16) & 31);
|
||||
// addi and the integer/float loads and stores.
|
||||
if ura == rd && matches!(uop, 14 | 32 | 34 | 36 | 40 | 44 | 48 | 50 | 52 | 54) {
|
||||
let low = (u & 0xffff) as u16 as i16 as i32;
|
||||
formed.push((j, high.wrapping_add(low as u32)));
|
||||
}
|
||||
if uop == 15 && urd == rd {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut found: Vec<u32> = Vec::new();
|
||||
for (k, &(i, address)) in formed.iter().enumerate() {
|
||||
let pair = formed[k.saturating_sub(64)..(k + 64).min(formed.len())]
|
||||
.iter()
|
||||
.any(|&(j, other)| other == address.wrapping_add(WEIGHTS_OFFSET) && i.abs_diff(j) <= PAIR_WINDOW);
|
||||
if pair && !found.contains(&address) && plausible(image, address) {
|
||||
found.push(address);
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
/// Unpacks `xex`, extracts the table and returns the game's JSON.
|
||||
pub fn json_from_xex(xex: &[u8]) -> Result<String, String> {
|
||||
let image = XexImage::parse(xex)?;
|
||||
let pca = extract(&image)?;
|
||||
Ok(pca.to_json(&crate::sha256::digest(&image.image)))
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! AES-128 decryption (FIPS-197 inverse cipher), only what XEX2 needs:
|
||||
//! ECB for the file key and CBC with a zero IV for the payload.
|
||||
|
||||
const SBOX: [u8; 256] = [
|
||||
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
|
||||
0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
|
||||
0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
|
||||
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
|
||||
0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
|
||||
0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
|
||||
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
|
||||
0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
|
||||
0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
|
||||
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
|
||||
0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
|
||||
0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
|
||||
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
|
||||
0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
|
||||
0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
|
||||
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
|
||||
];
|
||||
|
||||
fn xtime(x: u8) -> u8 {
|
||||
(x << 1) ^ if x & 0x80 != 0 { 0x1b } else { 0 }
|
||||
}
|
||||
|
||||
fn mul(mut a: u8, mut b: u8) -> u8 {
|
||||
let mut product = 0;
|
||||
while b != 0 {
|
||||
if b & 1 != 0 {
|
||||
product ^= a;
|
||||
}
|
||||
a = xtime(a);
|
||||
b >>= 1;
|
||||
}
|
||||
product
|
||||
}
|
||||
|
||||
pub(crate) struct Aes128 {
|
||||
round_keys: [[u8; 16]; 11],
|
||||
inverse_sbox: [u8; 256],
|
||||
}
|
||||
|
||||
impl Aes128 {
|
||||
pub(crate) fn new(key: [u8; 16]) -> Self {
|
||||
let mut words = [[0u8; 4]; 44];
|
||||
for (i, word) in words.iter_mut().take(4).enumerate() {
|
||||
word.copy_from_slice(&key[i * 4..i * 4 + 4]);
|
||||
}
|
||||
let mut rcon = 1u8;
|
||||
for i in 4..44 {
|
||||
let mut t = words[i - 1];
|
||||
if i % 4 == 0 {
|
||||
t = [SBOX[t[1] as usize] ^ rcon, SBOX[t[2] as usize], SBOX[t[3] as usize], SBOX[t[0] as usize]];
|
||||
rcon = xtime(rcon);
|
||||
}
|
||||
for j in 0..4 {
|
||||
words[i][j] = words[i - 4][j] ^ t[j];
|
||||
}
|
||||
}
|
||||
let mut round_keys = [[0u8; 16]; 11];
|
||||
for (round, key) in round_keys.iter_mut().enumerate() {
|
||||
for j in 0..4 {
|
||||
key[j * 4..j * 4 + 4].copy_from_slice(&words[round * 4 + j]);
|
||||
}
|
||||
}
|
||||
let mut inverse_sbox = [0u8; 256];
|
||||
for (i, &s) in SBOX.iter().enumerate() {
|
||||
inverse_sbox[s as usize] = i as u8;
|
||||
}
|
||||
Self { round_keys, inverse_sbox }
|
||||
}
|
||||
|
||||
pub(crate) fn decrypt_block(&self, block: &mut [u8; 16]) {
|
||||
let add = |block: &mut [u8; 16], key: &[u8; 16]| {
|
||||
for (b, k) in block.iter_mut().zip(key) {
|
||||
*b ^= k;
|
||||
}
|
||||
};
|
||||
add(block, &self.round_keys[10]);
|
||||
for round in (0..10).rev() {
|
||||
// Inverse ShiftRows (state is column-major: byte r + 4c).
|
||||
let s = *block;
|
||||
for c in 0..4 {
|
||||
for r in 0..4 {
|
||||
block[r + 4 * ((c + r) % 4)] = s[r + 4 * c];
|
||||
}
|
||||
}
|
||||
for b in block.iter_mut() {
|
||||
*b = self.inverse_sbox[*b as usize];
|
||||
}
|
||||
add(block, &self.round_keys[round]);
|
||||
if round > 0 {
|
||||
for c in 0..4 {
|
||||
let col = [block[4 * c], block[4 * c + 1], block[4 * c + 2], block[4 * c + 3]];
|
||||
for r in 0..4 {
|
||||
block[4 * c + r] = mul(col[r], 14)
|
||||
^ mul(col[(r + 1) % 4], 11)
|
||||
^ mul(col[(r + 2) % 4], 13)
|
||||
^ mul(col[(r + 3) % 4], 9);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// CBC with a zero IV over whole 16-byte blocks; a trailing partial
|
||||
/// block is left untouched.
|
||||
pub(crate) fn decrypt_cbc_zero_iv(&self, data: &mut [u8]) {
|
||||
let mut previous = [0u8; 16];
|
||||
for chunk in data.chunks_exact_mut(16) {
|
||||
let cipher: [u8; 16] = chunk.try_into().unwrap();
|
||||
let mut block = cipher;
|
||||
self.decrypt_block(&mut block);
|
||||
for (out, (plain, prev)) in chunk.iter_mut().zip(block.iter().zip(previous)) {
|
||||
*out = plain ^ prev;
|
||||
}
|
||||
previous = cipher;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fips_197_appendix_c1_vector_decrypts() {
|
||||
let key: [u8; 16] = core::array::from_fn(|i| i as u8);
|
||||
let mut block = [
|
||||
0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30, 0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a,
|
||||
];
|
||||
Aes128::new(key).decrypt_block(&mut block);
|
||||
let expected: [u8; 16] = core::array::from_fn(|i| (i as u8) * 0x11);
|
||||
assert_eq!(block, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cbc_chains_previous_ciphertext() {
|
||||
// NIST SP 800-38A F.2.2 (CBC-AES128.Decrypt), first two blocks, but
|
||||
// with a zero IV: the first plaintext is XORed with the real IV here.
|
||||
let key = [
|
||||
0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
|
||||
];
|
||||
let iv: [u8; 16] = core::array::from_fn(|i| i as u8);
|
||||
let mut data = [
|
||||
0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d,
|
||||
0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2,
|
||||
];
|
||||
Aes128::new(key).decrypt_cbc_zero_iv(&mut data);
|
||||
for (b, v) in data[..16].iter_mut().zip(iv) {
|
||||
*b ^= v;
|
||||
}
|
||||
assert_eq!(
|
||||
data,
|
||||
[
|
||||
0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
|
||||
0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
//! LZX decompression as used by XEX2 "normal" compression: one continuous
|
||||
//! stream, 32 KiB frames, no reset interval, no E8 translation.
|
||||
//! Follows the classic (non-DELTA) LZX decoder of libmspack's lzxd.c.
|
||||
|
||||
const FRAME: usize = 32768;
|
||||
const MIN_MATCH: usize = 2;
|
||||
const NUM_CHARS: usize = 256;
|
||||
const PRETREE: usize = 20;
|
||||
const LENGTH_TREE: usize = 249;
|
||||
const ALIGNED_TREE: usize = 8;
|
||||
const MAX_BITS: usize = 16;
|
||||
|
||||
/// 16-bit little-endian words, consumed most-significant bit first.
|
||||
struct Bits<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
buffer: u32,
|
||||
left: u32,
|
||||
}
|
||||
|
||||
impl<'a> Bits<'a> {
|
||||
fn new(data: &'a [u8]) -> Self {
|
||||
Self { data, pos: 0, buffer: 0, left: 0 }
|
||||
}
|
||||
fn byte(&mut self) -> u8 {
|
||||
// Past the end the stream reads as zeros, like libmspack's padding.
|
||||
let b = self.data.get(self.pos).copied().unwrap_or(0);
|
||||
self.pos += 1;
|
||||
b
|
||||
}
|
||||
fn ensure(&mut self, n: u32) {
|
||||
while self.left < n {
|
||||
let lo = u32::from(self.byte());
|
||||
let hi = u32::from(self.byte());
|
||||
self.buffer |= ((hi << 8) | lo) << (16 - self.left);
|
||||
self.left += 16;
|
||||
}
|
||||
}
|
||||
fn peek(&mut self, n: u32) -> u32 {
|
||||
self.ensure(n);
|
||||
self.buffer >> (32 - n)
|
||||
}
|
||||
fn remove(&mut self, n: u32) {
|
||||
self.buffer = if n >= 32 { 0 } else { self.buffer << n };
|
||||
self.left -= n;
|
||||
}
|
||||
fn read(&mut self, n: u32) -> u32 {
|
||||
if n == 0 {
|
||||
return 0;
|
||||
}
|
||||
if n > 16 {
|
||||
let hi = self.read(n - 16);
|
||||
return (hi << 16) | self.read(16);
|
||||
}
|
||||
let v = self.peek(n);
|
||||
self.remove(n);
|
||||
v
|
||||
}
|
||||
/// libmspack's start-of-uncompressed-block realignment.
|
||||
fn realign_to_bytes(&mut self) {
|
||||
self.ensure(16);
|
||||
if self.left > 16 {
|
||||
self.pos -= 2;
|
||||
}
|
||||
self.left = 0;
|
||||
self.buffer = 0;
|
||||
}
|
||||
/// End-of-frame realignment to a 16-bit boundary.
|
||||
fn realign_frame(&mut self) {
|
||||
if self.left > 0 {
|
||||
self.ensure(16);
|
||||
}
|
||||
if self.left & 15 != 0 {
|
||||
let n = self.left & 15;
|
||||
self.remove(n);
|
||||
}
|
||||
}
|
||||
fn le32(&mut self) -> u32 {
|
||||
u32::from_le_bytes([self.byte(), self.byte(), self.byte(), self.byte()])
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical Huffman decoder built from code lengths (MSB-first codes).
|
||||
struct Tree {
|
||||
/// Symbols sorted by (length, symbol).
|
||||
symbols: Vec<u16>,
|
||||
/// Per length: number of codes.
|
||||
counts: [u16; MAX_BITS + 1],
|
||||
empty: bool,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
fn new(lengths: &[u8]) -> Result<Self, String> {
|
||||
let mut counts = [0u16; MAX_BITS + 1];
|
||||
for &l in lengths {
|
||||
if usize::from(l) > MAX_BITS {
|
||||
return Err("LZX code length exceeds 16".into());
|
||||
}
|
||||
counts[usize::from(l)] += 1;
|
||||
}
|
||||
counts[0] = 0;
|
||||
let mut symbols: Vec<u16> = (0..lengths.len() as u16).filter(|&s| lengths[s as usize] != 0).collect();
|
||||
symbols.sort_by_key(|&s| (lengths[s as usize], s));
|
||||
// Over-subscribed trees are invalid; incomplete ones are tolerated
|
||||
// only when empty (libmspack accepts an all-zero tree).
|
||||
let mut left: i32 = 1;
|
||||
for len in 1..=MAX_BITS {
|
||||
left = (left << 1) - i32::from(counts[len]);
|
||||
if left < 0 {
|
||||
return Err("LZX Huffman tree is over-subscribed".into());
|
||||
}
|
||||
}
|
||||
let empty = symbols.is_empty();
|
||||
if !empty && left != 0 {
|
||||
return Err("LZX Huffman tree is incomplete".into());
|
||||
}
|
||||
Ok(Self { symbols, counts, empty })
|
||||
}
|
||||
|
||||
fn decode(&self, bits: &mut Bits<'_>) -> Result<usize, String> {
|
||||
if self.empty {
|
||||
return Err("LZX decode from an empty Huffman tree".into());
|
||||
}
|
||||
let mut code: i32 = 0;
|
||||
let mut first: i32 = 0;
|
||||
let mut index: i32 = 0;
|
||||
for len in 1..=MAX_BITS {
|
||||
code |= bits.read(1) as i32;
|
||||
let count = i32::from(self.counts[len]);
|
||||
if code - first < count {
|
||||
return Ok(usize::from(self.symbols[(index + code - first) as usize]));
|
||||
}
|
||||
index += count;
|
||||
first = (first + count) << 1;
|
||||
code <<= 1;
|
||||
}
|
||||
Err("LZX Huffman code not found".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn read_lengths(bits: &mut Bits<'_>, lengths: &mut [u8], first: usize, last: usize) -> Result<(), String> {
|
||||
let mut pre = [0u8; PRETREE];
|
||||
for l in &mut pre {
|
||||
*l = bits.read(4) as u8;
|
||||
}
|
||||
let pretree = Tree::new(&pre)?;
|
||||
let mut x = first;
|
||||
while x < last {
|
||||
let z = pretree.decode(bits)?;
|
||||
match z {
|
||||
17 => {
|
||||
let run = bits.read(4) as usize + 4;
|
||||
fill(lengths, &mut x, last, run, 0)?;
|
||||
}
|
||||
18 => {
|
||||
let run = bits.read(5) as usize + 20;
|
||||
fill(lengths, &mut x, last, run, 0)?;
|
||||
}
|
||||
19 => {
|
||||
let run = bits.read(1) as usize + 4;
|
||||
let z = pretree.decode(bits)?;
|
||||
let v = (i32::from(lengths[x]) - z as i32).rem_euclid(17) as u8;
|
||||
fill(lengths, &mut x, last, run, v)?;
|
||||
}
|
||||
_ => {
|
||||
lengths[x] = (i32::from(lengths[x]) - z as i32).rem_euclid(17) as u8;
|
||||
x += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fill(lengths: &mut [u8], x: &mut usize, last: usize, run: usize, value: u8) -> Result<(), String> {
|
||||
if *x + run > last {
|
||||
return Err("LZX length run overflows the tree".into());
|
||||
}
|
||||
lengths[*x..*x + run].fill(value);
|
||||
*x += run;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decompresses `output_len` bytes with a window of `window_size` bytes.
|
||||
pub(crate) fn decompress(input: &[u8], window_size: usize, output_len: usize) -> Result<Vec<u8>, String> {
|
||||
let window_bits = window_size.trailing_zeros();
|
||||
if !window_size.is_power_of_two() || !(15..=21).contains(&window_bits) {
|
||||
return Err(format!("Unsupported LZX window size {window_size}"));
|
||||
}
|
||||
let position_slots = match window_bits {
|
||||
20 => 42,
|
||||
21 => 50,
|
||||
b => (b * 2) as usize,
|
||||
};
|
||||
let mut extra_bits = [0u32; 51];
|
||||
let mut position_base = [0usize; 51];
|
||||
let mut j = 0;
|
||||
let mut i = 0;
|
||||
while i < 51 {
|
||||
extra_bits[i] = j;
|
||||
if i + 1 < 51 {
|
||||
extra_bits[i + 1] = j;
|
||||
}
|
||||
if i != 0 && j < 17 {
|
||||
j += 1;
|
||||
}
|
||||
i += 2;
|
||||
}
|
||||
let mut base = 0;
|
||||
for i in 0..51 {
|
||||
position_base[i] = base;
|
||||
base += 1 << extra_bits[i];
|
||||
}
|
||||
|
||||
let main_size = NUM_CHARS + position_slots * 8;
|
||||
let mut main_lengths = vec![0u8; main_size];
|
||||
let mut length_lengths = vec![0u8; LENGTH_TREE];
|
||||
let mut bits = Bits::new(input);
|
||||
let mut out = Vec::with_capacity(output_len);
|
||||
let mut r = [1usize; 3];
|
||||
|
||||
// Header: E8 translation flag (+ 32-bit file size when set).
|
||||
if bits.read(1) != 0 {
|
||||
return Err("LZX E8 translation is not supported for XEX images".into());
|
||||
}
|
||||
|
||||
let mut block_type = 0u32;
|
||||
let mut block_remaining = 0usize;
|
||||
let mut block_length = 0usize;
|
||||
let mut main_tree: Option<Tree> = None;
|
||||
let mut length_tree: Option<Tree> = None;
|
||||
let mut aligned_tree: Option<Tree> = None;
|
||||
|
||||
while out.len() < output_len {
|
||||
let frame_end = (out.len() + FRAME).min(output_len);
|
||||
while out.len() < frame_end {
|
||||
if block_remaining == 0 {
|
||||
if block_type == 3 && block_length & 1 == 1 {
|
||||
bits.byte();
|
||||
}
|
||||
block_type = bits.read(3);
|
||||
let hi = bits.read(16) as usize;
|
||||
let lo = bits.read(8) as usize;
|
||||
block_length = (hi << 8) | lo;
|
||||
block_remaining = block_length;
|
||||
match block_type {
|
||||
2 | 1 => {
|
||||
if block_type == 2 {
|
||||
let mut aligned = [0u8; ALIGNED_TREE];
|
||||
for l in &mut aligned {
|
||||
*l = bits.read(3) as u8;
|
||||
}
|
||||
aligned_tree = Some(Tree::new(&aligned)?);
|
||||
}
|
||||
read_lengths(&mut bits, &mut main_lengths, 0, NUM_CHARS)?;
|
||||
read_lengths(&mut bits, &mut main_lengths, NUM_CHARS, main_size)?;
|
||||
main_tree = Some(Tree::new(&main_lengths)?);
|
||||
read_lengths(&mut bits, &mut length_lengths, 0, LENGTH_TREE)?;
|
||||
length_tree = Some(Tree::new(&length_lengths)?);
|
||||
}
|
||||
3 => {
|
||||
bits.realign_to_bytes();
|
||||
for slot in &mut r {
|
||||
*slot = bits.le32() as usize;
|
||||
}
|
||||
}
|
||||
_ => return Err(format!("Invalid LZX block type {block_type}")),
|
||||
}
|
||||
if block_remaining == 0 {
|
||||
return Err("Empty LZX block".into());
|
||||
}
|
||||
}
|
||||
let this_run = block_remaining.min(frame_end - out.len());
|
||||
let target = out.len() + this_run;
|
||||
if block_type == 3 {
|
||||
for _ in 0..this_run {
|
||||
let b = bits.byte();
|
||||
out.push(b);
|
||||
}
|
||||
block_remaining -= this_run;
|
||||
continue;
|
||||
}
|
||||
let main = main_tree.as_ref().ok_or("LZX main tree missing")?;
|
||||
let start = out.len();
|
||||
while out.len() < target {
|
||||
let symbol = main.decode(&mut bits)?;
|
||||
if symbol < NUM_CHARS {
|
||||
out.push(symbol as u8);
|
||||
continue;
|
||||
}
|
||||
let symbol = symbol - NUM_CHARS;
|
||||
let mut length = symbol & 7;
|
||||
if length == 7 {
|
||||
length += length_tree.as_ref().ok_or("LZX length tree missing")?.decode(&mut bits)?;
|
||||
}
|
||||
length += MIN_MATCH;
|
||||
let slot = symbol >> 3;
|
||||
let offset = if slot > 2 {
|
||||
let extra = extra_bits[slot];
|
||||
let offset = if block_type == 1 {
|
||||
if slot == 3 {
|
||||
1
|
||||
} else {
|
||||
position_base[slot] - 2 + bits.read(extra) as usize
|
||||
}
|
||||
} else {
|
||||
let aligned = aligned_tree.as_ref().ok_or("LZX aligned tree missing")?;
|
||||
let mut offset = position_base[slot] - 2;
|
||||
if extra > 3 {
|
||||
offset += (bits.read(extra - 3) as usize) << 3;
|
||||
offset += aligned.decode(&mut bits)?;
|
||||
} else if extra == 3 {
|
||||
offset += aligned.decode(&mut bits)?;
|
||||
} else if extra > 0 {
|
||||
offset += bits.read(extra) as usize;
|
||||
} else {
|
||||
offset = 1;
|
||||
}
|
||||
offset
|
||||
};
|
||||
r[2] = r[1];
|
||||
r[1] = r[0];
|
||||
r[0] = offset;
|
||||
offset
|
||||
} else if slot == 0 {
|
||||
r[0]
|
||||
} else if slot == 1 {
|
||||
r.swap(0, 1);
|
||||
r[0]
|
||||
} else {
|
||||
r.swap(0, 2);
|
||||
r[0]
|
||||
};
|
||||
if offset == 0 || offset > out.len() || offset > window_size {
|
||||
return Err(format!("LZX match offset {offset} outside the window at {}", out.len()));
|
||||
}
|
||||
for _ in 0..length {
|
||||
let b = out[out.len() - offset];
|
||||
out.push(b);
|
||||
}
|
||||
}
|
||||
let decoded = out.len() - start;
|
||||
if decoded > block_remaining {
|
||||
return Err("LZX match overran its block".into());
|
||||
}
|
||||
block_remaining -= decoded;
|
||||
if out.len() > frame_end {
|
||||
return Err("LZX match overran its frame".into());
|
||||
}
|
||||
}
|
||||
bits.realign_frame();
|
||||
}
|
||||
out.truncate(output_len);
|
||||
Ok(out)
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Xbox 360 XEX2 executables: recover the mapped base image (the bytes the
|
||||
//! console loads at the image base) from a retail `default.xex`.
|
||||
//! Layout follows the public XEX2 documentation used by Xenia and xextool.
|
||||
//! Supports retail encryption and "normal" (LZX) or "basic" (raw) compression.
|
||||
mod aes;
|
||||
mod lzx;
|
||||
|
||||
/// The published Xbox 360 retail XEX key (it decrypts the per-file key).
|
||||
const RETAIL_KEY: [u8; 16] = [
|
||||
0x20, 0xb1, 0x85, 0xa5, 0x9d, 0x28, 0xfd, 0xc3, 0x40, 0x58, 0x3f, 0xbb, 0x08, 0x96, 0xbf, 0x91,
|
||||
];
|
||||
|
||||
const FILE_FORMAT_INFO: u32 = 0x0000_03ff;
|
||||
const IMAGE_BASE_ADDRESS: u32 = 0x0001_0201;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct XexImage {
|
||||
/// Virtual address of `image[0]`.
|
||||
pub base_address: u32,
|
||||
pub image: Vec<u8>,
|
||||
}
|
||||
|
||||
fn be16(d: &[u8], at: usize) -> Result<u16, String> {
|
||||
d.get(at..at + 2)
|
||||
.map(|b| u16::from_be_bytes([b[0], b[1]]))
|
||||
.ok_or_else(|| format!("Truncated XEX at {at}"))
|
||||
}
|
||||
fn be32(d: &[u8], at: usize) -> Result<u32, String> {
|
||||
d.get(at..at + 4)
|
||||
.map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]]))
|
||||
.ok_or_else(|| format!("Truncated XEX at {at}"))
|
||||
}
|
||||
fn span(d: &[u8], at: usize, n: usize) -> Result<&[u8], String> {
|
||||
d.get(at..at.checked_add(n).ok_or("XEX offset overflow")?)
|
||||
.ok_or_else(|| format!("Truncated XEX at {at} (need {n})"))
|
||||
}
|
||||
|
||||
impl XexImage {
|
||||
pub fn parse(xex: &[u8]) -> Result<Self, String> {
|
||||
if span(xex, 0, 4)? != b"XEX2" {
|
||||
return Err("Not an XEX2 executable".into());
|
||||
}
|
||||
let payload_offset = be32(xex, 8)? as usize;
|
||||
let security = be32(xex, 16)? as usize;
|
||||
let header_count = be32(xex, 20)? as usize;
|
||||
let mut format = None;
|
||||
let mut base_address = None;
|
||||
for i in 0..header_count {
|
||||
let key = be32(xex, 24 + i * 8)?;
|
||||
let value = be32(xex, 28 + i * 8)?;
|
||||
match key {
|
||||
FILE_FORMAT_INFO => format = Some(value as usize),
|
||||
IMAGE_BASE_ADDRESS => base_address = Some(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let format = format.ok_or("XEX has no file format header")?;
|
||||
let encryption = be16(xex, format + 4)?;
|
||||
let compression = be16(xex, format + 6)?;
|
||||
// Security info: image size at +4, the encrypted file key at +0x150,
|
||||
// the load address at +0x110.
|
||||
let image_size = be32(xex, security + 4)? as usize;
|
||||
let base_address = match base_address {
|
||||
Some(address) => address,
|
||||
None => be32(xex, security + 0x110)?,
|
||||
};
|
||||
let mut payload = span(xex, payload_offset, xex.len().saturating_sub(payload_offset))?.to_vec();
|
||||
match encryption {
|
||||
0 => {}
|
||||
1 => {
|
||||
let mut key: [u8; 16] = span(xex, security + 0x150, 16)?.try_into().unwrap();
|
||||
aes::Aes128::new(RETAIL_KEY).decrypt_block(&mut key);
|
||||
aes::Aes128::new(key).decrypt_cbc_zero_iv(&mut payload);
|
||||
}
|
||||
other => return Err(format!("Unsupported XEX encryption type {other}")),
|
||||
}
|
||||
let image = match compression {
|
||||
1 => basic(xex, format, &payload, image_size)?,
|
||||
2 => normal(xex, format, &payload, image_size)?,
|
||||
other => return Err(format!("Unsupported XEX compression type {other}")),
|
||||
};
|
||||
if image.get(..2) != Some(b"MZ") {
|
||||
return Err("Unpacked XEX image does not start with a PE header (wrong key?)".into());
|
||||
}
|
||||
Ok(Self { base_address, image })
|
||||
}
|
||||
|
||||
/// Bytes at a virtual address.
|
||||
pub fn at(&self, address: u32, len: usize) -> Option<&[u8]> {
|
||||
let offset = address.checked_sub(self.base_address)? as usize;
|
||||
self.image.get(offset..offset.checked_add(len)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Basic compression: (data size, zero size) pairs, zeros not stored.
|
||||
fn basic(xex: &[u8], format: usize, payload: &[u8], image_size: usize) -> Result<Vec<u8>, String> {
|
||||
let info_size = be32(xex, format)? as usize;
|
||||
let blocks = info_size.saturating_sub(8) / 8;
|
||||
let mut image = Vec::with_capacity(image_size);
|
||||
let mut at = 0;
|
||||
for i in 0..blocks {
|
||||
let data = be32(xex, format + 8 + i * 8)? as usize;
|
||||
let zeros = be32(xex, format + 12 + i * 8)? as usize;
|
||||
image.extend_from_slice(span(payload, at, data)?);
|
||||
at += data;
|
||||
image.resize(image.len() + zeros, 0);
|
||||
}
|
||||
image.resize(image_size, 0);
|
||||
Ok(image)
|
||||
}
|
||||
|
||||
/// Normal compression: a chain of blocks, each starting with the next
|
||||
/// block's size and hash, then u16-length-prefixed LZX chunks.
|
||||
fn normal(xex: &[u8], format: usize, payload: &[u8], image_size: usize) -> Result<Vec<u8>, String> {
|
||||
let window_size = be32(xex, format + 8)? as usize;
|
||||
let mut block_size = be32(xex, format + 12)? as usize;
|
||||
let mut at = 0;
|
||||
let mut stream = Vec::with_capacity(payload.len());
|
||||
while block_size != 0 {
|
||||
let block = span(payload, at, block_size)?;
|
||||
let next = be32(block, 0)? as usize;
|
||||
let mut p = 24;
|
||||
loop {
|
||||
let chunk = be16(block, p)? as usize;
|
||||
p += 2;
|
||||
if chunk == 0 {
|
||||
break;
|
||||
}
|
||||
stream.extend_from_slice(span(block, p, chunk)?);
|
||||
p += chunk;
|
||||
}
|
||||
at += block_size;
|
||||
block_size = next;
|
||||
}
|
||||
lzx::decompress(&stream, window_size, image_size)
|
||||
}
|
||||
@@ -60,6 +60,8 @@ fn main() -> bevy::app::AppExit {
|
||||
Ok(false) => {}
|
||||
}
|
||||
if let Some(code) = crash_report::entry() { std::process::exit(code); }
|
||||
// Setup tool mode: `--extract-ocean-pca <default.xex> <ocean-pca.json>`.
|
||||
if let Some(code) = extract_ocean_pca() { std::process::exit(code); }
|
||||
let _trace = match profiling::init() {
|
||||
Ok(guard) => guard,
|
||||
Err(error) => { eprintln!("{error}"); return bevy::app::AppExit::error(); }
|
||||
@@ -158,3 +160,30 @@ fn main() -> bevy::app::AppExit {
|
||||
#[cfg(test)]
|
||||
#[path = "tests/action_host.rs"]
|
||||
mod action_host_tests;
|
||||
|
||||
/// Writes the retail water/ocean animation table for setup (see
|
||||
/// skate_data::ocean_pca). Runs before any game initialization.
|
||||
fn extract_ocean_pca() -> Option<i32> {
|
||||
let mut args = std::env::args_os().skip(1);
|
||||
if args.next()? != "--extract-ocean-pca" {
|
||||
return None;
|
||||
}
|
||||
let (Some(xex), Some(out), None) = (args.next(), args.next(), args.next()) else {
|
||||
eprintln!("Usage: skate3rust --extract-ocean-pca <default.xex> <ocean-pca.json>");
|
||||
return Some(2);
|
||||
};
|
||||
let result = std::fs::read(&xex)
|
||||
.map_err(|e| format!("{}: {e}", std::path::Path::new(&xex).display()))
|
||||
.and_then(|bytes| skate_data::ocean_pca::json_from_xex(&bytes))
|
||||
.and_then(|json| std::fs::write(&out, json).map_err(|e| format!("{}: {e}", std::path::Path::new(&out).display())));
|
||||
match result {
|
||||
Ok(()) => {
|
||||
println!("OCEAN_PCA_READY");
|
||||
Some(0)
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("Ocean PCA extraction failed: {error}");
|
||||
Some(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
# Water (investigation in progress)
|
||||
|
||||
Branch: `gameplay/water`. Status: **water bail confirmed in play; water animation extraction added, awaiting in-play check.**
|
||||
|
||||
## Problem
|
||||
|
||||
Water does nothing in play: the skater rides over it or falls through it as if it
|
||||
were ordinary ground or empty space. The retail game has water-specific behaviour
|
||||
(the `IsInWater` motion-graph condition, the water ragdoll profile, and the
|
||||
"special surface" path in the wipeout state).
|
||||
|
||||
## Evidence so far
|
||||
|
||||
### 1. Water is a retail collision surface type, and it is in the converted maps
|
||||
|
||||
Retail collision units carry a 16-bit surface ID. The surface **type** is
|
||||
`(surface >> 7) & 31`; the engine already treats type 12 as water:
|
||||
|
||||
- `crates/skate-core/src/physics/board_ground.rs` — the board sets collision flag
|
||||
bit 25 (`Body872`) and records the contact height (`Body864`,
|
||||
`surface_twelve_height`) when any part touches type 12.
|
||||
- `crates/skate-game/src/physics/wipeout_states/prediction.rs` — the wipeout
|
||||
trajectory query tests `surface & 0xF80 == 0x600` (type 12).
|
||||
|
||||
New tool: `crates/skate-data/examples/water_surfaces.rs` lists the surface types
|
||||
in each map's embedded RWCM collision and details the type-12 triangles.
|
||||
|
||||
```
|
||||
cargo run --locked --release -p skate-data --example water_surfaces -- data/installations/<id>/maps/*.skate
|
||||
```
|
||||
|
||||
Results (installation `c82bd63f…`):
|
||||
|
||||
| Map | Water triangles | Surface ID | Heights | Notes |
|
||||
|---|---|---|---|---|
|
||||
| DownTown | 355 (330 flat, all one-sided) | 1591 only | 0.4 m … 48.3 m, many levels | fountains/pools spread over 26 tiles |
|
||||
| University | 322 (all flat, all one-sided) | 1591 only | 296 at 217.9 m, rest 67.9–71.0 m | reservoir plus smaller pools |
|
||||
| Industrial, all parks | 0 | — | — | — |
|
||||
|
||||
So water collision exists only in DownTown and University. Every collision
|
||||
stream is already converted (cities use their `cSim_*_high` tiles; parks use
|
||||
`cSim_Global`), and the volume filter from change 5 cannot drop water, because
|
||||
water triangles carry a surface ID.
|
||||
|
||||
Render materials agree: `water.alpha` / `water.flowingalpha` appear only in
|
||||
DownTown and `water.default` / `water.flowing` only in University. Industrial's
|
||||
harbour/sea is only the backdrop `ocean.reflection` mesh
|
||||
(`assets/private/native-backdrops/Industrial.skate`), with no water collision
|
||||
under it; the harbour bed is ordinary collision at about y = -3 (see change 4).
|
||||
|
||||
### 2. The skater never learns it is in water (the missing link)
|
||||
|
||||
The skater-side water signal comes from the collision output:
|
||||
|
||||
- `CollisionOutputFields.flag_3481` → `Processed.flags_2488` bit 30
|
||||
(`0x4000_0000`), and
|
||||
- `CollisionOutputFields.scalar_28` → `Processed.collision_scalar_2924`
|
||||
(water height)
|
||||
|
||||
(`crates/skate-core/src/player/input_phase/publication.rs:253-254`). The wipeout
|
||||
state reads both (`wipeout_states/lifecycle.rs:81-82`, `update.rs:24-26`) to
|
||||
enter the "special surface" path (ragdoll profile 10, `below_surface`, the
|
||||
`IsInWater` motion-graph condition).
|
||||
|
||||
**Nothing in the codebase ever writes `flag_3481` or `scalar_28`.** They stay
|
||||
at their defaults (0), so the water path can never start, even on DownTown and
|
||||
University where the data exists.
|
||||
|
||||
The board already computes the matching values (`collision_flags` bit 25 and
|
||||
`surface_twelve_height`), and the board manager (`offboard/board_manager/runtime.rs`
|
||||
`surface()`) and the player state (`player_state/publication.rs`) already read bit 25.
|
||||
The likely retail behaviour is that the physics output fill copies the board's
|
||||
`Body872` bit 25 → `Collision+3481` and `Body864` → `Collision+28`. **This
|
||||
is a hypothesis; the retail write site has not been found yet.**
|
||||
|
||||
### 3. The skater's own contacts also classify water
|
||||
|
||||
`SkeletonCollision` (82BD4A30, `crates/skate-core/src/physics/skeleton_body/collision_update.rs`)
|
||||
sets `flags.material_12` and `material_12_height` when any body part touches
|
||||
type 12. Its flags sit at native 4079/4080/4081 (materials 10/11/12); the
|
||||
collision output's 3479/3480/3481 use the same layout 600 bytes lower, and
|
||||
3479/3480 are already the material-10/11 results. This makes the skater
|
||||
body the most likely retail source of Collision+3481/+28.
|
||||
|
||||
### 4. Industrial's sea
|
||||
|
||||
The user remembers that falling into Industrial's sea respawned the player in
|
||||
retail, and suggests the sea may simply be out of reach in retail. Industrial's
|
||||
harbour floor uses ordinary surfaces (concrete and so on) and its sea has no
|
||||
collision of any kind on the disc (the proxy archives are visual only), so no
|
||||
data-driven water detection is possible there. Not handled by this change.
|
||||
|
||||
## Change
|
||||
|
||||
`PlayerInputRuntime::publish_water` (`crates/skate-game/src/physics/player_input/mod.rs`),
|
||||
called from `frame.rs` right after `publish_board` (after the packet reset and
|
||||
the skeleton contact feedback for the frame):
|
||||
|
||||
- skater body touching type 12 → `flag_3481 = 1`, `scalar_28 = material_12_height`;
|
||||
- otherwise board touching type 12 (`collision_flags` bit 25) → `flag_3481 = 1`,
|
||||
`scalar_28 = surface_twelve_height`;
|
||||
- otherwise both stay at their reset values (0).
|
||||
|
||||
Project choice: retail's writer is unconfirmed; the skater body wins over the
|
||||
board because the wipeout water path simulates the body. Water triangles stay
|
||||
solid one-sided collision, as before.
|
||||
|
||||
### Water bail (added after the first in-play test)
|
||||
|
||||
First in-play test (University fountain basin): the skater landed on the water
|
||||
and rode on it; no bail. Expected: the published flag only feeds the wipeout
|
||||
state, and nothing starts a bail on water. I searched every recovered bail check
|
||||
(`skate-core/src/player/wipeout/`, the state selector, all motion-graph conditions
|
||||
in the stock state XML) and found no water/type-12 trigger. The retail
|
||||
trigger is in code that hasn't been recovered, and no decrypted executable is
|
||||
available.
|
||||
|
||||
`physics/wipeout.rs` `check_after_physics` now requests a bail (request slot 33,
|
||||
`WATER_BAIL_REASON`, otherwise unused and not a runout reason) when the board
|
||||
(`collision_flags` bit 25) or the skater body (`flags.material_12`) touches
|
||||
water, unless the skater is already in WipeoutGround, Teleporting or Sleeping.
|
||||
It then takes the ordinary path: state flag 65 → `PhysicsWantsWipeOut` →
|
||||
WipeoutGround, where `Collision+3481` starts the water ragdoll and the
|
||||
respawn timer. Logged as `WATER_BAIL tick=… state=…`. **Project choice**, not
|
||||
recovered retail behaviour.
|
||||
|
||||
Not changed: water stays solid. The water buoyancy code
|
||||
(`wipeout_state::body::special_surface`, which pushes parts up when below
|
||||
height + 0.1) suggests that in retail the body sinks slightly into water, so
|
||||
water may not be solid for the body in retail. Left alone until play shows
|
||||
whether that matters.
|
||||
|
||||
### Water rendering: extract the ocean animation table from default.xex
|
||||
|
||||
Water (family 33) and ocean (family 31) shaders need `assets/private/ocean-pca.json`,
|
||||
which nothing in setup created (see open question 5), so 52 water/ocean materials
|
||||
fell back to static shading (black for `ocean.default`, which has no diffuse).
|
||||
|
||||
- `crates/skate-data/src/xex/`: XEX2 unpacker (retail AES-128 file-key and CBC
|
||||
payload decryption, "normal" LZX or "basic" decompression) producing the
|
||||
mapped base image. No new dependencies; AES is checked against FIPS-197 and
|
||||
NIST SP 800-38A vectors. The owned disc `default.xex` (normal encryption,
|
||||
LZX) unpacks to an 18,022,400-byte image at 0x82000000 in about 0.2 s
|
||||
(sha256 `ce1e3ae5…`, not the TU3 image `ocean_pca.py` expects).
|
||||
- `crates/skate-data/src/ocean_pca.rs`: locates cPCAWaterAnimationData's table
|
||||
the way the code addresses it (`lis` + D-form pairs forming two addresses
|
||||
0x168 apart within 32 instructions, data plausibility check), so it works for
|
||||
any build. The disc build has exactly one match: means at 0x82FC3978, weights
|
||||
at 0x82FC3AE0 (TU3: 0x830118D8 / 0x83011A40); frame 0 mean is
|
||||
(127.55, 251.83, 127.54), an "up" normal. JSON layout matches `ocean_pca.py`.
|
||||
- `skate3rust --extract-ocean-pca <default.xex> <out.json>` (`main.rs`, runs
|
||||
before any game initialization; prints `OCEAN_PCA_READY`).
|
||||
- Setup: `ocean_pca.convert` runs it in the `environment` group
|
||||
(`asset_exports.environment`, new `game_exe` argument; `versions.py` adds
|
||||
`ocean_pca.py` to the group). Failure is optional content, like other
|
||||
environment parts. Changing the environment recipe refreshes that group once.
|
||||
|
||||
## Verification
|
||||
|
||||
- Unit test `physics::player_input::tests::water_contact_prefers_the_skater_body_height`.
|
||||
- `cargo test --locked -p skate-game --release --bin skate3rust -- water_contact`: passes.
|
||||
- Release build staged into `bin\`; `--test-world --check-assets` → `SKATE_ASSETS_READY`.
|
||||
- In play, first build: rode on top of the water, no bail (led to the water bail above).
|
||||
- In play, second build: bails on University fountain basin (F2), University reservoir (F3) and DownTown fountain (F4). Log shows `WATER_BAIL ... state=KnownAir` on each drop.
|
||||
- Asset-backed wipeout tests (`--ignored wipeout`): 4 pass; `marker_reply_restores_on_foot`
|
||||
fails identically with these changes stashed (pre-existing).
|
||||
|
||||
## Open questions
|
||||
|
||||
1. Where does retail write `Collision+3481` / `+28`? The wiring above is the
|
||||
best match to the evidence; it is not confirmed against retail.
|
||||
2. Water triangles are solid one-sided surfaces in our collision world. Does the
|
||||
retail board/body pass through type 12, or rest on it? (In play: does the
|
||||
skater stand on the University reservoir?)
|
||||
3. Industrial's sea: retail respawned the player (user's memory), possibly
|
||||
out of reach in retail. No collision data exists for it; see section 4.
|
||||
5. Water rendering (user: static texture on the fountains, black on the
|
||||
University reservoir). Cause found: the log says "52 of 8546 world
|
||||
materials use an unsupported shader family and render as family 1". Water
|
||||
(family 33) and ocean (family 31) shaders require `assets/private/ocean-pca.json`
|
||||
(`MaterialTuning::supported`, `read_pca`). Nothing in setup (here or on
|
||||
upstream `60efdef`) creates that file; `tools/asset_pipeline/ocean_pca.py`
|
||||
needs a decrypted, decompressed TU3 executable image (hard-coded sha256 and
|
||||
table addresses 0x830118D8 / 0x83011A40). The disc `default.xex` is XEX2 with
|
||||
normal encryption and LZX compression, and setup never unpacks it. Fallback
|
||||
family 1 shows the static diffuse; `ocean.default` has no diffuse, hence black.
|
||||
This affects upstream users equally.
|
||||
4. What is surface type 13 (DownTown 88, University 17,790 triangles)? It may be
|
||||
related (shallow water or a splash surface) or something unrelated, such as grass.
|
||||
|
||||
## Files
|
||||
|
||||
- `crates/skate-data/examples/water_surfaces.rs` (new, diagnostic only; also prints water/ocean render mesh heights).
|
||||
- `crates/skate-game/src/physics/player_input/mod.rs` (`publish_water`, unit test).
|
||||
- `crates/skate-game/src/physics/frame.rs` (call site).
|
||||
- `crates/skate-game/src/physics/wipeout.rs` (water bail request).
|
||||
- `mods/water-test-teleport/` (dev-only test mod, F2/F3/F4 teleports; not committed, not for upstream).
|
||||
- `crates/skate-data/src/xex/{mod,aes,lzx}.rs`, `crates/skate-data/src/ocean_pca.rs`, `crates/skate-data/src/lib.rs`.
|
||||
- `crates/skate-data/examples/xex_unpack.rs` (diagnostic: unpack + locate table).
|
||||
- `crates/skate-game/src/main.rs` (`--extract-ocean-pca`).
|
||||
- `tools/asset_pipeline/ocean_pca.py` (`convert`), `asset_exports.py`, `install.py`, `versions.py`.
|
||||
@@ -14,6 +14,7 @@ Each document stands alone. File paths are relative to the repository root.
|
||||
| 3 | [SDL3 gamepad input](03-sdl3-gamepad-input.md) | Game (Rust), build scripts, vendored `sdl3-sys` | Done, verified in play with an Xbox Elite Series 2 |
|
||||
| 4 | [Authored map spawns](04-map-spawns.md) | Asset pipeline (Python) | Done, all 10 maps confirmed in play |
|
||||
| 5 | [Invisible collision volumes](05-collision-volumes.md) | Game data loader (Rust) | Done, all four affected maps confirmed in play |
|
||||
| 9 | [Water](09-water.md) | Game physics (Rust), diagnostics | Water flag wired up (DownTown, University); awaiting in-play test |
|
||||
|
||||
## Environment used for verification
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ def character(game_root, stage, work, report, log, converted=None):
|
||||
(private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
|
||||
|
||||
|
||||
def environment(game_root, stage, work, report, log, converted=None):
|
||||
def environment(game_root, stage, work, report, log, converted=None, game_exe=None):
|
||||
private=stage/"assets/private"
|
||||
stock=private/"stock"
|
||||
report('Preparing retail sky domes')
|
||||
@@ -78,6 +78,11 @@ def environment(game_root, stage, work, report, log, converted=None):
|
||||
attempt('skies',lambda assets:write_skies(game_root,assets,converted))
|
||||
from .render_parameters import convert as write_render_parameters
|
||||
attempt('parameters',lambda assets:write_render_parameters(assets,converted))
|
||||
if game_exe is not None:
|
||||
# Water/ocean animation table, held only in the executable.
|
||||
report('Extracting the original water animation')
|
||||
from .ocean_pca import convert as write_ocean_pca
|
||||
attempt('ocean',lambda assets:write_ocean_pca(game_exe,game_root/'default.xex',assets,log))
|
||||
report('Extracting original travel destinations and location names')
|
||||
from .teleports import convert as write_teleports
|
||||
attempt('teleports',lambda assets:write_teleports(game_root,assets,converted))
|
||||
|
||||
@@ -433,7 +433,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
|
||||
if 'character' in groups:
|
||||
exports.character(game_root,stage,work,report,log,converted)
|
||||
if 'environment' in groups:
|
||||
exports.environment(game_root,stage,work,report,log,converted)
|
||||
exports.environment(game_root,stage,work,report,log,converted,game_exe)
|
||||
if 'maps' in groups:
|
||||
report('Preparing authored movable-object models')
|
||||
from .dynamic_props import prepare_catalog
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
"""Extract TU3's built-in PCA ocean animation from an owned mapped image.
|
||||
"""Extract the built-in PCA ocean/water animation (assets/private/ocean-pca.json).
|
||||
|
||||
`convert` is what setup uses: the game executable unpacks the owned
|
||||
default.xex and locates the table in any build (skate_data::ocean_pca,
|
||||
`skate3rust --extract-ocean-pca`). Without this file every water and ocean
|
||||
material renders as a static fallback.
|
||||
|
||||
`extract` is the original TU3-only path from an already mapped image.
|
||||
Addresses are corroborated by cPCAWaterAnimationData::Init (827905B0) and
|
||||
its update routine (82790858). No game content is embedded in this tool.
|
||||
"""
|
||||
@@ -32,6 +38,28 @@ def extract(image, output, base=0x82000000):
|
||||
return len(frames)
|
||||
|
||||
|
||||
def convert(game_exe, xex, assets, log=None):
|
||||
"""Writes assets/private/ocean-pca.json from the owned default.xex."""
|
||||
import subprocess
|
||||
from .install import spawn
|
||||
output = Path(assets) / 'private/ocean-pca.json'
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with spawn([str(game_exe), '--extract-ocean-pca', str(xex), str(output)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process:
|
||||
text = process.stdout.read().decode('utf-8', errors='replace')
|
||||
code = process.wait()
|
||||
if log is not None:
|
||||
log.write(text)
|
||||
log.flush()
|
||||
if code or 'OCEAN_PCA_READY' not in text:
|
||||
output.unlink(missing_ok=True)
|
||||
raise RuntimeError('Ocean animation extraction failed: ' + (text.strip().splitlines() or ['no output'])[-1])
|
||||
frames = json.loads(output.read_text(encoding='utf-8'))['frames']
|
||||
if len(frames) != 30:
|
||||
raise ValueError('Ocean animation table must have 30 frames')
|
||||
return len(frames)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
|
||||
@@ -20,7 +20,8 @@ SOURCES = {
|
||||
'environment': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/sky.py', 'asset_pipeline/backdrop.py',
|
||||
'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py',
|
||||
'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py',
|
||||
'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py'),
|
||||
'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py',
|
||||
'asset_pipeline/ocean_pca.py'),
|
||||
'maps': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py',
|
||||
'asset_pipeline/environment.py', 'asset_pipeline/irradiance.py',
|
||||
'asset_pipeline/retail_material.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/sky.py'),
|
||||
|
||||
Reference in new issue
Block a user