diff --git a/crates/skate-game/src/retail_render.rs b/crates/skate-game/src/retail_render.rs index 3f84eff0..42fb52db 100644 --- a/crates/skate-game/src/retail_render.rs +++ b/crates/skate-game/src/retail_render.rs @@ -47,6 +47,14 @@ const MAX_PAGE_LAYERS: usize = 2048; /// scene.hlsl's retail world ALPHAREF is 30, not the portable 0.5. const ALPHA_REF: f32 = 30. / 255.; +/// Shader family of the retail `environment.transparent` materials +/// (`transparentenvironment_defaultPS` in `shaders_final.big`): chain-link +/// fences, wire mesh, grilles. Diffuse^2 times the shadowed lightmap, scaled by +/// the diffuse alpha (ALU 43), plus the lightmap-masked specular, with no normal +/// map and no `kd`; the output alpha is the diffuse alpha squared (ALU 53). See +/// the `fam == 16u` terms of `retail_world.wgsl`. +pub(crate) const TRANSPARENT_ENVIRONMENT_FAMILY: u32 = 16; + /// Shared per-frame state: shadow floor, animation clock and authored ocean PCA. /// Map-independent, so it is a fixed handle rather than a staged asset. pub(crate) const FRAME_STATE: Handle = @@ -467,6 +475,10 @@ impl Definition { "water.default" | "water.alpha" | "water.skatepark" ) { 33 + } else if stored_family == 0 && shader == "environment.transparent" { + // Map packages exported before the converter knew this family + // stored 0 and drew as opaque family 1; classify them on load. + TRANSPARENT_ENVIRONMENT_FAMILY } else { stored_family }; @@ -530,6 +542,7 @@ impl Definition { pub(crate) fn supported(&self, tuning: &MaterialTuning) -> bool { (1..=13).contains(&self.family) + || self.family == TRANSPARENT_ENVIRONMENT_FAMILY || match self.family { 14 | 32 => tuning.rows.get(&self.shader).is_some_and(|r| !r.is_empty()), 31 => { @@ -1816,4 +1829,59 @@ mod tests { ); } } + + /// A retail definition record with the shader name, stored family and + /// diffuse / lightmap / specular bindings, no parameters. + fn transparent_definition(shader: &str, family: u32) -> Vec { + fn text(out: &mut Vec, value: &str) { + out.extend_from_slice(&(value.len() as u32).to_le_bytes()); + out.extend_from_slice(value.as_bytes()); + } + let mut out = vec![0u8; 16]; + text(&mut out, shader); + for value in [family, 2, 3] { + out.extend_from_slice(&value.to_le_bytes()); + } + for (i, role) in ["diffuse", "lightmap", "specular"].into_iter().enumerate() { + text(&mut out, role); + for value in [i as u32 + 1, 0, 0, 0] { + out.extend_from_slice(&value.to_le_bytes()); + } + } + out.extend_from_slice(&0u32.to_le_bytes()); + text(&mut out, ""); + out + } + + #[test] + fn transparent_environment_takes_its_own_family() { + // Packages exported before the converter knew the family stored 0 and + // drew chain-link fences as opaque family 1. + for stored in [0, TRANSPARENT_ENVIRONMENT_FAMILY] { + let bytes = transparent_definition("environment.transparent", stored); + let definition = Definition::parse(&bytes).expect("parses"); + assert_eq!(definition.family, TRANSPARENT_ENVIRONMENT_FAMILY, "stored {stored}"); + assert!(definition.supported(&MaterialTuning::default())); + } + // Other shaders keep their stored family. + let reflective = Definition::parse(&transparent_definition("environment.reflective_trans", 13)).unwrap(); + assert_eq!(reflective.family, 13); + let unknown = Definition::parse(&transparent_definition("incandescent.transparent", 0)).unwrap(); + assert_eq!(unknown.family, 0); + } + + #[test] + fn transparent_environment_shader_matches_retail() { + let src = include_str!("retail_world.wgsl"); + // transparentenvironment_defaultPS 39, 43, 53: alpha-scaled light, alpha^2 out. + assert!(src.contains("if fam == 13u || fam == 16u { lin = lml*d*a.a; }")); + assert!(src.contains("if fam == 13u || fam == 16u { alpha *= alpha; }")); + assert!(src.contains("if fam >= 7u { alpha = a.a; }")); + // Shadowed lightmap and the m_params multiplier like the other world families. + assert!(src.contains("(fam<=8u || fam==13u || fam==16u)")); + assert!(src.contains("if fam <= 8u || fam == 13u || fam == 16u { fog_a *= p.surface.w; }")); + // No normal map read: the retail program fetches none. + let normal_read = src.lines().find(|l| l.contains("sample_normal_map(slot,i.uv,g)")).unwrap(); + assert!(!normal_read.contains("16u"), "{normal_read}"); + } } diff --git a/crates/skate-game/src/retail_world.wgsl b/crates/skate-game/src/retail_world.wgsl index 07781789..22f7833f 100644 --- a/crates/skate-game/src/retail_world.wgsl +++ b/crates/skate-game/src/retail_world.wgsl @@ -119,7 +119,7 @@ fn fragment(i: VertexOutput) -> @location(0) vec4 { var alpha = 1.0; var lin = vec3(0.0); var baked = lm*lm; - if frame_state.shadow.w>0.0 && (fam<=8u || fam==13u) { + if frame_state.shadow.w>0.0 && (fam<=8u || fam==13u || fam==16u) { let view_z=(frame::view.view_from_world*i.world_position).z; for (var light_id=0u; light_id @location(0) vec4 { } var lml = baked; lin = lml*kd*d; - if fam == 13u { lin = lml*d*a.a; } + // transparentenvironment_defaultPS 39, 43: diffuse^2 * lightmap * alpha, no kd. + if fam == 13u || fam == 16u { lin = lml*d*a.a; } if (flags & 16u) != 0u { let lit = vec3(-0.14,0.5,0.9); let reflected = lit-2.0*wn*dot(wn,lit); @@ -304,12 +305,13 @@ fn fragment(i: VertexOutput) -> @location(0) vec4 { lin += cube*lum*masks.z*1.5; } if fam >= 7u { alpha = a.a; } - if fam == 13u { alpha *= alpha; } + // transparentenvironment_defaultPS 53: oC0.w = alpha * alpha. + if fam == 13u || fam == 16u { alpha *= alpha; } } var f = saturate(length(rpos)*p.fog_ramp.x+p.fog_ramp.y); if p.fog_ramp.z != 1.0 { f = pow(max(f,1e-6),p.fog_ramp.z); } var fog_a = 1.0+p.fog_color.a*f; - if fam <= 8u || fam == 13u { fog_a *= p.surface.w; } + if fam <= 8u || fam == 13u || fam == 16u { fog_a *= p.surface.w; } var xe = max((lin*fog_a+p.fog_color.rgb*f)*p.mode.w,vec3(0.0)); // Reduced curve is the full curve with the linear input capped at one. if fam == 8u { xe = min(xe,vec3(1.0)); } diff --git a/docs/hails-additions/32-transparent-fences.md b/docs/hails-additions/32-transparent-fences.md new file mode 100644 index 00000000..db745d5a --- /dev/null +++ b/docs/hails-additions/32-transparent-fences.md @@ -0,0 +1,131 @@ +# See-through fences and grates: retail `environment.transparent` shading + +Branch `fix/transparent-materials` (off `main` `b3c96793`). First written on +`fix/fence-alpha` (off `hails-additions`); this branch is the port to `main` for +its own upstream PR (bundle O in [PULL-REQUESTS.md](PULL-REQUESTS.md)). + +## Problem + +Chain-link fences, wire mesh and grilles, for example the row of fence panels +behind the bus stop at the Industrial Ghetto Spot (locator `Z_Ind_GhettoSpot`, +about (-516.9, 3.6, -294.5)), draw as solid black panels. Retail draws the wire +mesh with the background visible through it. + +## Root cause + +- [data] The fence panels use the retail world shader `environment.transparent` + (Industrial materials 884 `0x15EA09769DB7638D_12`, 1179 `0xC50D7FD400B4BECD_17`, + 1293 `0x3A36F543A04A4DA9_26` within 30 m of the spot; the exported map stores + `alpha_mode 2`, so the draw is already blended). Game logs list 429 + `environment.transparent` materials in one district and 90 / 15 in others. +- [code] The converter had no family for this shader + (`tools/asset_pipeline/retail_material.py` `_retail_shader_family` fell + through to `return 0`, line 46 on this branch), so the renderer counted it as + unsupported and drew it as family 1 + (`crates/skate-game/src/retail_render.rs:803`, "unsupported family renders as + plain diffuse plus lightmap", logged at `:821`). +- [code] Family 1 is `environment.default`, an opaque program: the shader leaves + the output alpha at 1.0 (`crates/skate-game/src/retail_world.wgsl:119`, + `var alpha = 1.0`; only families 7 and up take the diffuse alpha, `:307`). + The blended draw is therefore fully opaque, and the wire texture's empty + texels (alpha 0, near-black colour) fill the panel. + +## Evidence (retail program) + +`transparentenvironment_defaultPS.fpo` from `data/big/shaders_final.big`, read +with the Xenos disassembler in `.claude/skills/living-world/tools/xenos_disasm.py` +(ALU slot numbers): + +- 23..26, 28, 31, 33, 34: four lightmap taps averaged (x 0.25); 36..38: + `min(lm^2, shadow + (0.05, 0.09, 0.13))`, the same shadowed lightmap as the + other world families. +- 9, 10, 14: diffuse squared; 39: `diffuse^2 * lightmap`. +- 30..42: specular from the pseudo light (-0.14, 0.5, 0.9) reflected about the + interpolated normal, `(2.1, 1.8, 1.5)`, masked by `lightmap.g` and the + specular texture; no normal-map fetch, no `kd` term. +- 43: `lit = diffuse^2 * lightmap * diffuse.a + specular`. +- 42, 44: fog with `m_params.y` (c10.y, last swizzle component rule) as the + multiplier; 45..53: exposure c9.x and the shared tone curve. +- 53: `oC0.w = diffuse.a * diffuse.a`. + +These are the terms our family 13 (`environment.reflective_trans`) already +implements, without its normal map and reflection cube. + +## Change + +- `tools/asset_pipeline/retail_material.py:16`: `environment.transparent` is + family 16. +- `crates/skate-game/src/retail_render.rs:56`: + `TRANSPARENT_ENVIRONMENT_FAMILY = 16`; `Definition::parse` (`:478`) upgrades + map packages that stored 0 for this shader, so existing installs need no + re-export; `supported` accepts it (`:545`). +- `crates/skate-game/src/retail_world.wgsl`: family 16 takes the shadowed + lightmap (`:122`), `lin = lightmap * diffuse^2 * alpha` (`:291`), + `alpha = diffuse.a^2` (`:309`) and the `m_params` multiplier (`:314`), like + family 13. It reads no normal map (`:83`), no macro overlay (`:265`, families + below 13 only) and no reflection cube (`:299`). + +## Files + +- `crates/skate-game/src/retail_render.rs` (family constant, parse upgrade, + `supported`, two tests) +- `crates/skate-game/src/retail_world.wgsl` (family 16 terms) +- `tools/asset_pipeline/retail_material.py` (family mapping) +- `tools/asset_pipeline/test_environment.py` + (`TransparentEnvironmentMaterialTests`) +- `docs/hails-additions/32-transparent-fences.md`, `README.md` index row, + `PULL-REQUESTS.md` row O + +## Moddability + +The family follows the material's shader name, which is map data: the +converter maps shader names to families in `retail_material.py` +(`_retail_shader_family`), and the renderer reads the family and the material +parameters (`m_params`, bindings) from each map package's definition records. +A custom map or mod material that names `environment.transparent` gets the same +retail shading, with its own diffuse alpha, lightmap and specular textures. +Nothing here is a per-object rule. + +Hard-coded values that could become data (open, not changed here): the +specular pseudo light `(-0.14, 0.5, 0.9)` and colour `(2.1, 1.8, 1.5)` and the +shadow floor are shader constants shared with the other world families +(`retail_world.wgsl:293`, `:297`), and the material multiplier +(`m_params[0].y`) is still the constant 1 for every world family, the same open +item as in the world radiance notes. + +## Retail parity and tests + +The shading is a term-by-term port of the retail pixel program +`transparentenvironment_defaultPS` (see Evidence); no value was tuned by eye. +Tests: + +- `retail_render::tests::transparent_environment_takes_its_own_family`: a + definition record naming `environment.transparent` gets family 16 whether the + package stored 0 (old exports) or 16, and is supported; other shaders keep + their stored family (`environment.reflective_trans` 13, + `incandescent.transparent` 0). +- `retail_render::tests::transparent_environment_shader_matches_retail`: the + WGSL holds the retail terms (ALU 39/43 alpha-scaled light, 53 alpha squared, + shadowed lightmap, `m_params` fog multiplier) and no normal-map read for + family 16. +- `test_environment.TransparentEnvironmentMaterialTests`: the converter maps + `environment.transparent` to 16 with the blended render flag, and keeps + `environment.reflective_trans` at 13. +- The existing naga validation tests of the retail world shader cover the + changed WGSL. + +## Verification + +Not run yet on this branch: the build was stopped before the tests ran (2026-10-08). The first version (on `fix/fence-alpha`, off `hails-additions`) passed 2/2 `transparent_environment` tests and the 46 retail shader validation tests; the Python converter tests pass here (6/6). The PR stays a draft until the tests above pass on this branch and the fences are checked in game. + +## Open questions + +- In-game check at the Ghetto Spot still to do. +- Retail render state for these draws (blend factors, depth write, cull mode) is + not read yet; we use the engine's existing blended class + (`AlphaMode::Blend`, `retail_render.rs:263`; back-face culling unless the + material is two-sided, `:412`). +- The specular mask channel and power use the same convention as the other + ported world families; the texture fetch destination swizzles were not decoded. +- `incandescent.transparent`, `advertisement.default`, `animated.flag` and the + traffic light shaders also still fall back to family 1. diff --git a/docs/hails-additions/PULL-REQUESTS.md b/docs/hails-additions/PULL-REQUESTS.md index 3811fc60..ab9ee644 100644 --- a/docs/hails-additions/PULL-REQUESTS.md +++ b/docs/hails-additions/PULL-REQUESTS.md @@ -20,6 +20,7 @@ verification, open questions). | G | Performance (setup) | Skip decoding duplicate stream copies; run the customiser beside the maps; parallel clothing library and pro roster, plus a native RefPack DLL in `Build.ps1` | [08](08-setup-performance.md) | `8327e94` (+ overlap hunks of `install.py` in `18f41a3`) | — | Draft upstream [#29](https://github.com/SK8-ENGINE/skate-3-rust-engine/pull/29) | | K | Gameplay fix (physics) | Board/skater solver: 50 constraint iterations (retail live value) instead of `RWMaxIterations` = 25 | [12](12-solver-iterations.md) | `physics/solver-iterations` (`3f6e989`, cherry-picked from `gameplay/audio`'s `7a9e6dc` onto `main` `7ae67f2`) | — (one-file change in `physics/settings.rs`) | Open upstream: [#35](https://github.com/SK8-ENGINE/skate-3-rust-engine/pull/35) (2026-10-03). #32 depends on it for retail-like pops and carries the same commit until #35 merges. Open question in the PR: the asset-gated `customiser_equipment_reaches_ground_force_and_torque` test (doc 12). Tests: 309 pass; the 2 failures (`pipelines_accept_…`, `sky_shader_validates`) also fail on `main`. | | L | Tooling | Development and research tools in `tools//` (regression checks, setup equivalence, collision / world-stream / audio-file / vault readers, audio e2e and bench; recomp trace and code search, reference only) | [14](14-published-tools.md) | uncommitted (on `gameplay/audio` working tree) | `audio-e2e` / `audio-bench` / `audio-file-inspect` need J's native audio and setup modules; `setup-equivalence/compare_refpack.py` needs G's RefPack DLL; `regression-checks/check_maps.py` uses F's `--validate-maps` (falls back to `--check-assets` without it) | Upstream [#37](https://github.com/SK8-ENGINE/skate-3-rust-engine/pull/37) (2026-10-04): the six non-audio tool folders, branch `tooling/published-tools` (`92d078f`); the audio tools stay in #32. | +| O | Rendering fix | Rendering: see-through fences and grates (retail `environment.transparent`): the converter maps the shader to new family 16, map packages that stored 0 are upgraded on load, and `retail_world.wgsl` ports `transparentenvironment_defaultPS` (alpha-scaled lightmapped diffuse, specular, alpha squared out); fences no longer draw as opaque black panels | [32](32-transparent-fences.md) | `fix/transparent-materials` (off `main` `b3c96793`; first written as `fix/fence-alpha` off `hails-additions`) | — (may conflict textually with other pending `retail_world.wgsl` edits, e.g. the ocean work; merge order only) | Draft upstream (2026-10-08). Tests: Rust tests not yet run on this branch; Python 6/6. In-game check at the Ghetto Spot pending. | Opened 2026-10-01 as drafts from per-PR branches cut from upstream `60efdef` (code only; `docs/hails-additions` stays in the fork). #29 (G) is stacked on diff --git a/docs/hails-additions/README.md b/docs/hails-additions/README.md index 7c128f86..f8593a18 100644 --- a/docs/hails-additions/README.md +++ b/docs/hails-additions/README.md @@ -24,6 +24,7 @@ Each document stands alone. File paths are relative to the repository root. | — | [Audio specs](audio-specs/README.md) | Reference for 11 and 15 (Markdown) | Published 2026-10-03: the behavioural specs (AEMS evaluator, voice graph, buses, MixMap, grain player, player components, world / NPC audio, prior work) that docs 11 / 15 and the code comments cite by section | | 16 | [Audio modding](16-audio-modding.md) | Mods (Lua API 2), native audio (Rust) | Part of #32 (the former #36 and #43): the modder's guide (`audio.json` content overlay, custom-map audio by sidecar / `.skate` tag / mods, retail posts / globals / MixMap watch, observe-only audio events, limits, `check_mod`), the implementation and its byte-identical proofs, then the design and research. Depends on #32. | | 25 | [Performance diagnostics and setup phases 4-5](25-perf-diagnostics.md) | Game (Rust), mod API (Lua), setup (Python) | Upstream #51; merged into `hails-additions` 2026-10-04 (frame-time row id 27 there). Frame-time counter and log, `sdk.snapshot.frame`, below-normal setup priority and budget overrides, threaded `write_map`, props beside collision and `write_map`; setup outputs byte-identical | +| 32 | [See-through fences and grates: retail `environment.transparent` shading](32-transparent-fences.md) | Game rendering (Rust, WGSL), setup (Python) | Fix, branch `fix/transparent-materials` (planned upstream PR O): fences drew as opaque black panels (shader fell back to family 1); new family 16 ports `transparentenvironment_defaultPS` (alpha-scaled lightmapped diffuse, alpha squared out); in-game check pending | ## Environment used for verification diff --git a/tools/asset_pipeline/retail_material.py b/tools/asset_pipeline/retail_material.py index 839a5ea0..ae897d0e 100644 --- a/tools/asset_pipeline/retail_material.py +++ b/tools/asset_pipeline/retail_material.py @@ -13,6 +13,10 @@ def _retail_shader_family(shader_name: str) -> int: return 3 if shader.startswith("environment.default"): return 1 + if shader == "environment.transparent": + # transparentenvironment_defaultPS: alpha-scaled lightmapped diffuse, + # alpha squared out (chain-link fences, wire mesh). + return 16 if shader.startswith("environmentsimple.alphatest"): return 7 if shader.startswith("environmentsimple.diffuse"): diff --git a/tools/asset_pipeline/test_environment.py b/tools/asset_pipeline/test_environment.py index a1e28e88..202c67bc 100644 --- a/tools/asset_pipeline/test_environment.py +++ b/tools/asset_pipeline/test_environment.py @@ -3,6 +3,7 @@ import struct import unittest from .environment import Collections, field, key_hash, sky_parameters, world_environment, fog_parameters from .render_parameters import exposure_parameters +from .retail_material import _retail_render_flags, _retail_shader_family def raw(*values): @@ -82,5 +83,13 @@ class EnvironmentTests(unittest.TestCase): sky_parameters(c, 'sky.missing') +class TransparentEnvironmentMaterialTests(unittest.TestCase): + def test_transparent_environment_has_its_own_family(self): + # transparentenvironment_defaultPS (chain-link fences), blended. + self.assertEqual(_retail_shader_family('environment.transparent'), 16) + self.assertEqual(_retail_render_flags('environment.transparent', 2) & 2, 2) + self.assertEqual(_retail_shader_family('environment.reflective_trans'), 13) + + if __name__ == '__main__': unittest.main()