Tooling: streaming map validator (--validate-maps) wired into setup

One skate3rust process validates TEST_WORLD and every converted map over
stdin/stdout (SKATE_MAP_CHECK lines). Existing checks still fail a map;
new spawn-support / neutral-startup / invisible-collision checks report
warnings. Setup falls back to per-map --check-assets with an older exe.
See docs/hails-additions/06-map-validator.md (PR F).

Note: install.py also carries PR G's customiser-overlap hunks
(available_memory, overlap_customiser, Background); they are interleaved
with the validator wiring in _install and are documented in doc 08.
This commit is contained in:
Hails committed 2026-10-01 00:13:53 -06:00
1 parent 810defa65e
commit 18f41a33c5
14 files changed
+825 -35

No files matched your search

+9 -1
View File
@@ -9,6 +9,8 @@ pub(crate) struct Config {
pub map_path: Option<PathBuf>,
pub difficulty: crate::difficulty::Difficulty,
pub check_assets: bool,
/// `--validate-maps`: validate map paths read from stdin (see map_validation).
pub validate_maps: bool,
pub start_paused: bool,
pub multiplayer: crate::multiplayer::Options,
pub map_fingerprint: u64,
@@ -24,6 +26,7 @@ impl Config {
map_path: None,
difficulty: crate::difficulty::Difficulty::default(),
check_assets: false,
validate_maps: false,
start_paused: false,
multiplayer: crate::multiplayer::Options::default(),
map_fingerprint: 0,
@@ -70,6 +73,8 @@ impl Config {
}
Some("--test-world") => { explicit_map = true; config.map = None; config.map_path = None; }
Some("--check-assets") => config.check_assets = true,
// Maps arrive on stdin; never load the saved default map.
Some("--validate-maps") => { config.validate_maps = true; explicit_map = true; }
Some("--start-paused") => config.start_paused = true,
Some("--teleport") => config.teleport = Some(args.next().ok_or("--teleport requires a destination ID")?.to_string_lossy().into_owned()),
Some("--difficulty") => {
@@ -85,7 +90,7 @@ impl Config {
}
_ => {
return Err(format!(
"Unknown argument {arg:?}. Usage: skate3rust [--assets DIRECTORY] [--map MAP.skate | --test-world] [--difficulty easy|normal|hardcore|motorized|custom] [--verify CAPTURE.png] [--check-assets] [--start-paused]"
"Unknown argument {arg:?}. Usage: skate3rust [--assets DIRECTORY] [--map MAP.skate | --test-world] [--difficulty easy|normal|hardcore|motorized|custom] [--verify CAPTURE.png] [--check-assets | --validate-maps] [--start-paused]"
));
}
}
@@ -104,6 +109,9 @@ impl Config {
config.map_path = Some(path.canonicalize().map_err(|e| e.to_string())?);
}
}
if config.validate_maps && (config.map.is_some() || config.check_assets) {
return Err("--validate-maps reads maps from stdin; do not combine it with --map or --check-assets".into());
}
config.map_fingerprint = map_fingerprint(config.map_path.as_deref())?;
if config.multiplayer.direct.is_some() && config.multiplayer.session==0 {return Err("Direct multiplayer requires --net-session (a nonzero number shared by both players)".into());}
if let Some(id) = &config.teleport {
+10
View File
@@ -51,6 +51,7 @@ mod replay;
mod world;
mod grind_world;
mod skate_world;
mod map_validation;
fn main() -> bevy::app::AppExit {
match updater::recover() {
@@ -95,6 +96,15 @@ fn main() -> bevy::app::AppExit {
return bevy::app::AppExit::error();
}
};
if config.validate_maps {
return match map_validation::run(&config, &graphs) {
Ok(()) => bevy::app::AppExit::Success,
Err(error) => {
eprintln!("{error}");
bevy::app::AppExit::error()
}
};
}
eprintln!("REPORT_META stage=map_validation");
if let Some(map) = &config.map {
if let Err(error) = skate_world::validate_runtime(map) {
+237
View File
@@ -0,0 +1,237 @@
//! `--validate-maps`: one process validates many converted maps.
//!
//! Setup streams one request per stdin line — a `.skate` path or `TEST_WORLD` —
//! and reads one `SKATE_MAP_CHECK {json}` stdout line back per request; EOF ends
//! the process. Stock data shared by every map (graphs, animation source,
//! controls, camera) loads once, so a map costs its own load instead of a whole
//! game startup (~7–8 s per map with `--check-assets`).
//!
//! `errors` are what `--check-assets` rejects (map load/validation, physics,
//! skater). `warnings` are physics::startup_check findings (spawn support, deck
//! pose, neutral startup); setup reports them without rejecting the map.
use crate::{
camera::CameraRuntime,
config::Config,
graph_runtime::StockGraphs,
physics::{startup_check, GamePhysics, PlayerControls, SkaterRuntime},
skater_animation::AnimationSource,
};
use skate_data::skate_map::SkateMap;
use std::{
io::{BufRead, Write},
panic::{catch_unwind, AssertUnwindSafe},
path::Path,
sync::Arc,
time::Instant,
};
pub(crate) const TEST_WORLD: &str = "TEST_WORLD";
pub(crate) const READY: &str = "SKATE_VALIDATOR_READY";
pub(crate) const RESULT: &str = "SKATE_MAP_CHECK";
/// Collision and render floors may differ slightly; a gap above this means
/// the board stands on collision with nothing visible there.
const VISIBLE_FLOOR_TOLERANCE: f32 = 1.0;
/// Heights of every render triangle covering the XZ point.
pub(crate) fn visible_heights(map: &SkateMap, x: f32, z: f32) -> Vec<f32> {
let vertices = &map.geometry.vertices;
let mut heights = Vec::new();
for tri in map.geometry.indices.chunks_exact(3) {
let (Some(a), Some(b), Some(c)) = (
vertices.get(tri[0] as usize),
vertices.get(tri[1] as usize),
vertices.get(tri[2] as usize),
) else {
continue;
};
let [a, b, c] = [a.position, b.position, c.position];
if x < a[0].min(b[0]).min(c[0]) || x > a[0].max(b[0]).max(c[0])
|| z < a[2].min(b[2]).min(c[2]) || z > a[2].max(b[2]).max(c[2])
{
continue;
}
let (abx, abz, acx, acz) = (b[0] - a[0], b[2] - a[2], c[0] - a[0], c[2] - a[2]);
let det = abx * acz - acx * abz;
if det.abs() < 1e-12 {
continue;
}
let (dx, dz) = (x - a[0], z - a[2]);
let u = (dx * acz - acx * dz) / det;
let w = (abx * dz - dx * abz) / det;
if u >= -1e-5 && w >= -1e-5 && u + w <= 1.0 + 1e-5 {
heights.push(a[1] + u * (b[1] - a[1]) + w * (c[1] - a[1]));
}
}
heights
}
#[derive(serde::Serialize, Default, Debug)]
pub(crate) struct MapCheck {
pub path: String,
pub ok: bool,
pub errors: Vec<String>,
pub warnings: Vec<String>,
pub collision_triangles: Option<usize>,
pub spawn: Option<[f32; 3]>,
pub support_drop: Option<f32>,
pub grounded_ticks: Option<usize>,
/// Smallest height difference between a render surface at the spawn's XZ
/// and the collision floor under the spawn.
pub visible_floor_gap: Option<f32>,
pub seconds: f64,
/// Seconds per step, in order (map, physics, skater, controls_camera, startup).
pub timings: Vec<(&'static str, f64)>,
}
struct Shared<'a> {
config: &'a Config,
graphs: &'a StockGraphs,
source: Arc<AnimationSource>,
}
/// Runs the request loop. Errors only for stock inputs every map needs.
pub(crate) fn run(config: &Config, graphs: &StockGraphs) -> Result<(), String> {
let start = Instant::now();
let source = AnimationSource::load(&config.asset_root)?;
// Fail fast on stock inputs, exactly as --check-assets does.
PlayerControls::load(&config.asset_root)?;
CameraRuntime::load(&config.asset_root)?;
let shared = Shared { config, graphs, source };
let mut out = std::io::stdout().lock();
writeln!(out, "{READY} seconds={:.3}", start.elapsed().as_secs_f64()).map_err(|e| e.to_string())?;
out.flush().map_err(|e| e.to_string())?;
for line in std::io::stdin().lock().lines() {
let line = line.map_err(|e| format!("stdin: {e}"))?;
let request = line.trim();
if request.is_empty() {
continue;
}
let result = check(&shared, request);
let json = serde_json::to_string(&result).map_err(|e| e.to_string())?;
writeln!(out, "{RESULT} {json}").map_err(|e| e.to_string())?;
out.flush().map_err(|e| e.to_string())?;
}
Ok(())
}
fn check(shared: &Shared, request: &str) -> MapCheck {
let start = Instant::now();
let mut result = MapCheck { path: request.to_owned(), ..MapCheck::default() };
// A defect in one map must become that map's result, not end the process.
match catch_unwind(AssertUnwindSafe(|| check_map(shared, request, &mut result))) {
Ok(Ok(())) => {}
Ok(Err(error)) => result.errors.push(error),
Err(panic) => result.errors.push(format!(
"panic: {}",
panic
.downcast_ref::<String>()
.map(String::as_str)
.or_else(|| panic.downcast_ref::<&str>().copied())
.unwrap_or("non-string panic payload")
)),
}
result.ok = result.errors.is_empty();
result.seconds = start.elapsed().as_secs_f64();
result
}
fn check_map(shared: &Shared, request: &str, result: &mut MapCheck) -> Result<(), String> {
let root = &shared.config.asset_root;
let difficulty = shared.config.difficulty;
let mut step = Instant::now();
let mut lap = |result: &mut MapCheck, name: &'static str| {
result.timings.push((name, step.elapsed().as_secs_f64()));
step = Instant::now();
};
let map = if request == TEST_WORLD {
None
} else {
let map = SkateMap::load(Path::new(request))?;
crate::skate_world::validate_runtime(&map)?;
Some(map)
};
lap(result, "map");
let mut physics = GamePhysics::load_with_difficulty(root, map.as_ref(), difficulty)?;
result.collision_triangles = Some(startup_check::collision_triangles(&physics));
lap(result, "physics");
let mut skater = SkaterRuntime::load_for_world(
root,
shared.graphs,
&physics,
difficulty.profile_key(),
Some(shared.source.clone()),
)?;
lap(result, "skater");
let Some(map) = map else { return Ok(()) };
result.spawn = Some(map.spawn);
let mut controls = PlayerControls::load(root)?;
let mut camera = CameraRuntime::load(root)?;
lap(result, "controls_camera");
match startup_check::check(
&mut physics,
&mut skater,
&mut controls,
&mut camera,
shared.graphs,
map.spawn,
map.heading,
) {
Ok(report) => {
result.support_drop = report.support_drop;
result.grounded_ticks = Some(report.grounded_ticks);
result.warnings = report.warnings;
if let Some(drop) = report.support_drop {
// The floor the board lands on must be visible: collision-only
// floors (e.g. Industrial's harbour bed) or volumes look like
// an invisible map with working collision.
let floor = map.spawn[1] - drop;
let gap = visible_heights(&map, map.spawn[0], map.spawn[2])
.into_iter()
.map(|height| (height - floor).abs())
.reduce(f32::min);
result.visible_floor_gap = gap;
if gap.is_none_or(|gap| gap > VISIBLE_FLOOR_TOLERANCE) {
result.warnings.push(format!(
"no visible surface within {VISIBLE_FLOOR_TOLERANCE} m of the collision floor under the spawn (invisible collision)"
));
}
}
}
// New checks report but never reject a map.
Err(error) => result.warnings.push(format!("startup: {error}")),
}
lap(result, "startup");
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn demo() -> SkateMap {
SkateMap::parse(include_bytes!("../../../maps/format-demo.skate")).unwrap()
}
#[test]
fn visible_heights_find_render_triangles_under_a_point_and_nothing_outside() {
let map = demo();
let tri = &map.geometry.indices[..3];
let [a, b, c] = [tri[0], tri[1], tri[2]].map(|i| map.geometry.vertices[i as usize].position);
let centre = [(a[0] + b[0] + c[0]) / 3.0, (a[1] + b[1] + c[1]) / 3.0, (a[2] + b[2] + c[2]) / 3.0];
let heights = visible_heights(&map, centre[0], centre[2]);
assert!(heights.iter().any(|h| (h - centre[1]).abs() < 1e-3), "{heights:?} vs {centre:?}");
assert!(visible_heights(&map, 1.0e7, -1.0e7).is_empty());
}
#[test]
fn result_lines_are_single_line_json_with_the_documented_fields() {
let result = MapCheck { path: "a\nb".into(), ok: true, timings: vec![("map", 0.5)], ..MapCheck::default() };
let json = serde_json::to_string(&result).unwrap();
assert!(!json.contains('\n'));
let value: serde_json::Value = serde_json::from_str(&json).unwrap();
for key in ["path", "ok", "errors", "warnings", "collision_triangles", "spawn", "support_drop",
"grounded_ticks", "visible_floor_gap", "seconds", "timings"] {
assert!(value.get(key).is_some(), "missing {key}");
}
}
}
+1
View File
@@ -42,6 +42,7 @@ mod animation_input;
mod animation_phase;
mod biped_ground;
mod frame;
pub(crate) mod startup_check;
#[cfg(debug_assertions)]
mod dev_trace;
mod grind;
@@ -0,0 +1,112 @@
//! Map startup checks for `--validate-maps` (see crate::map_validation).
//! Mirrors the private `map_startup` test: spawn support, deck pose and a short
//! neutral startup must reach supported wheels with finite state. Results are
//! reported, not asserted.
use super::*;
use crate::{camera::CameraRuntime, graph_runtime::StockGraphs};
use skate_core::input::gameplay_map::GameplayActions;
/// Neutral startup runs until the wheels stay supported for SETTLE_TICKS in a
/// row, up to STARTUP_TICKS (4 s at 60 Hz). Authored starts can sit up to ~1 m
/// above the floor (IndustrialSkatePark: 0.75 m), which a fixed 24-tick window
/// (the map_startup test's) does not always cover.
pub(crate) const STARTUP_TICKS: u64 = 240;
const SETTLE_TICKS: usize = 12;
/// Spawn support probe: from 1 m above the spawn to 10 m below it.
const SUPPORT_ABOVE: f32 = 1.0;
const SUPPORT_BELOW: f32 = 10.0;
pub(crate) struct StartupReport {
/// Distance from the spawn down to the first supporting surface, if any.
pub support_drop: Option<f32>,
pub grounded_ticks: usize,
pub warnings: Vec<String>,
}
pub(crate) fn collision_triangles(physics: &GamePhysics) -> usize {
physics.world.triangles().len()
}
/// Spawn support, deck pose and a neutral startup on an already loaded map.
pub(crate) fn check(
physics: &mut GamePhysics,
skater: &mut SkaterRuntime,
controls: &mut PlayerControls,
camera: &mut CameraRuntime,
graphs: &StockGraphs,
spawn: [f32; 3],
heading: f32,
) -> Result<StartupReport, String> {
let mut warnings = Vec::new();
let [x, y, z] = spawn;
let support_drop = physics
.world
.query_thin_line(Vector3::new(x, y + SUPPORT_ABOVE, z), Vector3::new(x, y - SUPPORT_BELOW, z))
.map_err(|e| format!("spawn support query: {e}"))?
.map(|hit| y - hit.geometry.position.y);
if support_drop.is_none() {
warnings.push(format!("no collision within {SUPPORT_BELOW} m below the spawn"));
}
let deck = physics.board.part_transforms()[skate_core::physics::board::BodyId::Deck.index()];
if (deck.translation.x - x).abs() > 0.01 || (deck.translation.z - z).abs() > 0.01 {
warnings.push(format!(
"deck spawned at ({:.3}, {:.3}) instead of ({x:.3}, {z:.3})",
deck.translation.x, deck.translation.z
));
}
let forward = deck.basis.columns[2];
let (sin, cos) = heading.sin_cos();
if (forward[0] - sin).abs() > 0.01 || (forward[2] - cos).abs() > 0.01 {
warnings.push(format!("deck heading {forward:?} does not match map heading {heading}"));
}
let mut grounded_ticks = 0;
let mut settled = 0;
let mut solved_wheel_contact = false;
for _ in 0..STARTUP_TICKS {
if settled >= SETTLE_TICKS {
break;
}
// No controller: the same zero actions the game publishes when no pad is ready.
let mut actions = GameplayActions::from_values([0.0; 18]);
controls.update_for_physics(&mut actions, physics, skater, camera)?;
frame::advance(physics, skater, controls, graphs, &mut actions, true, camera)
.map_err(|e| format!("tick {}: {e}", physics.ticks))?;
if physics.failed {
warnings.push(format!("physics failed at tick {}", physics.ticks));
break;
}
let finite = physics
.board
.bodies()
.iter()
.chain(skater.skeleton.bodies())
.all(|b| {
[b.rates.position.x, b.rates.position.y, b.rates.position.z].iter().all(|v| v.is_finite())
&& b.rates.basis.columns.iter().flatten().all(|v| v.is_finite())
});
if !finite {
warnings.push(format!("non-finite body state at tick {}", physics.ticks));
break;
}
let supported = physics.riding.ground.wheel_contact_count > 0;
grounded_ticks += usize::from(supported);
settled = if supported { settled + 1 } else { 0 };
for report in physics.board.contact_reports() {
let force = report.normal_force_on_a;
solved_wheel_contact |= report.part.index() < 4
&& matches!(report.other, skate_core::physics::board_step::CollisionBody::StaticWorld)
&& [force.x, force.y, force.z].iter().any(|v| *v != 0.0);
}
}
if grounded_ticks == 0 {
warnings.push(format!("wheels never found support in {STARTUP_TICKS} neutral ticks"));
} else if settled < SETTLE_TICKS {
warnings.push(format!("wheels did not stay supported for {SETTLE_TICKS} ticks within {STARTUP_TICKS} neutral ticks"));
}
if grounded_ticks > 0 && !solved_wheel_contact {
warnings.push("no solved wheel contact reaction during startup".into());
}
Ok(StartupReport { support_drop, grounded_ticks, warnings })
}
+1 -1
View File
@@ -616,7 +616,7 @@ mod tests {
world.insert_resource(crate::config::Config {
asset_root: "unused".into(), verification_capture: None,
map: None, map_path: None, difficulty: crate::difficulty::Difficulty::Hardcore,
check_assets: false, start_paused: false, teleport: None,
check_assets: false, validate_maps: false, start_paused: false, teleport: None,
multiplayer: Default::default(), map_fingerprint: 0,
});
world.insert_resource(crate::retail_render::RetailScene(false));
+1 -1
View File
@@ -38,7 +38,7 @@ pub(crate) struct AnimationSource {
evaluator: Arc<PoseEvaluator>,
}
impl AnimationSource {
fn load(root: &Path) -> Result<Arc<Self>, String> {
pub(crate) fn load(root: &Path) -> Result<Arc<Self>, String> {
let banks = skate_data::animation_banks::AnimationBanks::load(root)?;
let mut evaluator = PoseEvaluator::from_banks(&banks)?;
evaluator.load_authored_clips(root)?;
@@ -27,7 +27,7 @@ fn installed_worlds_prepare_commit_and_retire_without_simulation() {
};
let mut world = World::new();
world.insert_resource(Config { asset_root: root.clone(), verification_capture: None,
map: None, map_path: None, difficulty, check_assets: false, start_paused: false, teleport: None, multiplayer: Default::default(), map_fingerprint: crate::config::map_fingerprint(None).unwrap() });
map: None, map_path: None, difficulty, check_assets: false, validate_maps: false, start_paused: false, teleport: None, multiplayer: Default::default(), map_fingerprint: crate::config::map_fingerprint(None).unwrap() });
world.insert_resource(CurrentMap::from_package(None, None));
world.insert_resource(graphs);
world.insert_resource(physics);
+9 -4
View File
@@ -44,10 +44,15 @@ from the area-weighted centre of flat surfaces, and accept a point that is
the lowest surface at its XZ, has ≥ 2.5 m headroom, and has no flat surface
within 25 m (by triangle extent) more than 1 m lower.
**Regression found in play:** applied to city maps, "lowest surface" is
under the streets. Industrial's spawn became (−864.7, −3.0, 414.1); the player
stood on collision under the map and the world looked invisible (back faces
culled from below). Fix: only districts whose collision spans ≤ 500 m
**Regression found in play:** applied to city maps, the search starts from the
area-weighted centre of the whole district. For Industrial (a port) that point
is out in the harbour: the spawn became (−864.7, −3.0, 414.1), on a 1,604 m²
collision-only floor at y −4.0 with nothing above it (likely the harbour bed)
and no visible geometry nearby, so the world looked invisible while collision
held the player up. (An earlier version of this note said "under the streets";
collision analysis at that point showed otherwise.) The streaming map validator
(doc 6) now flags this case as "no visible surface … (invisible collision)".
Fix: only districts whose collision spans ≤ 500 m
(`GROUND_MAX_EXTENT`, i.e. the parks) use `ground_spawn`; larger districts keep
the original rule unchanged. A unit test pins this. Lesson recorded: a spawn
change moves every map's startup position, so all maps must be checked.
+124
View File
@@ -0,0 +1,124 @@
# 6. Streaming map validator (PR F, tooling)
## Problem
Setup validated every converted map by launching the game once per map
(`skate3rust --assets <stage> --map <map> --check-assets`, inside each
`map_job` subprocess), plus a `--test-world --check-assets` launch before and
after the map stage and one per recovered old map. Each launch cost ~7–8 s, of
which only 0.1–2 s depended on the map: ~6 s was the same stock loading
(skater runtime 3.2 s, controls + camera 2.5 s) repeated every time.
The check was also weak: a map passed as long as it loaded. It did not catch the
spawn problems found in play (skater falling into the void on MegaPark, standing
on an invisible collision-only floor in Industrial).
## Change
### Game: `skate3rust --assets <dir> --validate-maps`
New module `crates/skate-game/src/map_validation.rs`, dispatched from `main.rs`
after the stock loads (config, gameplay config, manifest, graphs):
- Loads once: animation source, controls and camera (fail fast, like
`--check-assets`). Prints `SKATE_VALIDATOR_READY seconds=…` on stdout.
- Reads one request per stdin line (a `.skate` path or `TEST_WORLD`) and answers
with one `SKATE_MAP_CHECK {json}` stdout line; exits 0 at EOF. All other game
output stays on stderr. Works through the crash supervisor, which forwards the
child's stdout/stderr unchanged and lets it inherit stdin.
- Per request:
- **errors** (the map is rejected, as before): map load,
`skate_world::validate_runtime`, physics with the map's collision, skater
runtime. A panic inside one map becomes that map's error (`catch_unwind`).
- **warnings** (reported, never rejecting): from
`physics/startup_check.rs` — no collision within 10 m below the spawn; deck
spawned away from the map spawn/heading; a neutral startup (zero actions,
as with no controller) must settle with wheel support for 12 consecutive
ticks within 240 ticks (4 s at 60 Hz), with finite body state, no physics
failure and a solved wheel contact; and the collision floor under the spawn
must have a visible (render) surface within 1 m.
- JSON fields: `path, ok, errors, warnings, collision_triangles, spawn,
support_drop, grounded_ticks, visible_floor_gap, seconds, timings`.
- `--check-assets` is unchanged.
### Setup (`tools/asset_pipeline/install.py`)
- `MapValidator` runs one validator process; `start_validator` falls back to
per-map `--check-assets` launches if it cannot start (e.g. an older game exe),
and `validate()` falls back for the rest of the run if the process dies.
- The pre-map stock check, every map (as soon as its conversion finishes, while
the remaining maps keep converting), recovered old maps and the final
test-world check all go to the same process. The per-map launch was removed
from `convert_map`.
- Process launching (packaged-exe DLL/PATH handling) moved from `run()` into a
shared `spawn()`; `run()` behaves as before.
- Warnings are written to `assets/private/map-status/<map>-validation.json`
(`status: "warning"`) by `record_validation`. New module
`validation_report.py` wraps `optional_content.summary` and adds them to
`setup-report.json` and the setup message; `install.py` and `tools/setup.py`
call the wrapper. `optional_content.py` itself is deliberately untouched: it is
part of the hud/environment/maps fingerprints **and** of every
character-customiser stage version (`customiser_cache.SHARED`), so editing it
would make every existing installation rebuild the customiser (~4 min), which
has no equivalence mechanism. (A first version edited it; the fingerprint
check that caught this compares against the committed tree, not a local
install.)
- `pipeline-equivalence.json`: removing validation from `convert_map` changes the
`maps` fingerprint without changing any output, so an old→new pair is added
(combined with PR G's stream change, see doc 8). Verified: an installation
built from the committed code rebuilds no group, and the customiser
fingerprint and all five stage versions are unchanged.
## Verification
- Validator on the test world and all 10 maps: all `ok`, no warnings; collision
triangle counts equal the post-volume-fix baseline.
- Negative cases (bad spawns written into copies of real maps):
MegaPark's old void spawn → "no collision within 10 m below the spawn" +
"wheels never found support"; Industrial's old harbour-bed spawn → "no visible
surface … (invisible collision)".
- Known limit: a spawn on a solid, visible roof (Industrial Skate Park's old
roof spawn) is not flagged — rooftops are legitimate skate spots; only
authored start data (doc 4) distinguishes them.
- Unit tests: `map_validation::tests` (visible-surface heights, single-line JSON
with the documented fields); Python `test_map_validator.py` with a fake
validator speaking the protocol (results, warnings into `setup-report.json`,
start failure fallback, process death, request injection, pipes closed under
`-W error::ResourceWarning`). Existing setup/pipeline suites pass (40 tests).
- Fingerprints: with the equivalence pairs, an existing installation rebuilds
no groups.
- Full setup with the real game: see "Setup run" below.
### Setup run
Full fresh setup from the extracted disc with the real game exe (Windows 11,
i7-14700KF, 3 map workers): **466 s, 0 warnings**; the validator was used
throughout (no fallback).
| Step | Before | After |
|---|---|---|
| Pre-map stock check (`--test-world --check-assets` → validator start + `TEST_WORLD`) | ~6 s | ~1 s |
| Per-map validation | ~7–9 s each, inside each map job | 0.28–1.79 s each, in the parent while other maps convert |
| Map stage (convert + validate 10 maps) | ~217 s | 183 s |
Per-map `phase_seconds.validate` in `maps.json`: University 1.54, DownTown 1.79,
Industrial 1.11, parks 0.28–0.37 s. Afterwards `check_maps`-style validation of
all maps: all ok, no warnings, collision triangles identical; baked spawns
identical to the authored values (doc 4).
## Notes
- The ignored private test `private_extracted_map_supports_production_gameplay_startup`
uses a fixed 24-tick window and fails on Industrial Skate Park, whose authored
start is 0.75 m above the floor (≈0.39 s ≈ 23–24 ticks of fall). The map is
fine in play; the validator uses the settle rule above instead. The test was
left unchanged.
- Without PR E (collection index) every per-map check would still take ~5.8 s.
## Files
- `crates/skate-game/src/map_validation.rs` (new), `physics/startup_check.rs` (new),
`main.rs`, `config.rs`, `physics.rs`, `skater_animation.rs` (`AnimationSource::load` visibility),
`retail_character.rs` + `tests/map_transition.rs` (new `Config` field in test literals)
- `tools/asset_pipeline/install.py`, `validation_report.py` (new), `pipeline-equivalence.json`,
`test_map_validator.py` (new), `tools/setup.py` (message wording)
+159 -24
View File
@@ -6,19 +6,47 @@ from tools.owned_game.big import BigArchive
TOOLS=Path(__file__).resolve().parents[1]
GIB=1024**3
def available_memory():
"""Available physical memory in bytes, or None where it cannot be queried."""
if os.name!='nt':return None
import ctypes
class Memory(ctypes.Structure):
_fields_=[('length',ctypes.c_ulong),('load',ctypes.c_ulong)]+[(name,ctypes.c_ulonglong) for name in
('total','available','page_total','page_available','virtual_total','virtual_available','extended')]
memory=Memory();memory.length=ctypes.sizeof(memory)
return memory.available if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)) else None
def map_workers():
count=min(3,max(1,(os.cpu_count() or 1)//2))
if os.name=='nt':
import ctypes
class Memory(ctypes.Structure):
_fields_=[('length',ctypes.c_ulong),('load',ctypes.c_ulong)]+[(name,ctypes.c_ulonglong) for name in
('total','available','page_total','page_available','virtual_total','virtual_available','extended')]
memory=Memory();memory.length=ctypes.sizeof(memory)
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)):
# Reserve memory for the desktop; each map and its loader can
# briefly hold several copies of geometry and textures.
count=min(count,max(1,(memory.available-2*1024**3)//(3*1024**3)))
available=available_memory()
if available is not None:
# Reserve memory for the desktop; each map and its loader can
# briefly hold several copies of geometry and textures.
count=min(count,max(1,(available-2*GIB)//(3*GIB)))
return count
def overlap_customiser(workers):
"""Run the character customiser beside the map jobs only with room for one
more 3 GiB slot on top of the map workers and the 2 GiB desktop reserve."""
available=available_memory()
return available is None or available>=2*GIB+(workers+1)*3*GIB
class Background:
"""Run an action on a thread; join() waits and re-raises its exception."""
def __init__(self,action,name):
import threading
self.error=None
def target():
try:action()
except BaseException as error:self.error=error
self.thread=threading.Thread(target=target,name=name,daemon=True)
self.thread.start()
def join(self):
self.thread.join()
if self.error is not None:raise self.error
XISO_URL='https://github.com/XboxDev/extract-xiso/releases/download/build-202505152050/extract-xiso-Win64_Release.zip'
XISO_SHA='fec88d03c7efd6205ab09be4abba70c0afd0eb27a5709f0a6235b828ba5ac11e'
@@ -107,12 +135,13 @@ def dependency(cache,name,url,sha,report):
if executable is None:raise RuntimeError('Missing downloaded tool: '+name)
return executable
def run(args,log,report):
def spawn(args,**popen):
kwargs={'creationflags':subprocess.CREATE_NO_WINDOW} if os.name=='nt' else {}
external=os.name=='nt' and getattr(sys,'frozen',False) and Path(args[0]).resolve()!=Path(sys.executable).resolve()
if external:
# External tools and the game must load their own libraries, not
# the setup bundle's DLL directory inherited by child processes.
# SetDllDirectoryW is process-wide: only spawn external tools from one thread.
import ctypes
ctypes.windll.kernel32.SetDllDirectoryW(None)
env=os.environ.copy()
@@ -121,15 +150,94 @@ def run(args,log,report):
if p and not Path(p).resolve().is_relative_to(bundle))
kwargs['env']=env
try:
child=subprocess.Popen([str(a) for a in args],stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
text=True,encoding='utf-8',errors='replace',**kwargs)
return subprocess.Popen([str(a) for a in args],text=True,encoding='utf-8',errors='replace',**kwargs,**popen)
finally:
if external:ctypes.windll.kernel32.SetDllDirectoryW(sys._MEIPASS)
def run(args,log,report):
child=spawn(args,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
with child as process:
for line in process.stdout:
log.write(line);log.flush()
if process.wait():raise RuntimeError('Conversion failed. See '+str(log.name))
class MapValidator:
"""One `skate3rust --validate-maps` process (crates/skate-game/src/map_validation.rs).
Shared stock data loads once, so each map costs well under a second instead
of a full --check-assets launch. Requests are one path (or TEST_WORLD) per
line; each answer is one `SKATE_MAP_CHECK {json}` stdout line. Game logs go
to stderr and are copied to the setup log. Use from one thread only.
"""
READY='SKATE_VALIDATOR_READY'
RESULT='SKATE_MAP_CHECK '
def __init__(self,args,log):
import threading
self.process=spawn(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
self.log=log;self.lock=threading.Lock()
def copy_stderr():
with self.process.stderr as stream:
for line in stream:
with self.lock:log.write(line);log.flush()
threading.Thread(target=copy_stderr,daemon=True).start()
ready=self.process.stdout.readline()
if not ready.startswith(self.READY):
self.close()
raise RuntimeError('Map validator did not start'+(f': {ready.strip()}' if ready.strip() else ''))
self.write(ready)
def write(self,line):
with self.lock:self.log.write(line if line.endswith('\n') else line+'\n');self.log.flush()
def check(self,request):
"""Return the result dict; RuntimeError if the validator is gone."""
if '\n' in str(request) or '\r' in str(request):raise ValueError('Invalid map path')
try:
self.process.stdin.write(f'{request}\n');self.process.stdin.flush()
while True:
line=self.process.stdout.readline()
if not line:break
self.write(line)
if line.startswith(self.RESULT):return json.loads(line[len(self.RESULT):])
except (OSError,ValueError) as error:
raise RuntimeError(f'Map validator failed: {error}') from error
raise RuntimeError(f'Map validator exited (code {self.process.poll()})')
def close(self):
try:self.process.stdin.close()
except OSError:pass # already exited: the pipe is broken
try:self.process.wait(timeout=30)
except subprocess.TimeoutExpired:
self.process.kill();self.process.wait()
self.process.stdout.close()
def start_validator(game_exe,assets,log,report):
"""MapValidator, or None (older game exe / start failure: use --check-assets)."""
try:
validator=MapValidator([game_exe,'--assets',assets,'--validate-maps'],log)
except (OSError,RuntimeError) as error:
report(f'Map validator unavailable ({error}); validating with --check-assets per map')
return None
import weakref
# Never leave the process behind if setup fails before closing it.
weakref.finalize(validator,validator.process.kill)
return validator
def record_validation(private,entry,result,report):
"""Store non-blocking findings as map-status/<map>-validation.json (status warning)."""
from .setup_state import atomic_json
path=private/'map-status'/(entry['name']+'-validation.json')
if result is None:return
entry.setdefault('phase_seconds',{})['validate']=round(result['seconds'],3)
if result['warnings']:
path.parent.mkdir(parents=True,exist_ok=True)
atomic_json(path,{'version':1,'component':entry['name']+' map checks','status':'warning',
'warnings':result['warnings']})
report(f"{entry['name']}: map check warnings: {'; '.join(result['warnings'])}")
else:
path.unlink(missing_ok=True)
def task(script,*args):
if getattr(sys,'frozen',False):return [sys.executable,'--task',str(script),*map(str,args)]
return [sys.executable,str(script),*map(str,args)]
@@ -196,9 +304,7 @@ def convert_map(archive,work,maps,stage,game_exe,log,report):
placed,unresolved=0,0
finished('props')
report(f'{label}: placed {placed} authored DMO instances, {unresolved} unresolved templates')
report('Checking converted map: '+label)
run([game_exe,'--assets',stage/'assets','--map',final,'--check-assets'],log,report)
finished('validate')
# Validation runs in the parent (_install, MapValidator) as each map finishes.
entry={'name':label,'path':'maps/'+final.name,'sha256':digest(final)}
remove_intermediate(district_work,work)
finished('hash_and_cleanup')
@@ -242,7 +348,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
with (base/'refresh-validation.log').open('w',encoding='utf-8') as log:
run([game_exe,'--assets',previous[0]/'assets','--test-world','--check-assets'],log,report)
if finalize:finalize(previous[0])
from .optional_content import summary
from .validation_report import summary
summary(previous[0])
atomic_json(base/'installation.json', {**previous[1], 'pipelines':target_versions,
'outputs':outputs(previous[0]), 'source':source})
@@ -339,7 +445,23 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
if (work/'dmo').exists():remove_intermediate(work/'dmo',work)
note(private/'native-props/props-availability.json','Movable props',error,report=report)
report('Validating skater, input and animation data')
run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
validator=start_validator(game_exe,stage/'assets',log,report)
customiser=None # Background customiser when overlapped with the map stage
def validate(request):
"""Today's checks raise (map rejected); new findings come back as warnings."""
nonlocal validator
if validator is not None:
try:result=validator.check(request)
except RuntimeError as error:
report(f'{error}; validating with --check-assets per map')
validator.close();validator=None
else:
if not result['ok']:raise RuntimeError('Map validation failed: '+'; '.join(result['errors']))
return result
target=['--test-world'] if request=='TEST_WORLD' else ['--map',request]
run([game_exe,'--assets',stage/'assets',*target,'--check-assets'],log,report)
return None
validate('TEST_WORLD')
if 'maps' in groups:
archives=list((game_root/'data/content').glob('worldDIST_*.big'))
archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower()))
@@ -351,6 +473,12 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
report(f'Authored map starts unavailable ({error}); using geometric spawns')
(work/'map-starts.json').write_text(json.dumps(starts),encoding='utf-8')
workers=map_workers()
# The customiser reads only stock, default-skater and disc data (all
# prepared above) and writes only assets/private/customisation, so it
# runs beside the map jobs instead of after them.
if finalize and overlap_customiser(workers):
report('Preparing the character customiser alongside the maps')
customiser=Background(lambda:finalize(stage),'customiser')
report(f'Converting {len(archives)} maps with {workers} workers')
def map_job(archive):
result=work/(archive.stem+'.json')
@@ -365,8 +493,12 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)}
for future in as_completed(futures):
archive=futures[future]
try:completed[archive.name]=future.result()
try:
completed[archive.name]=future.result()
# Validate while the remaining maps keep converting.
checked=validate(stage/completed[archive.name]['path'])
except CONTENT_ERRORS as error:
completed.pop(archive.name,None)
label=archive.stem.removeprefix('worldDIST_')
(maps/(label+'.skate')).unlink(missing_ok=True)
props=private/'native-props'/(label+'.skate')
@@ -374,6 +506,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
note(private/'map-status'/(label+'-availability.json'),label,error,report=report)
continue
(private/'map-status'/(completed[archive.name]['name']+'-availability.json')).unlink(missing_ok=True)
record_validation(private,completed[archive.name],checked,report)
report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}")
catalog=[completed[a.name] for a in archives if a.name in completed]
if previous:
@@ -384,7 +517,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
src=(previous[0]/old['path']).resolve()
if not src.is_relative_to((previous[0]/'maps').resolve()):raise ValueError('Invalid old map path')
if not src.is_file() or digest(src)!=old.get('sha256'):continue
try:run([game_exe,'--assets',stage/'assets','--map',src,'--check-assets'],log,report)
try:validate(src)
except CONTENT_ERRORS:continue
target=stage/old['path'];target.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,target)
catalog.append(old)
@@ -397,7 +530,8 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
if old['path'] not in valid_paths:(stage/old['path']).unlink(missing_ok=True)
if not catalog:raise RuntimeError('No playable map could be prepared or recovered. Restore at least one worldDIST_*.big archive beside default.xex and retry; the previous installation has been kept.')
report('Validating installed runtime inputs')
run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
validate('TEST_WORLD')
if validator is not None:validator.close()
settings=stage/'settings';settings.mkdir(exist_ok=True)
if not previous:
(settings/'default-map.json').write_text(json.dumps(next((m['path'] for m in catalog if m['name']=='University'),catalog[0]['path'])),encoding='utf-8')
@@ -407,8 +541,9 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
if selected.is_file() and not (stage/json.loads(selected.read_text())).is_file():
selected.write_text(json.dumps(catalog[0]['path']))
remove_intermediate(work,stage)
if finalize:finalize(stage)
from .optional_content import summary
if customiser is not None:customiser.join()
elif finalize:finalize(stage)
from .validation_report import summary
warnings=summary(stage)
# Publish after core validation and all optional outcomes have been recorded.
marker=base/'installation.json.new'
@@ -416,5 +551,5 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None
marker.replace(base/'installation.json')
remove_setup_logs(stage,report)
remove_stale_installations(base,stage,report)
report(f'Setup complete ({len(warnings)} unavailable/retained components; see setup-report.json)' if warnings else 'Setup complete')
report(f'Setup complete ({len(warnings)} warnings or unavailable/retained components; see setup-report.json)' if warnings else 'Setup complete')
return stage
+126
View File
@@ -0,0 +1,126 @@
import io
import json
import sys
import tempfile
import textwrap
import unittest
from pathlib import Path
from . import install
from .validation_report import summary
# Speaks the skate3rust --validate-maps protocol (crates/skate-game/src/map_validation.rs).
FAKE = textwrap.dedent('''
import json, sys
mode = sys.argv[1]
if mode == 'no-ready':
print('Unknown argument "--validate-maps"', file=sys.stderr); sys.exit(1)
print('SKATE_VALIDATOR_READY seconds=0.001', flush=True)
for line in sys.stdin:
path = line.strip()
print('SKATE_RWCM_READY triangles=12 (log line)', file=sys.stderr, flush=True)
if 'die' in path:
sys.exit(3)
result = {'path': path, 'ok': 'error' not in path, 'seconds': 0.25,
'errors': ['bad map'] if 'error' in path else [],
'warnings': ['no collision within 10 m below the spawn'] if 'warn' in path else []}
print('noise before the result', flush=True)
print('SKATE_MAP_CHECK ' + json.dumps(result), flush=True)
''')
class MapValidatorTests(unittest.TestCase):
def setUp(self):
self.work = tempfile.TemporaryDirectory()
self.root = Path(self.work.name)
self.fake = self.root / 'fake_validator.py'
self.fake.write_text(FAKE, encoding='utf-8')
self.log = io.StringIO()
self.reports = []
def tearDown(self):
self.work.cleanup()
def start(self, mode='ok'):
return install.MapValidator([sys.executable, self.fake, mode], self.log)
def test_results_warnings_and_log_copy(self):
validator = self.start()
try:
clean = validator.check(self.root / 'maps/Clean.skate')
warned = validator.check(self.root / 'maps/warn.skate')
failed = validator.check('error.skate')
finally:
validator.close()
self.assertTrue(clean['ok'])
self.assertEqual(clean['warnings'], [])
self.assertEqual(warned['warnings'], ['no collision within 10 m below the spawn'])
self.assertFalse(failed['ok'])
self.assertIn('SKATE_VALIDATOR_READY', self.log.getvalue())
self.assertIn('SKATE_MAP_CHECK', self.log.getvalue())
self.assertIn('SKATE_RWCM_READY', self.log.getvalue())
self.assertEqual(validator.process.returncode, 0)
def test_validator_that_never_becomes_ready_falls_back(self):
with self.assertRaisesRegex(RuntimeError, 'did not start'):
self.start('no-ready')
self.assertIsNone(install.start_validator(self.root / 'missing.exe', self.root, self.log, self.reports.append))
self.assertIn('validating with --check-assets per map', self.reports[0])
def test_validator_death_is_a_runtime_error(self):
validator = self.start()
with self.assertRaisesRegex(RuntimeError, 'exited'):
validator.check('die.skate')
validator.close()
def test_paths_cannot_inject_extra_requests(self):
validator = self.start()
try:
with self.assertRaises(ValueError):
validator.check('a.skate\nb.skate')
finally:
validator.close()
def test_warnings_are_recorded_and_reported_and_clean_runs_clear_them(self):
stage = self.root / 'stage'
private = stage / 'assets/private'
entry = {'name': 'MegaPark', 'path': 'maps/MegaPark.skate', 'phase_seconds': {'prepare': 1.0}}
install.record_validation(private, entry, {'ok': True, 'seconds': 0.25, 'warnings': ['spawn over void']},
self.reports.append)
self.assertEqual(entry['phase_seconds']['validate'], 0.25)
warnings = summary(stage)
self.assertEqual([w['status'] for w in warnings], ['warning'])
self.assertEqual(warnings[0]['warnings'], ['spawn over void'])
self.assertTrue((stage / 'setup-report.json').is_file())
install.record_validation(private, entry, {'ok': True, 'seconds': 0.2, 'warnings': []}, self.reports.append)
self.assertEqual(summary(stage), [])
self.assertFalse((stage / 'setup-report.json').exists())
# Fallback path (no validator) records nothing and keeps timings intact.
install.record_validation(private, entry, None, self.reports.append)
self.assertEqual(entry['phase_seconds']['validate'], 0.2)
class CustomiserOverlapTests(unittest.TestCase):
def test_background_join_waits_and_reraises(self):
done = []
install.Background(lambda: done.append(1), 'ok').join()
self.assertEqual(done, [1])
failing = install.Background(lambda: (_ for _ in ()).throw(RuntimeError('customiser failed')), 'bad')
with self.assertRaisesRegex(RuntimeError, 'customiser failed'):
failing.join()
def test_overlap_needs_one_more_slot_than_the_map_workers(self):
original = install.available_memory
try:
install.available_memory = lambda: (2 + 4 * 3) * install.GIB # 3 workers + customiser fit
self.assertTrue(install.overlap_customiser(3))
install.available_memory = lambda: (2 + 4 * 3) * install.GIB - 1 # one byte short
self.assertFalse(install.overlap_customiser(3))
install.available_memory = lambda: None # cannot query: allowed
self.assertTrue(install.overlap_customiser(3))
finally:
install.available_memory = original
if __name__ == '__main__':
unittest.main()
+32
View File
@@ -0,0 +1,32 @@
"""Setup report including non-blocking map check warnings.
Wraps optional_content.summary instead of changing it: optional_content.py is
part of several asset-group fingerprints and of every character-customiser
stage version, so editing it would force existing installations to rebuild.
This module's name matches no fingerprint pattern (see versions.py and
customiser_setup.fingerprint).
"""
import json
from .optional_content import summary as optional_summary
from .setup_state import atomic_json
def map_check_warnings(stage):
"""Items written by install.record_validation (status 'warning')."""
found = []
for path in sorted((stage/'assets/private/map-status').glob('*-validation.json')):
try:
item = json.loads(path.read_text())
except (OSError, ValueError):
continue
if isinstance(item, dict) and item.get('status') == 'warning':
found.append(item)
return found
def summary(stage):
"""optional_content.summary plus map check warnings, in one setup-report.json."""
warnings = optional_summary(stage) + map_check_warnings(stage)
if warnings:
atomic_json(stage/'setup-report.json', {'version': 1, 'warnings': warnings})
return warnings
+3 -3
View File
@@ -97,9 +97,9 @@ def main():
def work():
try:
installed_root=install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating)
from tools.asset_pipeline.optional_content import summary
from tools.asset_pipeline.validation_report import summary
warnings=summary(installed_root)
messages.put(('done',f'Ready with {len(warnings)} unavailable components. Details: {installed_root / "setup-report.json"}' if warnings else 'Ready'))
messages.put(('done',f'Ready with {len(warnings)} warnings or unavailable components. Details: {installed_root / "setup-report.json"}' if warnings else 'Ready'))
except Exception as error:
args.base.mkdir(parents=True,exist_ok=True)
(args.base/'setup-error.log').write_text(traceback.format_exc(),encoding='utf-8')
@@ -123,7 +123,7 @@ def main():
while not messages.empty():
kind,text=messages.get_nowait();status.set(text)
if kind=='done':
if text!='Ready':messagebox.showwarning('Setup completed with unavailable content',text,parent=window)
if text!='Ready':messagebox.showwarning('Setup completed with warnings',text,parent=window)
running=False;success=True;progress.stop();window.destroy();return
if kind=='error':
running=False;progress.stop();button.config(state='normal')