From 18f41a33c52ba08835765dd5e7b55bba009b4471 Mon Sep 17 00:00:00 2001 From: Hails <26784904+Hailey-Ross@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:13:53 -0600 Subject: [PATCH] Tooling: streaming map validator (--validate-maps) wired into setup One skate3rust process validates TEST_WORLD and every converted map over stdin/stdout (SKATE_MAP_CHECK lines). Existing checks still fail a map; new spawn-support / neutral-startup / invisible-collision checks report warnings. Setup falls back to per-map --check-assets with an older exe. See docs/hails-additions/06-map-validator.md (PR F). Note: install.py also carries PR G's customiser-overlap hunks (available_memory, overlap_customiser, Background); they are interleaved with the validator wiring in _install and are documented in doc 08. --- crates/skate-game/src/config.rs | 10 +- crates/skate-game/src/main.rs | 10 + crates/skate-game/src/map_validation.rs | 237 ++++++++++++++++++ crates/skate-game/src/physics.rs | 1 + .../skate-game/src/physics/startup_check.rs | 112 +++++++++ crates/skate-game/src/retail_character.rs | 2 +- crates/skate-game/src/skater_animation.rs | 2 +- crates/skate-game/src/tests/map_transition.rs | 2 +- docs/hails-additions/04-map-spawns.md | 13 +- docs/hails-additions/06-map-validator.md | 124 +++++++++ tools/asset_pipeline/install.py | 183 ++++++++++++-- tools/asset_pipeline/test_map_validator.py | 126 ++++++++++ tools/asset_pipeline/validation_report.py | 32 +++ tools/setup.py | 6 +- 14 files changed, 825 insertions(+), 35 deletions(-) create mode 100644 crates/skate-game/src/map_validation.rs create mode 100644 crates/skate-game/src/physics/startup_check.rs create mode 100644 docs/hails-additions/06-map-validator.md create mode 100644 tools/asset_pipeline/test_map_validator.py create mode 100644 tools/asset_pipeline/validation_report.py diff --git a/crates/skate-game/src/config.rs b/crates/skate-game/src/config.rs index 9348bb25..c88a6051 100644 --- a/crates/skate-game/src/config.rs +++ b/crates/skate-game/src/config.rs @@ -9,6 +9,8 @@ pub(crate) struct Config { pub map_path: Option, pub difficulty: crate::difficulty::Difficulty, pub check_assets: bool, + /// `--validate-maps`: validate map paths read from stdin (see map_validation). + pub validate_maps: bool, pub start_paused: bool, pub multiplayer: crate::multiplayer::Options, pub map_fingerprint: u64, @@ -24,6 +26,7 @@ impl Config { map_path: None, difficulty: crate::difficulty::Difficulty::default(), check_assets: false, + validate_maps: false, start_paused: false, multiplayer: crate::multiplayer::Options::default(), map_fingerprint: 0, @@ -70,6 +73,8 @@ impl Config { } Some("--test-world") => { explicit_map = true; config.map = None; config.map_path = None; } Some("--check-assets") => config.check_assets = true, + // Maps arrive on stdin; never load the saved default map. + Some("--validate-maps") => { config.validate_maps = true; explicit_map = true; } Some("--start-paused") => config.start_paused = true, Some("--teleport") => config.teleport = Some(args.next().ok_or("--teleport requires a destination ID")?.to_string_lossy().into_owned()), Some("--difficulty") => { @@ -85,7 +90,7 @@ impl Config { } _ => { return Err(format!( - "Unknown argument {arg:?}. Usage: skate3rust [--assets DIRECTORY] [--map MAP.skate | --test-world] [--difficulty easy|normal|hardcore|motorized|custom] [--verify CAPTURE.png] [--check-assets] [--start-paused]" + "Unknown argument {arg:?}. Usage: skate3rust [--assets DIRECTORY] [--map MAP.skate | --test-world] [--difficulty easy|normal|hardcore|motorized|custom] [--verify CAPTURE.png] [--check-assets | --validate-maps] [--start-paused]" )); } } @@ -104,6 +109,9 @@ impl Config { config.map_path = Some(path.canonicalize().map_err(|e| e.to_string())?); } } + if config.validate_maps && (config.map.is_some() || config.check_assets) { + return Err("--validate-maps reads maps from stdin; do not combine it with --map or --check-assets".into()); + } config.map_fingerprint = map_fingerprint(config.map_path.as_deref())?; if config.multiplayer.direct.is_some() && config.multiplayer.session==0 {return Err("Direct multiplayer requires --net-session (a nonzero number shared by both players)".into());} if let Some(id) = &config.teleport { diff --git a/crates/skate-game/src/main.rs b/crates/skate-game/src/main.rs index 61da4b57..8782d122 100644 --- a/crates/skate-game/src/main.rs +++ b/crates/skate-game/src/main.rs @@ -51,6 +51,7 @@ mod replay; mod world; mod grind_world; mod skate_world; +mod map_validation; fn main() -> bevy::app::AppExit { match updater::recover() { @@ -95,6 +96,15 @@ fn main() -> bevy::app::AppExit { return bevy::app::AppExit::error(); } }; + if config.validate_maps { + return match map_validation::run(&config, &graphs) { + Ok(()) => bevy::app::AppExit::Success, + Err(error) => { + eprintln!("{error}"); + bevy::app::AppExit::error() + } + }; + } eprintln!("REPORT_META stage=map_validation"); if let Some(map) = &config.map { if let Err(error) = skate_world::validate_runtime(map) { diff --git a/crates/skate-game/src/map_validation.rs b/crates/skate-game/src/map_validation.rs new file mode 100644 index 00000000..09692644 --- /dev/null +++ b/crates/skate-game/src/map_validation.rs @@ -0,0 +1,237 @@ +//! `--validate-maps`: one process validates many converted maps. +//! +//! Setup streams one request per stdin line — a `.skate` path or `TEST_WORLD` — +//! and reads one `SKATE_MAP_CHECK {json}` stdout line back per request; EOF ends +//! the process. Stock data shared by every map (graphs, animation source, +//! controls, camera) loads once, so a map costs its own load instead of a whole +//! game startup (~7–8 s per map with `--check-assets`). +//! +//! `errors` are what `--check-assets` rejects (map load/validation, physics, +//! skater). `warnings` are physics::startup_check findings (spawn support, deck +//! pose, neutral startup); setup reports them without rejecting the map. +use crate::{ + camera::CameraRuntime, + config::Config, + graph_runtime::StockGraphs, + physics::{startup_check, GamePhysics, PlayerControls, SkaterRuntime}, + skater_animation::AnimationSource, +}; +use skate_data::skate_map::SkateMap; +use std::{ + io::{BufRead, Write}, + panic::{catch_unwind, AssertUnwindSafe}, + path::Path, + sync::Arc, + time::Instant, +}; + +pub(crate) const TEST_WORLD: &str = "TEST_WORLD"; +pub(crate) const READY: &str = "SKATE_VALIDATOR_READY"; +pub(crate) const RESULT: &str = "SKATE_MAP_CHECK"; +/// Collision and render floors may differ slightly; a gap above this means +/// the board stands on collision with nothing visible there. +const VISIBLE_FLOOR_TOLERANCE: f32 = 1.0; + +/// Heights of every render triangle covering the XZ point. +pub(crate) fn visible_heights(map: &SkateMap, x: f32, z: f32) -> Vec { + let vertices = &map.geometry.vertices; + let mut heights = Vec::new(); + for tri in map.geometry.indices.chunks_exact(3) { + let (Some(a), Some(b), Some(c)) = ( + vertices.get(tri[0] as usize), + vertices.get(tri[1] as usize), + vertices.get(tri[2] as usize), + ) else { + continue; + }; + let [a, b, c] = [a.position, b.position, c.position]; + if x < a[0].min(b[0]).min(c[0]) || x > a[0].max(b[0]).max(c[0]) + || z < a[2].min(b[2]).min(c[2]) || z > a[2].max(b[2]).max(c[2]) + { + continue; + } + let (abx, abz, acx, acz) = (b[0] - a[0], b[2] - a[2], c[0] - a[0], c[2] - a[2]); + let det = abx * acz - acx * abz; + if det.abs() < 1e-12 { + continue; + } + let (dx, dz) = (x - a[0], z - a[2]); + let u = (dx * acz - acx * dz) / det; + let w = (abx * dz - dx * abz) / det; + if u >= -1e-5 && w >= -1e-5 && u + w <= 1.0 + 1e-5 { + heights.push(a[1] + u * (b[1] - a[1]) + w * (c[1] - a[1])); + } + } + heights +} + +#[derive(serde::Serialize, Default, Debug)] +pub(crate) struct MapCheck { + pub path: String, + pub ok: bool, + pub errors: Vec, + pub warnings: Vec, + pub collision_triangles: Option, + pub spawn: Option<[f32; 3]>, + pub support_drop: Option, + pub grounded_ticks: Option, + /// Smallest height difference between a render surface at the spawn's XZ + /// and the collision floor under the spawn. + pub visible_floor_gap: Option, + pub seconds: f64, + /// Seconds per step, in order (map, physics, skater, controls_camera, startup). + pub timings: Vec<(&'static str, f64)>, +} + +struct Shared<'a> { + config: &'a Config, + graphs: &'a StockGraphs, + source: Arc, +} + +/// Runs the request loop. Errors only for stock inputs every map needs. +pub(crate) fn run(config: &Config, graphs: &StockGraphs) -> Result<(), String> { + let start = Instant::now(); + let source = AnimationSource::load(&config.asset_root)?; + // Fail fast on stock inputs, exactly as --check-assets does. + PlayerControls::load(&config.asset_root)?; + CameraRuntime::load(&config.asset_root)?; + let shared = Shared { config, graphs, source }; + let mut out = std::io::stdout().lock(); + writeln!(out, "{READY} seconds={:.3}", start.elapsed().as_secs_f64()).map_err(|e| e.to_string())?; + out.flush().map_err(|e| e.to_string())?; + for line in std::io::stdin().lock().lines() { + let line = line.map_err(|e| format!("stdin: {e}"))?; + let request = line.trim(); + if request.is_empty() { + continue; + } + let result = check(&shared, request); + let json = serde_json::to_string(&result).map_err(|e| e.to_string())?; + writeln!(out, "{RESULT} {json}").map_err(|e| e.to_string())?; + out.flush().map_err(|e| e.to_string())?; + } + Ok(()) +} + +fn check(shared: &Shared, request: &str) -> MapCheck { + let start = Instant::now(); + let mut result = MapCheck { path: request.to_owned(), ..MapCheck::default() }; + // A defect in one map must become that map's result, not end the process. + match catch_unwind(AssertUnwindSafe(|| check_map(shared, request, &mut result))) { + Ok(Ok(())) => {} + Ok(Err(error)) => result.errors.push(error), + Err(panic) => result.errors.push(format!( + "panic: {}", + panic + .downcast_ref::() + .map(String::as_str) + .or_else(|| panic.downcast_ref::<&str>().copied()) + .unwrap_or("non-string panic payload") + )), + } + result.ok = result.errors.is_empty(); + result.seconds = start.elapsed().as_secs_f64(); + result +} + +fn check_map(shared: &Shared, request: &str, result: &mut MapCheck) -> Result<(), String> { + let root = &shared.config.asset_root; + let difficulty = shared.config.difficulty; + let mut step = Instant::now(); + let mut lap = |result: &mut MapCheck, name: &'static str| { + result.timings.push((name, step.elapsed().as_secs_f64())); + step = Instant::now(); + }; + let map = if request == TEST_WORLD { + None + } else { + let map = SkateMap::load(Path::new(request))?; + crate::skate_world::validate_runtime(&map)?; + Some(map) + }; + lap(result, "map"); + let mut physics = GamePhysics::load_with_difficulty(root, map.as_ref(), difficulty)?; + result.collision_triangles = Some(startup_check::collision_triangles(&physics)); + lap(result, "physics"); + let mut skater = SkaterRuntime::load_for_world( + root, + shared.graphs, + &physics, + difficulty.profile_key(), + Some(shared.source.clone()), + )?; + lap(result, "skater"); + let Some(map) = map else { return Ok(()) }; + result.spawn = Some(map.spawn); + let mut controls = PlayerControls::load(root)?; + let mut camera = CameraRuntime::load(root)?; + lap(result, "controls_camera"); + match startup_check::check( + &mut physics, + &mut skater, + &mut controls, + &mut camera, + shared.graphs, + map.spawn, + map.heading, + ) { + Ok(report) => { + result.support_drop = report.support_drop; + result.grounded_ticks = Some(report.grounded_ticks); + result.warnings = report.warnings; + if let Some(drop) = report.support_drop { + // The floor the board lands on must be visible: collision-only + // floors (e.g. Industrial's harbour bed) or volumes look like + // an invisible map with working collision. + let floor = map.spawn[1] - drop; + let gap = visible_heights(&map, map.spawn[0], map.spawn[2]) + .into_iter() + .map(|height| (height - floor).abs()) + .reduce(f32::min); + result.visible_floor_gap = gap; + if gap.is_none_or(|gap| gap > VISIBLE_FLOOR_TOLERANCE) { + result.warnings.push(format!( + "no visible surface within {VISIBLE_FLOOR_TOLERANCE} m of the collision floor under the spawn (invisible collision)" + )); + } + } + } + // New checks report but never reject a map. + Err(error) => result.warnings.push(format!("startup: {error}")), + } + lap(result, "startup"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn demo() -> SkateMap { + SkateMap::parse(include_bytes!("../../../maps/format-demo.skate")).unwrap() + } + + #[test] + fn visible_heights_find_render_triangles_under_a_point_and_nothing_outside() { + let map = demo(); + let tri = &map.geometry.indices[..3]; + let [a, b, c] = [tri[0], tri[1], tri[2]].map(|i| map.geometry.vertices[i as usize].position); + let centre = [(a[0] + b[0] + c[0]) / 3.0, (a[1] + b[1] + c[1]) / 3.0, (a[2] + b[2] + c[2]) / 3.0]; + let heights = visible_heights(&map, centre[0], centre[2]); + assert!(heights.iter().any(|h| (h - centre[1]).abs() < 1e-3), "{heights:?} vs {centre:?}"); + assert!(visible_heights(&map, 1.0e7, -1.0e7).is_empty()); + } + + #[test] + fn result_lines_are_single_line_json_with_the_documented_fields() { + let result = MapCheck { path: "a\nb".into(), ok: true, timings: vec![("map", 0.5)], ..MapCheck::default() }; + let json = serde_json::to_string(&result).unwrap(); + assert!(!json.contains('\n')); + let value: serde_json::Value = serde_json::from_str(&json).unwrap(); + for key in ["path", "ok", "errors", "warnings", "collision_triangles", "spawn", "support_drop", + "grounded_ticks", "visible_floor_gap", "seconds", "timings"] { + assert!(value.get(key).is_some(), "missing {key}"); + } + } +} diff --git a/crates/skate-game/src/physics.rs b/crates/skate-game/src/physics.rs index ec41d4ee..5d77da4f 100644 --- a/crates/skate-game/src/physics.rs +++ b/crates/skate-game/src/physics.rs @@ -42,6 +42,7 @@ mod animation_input; mod animation_phase; mod biped_ground; mod frame; +pub(crate) mod startup_check; #[cfg(debug_assertions)] mod dev_trace; mod grind; diff --git a/crates/skate-game/src/physics/startup_check.rs b/crates/skate-game/src/physics/startup_check.rs new file mode 100644 index 00000000..c2b65f0a --- /dev/null +++ b/crates/skate-game/src/physics/startup_check.rs @@ -0,0 +1,112 @@ +//! Map startup checks for `--validate-maps` (see crate::map_validation). +//! Mirrors the private `map_startup` test: spawn support, deck pose and a short +//! neutral startup must reach supported wheels with finite state. Results are +//! reported, not asserted. +use super::*; +use crate::{camera::CameraRuntime, graph_runtime::StockGraphs}; +use skate_core::input::gameplay_map::GameplayActions; + +/// Neutral startup runs until the wheels stay supported for SETTLE_TICKS in a +/// row, up to STARTUP_TICKS (4 s at 60 Hz). Authored starts can sit up to ~1 m +/// above the floor (IndustrialSkatePark: 0.75 m), which a fixed 24-tick window +/// (the map_startup test's) does not always cover. +pub(crate) const STARTUP_TICKS: u64 = 240; +const SETTLE_TICKS: usize = 12; +/// Spawn support probe: from 1 m above the spawn to 10 m below it. +const SUPPORT_ABOVE: f32 = 1.0; +const SUPPORT_BELOW: f32 = 10.0; + +pub(crate) struct StartupReport { + /// Distance from the spawn down to the first supporting surface, if any. + pub support_drop: Option, + pub grounded_ticks: usize, + pub warnings: Vec, +} + +pub(crate) fn collision_triangles(physics: &GamePhysics) -> usize { + physics.world.triangles().len() +} + +/// Spawn support, deck pose and a neutral startup on an already loaded map. +pub(crate) fn check( + physics: &mut GamePhysics, + skater: &mut SkaterRuntime, + controls: &mut PlayerControls, + camera: &mut CameraRuntime, + graphs: &StockGraphs, + spawn: [f32; 3], + heading: f32, +) -> Result { + let mut warnings = Vec::new(); + let [x, y, z] = spawn; + let support_drop = physics + .world + .query_thin_line(Vector3::new(x, y + SUPPORT_ABOVE, z), Vector3::new(x, y - SUPPORT_BELOW, z)) + .map_err(|e| format!("spawn support query: {e}"))? + .map(|hit| y - hit.geometry.position.y); + if support_drop.is_none() { + warnings.push(format!("no collision within {SUPPORT_BELOW} m below the spawn")); + } + + let deck = physics.board.part_transforms()[skate_core::physics::board::BodyId::Deck.index()]; + if (deck.translation.x - x).abs() > 0.01 || (deck.translation.z - z).abs() > 0.01 { + warnings.push(format!( + "deck spawned at ({:.3}, {:.3}) instead of ({x:.3}, {z:.3})", + deck.translation.x, deck.translation.z + )); + } + let forward = deck.basis.columns[2]; + let (sin, cos) = heading.sin_cos(); + if (forward[0] - sin).abs() > 0.01 || (forward[2] - cos).abs() > 0.01 { + warnings.push(format!("deck heading {forward:?} does not match map heading {heading}")); + } + + let mut grounded_ticks = 0; + let mut settled = 0; + let mut solved_wheel_contact = false; + for _ in 0..STARTUP_TICKS { + if settled >= SETTLE_TICKS { + break; + } + // No controller: the same zero actions the game publishes when no pad is ready. + let mut actions = GameplayActions::from_values([0.0; 18]); + controls.update_for_physics(&mut actions, physics, skater, camera)?; + frame::advance(physics, skater, controls, graphs, &mut actions, true, camera) + .map_err(|e| format!("tick {}: {e}", physics.ticks))?; + if physics.failed { + warnings.push(format!("physics failed at tick {}", physics.ticks)); + break; + } + let finite = physics + .board + .bodies() + .iter() + .chain(skater.skeleton.bodies()) + .all(|b| { + [b.rates.position.x, b.rates.position.y, b.rates.position.z].iter().all(|v| v.is_finite()) + && b.rates.basis.columns.iter().flatten().all(|v| v.is_finite()) + }); + if !finite { + warnings.push(format!("non-finite body state at tick {}", physics.ticks)); + break; + } + let supported = physics.riding.ground.wheel_contact_count > 0; + grounded_ticks += usize::from(supported); + settled = if supported { settled + 1 } else { 0 }; + for report in physics.board.contact_reports() { + let force = report.normal_force_on_a; + solved_wheel_contact |= report.part.index() < 4 + && matches!(report.other, skate_core::physics::board_step::CollisionBody::StaticWorld) + && [force.x, force.y, force.z].iter().any(|v| *v != 0.0); + } + } + if grounded_ticks == 0 { + warnings.push(format!("wheels never found support in {STARTUP_TICKS} neutral ticks")); + } else if settled < SETTLE_TICKS { + warnings.push(format!("wheels did not stay supported for {SETTLE_TICKS} ticks within {STARTUP_TICKS} neutral ticks")); + } + if grounded_ticks > 0 && !solved_wheel_contact { + warnings.push("no solved wheel contact reaction during startup".into()); + } + Ok(StartupReport { support_drop, grounded_ticks, warnings }) +} diff --git a/crates/skate-game/src/retail_character.rs b/crates/skate-game/src/retail_character.rs index b40c6d81..68e0d346 100644 --- a/crates/skate-game/src/retail_character.rs +++ b/crates/skate-game/src/retail_character.rs @@ -616,7 +616,7 @@ mod tests { world.insert_resource(crate::config::Config { asset_root: "unused".into(), verification_capture: None, map: None, map_path: None, difficulty: crate::difficulty::Difficulty::Hardcore, - check_assets: false, start_paused: false, teleport: None, + check_assets: false, validate_maps: false, start_paused: false, teleport: None, multiplayer: Default::default(), map_fingerprint: 0, }); world.insert_resource(crate::retail_render::RetailScene(false)); diff --git a/crates/skate-game/src/skater_animation.rs b/crates/skate-game/src/skater_animation.rs index b1b17044..eb24b7d9 100644 --- a/crates/skate-game/src/skater_animation.rs +++ b/crates/skate-game/src/skater_animation.rs @@ -38,7 +38,7 @@ pub(crate) struct AnimationSource { evaluator: Arc, } impl AnimationSource { - fn load(root: &Path) -> Result, String> { + pub(crate) fn load(root: &Path) -> Result, String> { let banks = skate_data::animation_banks::AnimationBanks::load(root)?; let mut evaluator = PoseEvaluator::from_banks(&banks)?; evaluator.load_authored_clips(root)?; diff --git a/crates/skate-game/src/tests/map_transition.rs b/crates/skate-game/src/tests/map_transition.rs index b3127fb7..22af473a 100644 --- a/crates/skate-game/src/tests/map_transition.rs +++ b/crates/skate-game/src/tests/map_transition.rs @@ -27,7 +27,7 @@ fn installed_worlds_prepare_commit_and_retire_without_simulation() { }; let mut world = World::new(); world.insert_resource(Config { asset_root: root.clone(), verification_capture: None, - map: None, map_path: None, difficulty, check_assets: false, start_paused: false, teleport: None, multiplayer: Default::default(), map_fingerprint: crate::config::map_fingerprint(None).unwrap() }); + map: None, map_path: None, difficulty, check_assets: false, validate_maps: false, start_paused: false, teleport: None, multiplayer: Default::default(), map_fingerprint: crate::config::map_fingerprint(None).unwrap() }); world.insert_resource(CurrentMap::from_package(None, None)); world.insert_resource(graphs); world.insert_resource(physics); diff --git a/docs/hails-additions/04-map-spawns.md b/docs/hails-additions/04-map-spawns.md index 9d85a9d6..ee3ddd69 100644 --- a/docs/hails-additions/04-map-spawns.md +++ b/docs/hails-additions/04-map-spawns.md @@ -44,10 +44,15 @@ from the area-weighted centre of flat surfaces, and accept a point that is the lowest surface at its XZ, has ≥ 2.5 m headroom, and has no flat surface within 25 m (by triangle extent) more than 1 m lower. -**Regression found in play:** applied to city maps, "lowest surface" is -under the streets. Industrial's spawn became (−864.7, −3.0, 414.1); the player -stood on collision under the map and the world looked invisible (back faces -culled from below). Fix: only districts whose collision spans ≤ 500 m +**Regression found in play:** applied to city maps, the search starts from the +area-weighted centre of the whole district. For Industrial (a port) that point +is out in the harbour: the spawn became (−864.7, −3.0, 414.1), on a 1,604 m² +collision-only floor at y −4.0 with nothing above it (likely the harbour bed) +and no visible geometry nearby, so the world looked invisible while collision +held the player up. (An earlier version of this note said "under the streets"; +collision analysis at that point showed otherwise.) The streaming map validator +(doc 6) now flags this case as "no visible surface … (invisible collision)". +Fix: only districts whose collision spans ≤ 500 m (`GROUND_MAX_EXTENT`, i.e. the parks) use `ground_spawn`; larger districts keep the original rule unchanged. A unit test pins this. Lesson recorded: a spawn change moves every map's startup position, so all maps must be checked. diff --git a/docs/hails-additions/06-map-validator.md b/docs/hails-additions/06-map-validator.md new file mode 100644 index 00000000..27a9a045 --- /dev/null +++ b/docs/hails-additions/06-map-validator.md @@ -0,0 +1,124 @@ +# 6. Streaming map validator (PR F, tooling) + +## Problem + +Setup validated every converted map by launching the game once per map +(`skate3rust --assets --map --check-assets`, inside each +`map_job` subprocess), plus a `--test-world --check-assets` launch before and +after the map stage and one per recovered old map. Each launch cost ~7–8 s, of +which only 0.1–2 s depended on the map: ~6 s was the same stock loading +(skater runtime 3.2 s, controls + camera 2.5 s) repeated every time. + +The check was also weak: a map passed as long as it loaded. It did not catch the +spawn problems found in play (skater falling into the void on MegaPark, standing +on an invisible collision-only floor in Industrial). + +## Change + +### Game: `skate3rust --assets --validate-maps` + +New module `crates/skate-game/src/map_validation.rs`, dispatched from `main.rs` +after the stock loads (config, gameplay config, manifest, graphs): + +- Loads once: animation source, controls and camera (fail fast, like + `--check-assets`). Prints `SKATE_VALIDATOR_READY seconds=…` on stdout. +- Reads one request per stdin line (a `.skate` path or `TEST_WORLD`) and answers + with one `SKATE_MAP_CHECK {json}` stdout line; exits 0 at EOF. All other game + output stays on stderr. Works through the crash supervisor, which forwards the + child's stdout/stderr unchanged and lets it inherit stdin. +- Per request: + - **errors** (the map is rejected, as before): map load, + `skate_world::validate_runtime`, physics with the map's collision, skater + runtime. A panic inside one map becomes that map's error (`catch_unwind`). + - **warnings** (reported, never rejecting): from + `physics/startup_check.rs` — no collision within 10 m below the spawn; deck + spawned away from the map spawn/heading; a neutral startup (zero actions, + as with no controller) must settle with wheel support for 12 consecutive + ticks within 240 ticks (4 s at 60 Hz), with finite body state, no physics + failure and a solved wheel contact; and the collision floor under the spawn + must have a visible (render) surface within 1 m. + - JSON fields: `path, ok, errors, warnings, collision_triangles, spawn, + support_drop, grounded_ticks, visible_floor_gap, seconds, timings`. +- `--check-assets` is unchanged. + +### Setup (`tools/asset_pipeline/install.py`) + +- `MapValidator` runs one validator process; `start_validator` falls back to + per-map `--check-assets` launches if it cannot start (e.g. an older game exe), + and `validate()` falls back for the rest of the run if the process dies. +- The pre-map stock check, every map (as soon as its conversion finishes, while + the remaining maps keep converting), recovered old maps and the final + test-world check all go to the same process. The per-map launch was removed + from `convert_map`. +- Process launching (packaged-exe DLL/PATH handling) moved from `run()` into a + shared `spawn()`; `run()` behaves as before. +- Warnings are written to `assets/private/map-status/-validation.json` + (`status: "warning"`) by `record_validation`. New module + `validation_report.py` wraps `optional_content.summary` and adds them to + `setup-report.json` and the setup message; `install.py` and `tools/setup.py` + call the wrapper. `optional_content.py` itself is deliberately untouched: it is + part of the hud/environment/maps fingerprints **and** of every + character-customiser stage version (`customiser_cache.SHARED`), so editing it + would make every existing installation rebuild the customiser (~4 min), which + has no equivalence mechanism. (A first version edited it; the fingerprint + check that caught this compares against the committed tree, not a local + install.) +- `pipeline-equivalence.json`: removing validation from `convert_map` changes the + `maps` fingerprint without changing any output, so an old→new pair is added + (combined with PR G's stream change, see doc 8). Verified: an installation + built from the committed code rebuilds no group, and the customiser + fingerprint and all five stage versions are unchanged. + +## Verification + +- Validator on the test world and all 10 maps: all `ok`, no warnings; collision + triangle counts equal the post-volume-fix baseline. +- Negative cases (bad spawns written into copies of real maps): + MegaPark's old void spawn → "no collision within 10 m below the spawn" + + "wheels never found support"; Industrial's old harbour-bed spawn → "no visible + surface … (invisible collision)". +- Known limit: a spawn on a solid, visible roof (Industrial Skate Park's old + roof spawn) is not flagged — rooftops are legitimate skate spots; only + authored start data (doc 4) distinguishes them. +- Unit tests: `map_validation::tests` (visible-surface heights, single-line JSON + with the documented fields); Python `test_map_validator.py` with a fake + validator speaking the protocol (results, warnings into `setup-report.json`, + start failure fallback, process death, request injection, pipes closed under + `-W error::ResourceWarning`). Existing setup/pipeline suites pass (40 tests). +- Fingerprints: with the equivalence pairs, an existing installation rebuilds + no groups. +- Full setup with the real game: see "Setup run" below. + +### Setup run + +Full fresh setup from the extracted disc with the real game exe (Windows 11, +i7-14700KF, 3 map workers): **466 s, 0 warnings**; the validator was used +throughout (no fallback). + +| Step | Before | After | +|---|---|---| +| Pre-map stock check (`--test-world --check-assets` → validator start + `TEST_WORLD`) | ~6 s | ~1 s | +| Per-map validation | ~7–9 s each, inside each map job | 0.28–1.79 s each, in the parent while other maps convert | +| Map stage (convert + validate 10 maps) | ~217 s | 183 s | + +Per-map `phase_seconds.validate` in `maps.json`: University 1.54, DownTown 1.79, +Industrial 1.11, parks 0.28–0.37 s. Afterwards `check_maps`-style validation of +all maps: all ok, no warnings, collision triangles identical; baked spawns +identical to the authored values (doc 4). + +## Notes + +- The ignored private test `private_extracted_map_supports_production_gameplay_startup` + uses a fixed 24-tick window and fails on Industrial Skate Park, whose authored + start is 0.75 m above the floor (≈0.39 s ≈ 23–24 ticks of fall). The map is + fine in play; the validator uses the settle rule above instead. The test was + left unchanged. +- Without PR E (collection index) every per-map check would still take ~5.8 s. + +## Files + +- `crates/skate-game/src/map_validation.rs` (new), `physics/startup_check.rs` (new), + `main.rs`, `config.rs`, `physics.rs`, `skater_animation.rs` (`AnimationSource::load` visibility), + `retail_character.rs` + `tests/map_transition.rs` (new `Config` field in test literals) +- `tools/asset_pipeline/install.py`, `validation_report.py` (new), `pipeline-equivalence.json`, + `test_map_validator.py` (new), `tools/setup.py` (message wording) diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index 27f41b35..db36ac64 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -6,19 +6,47 @@ from tools.owned_game.big import BigArchive TOOLS=Path(__file__).resolve().parents[1] +GIB=1024**3 + +def available_memory(): + """Available physical memory in bytes, or None where it cannot be queried.""" + if os.name!='nt':return None + import ctypes + class Memory(ctypes.Structure): + _fields_=[('length',ctypes.c_ulong),('load',ctypes.c_ulong)]+[(name,ctypes.c_ulonglong) for name in + ('total','available','page_total','page_available','virtual_total','virtual_available','extended')] + memory=Memory();memory.length=ctypes.sizeof(memory) + return memory.available if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)) else None + def map_workers(): count=min(3,max(1,(os.cpu_count() or 1)//2)) - if os.name=='nt': - import ctypes - class Memory(ctypes.Structure): - _fields_=[('length',ctypes.c_ulong),('load',ctypes.c_ulong)]+[(name,ctypes.c_ulonglong) for name in - ('total','available','page_total','page_available','virtual_total','virtual_available','extended')] - memory=Memory();memory.length=ctypes.sizeof(memory) - if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory)): - # Reserve memory for the desktop; each map and its loader can - # briefly hold several copies of geometry and textures. - count=min(count,max(1,(memory.available-2*1024**3)//(3*1024**3))) + available=available_memory() + if available is not None: + # Reserve memory for the desktop; each map and its loader can + # briefly hold several copies of geometry and textures. + count=min(count,max(1,(available-2*GIB)//(3*GIB))) return count + +def overlap_customiser(workers): + """Run the character customiser beside the map jobs only with room for one + more 3 GiB slot on top of the map workers and the 2 GiB desktop reserve.""" + available=available_memory() + return available is None or available>=2*GIB+(workers+1)*3*GIB + +class Background: + """Run an action on a thread; join() waits and re-raises its exception.""" + def __init__(self,action,name): + import threading + self.error=None + def target(): + try:action() + except BaseException as error:self.error=error + self.thread=threading.Thread(target=target,name=name,daemon=True) + self.thread.start() + + def join(self): + self.thread.join() + if self.error is not None:raise self.error XISO_URL='https://github.com/XboxDev/extract-xiso/releases/download/build-202505152050/extract-xiso-Win64_Release.zip' XISO_SHA='fec88d03c7efd6205ab09be4abba70c0afd0eb27a5709f0a6235b828ba5ac11e' @@ -107,12 +135,13 @@ def dependency(cache,name,url,sha,report): if executable is None:raise RuntimeError('Missing downloaded tool: '+name) return executable -def run(args,log,report): +def spawn(args,**popen): kwargs={'creationflags':subprocess.CREATE_NO_WINDOW} if os.name=='nt' else {} external=os.name=='nt' and getattr(sys,'frozen',False) and Path(args[0]).resolve()!=Path(sys.executable).resolve() if external: # External tools and the game must load their own libraries, not # the setup bundle's DLL directory inherited by child processes. + # SetDllDirectoryW is process-wide: only spawn external tools from one thread. import ctypes ctypes.windll.kernel32.SetDllDirectoryW(None) env=os.environ.copy() @@ -121,15 +150,94 @@ def run(args,log,report): if p and not Path(p).resolve().is_relative_to(bundle)) kwargs['env']=env try: - child=subprocess.Popen([str(a) for a in args],stdout=subprocess.PIPE,stderr=subprocess.STDOUT, - text=True,encoding='utf-8',errors='replace',**kwargs) + return subprocess.Popen([str(a) for a in args],text=True,encoding='utf-8',errors='replace',**kwargs,**popen) finally: if external:ctypes.windll.kernel32.SetDllDirectoryW(sys._MEIPASS) + +def run(args,log,report): + child=spawn(args,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) with child as process: for line in process.stdout: log.write(line);log.flush() if process.wait():raise RuntimeError('Conversion failed. See '+str(log.name)) +class MapValidator: + """One `skate3rust --validate-maps` process (crates/skate-game/src/map_validation.rs). + + Shared stock data loads once, so each map costs well under a second instead + of a full --check-assets launch. Requests are one path (or TEST_WORLD) per + line; each answer is one `SKATE_MAP_CHECK {json}` stdout line. Game logs go + to stderr and are copied to the setup log. Use from one thread only. + """ + READY='SKATE_VALIDATOR_READY' + RESULT='SKATE_MAP_CHECK ' + + def __init__(self,args,log): + import threading + self.process=spawn(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + self.log=log;self.lock=threading.Lock() + def copy_stderr(): + with self.process.stderr as stream: + for line in stream: + with self.lock:log.write(line);log.flush() + threading.Thread(target=copy_stderr,daemon=True).start() + ready=self.process.stdout.readline() + if not ready.startswith(self.READY): + self.close() + raise RuntimeError('Map validator did not start'+(f': {ready.strip()}' if ready.strip() else '')) + self.write(ready) + + def write(self,line): + with self.lock:self.log.write(line if line.endswith('\n') else line+'\n');self.log.flush() + + def check(self,request): + """Return the result dict; RuntimeError if the validator is gone.""" + if '\n' in str(request) or '\r' in str(request):raise ValueError('Invalid map path') + try: + self.process.stdin.write(f'{request}\n');self.process.stdin.flush() + while True: + line=self.process.stdout.readline() + if not line:break + self.write(line) + if line.startswith(self.RESULT):return json.loads(line[len(self.RESULT):]) + except (OSError,ValueError) as error: + raise RuntimeError(f'Map validator failed: {error}') from error + raise RuntimeError(f'Map validator exited (code {self.process.poll()})') + + def close(self): + try:self.process.stdin.close() + except OSError:pass # already exited: the pipe is broken + try:self.process.wait(timeout=30) + except subprocess.TimeoutExpired: + self.process.kill();self.process.wait() + self.process.stdout.close() + +def start_validator(game_exe,assets,log,report): + """MapValidator, or None (older game exe / start failure: use --check-assets).""" + try: + validator=MapValidator([game_exe,'--assets',assets,'--validate-maps'],log) + except (OSError,RuntimeError) as error: + report(f'Map validator unavailable ({error}); validating with --check-assets per map') + return None + import weakref + # Never leave the process behind if setup fails before closing it. + weakref.finalize(validator,validator.process.kill) + return validator + +def record_validation(private,entry,result,report): + """Store non-blocking findings as map-status/-validation.json (status warning).""" + from .setup_state import atomic_json + path=private/'map-status'/(entry['name']+'-validation.json') + if result is None:return + entry.setdefault('phase_seconds',{})['validate']=round(result['seconds'],3) + if result['warnings']: + path.parent.mkdir(parents=True,exist_ok=True) + atomic_json(path,{'version':1,'component':entry['name']+' map checks','status':'warning', + 'warnings':result['warnings']}) + report(f"{entry['name']}: map check warnings: {'; '.join(result['warnings'])}") + else: + path.unlink(missing_ok=True) + def task(script,*args): if getattr(sys,'frozen',False):return [sys.executable,'--task',str(script),*map(str,args)] return [sys.executable,str(script),*map(str,args)] @@ -196,9 +304,7 @@ def convert_map(archive,work,maps,stage,game_exe,log,report): placed,unresolved=0,0 finished('props') report(f'{label}: placed {placed} authored DMO instances, {unresolved} unresolved templates') - report('Checking converted map: '+label) - run([game_exe,'--assets',stage/'assets','--map',final,'--check-assets'],log,report) - finished('validate') + # Validation runs in the parent (_install, MapValidator) as each map finishes. entry={'name':label,'path':'maps/'+final.name,'sha256':digest(final)} remove_intermediate(district_work,work) finished('hash_and_cleanup') @@ -242,7 +348,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None with (base/'refresh-validation.log').open('w',encoding='utf-8') as log: run([game_exe,'--assets',previous[0]/'assets','--test-world','--check-assets'],log,report) if finalize:finalize(previous[0]) - from .optional_content import summary + from .validation_report import summary summary(previous[0]) atomic_json(base/'installation.json', {**previous[1], 'pipelines':target_versions, 'outputs':outputs(previous[0]), 'source':source}) @@ -339,7 +445,23 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None if (work/'dmo').exists():remove_intermediate(work/'dmo',work) note(private/'native-props/props-availability.json','Movable props',error,report=report) report('Validating skater, input and animation data') - run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) + validator=start_validator(game_exe,stage/'assets',log,report) + customiser=None # Background customiser when overlapped with the map stage + def validate(request): + """Today's checks raise (map rejected); new findings come back as warnings.""" + nonlocal validator + if validator is not None: + try:result=validator.check(request) + except RuntimeError as error: + report(f'{error}; validating with --check-assets per map') + validator.close();validator=None + else: + if not result['ok']:raise RuntimeError('Map validation failed: '+'; '.join(result['errors'])) + return result + target=['--test-world'] if request=='TEST_WORLD' else ['--map',request] + run([game_exe,'--assets',stage/'assets',*target,'--check-assets'],log,report) + return None + validate('TEST_WORLD') if 'maps' in groups: archives=list((game_root/'data/content').glob('worldDIST_*.big')) archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower())) @@ -351,6 +473,12 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None report(f'Authored map starts unavailable ({error}); using geometric spawns') (work/'map-starts.json').write_text(json.dumps(starts),encoding='utf-8') workers=map_workers() + # The customiser reads only stock, default-skater and disc data (all + # prepared above) and writes only assets/private/customisation, so it + # runs beside the map jobs instead of after them. + if finalize and overlap_customiser(workers): + report('Preparing the character customiser alongside the maps') + customiser=Background(lambda:finalize(stage),'customiser') report(f'Converting {len(archives)} maps with {workers} workers') def map_job(archive): result=work/(archive.stem+'.json') @@ -365,8 +493,12 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)} for future in as_completed(futures): archive=futures[future] - try:completed[archive.name]=future.result() + try: + completed[archive.name]=future.result() + # Validate while the remaining maps keep converting. + checked=validate(stage/completed[archive.name]['path']) except CONTENT_ERRORS as error: + completed.pop(archive.name,None) label=archive.stem.removeprefix('worldDIST_') (maps/(label+'.skate')).unlink(missing_ok=True) props=private/'native-props'/(label+'.skate') @@ -374,6 +506,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None note(private/'map-status'/(label+'-availability.json'),label,error,report=report) continue (private/'map-status'/(completed[archive.name]['name']+'-availability.json')).unlink(missing_ok=True) + record_validation(private,completed[archive.name],checked,report) report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}") catalog=[completed[a.name] for a in archives if a.name in completed] if previous: @@ -384,7 +517,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None src=(previous[0]/old['path']).resolve() if not src.is_relative_to((previous[0]/'maps').resolve()):raise ValueError('Invalid old map path') if not src.is_file() or digest(src)!=old.get('sha256'):continue - try:run([game_exe,'--assets',stage/'assets','--map',src,'--check-assets'],log,report) + try:validate(src) except CONTENT_ERRORS:continue target=stage/old['path'];target.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,target) catalog.append(old) @@ -397,7 +530,8 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None if old['path'] not in valid_paths:(stage/old['path']).unlink(missing_ok=True) if not catalog:raise RuntimeError('No playable map could be prepared or recovered. Restore at least one worldDIST_*.big archive beside default.xex and retry; the previous installation has been kept.') report('Validating installed runtime inputs') - run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) + validate('TEST_WORLD') + if validator is not None:validator.close() settings=stage/'settings';settings.mkdir(exist_ok=True) if not previous: (settings/'default-map.json').write_text(json.dumps(next((m['path'] for m in catalog if m['name']=='University'),catalog[0]['path'])),encoding='utf-8') @@ -407,8 +541,9 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None if selected.is_file() and not (stage/json.loads(selected.read_text())).is_file(): selected.write_text(json.dumps(catalog[0]['path'])) remove_intermediate(work,stage) - if finalize:finalize(stage) - from .optional_content import summary + if customiser is not None:customiser.join() + elif finalize:finalize(stage) + from .validation_report import summary warnings=summary(stage) # Publish after core validation and all optional outcomes have been recorded. marker=base/'installation.json.new' @@ -416,5 +551,5 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None marker.replace(base/'installation.json') remove_setup_logs(stage,report) remove_stale_installations(base,stage,report) - report(f'Setup complete ({len(warnings)} unavailable/retained components; see setup-report.json)' if warnings else 'Setup complete') + report(f'Setup complete ({len(warnings)} warnings or unavailable/retained components; see setup-report.json)' if warnings else 'Setup complete') return stage diff --git a/tools/asset_pipeline/test_map_validator.py b/tools/asset_pipeline/test_map_validator.py new file mode 100644 index 00000000..c24add7c --- /dev/null +++ b/tools/asset_pipeline/test_map_validator.py @@ -0,0 +1,126 @@ +import io +import json +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +from . import install +from .validation_report import summary + +# Speaks the skate3rust --validate-maps protocol (crates/skate-game/src/map_validation.rs). +FAKE = textwrap.dedent(''' + import json, sys + mode = sys.argv[1] + if mode == 'no-ready': + print('Unknown argument "--validate-maps"', file=sys.stderr); sys.exit(1) + print('SKATE_VALIDATOR_READY seconds=0.001', flush=True) + for line in sys.stdin: + path = line.strip() + print('SKATE_RWCM_READY triangles=12 (log line)', file=sys.stderr, flush=True) + if 'die' in path: + sys.exit(3) + result = {'path': path, 'ok': 'error' not in path, 'seconds': 0.25, + 'errors': ['bad map'] if 'error' in path else [], + 'warnings': ['no collision within 10 m below the spawn'] if 'warn' in path else []} + print('noise before the result', flush=True) + print('SKATE_MAP_CHECK ' + json.dumps(result), flush=True) +''') + + +class MapValidatorTests(unittest.TestCase): + def setUp(self): + self.work = tempfile.TemporaryDirectory() + self.root = Path(self.work.name) + self.fake = self.root / 'fake_validator.py' + self.fake.write_text(FAKE, encoding='utf-8') + self.log = io.StringIO() + self.reports = [] + + def tearDown(self): + self.work.cleanup() + + def start(self, mode='ok'): + return install.MapValidator([sys.executable, self.fake, mode], self.log) + + def test_results_warnings_and_log_copy(self): + validator = self.start() + try: + clean = validator.check(self.root / 'maps/Clean.skate') + warned = validator.check(self.root / 'maps/warn.skate') + failed = validator.check('error.skate') + finally: + validator.close() + self.assertTrue(clean['ok']) + self.assertEqual(clean['warnings'], []) + self.assertEqual(warned['warnings'], ['no collision within 10 m below the spawn']) + self.assertFalse(failed['ok']) + self.assertIn('SKATE_VALIDATOR_READY', self.log.getvalue()) + self.assertIn('SKATE_MAP_CHECK', self.log.getvalue()) + self.assertIn('SKATE_RWCM_READY', self.log.getvalue()) + self.assertEqual(validator.process.returncode, 0) + + def test_validator_that_never_becomes_ready_falls_back(self): + with self.assertRaisesRegex(RuntimeError, 'did not start'): + self.start('no-ready') + self.assertIsNone(install.start_validator(self.root / 'missing.exe', self.root, self.log, self.reports.append)) + self.assertIn('validating with --check-assets per map', self.reports[0]) + + def test_validator_death_is_a_runtime_error(self): + validator = self.start() + with self.assertRaisesRegex(RuntimeError, 'exited'): + validator.check('die.skate') + validator.close() + + def test_paths_cannot_inject_extra_requests(self): + validator = self.start() + try: + with self.assertRaises(ValueError): + validator.check('a.skate\nb.skate') + finally: + validator.close() + + def test_warnings_are_recorded_and_reported_and_clean_runs_clear_them(self): + stage = self.root / 'stage' + private = stage / 'assets/private' + entry = {'name': 'MegaPark', 'path': 'maps/MegaPark.skate', 'phase_seconds': {'prepare': 1.0}} + install.record_validation(private, entry, {'ok': True, 'seconds': 0.25, 'warnings': ['spawn over void']}, + self.reports.append) + self.assertEqual(entry['phase_seconds']['validate'], 0.25) + warnings = summary(stage) + self.assertEqual([w['status'] for w in warnings], ['warning']) + self.assertEqual(warnings[0]['warnings'], ['spawn over void']) + self.assertTrue((stage / 'setup-report.json').is_file()) + install.record_validation(private, entry, {'ok': True, 'seconds': 0.2, 'warnings': []}, self.reports.append) + self.assertEqual(summary(stage), []) + self.assertFalse((stage / 'setup-report.json').exists()) + # Fallback path (no validator) records nothing and keeps timings intact. + install.record_validation(private, entry, None, self.reports.append) + self.assertEqual(entry['phase_seconds']['validate'], 0.2) + + +class CustomiserOverlapTests(unittest.TestCase): + def test_background_join_waits_and_reraises(self): + done = [] + install.Background(lambda: done.append(1), 'ok').join() + self.assertEqual(done, [1]) + failing = install.Background(lambda: (_ for _ in ()).throw(RuntimeError('customiser failed')), 'bad') + with self.assertRaisesRegex(RuntimeError, 'customiser failed'): + failing.join() + + def test_overlap_needs_one_more_slot_than_the_map_workers(self): + original = install.available_memory + try: + install.available_memory = lambda: (2 + 4 * 3) * install.GIB # 3 workers + customiser fit + self.assertTrue(install.overlap_customiser(3)) + install.available_memory = lambda: (2 + 4 * 3) * install.GIB - 1 # one byte short + self.assertFalse(install.overlap_customiser(3)) + install.available_memory = lambda: None # cannot query: allowed + self.assertTrue(install.overlap_customiser(3)) + finally: + install.available_memory = original + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/asset_pipeline/validation_report.py b/tools/asset_pipeline/validation_report.py new file mode 100644 index 00000000..f055b080 --- /dev/null +++ b/tools/asset_pipeline/validation_report.py @@ -0,0 +1,32 @@ +"""Setup report including non-blocking map check warnings. + +Wraps optional_content.summary instead of changing it: optional_content.py is +part of several asset-group fingerprints and of every character-customiser +stage version, so editing it would force existing installations to rebuild. +This module's name matches no fingerprint pattern (see versions.py and +customiser_setup.fingerprint). +""" +import json +from .optional_content import summary as optional_summary +from .setup_state import atomic_json + + +def map_check_warnings(stage): + """Items written by install.record_validation (status 'warning').""" + found = [] + for path in sorted((stage/'assets/private/map-status').glob('*-validation.json')): + try: + item = json.loads(path.read_text()) + except (OSError, ValueError): + continue + if isinstance(item, dict) and item.get('status') == 'warning': + found.append(item) + return found + + +def summary(stage): + """optional_content.summary plus map check warnings, in one setup-report.json.""" + warnings = optional_summary(stage) + map_check_warnings(stage) + if warnings: + atomic_json(stage/'setup-report.json', {'version': 1, 'warnings': warnings}) + return warnings diff --git a/tools/setup.py b/tools/setup.py index 93a85ca6..84d11e03 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -97,9 +97,9 @@ def main(): def work(): try: installed_root=install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating) - from tools.asset_pipeline.optional_content import summary + from tools.asset_pipeline.validation_report import summary warnings=summary(installed_root) - messages.put(('done',f'Ready with {len(warnings)} unavailable components. Details: {installed_root / "setup-report.json"}' if warnings else 'Ready')) + messages.put(('done',f'Ready with {len(warnings)} warnings or unavailable components. Details: {installed_root / "setup-report.json"}' if warnings else 'Ready')) except Exception as error: args.base.mkdir(parents=True,exist_ok=True) (args.base/'setup-error.log').write_text(traceback.format_exc(),encoding='utf-8') @@ -123,7 +123,7 @@ def main(): while not messages.empty(): kind,text=messages.get_nowait();status.set(text) if kind=='done': - if text!='Ready':messagebox.showwarning('Setup completed with unavailable content',text,parent=window) + if text!='Ready':messagebox.showwarning('Setup completed with warnings',text,parent=window) running=False;success=True;progress.stop();window.destroy();return if kind=='error': running=False;progress.stop();button.config(state='normal')