From ea7b9a82189ffc3fe7acad8e8ebde9046d134529 Mon Sep 17 00:00:00 2001 From: chazmm Date: Thu, 10 Sep 2026 02:36:05 +0100 Subject: [PATCH] Keep setup playable when optional disc content is missing --- crates/skate-game/src/customiser_parts.rs | 5 + crates/skate-game/src/setup.rs | 39 ++++++++ docs/character-customisation.md | 4 +- docs/installation.md | 27 +++++- scripts/Build-Release.ps1 | 2 +- tools/asset_pipeline/asset_exports.py | 21 +++-- tools/asset_pipeline/customisation_library.py | 86 ++++++++++------- tools/asset_pipeline/customiser_cache.py | 4 +- tools/asset_pipeline/customiser_lighting.py | 39 ++++---- tools/asset_pipeline/customiser_setup.py | 63 +++++++++++-- tools/asset_pipeline/group_receipts.py | 9 +- tools/asset_pipeline/install.py | 77 +++++++++++++--- tools/asset_pipeline/native_roster.py | 7 +- tools/asset_pipeline/optional_content.py | 48 ++++++++++ tools/asset_pipeline/setup_state.py | 10 +- tools/asset_pipeline/test_customiser_setup.py | 9 +- tools/asset_pipeline/test_optional_content.py | 92 +++++++++++++++++++ tools/asset_pipeline/test_versions.py | 2 +- tools/asset_pipeline/versions.py | 6 +- tools/install_prepared_hud.py | 1 + tools/prepare_runtime_huds.py | 25 ++++- tools/setup.py | 7 +- tools/test_setup_assets.py | 61 ++++++++---- 23 files changed, 517 insertions(+), 127 deletions(-) create mode 100644 tools/asset_pipeline/optional_content.py create mode 100644 tools/asset_pipeline/test_optional_content.py diff --git a/crates/skate-game/src/customiser_parts.rs b/crates/skate-game/src/customiser_parts.rs index 1eb151ac..1358fa39 100644 --- a/crates/skate-game/src/customiser_parts.rs +++ b/crates/skate-game/src/customiser_parts.rs @@ -74,6 +74,11 @@ pub(crate) struct Parts { pub(crate) struct PartRoot(pub String); pub(crate) fn asset_directory(assets: &std::path::Path) -> std::path::PathBuf { let base = assets.join("private/customisation"); + if std::fs::read(base.join("customiser-availability.json")).ok() + .and_then(|bytes|serde_json::from_slice::(&bytes).ok()) + .is_some_and(|v|v["version"]==1 && v["status"]=="unavailable") { + return base.join("unavailable"); + } if let Some(set) = std::fs::read(base.join("current.json")).ok() .and_then(|b| serde_json::from_slice::(&b).ok()) .and_then(|v| v["set"].as_str().map(str::to_owned)) diff --git a/crates/skate-game/src/setup.rs b/crates/skate-game/src/setup.rs index aa558807..8aaf8ca6 100644 --- a/crates/skate-game/src/setup.rs +++ b/crates/skate-game/src/setup.rs @@ -76,6 +76,19 @@ pub(crate) fn asset_root() -> Result { fn customiser_current(assets: &Path, expected: Option<&str>) -> bool { expected.is_none_or(|expected| { + let degraded = std::fs::read(assets.join("private/customisation/customiser-availability.json")).ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + if let Some(v) = degraded.filter(|v| v["version"].as_u64()==Some(1) + && v["fingerprint"].as_str()==Some(expected)) { + if v["status"]=="unavailable" { return true; } + if v["status"]=="retained" { + let directory=crate::customiser_parts::asset_directory(assets); + return ["catalog","library","menu","lighting","roster"].iter().all(|stage| + std::fs::read(directory.join(format!("{stage}-complete.json"))).ok() + .and_then(|bytes|serde_json::from_slice::(&bytes).ok()) + .is_some_and(|v|receipt_present(&directory,&v["files"]))); + } + } std::fs::read(assets.join("private/customisation/current.json")).ok() .and_then(|b| serde_json::from_slice::(&b).ok()) .is_some_and(|v| v["version"].as_u64() == Some(1) && v["fingerprint"].as_str() == Some(expected) @@ -107,6 +120,32 @@ fn receipt_present(root: &Path, files: &serde_json::Value) -> bool { mod tests { use super::*; #[test] + fn acknowledged_optional_failure_is_versioned_and_retained_data_is_checked() { + let root = std::env::temp_dir().join(format!("sk8-availability-{}-{}", std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos())); + let base = root.join("private/customisation"); + std::fs::create_dir_all(&base).unwrap(); + let availability = base.join("customiser-availability.json"); + std::fs::write(&availability, br#"{"version":1,"fingerprint":"new","status":"unavailable"}"#).unwrap(); + assert!(customiser_current(&root, Some("new"))); + assert!(!customiser_current(&root, Some("future"))); + assert_eq!(crate::customiser_parts::asset_directory(&root), base.join("unavailable")); + std::fs::write(&availability, br#"{"version":1,"fingerprint":"new","status":"retained"}"#).unwrap(); + assert!(!customiser_current(&root, Some("new"))); + std::fs::write(base.join("current.json"), br#"{"set":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}"#).unwrap(); + let directory = base.join("sets/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join("payload"), b"ok").unwrap(); + for stage in ["catalog", "library", "menu", "lighting", "roster"] { + std::fs::write(directory.join(format!("{stage}-complete.json")), + br#"{"files":{"payload":{"size":2}}}"#).unwrap(); + } + assert!(customiser_current(&root, Some("new"))); + std::fs::remove_file(directory.join("payload")).unwrap(); + assert!(!customiser_current(&root, Some("new"))); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn receipts_reject_missing_files_empty_lists_and_traversal() { let root = std::env::temp_dir(); assert!(!receipt_present(&root, &serde_json::json!({}))); diff --git a/docs/character-customisation.md b/docs/character-customisation.md index 560bb957..84c3f64f 100644 --- a/docs/character-customisation.md +++ b/docs/character-customisation.md @@ -7,8 +7,8 @@ steps previously ran only in private feature-test folders. Character additions have their own release fingerprint. Updating an already prepared copy can build these assets from its original Xbox source without -reconverting maps. Setup publishes a new character generation only after the -library, lighting and native roster all succeed. Existing imported models and +reconverting maps. Setup publishes validated character stages and records unavailable extras. +If the base library fails, it keeps a verified previous generation or the stock skater. Existing imported models and saved profiles are preserved; interrupted preparation can reuse its own cache. Players do not need Python, Blender or development tools. diff --git a/docs/installation.md b/docs/installation.md index bf08181a..c7fca2c3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -120,9 +120,9 @@ Core refreshes create a new installation before publication. Unchanged prepared maps and immutable character generations use hardlinks (copy fallback on filesystems without support); mutable user files and outputs being rebuilt have independent storage. Customiser-only refreshes keep the current core installation and publish a new character generation. Custom -models, profiles, settings, selections and mods are retained. Every required -stage, including the complete character library and pro roster, must succeed -before the installation record changes. Failures preserve the previous record. +models, profiles, settings, selections and mods are retained. Core gameplay inputs and at least one map must pass validation before the +installation record changes. Missing optional content is recorded separately; +fatal failures preserve the previous record. A kernel lock prevents concurrent setup and releases automatically on process exit, including a crash; the harmless `setup.lock` file can remain on disk. @@ -142,3 +142,24 @@ Installations from releases that used the old global asset store need one setup in the new per-copy layout. There is deliberately no automatic global migration. Manually unpacking a ZIP over the same folder retains that folder's `data` and behaves like an in-place update; unpack into a new folder for a fresh setup. + +## Missing content + +Setup isolates missing or malformed optional items. Clothing variants and tattoos +that cannot be decoded are excluded from the customiser. Missing pro characters, +lighting, HUD banks, environment extras and movable props do not block core setup. +Scoring and session-marker artwork are prepared independently. Verified previous +HUDs and customiser generations are kept when a replacement fails; otherwise the +stock skater and available features remain usable. + +Each district converts independently. A failed or absent source district can use +its previous converted map only after checksum and engine validation. Initial +setup can select another available district if University is absent. Missing core +animation, physics, the stock skater, or every playable map still requires repairing +the source. Missing original artwork cannot be recreated by ignoring the error. + +Setup displays a notice and writes `setup-report.json` in the active installation, +with unavailable items and error details. Availability records participate in +output receipts so acknowledged missing extras do not cause a setup loop. A new +extractor fingerprint requests preparation again. Disk and permission failures +in in-process conversion still abort rather than publishing partial writes. diff --git a/scripts/Build-Release.ps1 b/scripts/Build-Release.ps1 index 02f7c402..0471dd87 100644 --- a/scripts/Build-Release.ps1 +++ b/scripts/Build-Release.ps1 @@ -16,7 +16,7 @@ try { } & $packagePython -m pip install -r tools/requirements-setup.txt if ($LASTEXITCODE -ne 0) { throw 'Could not install packaging dependencies' } - & $packagePython -m unittest tools.test_setup_assets tools.asset_pipeline.test_marquee_assets tools.asset_pipeline.test_customiser_setup tools.asset_pipeline.test_setup_recovery + & $packagePython -m unittest tools.test_setup_assets tools.asset_pipeline.test_marquee_assets tools.asset_pipeline.test_customiser_setup tools.asset_pipeline.test_setup_recovery tools.asset_pipeline.test_versions tools.asset_pipeline.test_optional_content if ($LASTEXITCODE -ne 0) { throw 'Setup extraction regression checks failed' } $env:CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS = '-C target-feature=+crt-static' $stamp = Get-Date -Format 'yyyyMMdd-HHmmss-ffff' diff --git a/tools/asset_pipeline/asset_exports.py b/tools/asset_pipeline/asset_exports.py index b0740c14..bfdf66cd 100644 --- a/tools/asset_pipeline/asset_exports.py +++ b/tools/asset_pipeline/asset_exports.py @@ -30,8 +30,6 @@ def core(game_root, stage, work, report, log, converted=None): def hud(game_root, stage, work, report, log, converted=None): private=stage/"assets/private" stock=private/"stock" - for folder in (private/'hud',private/'session-marker'): - if folder.is_dir():engine.remove_intermediate(folder,stage) report('Preparing original scoring and session-marker HUD assets') engine.run(engine.task(engine.TOOLS/'prepare_runtime_huds.py', '--game', game_root, '--assets', stage/'assets', '--work', work/'hud'), log, report) @@ -66,12 +64,23 @@ def environment(game_root, stage, work, report, log, converted=None): stock=private/"stock" report('Preparing retail sky domes') from .sky import convert as write_skies - write_skies(game_root,stage/'assets',converted) + def attempt(name, action): + from .optional_content import CONTENT_ERRORS, note + target=work/('environment-'+name)/'assets' + target.mkdir(parents=True,exist_ok=True) + availability=private/'environment-status'/(name+'-availability.json') + try:action(target) + except CONTENT_ERRORS as error: + note(availability,'Environment '+name,error,report=report) + return + shutil.copytree(target,stage/'assets',dirs_exist_ok=True) + availability.unlink(missing_ok=True) + attempt('skies',lambda assets:write_skies(game_root,assets,converted)) from .render_parameters import convert as write_render_parameters - write_render_parameters(stage/'assets',converted) + attempt('parameters',lambda assets:write_render_parameters(assets,converted)) report('Extracting original travel destinations and location names') from .teleports import convert as write_teleports - write_teleports(game_root,stage/'assets',converted) + attempt('teleports',lambda assets:write_teleports(game_root,assets,converted)) report('Preparing global foliage backdrops') from .backdrop import convert as write_backdrops - write_backdrops(game_root,stage/'assets',converted) + attempt('backdrops',lambda assets:write_backdrops(game_root,assets,converted)) diff --git a/tools/asset_pipeline/customisation_library.py b/tools/asset_pipeline/customisation_library.py index 3787c644..39b253a6 100644 --- a/tools/asset_pipeline/customisation_library.py +++ b/tools/asset_pipeline/customisation_library.py @@ -6,6 +6,7 @@ from PIL import Image,ImageChops from tools.asset_pipeline.customisation_worker import ROOT,default_profile,flag from tools.asset_pipeline.customisation_catalog import write_private from tools.owned_game.big import BigArchive +from tools.asset_pipeline.optional_content import CONTENT_ERRORS def friendly(name): text=name.lower().replace('_',' ') @@ -48,6 +49,17 @@ def variant_label(slot, model_name, native_name): return label+(' (Worn)' if worn else '') +def prune_unavailable(models, materials, errors): + """Never expose a model selection that points at an absent material.""" + for key, model in list(models.items()): + model['groups'] = [[mid for mid in group if mid in materials] for group in model['groups']] + if not model['groups'] or any(not group for group in model['groups']): + del models[key] + errors.append(dict(model=key, error='No available material for a mesh group')) + else: + model['materials'] = model['groups'][0] + + def prepare(config): from tools.extract_default_skater import import_rx2_parser,decode_texture,parse_fallback_recipe from tools.asset_pipeline.retail_character import RX2,decode_dense_morphs,AnimSource,SkeletonSet @@ -91,8 +103,9 @@ def prepare(config): for part in catalog['components']: for model in part['models']: if part['slot']=='Misc':continue # Non-geometric stamp catalogue, indexed below. - slot=part['slot'];lod=next(l for l in model['lods'] if l['index']==0);key=model['id'];path=out/(key+'_v4.glb') + slot=part['slot'];key=model['id'];path=out/(key+'_v4.glb') try: + lod=next(l for l in model['lods'] if l['index']==0) if not path.exists(): source=extract(lod['path']);folder=out/'work'/key/'models'/slot;folder.mkdir(parents=True,exist_ok=True) mesh_file=folder/source.name @@ -109,39 +122,48 @@ def prepare(config): groups=[[v['id'] for v in group] for group in lod['material_instances']] model_data[key]=dict(slot=slot,name=friendly(model['name']),flags={k[4:]:v for k,v in model['flags'].items() if v}, materials=groups[0],groups=groups,scene=path.relative_to(assets).as_posix()) - for v in lod['material_instances'][0]: - mid=v['id'] - if mid in mat_data:continue - mat=catalog['materials'][mid];tex={t['channel']:t['id'] for t in mat['textures']} - if 'diffuse' not in tex:continue - diffuse=texture(tex['diffuse']) - if 'alpha' in tex and slot!='Hair': - alpha_path=assets/texture(tex['alpha']);combined=out/'textures'/(tex['diffuse']+'_'+tex['alpha']+'.png') - if not combined.exists(): - im=Image.open(assets/diffuse).convert('RGBA');alpha=Image.open(alpha_path).convert('RGBA') - alpha=alpha.resize(im.size,Image.Resampling.LANCZOS);mask=alpha.getchannel('A') - if mask.getextrema()==(255,255):mask=alpha.convert('L') - im.putalpha(ImageChops.multiply(im.getchannel('A'),mask));im.save(combined) - diffuse=combined.relative_to(assets).as_posix() - mat_data[mid]=dict(name=variant_label(slot,model['name'],v['name']),flags={k[4:]:val for k,val in mat['flags'].items() if val}, - diffuse=diffuse,normal=texture(tex['normal'],'normal') if 'normal' in tex else None, - rough=texture(tex['specular'],'rough') if 'specular' in tex else None, - opacity=texture(tex['alpha']) if slot=='Hair' and 'alpha' in tex else None, - alpha='alpha' in tex,tint=defaults.get(mid,{}).get('tint', - [0.72,0.57,0.49] if mat['flags'].get('cas.SkinTone')=='light' else - [0.33,0.26,0.23] if mat['flags'].get('cas.SkinTone')=='dark' else - [0.02,0.01,0.01] if slot=='Hair' else [1.,1.,1.]), - metallic=.65 if slot=='SkateTruck' else 0.,roughness=.48 if slot in {'SkateTruck','SkateWheel'} else .72) - except Exception as e:errors.append(dict(model=key,slot=slot,error=str(e))) + for v in (v for group in lod['material_instances'] for v in group): + try: + mid=v['id'] + if mid in mat_data:continue + mat=catalog['materials'][mid];tex={t['channel']:t['id'] for t in mat['textures']} + if 'diffuse' not in tex:continue + diffuse=texture(tex['diffuse']) + if 'alpha' in tex and slot!='Hair': + alpha_path=assets/texture(tex['alpha']);combined=out/'textures'/(tex['diffuse']+'_'+tex['alpha']+'.png') + if not combined.exists(): + im=Image.open(assets/diffuse).convert('RGBA');alpha=Image.open(alpha_path).convert('RGBA') + alpha=alpha.resize(im.size,Image.Resampling.LANCZOS);mask=alpha.getchannel('A') + if mask.getextrema()==(255,255):mask=alpha.convert('L') + im.putalpha(ImageChops.multiply(im.getchannel('A'),mask));im.save(combined) + diffuse=combined.relative_to(assets).as_posix() + mat_data[mid]=dict(name=variant_label(slot,model['name'],v['name']),flags={k[4:]:val for k,val in mat['flags'].items() if val}, + diffuse=diffuse,normal=texture(tex['normal'],'normal') if 'normal' in tex else None, + rough=texture(tex['specular'],'rough') if 'specular' in tex else None, + opacity=texture(tex['alpha']) if slot=='Hair' and 'alpha' in tex else None, + alpha='alpha' in tex,tint=defaults.get(mid,{}).get('tint', + [0.72,0.57,0.49] if mat['flags'].get('cas.SkinTone')=='light' else + [0.33,0.26,0.23] if mat['flags'].get('cas.SkinTone')=='dark' else + [0.02,0.01,0.01] if slot=='Hair' else [1.,1.,1.]), + metallic=.65 if slot=='SkateTruck' else 0.,roughness=.48 if slot in {'SkateTruck','SkateWheel'} else .72) + except CONTENT_ERRORS as error: + errors.append(dict(model=key,material=v.get('id'),slot=slot,error=str(error))) + except CONTENT_ERRORS as e:errors.append(dict(model=key,slot=slot,error=str(e))) print('Prepared',part['slot'],len(model_data),'models',len(mat_data),'materials',flush=True) tattoos={} - misc=next(p for p in catalog['components'] if p['slot']=='Misc') - for variant in misc['models'][0]['lods'][0]['material_instances'][0]: - mid=variant['id'];mat=catalog['materials'][mid] - if not mat['flags'].get('cas.TattooCategory'):continue - tex={t['channel']:t['id'] for t in mat['textures']} - tattoos[mid]=dict(name=friendly(variant['name']),texture=texture(tex['decal']), - bounds=[float(v) for v in mat['flags']['cas.StampBorderConstraint'].split(',')]) + misc=next((p for p in catalog['components'] if p['slot']=='Misc'),None) + variants=(v for m in (misc or {}).get('models',[]) for lod in m.get('lods',[]) + for group in lod.get('material_instances',[]) for v in group) + for variant in variants: + try: + mid=variant['id'];mat=catalog['materials'][mid] + if not mat['flags'].get('cas.TattooCategory'):continue + tex={t['channel']:t['id'] for t in mat['textures']} + tattoos[mid]=dict(name=friendly(variant['name']),texture=texture(tex['decal']), + bounds=[float(v) for v in mat['flags']['cas.StampBorderConstraint'].split(',')]) + except CONTENT_ERRORS as error: + errors.append(dict(tattoo=variant.get('id'),error=str(error))) + prune_unavailable(model_data,mat_data,errors) male=default_profile();male['gender']='male' female=copy.deepcopy(male);female['gender']='female';female['selections']={} raw=parse_fallback_recipe(assets/'private/stock/data/cacrecipes/SavedRecipeFallbackFemale.bin',16) diff --git a/tools/asset_pipeline/customiser_cache.py b/tools/asset_pipeline/customiser_cache.py index 7faff60f..f9d0bc16 100644 --- a/tools/asset_pipeline/customiser_cache.py +++ b/tools/asset_pipeline/customiser_cache.py @@ -8,7 +8,7 @@ from .setup_state import atomic_json, receipt, valid_receipt SHARED = ('owned_game/**/*.py', 'vendor/utt/**/*.py', 'vendor/utt/**/*.json', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/refpack_native.rs', - 'requirements-setup.txt') + 'requirements-setup.txt', 'asset_pipeline/optional_content.py') GEOMETRY = ('asset_pipeline/character_glb.py', 'asset_pipeline/retail_character.py', 'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py', 'extract_default_skater.py', 'default_skater_retail_manifest.json') @@ -25,7 +25,7 @@ OUTPUTS = { 'catalog': ('catalog.json', 'native.json', 'database', 'source/data/content/recipe'), 'library': ('library', 'decoded', 'source/data/content/createacharacter', 'library-base.json'), 'menu': ('extra-menu.json',), - 'lighting': ('specular', 'native-lighting.json', 'library-v3.json'), + 'lighting': ('specular', 'native-lighting.json', 'library-v3.json', 'lighting-availability.json'), 'roster': ('native-roster',), } REQUIRED = { diff --git a/tools/asset_pipeline/customiser_lighting.py b/tools/asset_pipeline/customiser_lighting.py index f5b6e747..bd773a64 100644 --- a/tools/asset_pipeline/customiser_lighting.py +++ b/tools/asset_pipeline/customiser_lighting.py @@ -6,7 +6,8 @@ import sys from PIL import Image from .environment import Collections, key_hash from .native_roster import roster -from .marquee_assets import Resources, MissingMarqueeAsset +from .marquee_assets import Resources +from .optional_content import CONTENT_ERRORS, note from tools.owned_game.big import BigArchive from tools.extract_default_skater import import_rx2_parser, decode_texture @@ -34,9 +35,9 @@ def prepare(game, assets, directory, library): collections = Collections(converted) catalog = json.loads((directory/'catalog.json').read_text()) cac = BigArchive(game/'data/content/createacharacter.big') - marquee = BigArchive(game/'data/content/marquee.big') + marquee = None cac_entries = {e.path.lower(): e for e in cac.entries} - resources = Resources(marquee) + resources = None masks = directory/'specular'; masks.mkdir(exist_ok=True) def mask(archive, entries, prefix, tid): @@ -73,24 +74,30 @@ def prepare(game, assets, directory, library): material['lighting'] = data native = {} + try: + marquee = BigArchive(game/'data/content/marquee.big') + resources = Resources(marquee) + except CONTENT_ERRORS as error: + note(directory/'lighting-availability.json', 'Pro character lighting', error) for item in roster(converted['collections']): + if resources is None:break recipe = item['recipe'] try: root = resources.recipe(recipe) - except MissingMarqueeAsset as error: - print(f'Unavailable optional character {item["name"]}: {error}', flush=True) + definitions = {m.attrib['id']: m for m in root.findall('mat')} + materials = {} + for comp in root.findall('comp'): + slot = comp.attrib['n'] + lod = next(l for l in comp.find('mod').findall('lod') if l.get('idx') == '0') + raw = resources.read(f'data/content/marquee/model/{recipe}/{slot}/{lod.attrib["arenaid"]}.rx2') + mat = definitions[lod.find('matinst/matvar').attrib['id']] + textures = {s.attrib['chn']: s.attrib['id'] for s in mat.findall('sp')} + materials['Retail_'+slot] = material_data(raw, collections, mdl_parser, + mask(marquee, resources.entries, 'marquee', textures.get('specular'))) + native[item['key']] = materials + except CONTENT_ERRORS as error: + note(directory/'lighting-availability.json', 'Pro character lighting', error) continue - definitions = {m.attrib['id']: m for m in root.findall('mat')} - materials = {} - for comp in root.findall('comp'): - slot = comp.attrib['n'] - lod = next(l for l in comp.find('mod').findall('lod') if l.get('idx') == '0') - raw = resources.read(f'data/content/marquee/model/{recipe}/{slot}/{lod.attrib["arenaid"]}.rx2') - mat = definitions[lod.find('matinst/matvar').attrib['id']] - textures = {s.attrib['chn']: s.attrib['id'] for s in mat.findall('sp')} - materials['Retail_'+slot] = material_data(raw, collections, mdl_parser, - mask(marquee, resources.entries, 'marquee', textures.get('specular'))) - native[item['key']] = materials (directory/'native-lighting.json').write_text(json.dumps(native)) (directory/'library-v3.json').write_text(json.dumps(library, separators=(',', ':'))) return native diff --git a/tools/asset_pipeline/customiser_setup.py b/tools/asset_pipeline/customiser_setup.py index 6dfc9651..518ddbb0 100644 --- a/tools/asset_pipeline/customiser_setup.py +++ b/tools/asset_pipeline/customiser_setup.py @@ -12,7 +12,7 @@ def fingerprint(tools=None): paths = {p for pattern in ('asset_pipeline/customis*.py', 'asset_pipeline/setup_state.py', 'asset_pipeline/native_roster.py', 'asset_pipeline/character_glb.py', 'asset_pipeline/retail_character.py', 'asset_pipeline/vlt.py', 'asset_pipeline/environment.py', 'asset_pipeline/marquee_assets.py', - 'asset_pipeline/names.txt', 'extract_default_skater.py', + 'asset_pipeline/names.txt', 'asset_pipeline/optional_content.py', 'extract_default_skater.py', 'owned_game/**/*.py', 'vendor/utt/**/*.py', 'vendor/utt/**/*.json', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/refpack_native.rs', 'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py', @@ -30,12 +30,39 @@ def source_fingerprint(game): digest = hashlib.sha256() for name in ('data/content/createacharacter.big', 'data/content/marquee.big', 'data/big/db.big', 'data/big/miscload.big', 'default.xex'): + if name == 'data/content/marquee.big' and not (game/name).is_file(): + digest.update(b'missing optional archive') + continue with (game/name).open('rb') as source: digest.update(hashlib.file_digest(source, 'sha256').digest()) return digest.hexdigest() def prepare(game, assets, report=print): + from .optional_content import CONTENT_ERRORS, note + from .setup_state import atomic_json + base = assets/'private/customisation' + availability = base/'customiser-availability.json' + try: + _prepare(game, assets, report) + availability.unlink(missing_ok=True) + except CONTENT_ERRORS as error: + # Each generation is published atomically. Do not replace working + # profiles/generation pointers when an optional preparation stage fails. + from .customiser_cache import read, complete + saved = read(base/'current.json') + retained = (isinstance(saved.get('set'), str) and re.fullmatch('[0-9a-f]{32}', saved['set']) + and complete(base/'sets'/saved['set'])) + result = note(availability, 'Character customiser', error, bool(retained), report) + result['fingerprint'] = fingerprint() + if retained:result['set'] = saved['set'] + atomic_json(availability, result) + if not retained: + # Remove only the pointer, not the user's profiles or old files. + (base/'current.json').unlink(missing_ok=True) + + +def _prepare(game, assets, report=print): from tools.asset_pipeline.customisation_catalog import prepare as catalog from tools.asset_pipeline.customisation_library import prepare as library from tools.asset_pipeline.customisation_profiles import generate @@ -79,12 +106,35 @@ def prepare(game, assets, report=print): report('Preparing character customiser clothing, bodies and textures') stage_versions = cache.versions() def run_stage(name, action): - cache.stage(directory, old_directory, name, stage_versions[name], source, action, assets, report) + def optional_action(): + from .optional_content import CONTENT_ERRORS, note + try:action() + except CONTENT_ERRORS as error: + if name not in ('lighting','roster'):raise + # Failed optional output is never published as complete artwork. + import shutil + for relative in cache.OUTPUTS[name]: + path=directory/relative + if not path.resolve().is_relative_to(directory.resolve()): + raise ValueError('Optional output escaped character generation') + if path.is_dir():shutil.rmtree(path) + else:path.unlink(missing_ok=True) + if name=='lighting': + (directory/'native-lighting.json').write_text('{}') + shutil.copy2(directory/'library-base.json',directory/'library-v3.json') + availability=directory/'lighting-availability.json' + else: + (directory/'native-roster').mkdir(exist_ok=True) + (directory/'native-roster/complete.json').write_text('{"characters":0}') + availability=directory/'native-roster/roster-availability.json' + note(availability,'Character '+name,error,report=report) + cache.stage(directory, old_directory, name, stage_versions[name], source, optional_action, assets, report) run_stage('catalog', lambda: catalog(game, directory)) def build_library(): data = library(dict(assets=str(assets), game_root=str(game), directory=str(directory), library_index='library-base.json')) - if data['errors'] or not data['models']: + if not data['models']: raise RuntimeError('Character library preparation failed: '+str(data['errors'])[:1000]) + for error in data['errors']:report('Unavailable customiser item: '+str(error)) for profile in data['defaults'].values(): for selection in profile['selections'].values(): if selection['asset_id'] not in data['models'] or selection['material_id'] not in data['materials']: @@ -98,11 +148,10 @@ def prepare(game, assets, report=print): # Work textures were not published or receipted; discard failed work. import shutil if (directory/'roster-work').exists():shutil.rmtree(directory/'roster-work') + (directory/'native-roster').mkdir(exist_ok=True) roster = native_roster(game, assets, directory/'native-roster', directory/'database/collections.json', directory/'roster-work') - if not any(item['status'] == 'ready' for item in roster) or any(item['status'] not in ('ready','unavailable') for item in roster): - raise RuntimeError('Native character roster preparation did not complete') - unavailable = [item for item in roster if item['status'] == 'unavailable'] + unavailable = [item for item in roster if item['status'] != 'ready'] for item in unavailable:report(f'Unavailable optional character {item["name"]}: {item["error"]}') (directory/'native-roster/complete.json').write_text(json.dumps({'characters': sum(item['status']=='ready' for item in roster), 'unavailable': unavailable})) run_stage('roster', build_roster) @@ -128,7 +177,7 @@ def install(iso, base, game_exe, report, refresh=False): with (Path(temp)/'extract.log').open('w') as log: core.run([extractor, '-x', selected, '-d', source], log, report) else: - source = source_directory(selected) + source = source_directory(selected, require_core=not refresh) stage = core._install(iso, base, game_exe, report, game_root=source, refresh=refresh, finalize=lambda stage: prepare(source, stage/'assets', report)) return stage diff --git a/tools/asset_pipeline/group_receipts.py b/tools/asset_pipeline/group_receipts.py index 1b44b0e7..3f7a05c1 100644 --- a/tools/asset_pipeline/group_receipts.py +++ b/tools/asset_pipeline/group_receipts.py @@ -6,13 +6,14 @@ ROOTS = { 'core': ('assets/private/stock/data/anim', 'assets/private/stock/data/state', 'assets/private/stock/data/config', 'assets/private/stock/data/cacrecipes', 'assets/private/stock/skater-collections.json', 'assets/private/stock/physics-skeletons.json'), - 'hud': ('assets/private/hud', 'assets/private/session-marker'), + 'hud': ('assets/private/hud', 'assets/private/session-marker', + 'assets/private/hud-availability.json', 'assets/private/session-marker-availability.json'), 'character': ('assets/private/game.json', 'assets/private/skater.glb', 'assets/private/default_skater', 'assets/private/native-character', 'assets/private/character-lighting.json'), 'environment': ('assets/private/native-skies', 'assets/private/native-backdrops', 'assets/private/render-parameters.json', 'assets/private/exposure.json', - 'assets/private/exposure-profiles.json', 'assets/private/teleports.json'), - 'maps': ('assets/private/native-props', 'maps.json'), + 'assets/private/exposure-profiles.json', 'assets/private/teleports.json', 'assets/private/environment-status'), + 'maps': ('assets/private/native-props', 'maps.json', 'assets/private/map-status'), } @@ -49,7 +50,7 @@ def damaged(root, marker, exclude=()): # Legacy packages have no receipts: require their declared roots # and verify the map checksums they already recorded. The runtime # validator also checks legacy core inputs before publication. - if any(not (root/name).exists() for name in roots):result.add(group) + if any(not (root/name).exists() for name in roots if not name.endswith(('-availability.json', '-status'))):result.add(group) if group == 'hud' and group not in result: from tools.install_prepared_hud import runtime_files try: diff --git a/tools/asset_pipeline/install.py b/tools/asset_pipeline/install.py index bebe5577..be4755a0 100644 --- a/tools/asset_pipeline/install.py +++ b/tools/asset_pipeline/install.py @@ -134,10 +134,16 @@ def convert_map(archive,work,maps,stage,game_exe,log,report): finished('write_map') from .dynamic_props import export as write_props caches=list((work/'dmo/cache').glob('DMO_*')) - if not caches: - raise RuntimeError('Missing prepared DMO catalog') - placed, unresolved=write_props(manifest_path,caches,stage/'assets/private/native-props'/(label+'.skate'), - catalog_path=work/'dmo/catalog.json') + from .optional_content import CONTENT_ERRORS, note + props=stage/'assets/private/native-props'/(label+'.skate') + try: + if not caches:raise RuntimeError('Movable-object source catalog is unavailable') + placed, unresolved=write_props(manifest_path,caches,props,catalog_path=work/'dmo/catalog.json') + (stage/'assets/private/native-props'/(label+'-availability.json')).unlink(missing_ok=True) + except CONTENT_ERRORS as error: + if props.is_dir():remove_intermediate(props,stage) + note(stage/'assets/private/native-props'/(label+'-availability.json'),label+' movable props',error,report=report) + placed,unresolved=0,0 finished('props') report(f'{label}: placed {placed} authored DMO instances, {unresolved} unresolved templates') report('Checking converted map: '+label) @@ -175,7 +181,7 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None base=base.resolve();base.mkdir(parents=True,exist_ok=True) from .setup_state import source_directory if game_root is not None: - game_root=source_directory(game_root) + game_root=source_directory(game_root, require_core=not previous) if previous and previous[1].get('source_hash') not in (None,digest(game_root/'default.xex')): raise RuntimeError('Select the same Xbox game edition used to set up this copy') if previous:groups.update(damaged(*previous, exclude=groups)) @@ -186,6 +192,8 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None with (base/'refresh-validation.log').open('w',encoding='utf-8') as log: run([game_exe,'--assets',previous[0]/'assets','--test-world','--check-assets'],log,report) if finalize:finalize(previous[0]) + from .optional_content import summary + summary(previous[0]) atomic_json(base/'installation.json', {**previous[1], 'pipelines':target_versions, 'outputs':outputs(previous[0]), 'source':source}) report('Game assets are current') @@ -235,8 +243,10 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None report('Extracting your ISO') run([extractor,'-x',iso,'-d',game_root],log,report) else:game_root=game_root.resolve() - for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big', - 'data/content/createacharacter.big','data/content/worldDIST_University.big']: + required_files=['default.xex'] + if 'core' in groups:required_files += ['data/big/miscload.big','data/big/miscboot.big','data/big/db.big'] + if 'character' in groups:required_files += ['data/content/createacharacter.big'] + for required in required_files: if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required) source_hash=digest(game_root/'default.xex') if previous and previous[1].get('source_hash',source_hash)!=source_hash: @@ -255,7 +265,13 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None if 'maps' in groups: report('Preparing authored movable-object models') from .dynamic_props import prepare_catalog - prepare_catalog(game_root,work/'dmo') + from .optional_content import CONTENT_ERRORS, note + try: + prepare_catalog(game_root,work/'dmo') + (private/'native-props/props-availability.json').unlink(missing_ok=True) + except CONTENT_ERRORS as error: + if (work/'dmo').exists():remove_intermediate(work/'dmo',work) + note(private/'native-props/props-availability.json','Movable props',error,report=report) report('Validating skater, input and animation data') run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) if 'maps' in groups: @@ -275,22 +291,55 @@ def _install(iso,base,game_exe,report,game_root=None,refresh=False,finalize=None # remain queued behind a string of small parks. futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)} for future in as_completed(futures): - archive=futures[future];completed[archive.name]=future.result() + archive=futures[future] + try:completed[archive.name]=future.result() + except CONTENT_ERRORS as error: + label=archive.stem.removeprefix('worldDIST_') + (maps/(label+'.skate')).unlink(missing_ok=True) + props=private/'native-props'/(label+'.skate') + if props.is_dir():remove_intermediate(props,private) + note(private/'map-status'/(label+'-availability.json'),label,error,report=report) + continue + (private/'map-status'/(completed[archive.name]['name']+'-availability.json')).unlink(missing_ok=True) report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}") - catalog=[completed[a.name] for a in archives] - if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted') + catalog=[completed[a.name] for a in archives if a.name in completed] + if previous: + # An absent/failed source district may still have a usable old + # converted copy. Validate its bytes AND load with this engine. + for old in json.loads((previous[0]/'maps.json').read_text()): + if any(item['path']==old['path'] for item in catalog):continue + src=(previous[0]/old['path']).resolve() + if not src.is_relative_to((previous[0]/'maps').resolve()):raise ValueError('Invalid old map path') + if not src.is_file() or digest(src)!=old.get('sha256'):continue + try:run([game_exe,'--assets',stage/'assets','--map',src,'--check-assets'],log,report) + except CONTENT_ERRORS:continue + target=stage/old['path'];target.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,target) + catalog.append(old) + note(private/'map-status'/(old['name']+'-availability.json'),old['name'], + RuntimeError('New source map unavailable; previous map passed validation'),retained=True,report=report) + # Exclude invalid old retail maps from the runtime directory scan. + valid_paths={item['path'] for item in catalog} + if previous: + for old in json.loads((previous[0]/'maps.json').read_text()): + if old['path'] not in valid_paths:(stage/old['path']).unlink(missing_ok=True) + if not catalog:raise RuntimeError('No playable map could be prepared or recovered. Restore at least one worldDIST_*.big archive beside default.xex and retry; the previous installation has been kept.') report('Validating installed runtime inputs') run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report) settings=stage/'settings';settings.mkdir(exist_ok=True) if not previous: - (settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8') + (settings/'default-map.json').write_text(json.dumps(next((m['path'] for m in catalog if m['name']=='University'),catalog[0]['path'])),encoding='utf-8') if 'maps' in groups: (stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8') + selected=settings/'default-map.json' + if selected.is_file() and not (stage/json.loads(selected.read_text())).is_file(): + selected.write_text(json.dumps(catalog[0]['path'])) remove_intermediate(work,stage) if finalize:finalize(stage) - # Publish only after every conversion, including the customiser, succeeds. + from .optional_content import summary + warnings=summary(stage) + # Publish after core validation and all optional outcomes have been recorded. marker=base/'installation.json.new' marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id,'source':source,'source_hash':source_hash,'pipelines':target_versions,'outputs':outputs(stage)}),encoding='utf-8') marker.replace(base/'installation.json') - report('Setup complete') + report(f'Setup complete ({len(warnings)} unavailable/retained components; see setup-report.json)' if warnings else 'Setup complete') return stage diff --git a/tools/asset_pipeline/native_roster.py b/tools/asset_pipeline/native_roster.py index bd0db6a5..642d0388 100644 --- a/tools/asset_pipeline/native_roster.py +++ b/tools/asset_pipeline/native_roster.py @@ -8,7 +8,8 @@ from tools.owned_game.big import BigArchive from tools.extract_default_skater import import_rx2_parser, decode_texture from tools.asset_pipeline.character_glb import Glb, convert from tools.asset_pipeline.retail_character import RX2, decode_dense_morphs -from tools.asset_pipeline.marquee_assets import Resources, MissingMarqueeAsset +from tools.asset_pipeline.marquee_assets import Resources +from tools.asset_pipeline.optional_content import CONTENT_ERRORS # GetCACSettings 82590BE0..82590DBC, TU3: six named styles, Aggressive otherwise. STYLES={'danny_way':'DannyWay','mike_carroll':'MikeCarroll','pj_ladd':'PJLadd', @@ -73,7 +74,7 @@ def prepare(game,assets,library,collections,work,only=None): if only and item['key'] not in only:continue try: resources.recipe(item['recipe']) - except MissingMarqueeAsset as error: + except CONTENT_ERRORS as error: report.append({**item,'status':'unavailable','error':str(error)}) print(f'Unavailable optional character {item["name"]}: {error}',flush=True) continue @@ -121,7 +122,7 @@ def prepare(game,assets,library,collections,work,only=None): stage.rename(target) report.append({**item,'id':identity,'status':'ready'}) print('READY',key,flush=True) - except Exception as e: + except CONTENT_ERRORS as e: report.append({**item,'status':'error','error':str(e)});print('ERROR',key,str(e),flush=True) (work/'report.json').write_text(json.dumps(report,indent=2)) return report diff --git a/tools/asset_pipeline/optional_content.py b/tools/asset_pipeline/optional_content.py new file mode 100644 index 00000000..7cdf4745 --- /dev/null +++ b/tools/asset_pipeline/optional_content.py @@ -0,0 +1,48 @@ +"""Isolate optional content failures while keeping validated gameplay mandatory.""" +import json +import traceback +import re +import struct +from xml.etree.ElementTree import ParseError +from .setup_state import atomic_json + +# Do not downgrade disk-full/permission errors, MemoryError, cancellation or +# arbitrary programming exceptions into an incomplete but successful install. +CONTENT_ERRORS = (FileNotFoundError, KeyError, ValueError, RuntimeError, StopIteration, EOFError, struct.error, ParseError) + + +def note(path, component, error, retained=False, report=print): + path.parent.mkdir(parents=True, exist_ok=True) + value = {'version': 1, 'component': component, + 'status': 'retained' if retained else 'unavailable', + 'error': str(error), 'traceback': ''.join(traceback.format_exception(type(error), error, error.__traceback__))} + atomic_json(path, value) + report(f'{component}: {"keeping previous content" if retained else "unavailable"}: {error}') + return value + + +def summary(stage): + warnings = [] + private = stage/'assets/private' + custom = private/'customisation' + current = {} + try:current = json.loads((custom/'current.json').read_text()) + except (OSError, ValueError):pass + identity = current.get('set', '') if isinstance(current, dict) else '' + active = custom/'sets'/identity if re.fullmatch('[0-9a-f]{32}', identity) else None + for path in private.rglob('*-availability.json'): + if path.is_relative_to(custom/'sets') and (active is None or not path.is_relative_to(active)): + continue + try: + item = json.loads(path.read_text()) + if isinstance(item, dict) and item.get('status') in ('retained', 'unavailable'): + warnings.append(item) + except (OSError, ValueError):continue + if active: + for filename, key, component in [('library-v3.json', 'errors', 'Customiser items'), + ('native-roster/complete.json', 'unavailable', 'Pro characters')]: + try:items = json.loads((active/filename).read_text()).get(key, []) + except (OSError, ValueError):continue + if items:warnings.append(dict(component=component, status='unavailable', items=items)) + atomic_json(stage/'setup-report.json', {'version': 1, 'warnings': warnings}) + return warnings diff --git a/tools/asset_pipeline/setup_state.py b/tools/asset_pipeline/setup_state.py index e2309e59..28f3ed71 100644 --- a/tools/asset_pipeline/setup_state.py +++ b/tools/asset_pipeline/setup_state.py @@ -6,15 +6,17 @@ import os from pathlib import Path -def source_directory(selected): +def source_directory(selected, require_core=True): selected = Path(selected).resolve() if selected.is_file() and selected.name.lower() == 'default.xex': selected = selected.parent elif not selected.is_dir(): raise RuntimeError('Select your Skate 3 default.xex or Xbox 360 ISO') - for name in ('default.xex', 'data/big/miscload.big', 'data/big/miscboot.big', - 'data/big/db.big', 'data/content/createacharacter.big', - 'data/content/marquee.big', 'data/content/worldDIST_University.big'): + required = ('default.xex',) + if require_core: + required += ('data/big/miscload.big', 'data/big/miscboot.big', + 'data/big/db.big', 'data/content/createacharacter.big') + for name in required: if not (selected/name).is_file(): raise RuntimeError('Keep the Skate 3 game content beside default.xex; missing '+name) return selected diff --git a/tools/asset_pipeline/test_customiser_setup.py b/tools/asset_pipeline/test_customiser_setup.py index 6380bfca..df1bc5e6 100644 --- a/tools/asset_pipeline/test_customiser_setup.py +++ b/tools/asset_pipeline/test_customiser_setup.py @@ -29,7 +29,7 @@ class CharacterSetup(unittest.TestCase): old = {'set': 'a'*32, 'fingerprint': 'old'} (base/'current.json').write_text(json.dumps(old)) with patch('tools.asset_pipeline.customisation_catalog.prepare', side_effect=RuntimeError('bad source')): - with self.assertRaises(RuntimeError): s.prepare(Path(temp)/'source', assets, lambda _: None) + with self.assertRaises(RuntimeError): s._prepare(Path(temp)/'source', assets, lambda _: None) self.assertEqual(json.loads((base/'current.json').read_text()), old) def test_crash_after_publication_cannot_resume_inside_the_live_generation(self): @@ -42,7 +42,7 @@ class CharacterSetup(unittest.TestCase): (base/'pending.json').write_text(json.dumps(old)) with patch.object(s,'fingerprint',return_value='v'), \ patch('tools.asset_pipeline.customisation_catalog.prepare',side_effect=RuntimeError('failed')): - with self.assertRaises(RuntimeError):s.prepare(Path(temp)/'source',assets,lambda _:None) + with self.assertRaises(RuntimeError):s._prepare(Path(temp)/'source',assets,lambda _:None) self.assertEqual((live/'catalog.json').read_text(),'working output') self.assertEqual(json.loads((base/'current.json').read_text()),old) self.assertNotEqual(json.loads((base/'pending.json').read_text())['set'],old['set']) @@ -66,16 +66,13 @@ class CharacterSetup(unittest.TestCase): def roster(game, assets, library, collections, work): library.mkdir(exist_ok=True) return roster_results.pop(0) - roster_results = [[], [{'status': 'ready', 'key': 'pro'}, + roster_results = [[{'status': 'ready', 'key': 'pro'}, {'status': 'unavailable', 'key': 'dem_bones', 'name': 'Dem Bones', 'error': 'missing head'}]] with patch('tools.asset_pipeline.customisation_catalog.prepare', side_effect=catalog), \ patch('tools.asset_pipeline.customisation_library.prepare', side_effect=library), \ patch('tools.asset_pipeline.customisation_profiles.generate', return_value=[]), \ patch('tools.asset_pipeline.customiser_lighting.prepare', side_effect=lighting), \ patch('tools.asset_pipeline.native_roster.prepare', side_effect=roster): - with self.assertRaisesRegex(RuntimeError, 'roster preparation'): - s.prepare(Path(temp)/'source', assets, lambda _: None) - self.assertFalse((assets/'private/customisation/current.json').exists()) s.prepare(Path(temp)/'source', assets, lambda _: None) base = assets/'private/customisation' current = json.loads((base/'current.json').read_text()) diff --git a/tools/asset_pipeline/test_optional_content.py b/tools/asset_pipeline/test_optional_content.py new file mode 100644 index 00000000..c19ec1ca --- /dev/null +++ b/tools/asset_pipeline/test_optional_content.py @@ -0,0 +1,92 @@ +"""Optional-content failure isolation and publication regression checks.""" +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from tools.asset_pipeline import customiser_setup as character, optional_content +from tools.asset_pipeline import test_versions +from tools.asset_pipeline import versions +from tools.asset_pipeline.install import install, digest + + +class OptionalContent(unittest.TestCase): + def test_missing_unchanged_source_archives_do_not_block_refresh(self): + with tempfile.TemporaryDirectory() as tmp: + base,old,source,current,saved=test_versions.AssetVersions().fixture(Path(tmp)) + for name in ('data/content/createacharacter.big','data/content/marquee.big','data/big/miscload.big'): + (source/name).unlink() + with patch.object(versions,'fingerprints',return_value=current),patch('tools.asset_pipeline.install.run'): + result=install(source,base,Path('unused.exe'),lambda _:None,refresh=True) + self.assertTrue((result/'maps/University.skate').is_file()) + + def test_failed_customiser_retains_valid_generation_and_profiles(self): + with tempfile.TemporaryDirectory() as tmp: + assets=Path(tmp)/'assets';base=assets/'private/customisation';base.mkdir(parents=True) + old={'set':'a'*32,'fingerprint':'old'} + (base/'current.json').write_text(json.dumps(old));(base/'profile.json').write_text('user choices') + with patch.object(character,'_prepare',side_effect=KeyError('missing CAC item')),patch('tools.asset_pipeline.customiser_cache.complete',return_value=True): + character.prepare(Path(tmp),assets,lambda _:None) + self.assertEqual(json.loads((base/'current.json').read_text()),old) + self.assertEqual((base/'profile.json').read_text(),'user choices') + self.assertEqual(json.loads((base/'customiser-availability.json').read_text())['status'],'retained') + + def test_missing_customiser_records_unavailable_but_disk_error_propagates(self): + with tempfile.TemporaryDirectory() as tmp: + assets=Path(tmp)/'assets' + with patch.object(character,'_prepare',side_effect=FileNotFoundError('missing CAC')): + character.prepare(Path(tmp),assets,lambda _:None) + self.assertEqual(json.loads((assets/'private/customisation/customiser-availability.json').read_text())['status'],'unavailable') + with patch.object(character,'_prepare',side_effect=OSError('disk full')): + with self.assertRaises(OSError):character.prepare(Path(tmp),assets,lambda _:None) + + def test_summary_ignores_abandoned_generations_and_includes_item_errors(self): + with tempfile.TemporaryDirectory() as tmp: + stage=Path(tmp);base=stage/'assets/private/customisation';base.mkdir(parents=True) + (base/'current.json').write_text(json.dumps({'set':'a'*32})) + for key in ('a','b'): + directory=base/'sets'/(key*32);directory.mkdir(parents=True) + optional_content.note(directory/'lighting-availability.json',key,ValueError('missing'),report=lambda _:None) + (base/'sets'/('a'*32)/'library-v3.json').write_text(json.dumps({'errors':[{'model':'broken'}]})) + warnings=optional_content.summary(stage) + self.assertEqual([w['component'] for w in warnings],['a','Customiser items']) + + def test_no_dangling_material_selections(self): + from tools.asset_pipeline.customisation_library import prune_unavailable + models={'good':{'groups':[['ok','absent']], 'materials':['ok','absent']}, + 'bad':{'groups':[['absent']], 'materials':['absent']}} + errors=[];prune_unavailable(models,{'ok':{}},errors) + self.assertEqual(list(models),['good']);self.assertEqual(models['good']['materials'],['ok']) + self.assertEqual(errors[0]['model'],'bad') + + def test_failed_map_uses_validated_previous_map_and_keeps_other_new_map(self): + self.map_refresh(recover=True) + + def test_no_playable_map_preserves_installation_record(self): + self.map_refresh(recover=False) + + def map_refresh(self,recover): + with tempfile.TemporaryDirectory() as tmp: + base,old,source,current,saved=test_versions.AssetVersions().fixture(Path(tmp)) + previous={'name':'University','path':'maps/University.skate','sha256':digest(old/'maps/University.skate')} + (old/'maps.json').write_text(json.dumps([previous]));saved['pipelines']=dict(current,maps='old') + from tools.asset_pipeline.group_receipts import record + saved['outputs']['maps']=record(old,'maps') + (base/'installation.json').write_text(json.dumps(saved)) + if recover:(source/'data/content/worldDIST_Test.big').write_bytes(b'other map') + def run(args,*rest): + args=list(map(str,args)) + if any('map_job.py' in a for a in args): + archive=Path(args[args.index('--archive')+1]) + if archive.stem.endswith('University'):raise RuntimeError('bad district') + stage=Path(args[args.index('--stage')+1]);target=stage/'maps/Test.skate';target.write_bytes(b'valid converted map') + Path(args[args.index('--result')+1]).write_text(json.dumps(dict(name='Test',path='maps/Test.skate',sha256=digest(target)))) + elif '--map' in args and not recover:raise RuntimeError('old map incompatible') + with patch.object(versions,'fingerprints',return_value=current),patch('tools.asset_pipeline.install.run',side_effect=run),patch('tools.asset_pipeline.dynamic_props.prepare_catalog',side_effect=FileNotFoundError('no props')): + if recover: + result=install(source,base,Path('unused.exe'),lambda _:None,refresh=True) + self.assertEqual({m['name'] for m in json.loads((result/'maps.json').read_text())},{'Test','University'}) + self.assertEqual((result/'maps/University.skate').read_bytes(),b'unchanged map') + else: + with self.assertRaisesRegex(RuntimeError,'No playable map'):install(source,base,Path('unused.exe'),lambda _:None,refresh=True) + self.assertEqual(json.loads((base/'installation.json').read_text()),saved) diff --git a/tools/asset_pipeline/test_versions.py b/tools/asset_pipeline/test_versions.py index 0efceae9..cc4245cd 100644 --- a/tools/asset_pipeline/test_versions.py +++ b/tools/asset_pipeline/test_versions.py @@ -88,7 +88,7 @@ class AssetVersions(unittest.TestCase): self.assertEqual((new/'maps/University.skate').read_bytes(),b'unchanged map') self.assertEqual((new/'settings/default-map.json').read_text(),'"custom.skate"') self.assertTrue((old/'assets/private/hud/old.txt').is_file()) - self.assertFalse((new/'assets/private/hud/old.txt').exists()) + self.assertTrue((new/'assets/private/hud/old.txt').exists()) # Kept until replacement validates. self.assertEqual(v.installed(base)[1]['pipelines'],current) self.assertTrue(any(any('prepare_runtime_huds.py' in a for a in args) for args in calls)) self.assertFalse(any(any('map_job.py' in a for a in args) for args in calls)) diff --git a/tools/asset_pipeline/versions.py b/tools/asset_pipeline/versions.py index 2247be69..921a554c 100644 --- a/tools/asset_pipeline/versions.py +++ b/tools/asset_pipeline/versions.py @@ -12,16 +12,16 @@ COMMON = ('owned_game/**/*.py', 'asset_pipeline/fast_refpack.py', 'asset_pipelin PARSERS = ('vendor/utt/**/*.py', 'vendor/university/**/*.py', 'vendor/utt/**/*.json', 'vendor/university/**/*.json') SOURCES = { 'core': ('asset_pipeline/vlt.py', 'asset_pipeline/names.txt', 'asset_pipeline/physics_skeleton.py'), - 'hud': ('prepare_hud.py', 'prepare_runtime_huds.py', 'install_prepared_hud.py', + 'hud': ('asset_pipeline/optional_content.py', 'prepare_hud.py', 'prepare_runtime_huds.py', 'install_prepared_hud.py', 'extract_session_marker.py', 'vendor/skate3_ui/**/*.py', 'vendor/skate3_ui/**/*.json'), 'character': PARSERS + ('extract_default_skater.py', 'default_skater_retail_manifest.json', 'asset_pipeline/character*.py', 'asset_pipeline/retail_character.py', 'vendor/skate3_anim/abin_importer.py', 'vendor/skate3_anim/rx2_skeleton.py'), - 'environment': PARSERS + ('asset_pipeline/sky.py', 'asset_pipeline/backdrop.py', + 'environment': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/sky.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py', 'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py', 'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py'), - 'maps': PARSERS + ('asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py', + 'maps': PARSERS + ('asset_pipeline/optional_content.py', 'asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py', 'asset_pipeline/environment.py', 'asset_pipeline/irradiance.py', 'asset_pipeline/retail_material.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/sky.py'), } diff --git a/tools/install_prepared_hud.py b/tools/install_prepared_hud.py index bba8ba50..b654e26c 100644 --- a/tools/install_prepared_hud.py +++ b/tools/install_prepared_hud.py @@ -59,6 +59,7 @@ def install(assets, scoring_cache, marker_cache, replace=False): plan = [] for source, name, marker in [(scoring_cache, "hud", False), (marker_cache, "session-marker", True)]: + if source is None:continue destination = assets / "private" / name for relative, sha256 in runtime_files(source, marker).items(): target = contained(destination, relative) diff --git a/tools/prepare_runtime_huds.py b/tools/prepare_runtime_huds.py index e938bc2f..46a6969c 100644 --- a/tools/prepare_runtime_huds.py +++ b/tools/prepare_runtime_huds.py @@ -21,11 +21,30 @@ def prepare(game, assets, work): def prepare_in_workspace(game, assets, work): + from asset_pipeline.optional_content import CONTENT_ERRORS, note + from install_prepared_hud import runtime_files scoring = work / 'scoring' marker = work / 'session-marker' - prepare_scoring(game, scoring, assets / 'private/stock/skater-collections.json') - prepare_marker(game, marker) - files = install(assets, scoring, marker) + files = {} + for name, source, action, is_marker in ( + ('hud', scoring, lambda: prepare_scoring(game, scoring, assets/'private/stock/skater-collections.json'), False), + ('session-marker', marker, lambda: prepare_marker(game, marker), True)): + availability=assets/'private'/(name+'-availability.json') + try: + action() + runtime_files(source, is_marker) + except CONTENT_ERRORS as error: + try: + runtime_files(assets/'private'/name,is_marker) + retained=True + except CONTENT_ERRORS: + retained=False + # Broken artwork must not be loaded as if it were complete. + (assets/'private'/name/('hud.json' if is_marker else 'runtime/trickdisplay.json')).unlink(missing_ok=True) + note(availability,name,error,retained) + continue + files.update(install(assets,None if is_marker else source,source if is_marker else None,replace=True)) + availability.unlink(missing_ok=True) print(f'Original runtime HUDs ready: {len(files)} verified files', flush=True) return files diff --git a/tools/setup.py b/tools/setup.py index d635d24f..6962ae14 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -55,8 +55,10 @@ def main(): button.config(state='disabled');running=True;progress.start() def work(): try: - install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating) - messages.put(('done','Ready')) + installed_root=install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating) + from tools.asset_pipeline.optional_content import summary + warnings=summary(installed_root) + messages.put(('done',f'Ready with {len(warnings)} unavailable components. Details: {installed_root / "setup-report.json"}' if warnings else 'Ready')) except Exception as error: args.base.mkdir(parents=True,exist_ok=True) (args.base/'setup-error.log').write_text(traceback.format_exc(),encoding='utf-8') @@ -73,6 +75,7 @@ def main(): while not messages.empty(): kind,text=messages.get_nowait();status.set(text) if kind=='done': + if text!='Ready':messagebox.showwarning('Setup completed with unavailable content',text,parent=window) running=False;success=True;progress.stop();window.destroy();return if kind=='error': running=False;progress.stop();button.config(state='normal') diff --git a/tools/test_setup_assets.py b/tools/test_setup_assets.py index 8ce872f1..8c3a0a71 100644 --- a/tools/test_setup_assets.py +++ b/tools/test_setup_assets.py @@ -1,37 +1,62 @@ -"""Setup output placement checks; no game or retail data required.""" +"""Independent HUD installation checks without retail data.""" +import json import sys from pathlib import Path import tempfile import unittest from unittest.mock import patch - sys.path.insert(0,str(Path(__file__).resolve().parent)) import prepare_runtime_huds as huds +def scoring(game, output, *args): + (output/'runtime').mkdir(parents=True) + (output/'runtime/trickdisplay.json').write_text(json.dumps(dict( + version=1, format='skate3-scoring-hud', shapes={}, fonts={}))) + + +def marker(game, output): + output.mkdir(parents=True) + (output/'hud.json').write_text(json.dumps(dict(version=1, canvas=[1280,720], meshes=[{}], textures=[]))) + + class HudWorkspace(unittest.TestCase): def test_long_install_uses_short_temporary_workspace_and_keeps_outputs(self): with tempfile.TemporaryDirectory() as temp: - root=Path(temp) - assets=root/'assets' + root=Path(temp); assets=root/'assets'; seen=[] long_work=root/('nested-install-'*12)/'conversion/hud' - seen=[] def prepare(game, output, *args): seen.append(output) self.assertNotIn(long_work,output.parents) self.assertLess(len(str(output/'source-cache/assets/data/fe/source/images/buttons/xbox360/buttons/0000_button_DPad_Down_hud.Texture.texture.bin')),260) - output.mkdir(parents=True) - def install(destination,scoring,marker): - destination.mkdir() - (destination/'verified').write_bytes(b'original artwork') - return ['verified'] - with patch.object(huds,'prepare_scoring',side_effect=prepare),patch.object(huds,'prepare_marker',side_effect=prepare),patch.object(huds,'install',side_effect=install): - self.assertEqual(huds.prepare(root/'game',assets,long_work),['verified']) - self.assertTrue((assets/'verified').is_file()) + scoring(game,output,*args) + with patch.object(huds,'prepare_scoring',side_effect=prepare),patch.object(huds,'prepare_marker',side_effect=marker): + self.assertEqual(len(huds.prepare(root/'game',assets,long_work)),2) + self.assertTrue((assets/'private/hud/runtime/trickdisplay.json').is_file()) self.assertTrue(all(not p.exists() for p in seen)) - def test_extraction_failure_never_publishes(self): - with patch.object(huds,'prepare_scoring',side_effect=RuntimeError('decode failed')),patch.object(huds,'install') as install: - with self.assertRaisesRegex(RuntimeError,'decode failed'): - huds.prepare(Path('game'),Path('assets'),Path('work')) - install.assert_not_called() + def test_one_missing_hud_still_installs_other_and_receipts_allow_reuse(self): + from tools.asset_pipeline.group_receipts import record, damaged + with tempfile.TemporaryDirectory() as temp: + root=Path(temp); assets=root/'assets' + with patch.object(huds,'prepare_scoring',side_effect=KeyError('missing font')),patch.object(huds,'prepare_marker',side_effect=marker): + self.assertEqual(len(huds.prepare(root/'game',assets,root/'work')),1) + self.assertEqual(json.loads((assets/'private/hud-availability.json').read_text())['status'],'unavailable') + receipt=record(root,'hud') + self.assertTrue(receipt) + self.assertEqual(damaged(root,{'outputs':{'hud':receipt}},exclude=('core','character','environment','maps')),set()) + + def test_failure_keeps_verified_existing_huds(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);assets=root/'assets' + scoring(None,assets/'private/hud');marker(None,assets/'private/session-marker') + before=(assets/'private/hud/runtime/trickdisplay.json').read_bytes() + with patch.object(huds,'prepare_scoring',side_effect=RuntimeError('decode failed')),patch.object(huds,'prepare_marker',side_effect=FileNotFoundError('missing bank')): + self.assertEqual(huds.prepare(root/'game',assets,root/'work'),{}) + self.assertEqual((assets/'private/hud/runtime/trickdisplay.json').read_bytes(),before) + for name in ('hud','session-marker'): + self.assertEqual(json.loads((assets/'private'/(name+'-availability.json')).read_text())['status'],'retained') + + def test_disk_errors_still_abort(self): + with tempfile.TemporaryDirectory() as temp,patch.object(huds,'prepare_scoring',side_effect=PermissionError('read denied')): + with self.assertRaises(PermissionError):huds.prepare(Path(temp),Path(temp)/'assets',Path(temp)/'work')