Give each portable copy its own setup and refresh changed asset groups

This commit is contained in:
chazmm committed 2026-09-09 03:00:59 +01:00
1 parent 3ea9b4aa51
commit a4a82f5705
10 files changed
+339 -105

No files matched your search

+106 -78
View File
@@ -152,7 +152,15 @@ def convert_map(archive,work,maps,stage,game_exe,log,report):
return entry
def install(iso,base,game_exe,report,game_root=None):
def install(iso,base,game_exe,report,game_root=None,refresh=False):
from .versions import fingerprints, changed_groups, installed, GROUPS
target_versions=fingerprints()
previous=installed(base) if refresh else None
groups=changed_groups(previous[1].get('pipelines',{}),target_versions) if previous else set(GROUPS)
if not groups:
report('Game assets are already current')
return previous[0]
source=str((iso if iso is not None else game_root).resolve())
if game_root is None and iso is not None:
selected=iso.resolve()
if selected.is_dir():game_root=selected
@@ -169,8 +177,13 @@ def install(iso,base,game_exe,report,game_root=None):
install_id=uuid.uuid4().hex
stage=base/'installations'/install_id
stage.mkdir(parents=True)
private=stage/'assets/private';private.mkdir(parents=True)
maps=stage/'maps';maps.mkdir()
if previous:
report('Preparing an asset update; keeping the previous installation until it succeeds')
for name in ('assets','maps','settings'):
if (previous[0]/name).is_dir():shutil.copytree(previous[0]/name,stage/name)
shutil.copy2(previous[0]/'maps.json',stage/'maps.json')
private=stage/'assets/private';private.mkdir(parents=True,exist_ok=True)
maps=stage/'maps';maps.mkdir(exist_ok=True)
work=stage/'conversion';work.mkdir()
with (stage/'setup.log').open('w',encoding='utf-8') as log:
if game_root is None:
@@ -184,89 +197,104 @@ def install(iso,base,game_exe,report,game_root=None):
for required in ['default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big',
'data/content/createacharacter.big','data/content/worldDIST_University.big']:
if not (game_root/required).is_file():raise RuntimeError('This is not a supported Skate 3 disc: missing '+required)
report('Extracting animation banks, graphs and gameplay inputs')
source_hash=digest(game_root/'default.xex')
if previous and previous[1].get('source_hash',source_hash)!=source_hash:
raise RuntimeError('Select the same Xbox game edition used to set up this copy')
stock=private/'stock'
extract(game_root/'data/big/miscload.big',stock)
extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg')
# Some disc banks are loose files rather than members of miscload.
loose=game_root/'data/anim'
if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True)
report('Converting physics and difficulty settings')
database=work/'database'
extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in {
'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'})
names=(TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines()
converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names)
(stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8')
skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin')
(stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8')
report('Preparing original scoring and session-marker HUD assets')
run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root,
'--assets', stage/'assets', '--work', work/'hud'), log, report)
report('Preparing the skater model and textures')
manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text())
needed=set()
for c in manifest['components']:
needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower())
needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values())
extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed)
character=work/'character'
run(task(TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character,
'--private-root',private/'default_skater','--utt-root',TOOLS/'vendor/utt'),log,report)
report('Building the skater model and rig')
from .character_glb import convert as write_character
write_character(character/'selected/models',private,manifest)
from .character_lighting import convert as write_character_lighting
write_character_lighting(character/'selected/models',private,converted)
game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000',
'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph',
'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'}
(private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
report('Preparing retail sky domes')
from .sky import convert as write_skies
write_skies(game_root,stage/'assets',converted)
from .render_parameters import convert as write_render_parameters
write_render_parameters(stage/'assets',converted)
report('Extracting original travel destinations and location names')
from .teleports import convert as write_teleports
write_teleports(game_root,stage/'assets',converted)
report('Preparing global foliage backdrops')
from .backdrop import convert as write_backdrops
write_backdrops(game_root,stage/'assets',converted)
report('Preparing authored movable-object models')
from .dynamic_props import prepare_catalog
prepare_catalog(game_root,work/'dmo')
if 'core' in groups:
report('Extracting animation banks, graphs and gameplay inputs')
extract(game_root/'data/big/miscload.big',stock)
extract(game_root/'data/big/miscboot.big',stock,lambda e:e.path.lower()=='data/config/input.cfg')
# Some disc banks are loose files rather than members of miscload.
loose=game_root/'data/anim'
if loose.is_dir():shutil.copytree(loose,stock/'data/anim',dirs_exist_ok=True)
report('Converting physics and difficulty settings')
database=work/'database'
extract(game_root/'data/big/db.big',database,lambda e:Path(e.path).name.lower() in {
'skaterschema.bin','skaterschema.vlt','skatercollections.bin','skatercollections.vlt'})
names=(TOOLS/'asset_pipeline/names.txt').read_text(encoding='utf-8').splitlines()
converted=convert_vlt(database/'data/db/skaterschema',database/'data/db/skatercollections',names)
(stock/'skater-collections.json').write_text(json.dumps(converted),encoding='utf-8')
skeleton=convert_skeleton(stock/'data/anim/OnBoard.abin')
(stock/'physics-skeletons.json').write_text(json.dumps(skeleton),encoding='utf-8')
else:
converted=json.loads((stock/'skater-collections.json').read_text(encoding='utf-8'))
if 'hud' in groups:
for folder in (private/'hud',private/'session-marker'):
if folder.is_dir():remove_intermediate(folder,stage)
report('Preparing original scoring and session-marker HUD assets')
run(task(TOOLS/'prepare_runtime_huds.py', '--game', game_root,
'--assets', stage/'assets', '--work', work/'hud'), log, report)
if 'character' in groups:
report('Preparing the skater model and textures')
manifest=json.loads((TOOLS/'default_skater_retail_manifest.json').read_text())
needed=set()
for c in manifest['components']:
needed.add(f"data/content/createacharacter/model/cas_db/{c['slot']}/0x{c['model_id']}.rx2".lower())
needed.update(f'data/content/createacharacter/texture/0x{x}.rx2'.lower() for x in c['textures'].values())
extract(game_root/'data/content/createacharacter.big',stock,lambda e:e.path.lower() in needed)
character=work/'character'
run(task(TOOLS/'extract_default_skater.py','--owned-data-root',stock,'--work-root',character,
'--private-root',private/'default_skater','--utt-root',TOOLS/'vendor/utt'),log,report)
report('Building the skater model and rig')
from .character_glb import convert as write_character
write_character(character/'selected/models',private,manifest)
from .character_lighting import convert as write_character_lighting
write_character_lighting(character/'selected/models',private,converted)
game_manifest={'version':1,'character_scene':'private/skater.glb','initial_animation':'R_IDLE_HCOM_000',
'action_graph':'private/stock/data/state/ActionGraph_OnBoard.stategraph',
'motion_graph':'private/stock/data/state/MotionGraph_OnBoard.stategraph'}
(private/'game.json').write_text(json.dumps(game_manifest),encoding='utf-8')
if 'environment' in groups:
report('Preparing retail sky domes')
from .sky import convert as write_skies
write_skies(game_root,stage/'assets',converted)
from .render_parameters import convert as write_render_parameters
write_render_parameters(stage/'assets',converted)
report('Extracting original travel destinations and location names')
from .teleports import convert as write_teleports
write_teleports(game_root,stage/'assets',converted)
report('Preparing global foliage backdrops')
from .backdrop import convert as write_backdrops
write_backdrops(game_root,stage/'assets',converted)
if 'maps' in groups:
report('Preparing authored movable-object models')
from .dynamic_props import prepare_catalog
prepare_catalog(game_root,work/'dmo')
report('Validating skater, input and animation data')
run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
archives=list((game_root/'data/content').glob('worldDIST_*.big'))
archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower()))
workers=map_workers()
report(f'Converting {len(archives)} maps with {workers} workers')
def map_job(archive):
result=work/(archive.stem+'.json')
with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log:
run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage,
'--game-exe',game_exe,'--result',result),map_log,report)
return json.loads(result.read_text(encoding='utf-8'))
completed={}
with ThreadPoolExecutor(max_workers=workers) as pool:
# Start the expensive districts together so one does not
# remain queued behind a string of small parks.
futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)}
for future in as_completed(futures):
archive=futures[future];completed[archive.name]=future.result()
report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}")
catalog=[completed[a.name] for a in archives]
if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted')
if 'maps' in groups:
archives=list((game_root/'data/content').glob('worldDIST_*.big'))
archives.sort(key=lambda p:(p.stem!='worldDIST_University',p.name.lower()))
workers=map_workers()
report(f'Converting {len(archives)} maps with {workers} workers')
def map_job(archive):
result=work/(archive.stem+'.json')
with (stage/(archive.stem+'-conversion.log')).open('w',encoding='utf-8') as map_log:
run(task(TOOLS/'asset_pipeline/map_job.py','--archive',archive,'--stage',stage,
'--game-exe',game_exe,'--result',result),map_log,report)
return json.loads(result.read_text(encoding='utf-8'))
completed={}
with ThreadPoolExecutor(max_workers=workers) as pool:
# Start the expensive districts together so one does not
# remain queued behind a string of small parks.
futures={pool.submit(map_job,a):a for a in sorted(archives,key=lambda p:-p.stat().st_size)}
for future in as_completed(futures):
archive=futures[future];completed[archive.name]=future.result()
report(f"Converted {len(completed)}/{len(archives)} maps: {completed[archive.name]['name']}")
catalog=[completed[a.name] for a in archives]
if not any(m['name']=='University' for m in catalog):raise RuntimeError('University was not converted')
report('Validating installed runtime inputs')
run([game_exe,'--assets',stage/'assets','--test-world','--check-assets'],log,report)
settings=stage/'settings';settings.mkdir()
(settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8')
(stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8')
settings=stage/'settings';settings.mkdir(exist_ok=True)
if not previous:
(settings/'default-map.json').write_text(json.dumps('maps/University.skate'),encoding='utf-8')
if 'maps' in groups:
(stage/'maps.json').write_text(json.dumps(catalog,indent=2),encoding='utf-8')
remove_intermediate(work,stage)
# Publish only after every conversion and the runtime's own load succeeds.
marker=base/'installation.json.new'
marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id}),encoding='utf-8')
marker.write_text(json.dumps({'version':1,'directory':'installations/'+install_id,'source':source,'source_hash':source_hash,'pipelines':target_versions}),encoding='utf-8')
marker.replace(base/'installation.json')
report('Setup complete')
return stage
+72
View File
@@ -0,0 +1,72 @@
"""Installation selection and refresh checks using synthetic files only."""
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from tools.asset_pipeline import versions as v
from tools.asset_pipeline.install import install
class AssetVersions(unittest.TestCase):
def test_hud_change_does_not_reconvert_maps(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp)
(root/'prepare_hud.py').write_text('before')
before=v.fingerprints(root)
(root/'prepare_hud.py').write_text('after')
self.assertEqual(v.changed_groups(before,v.fingerprints(root)), {'hud'})
(root/'asset_pipeline').mkdir()
(root/'asset_pipeline/vlt.py').write_text('new decoder')
self.assertEqual(v.changed_groups(before,v.fingerprints(root)),set(v.GROUPS))
def test_fresh_copy_never_adopts_other_installation(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp)
(root/'installation.json').write_text(json.dumps({'version':1,'directory':'../old'}))
with self.assertRaises(ValueError):v.installed(root)
self.assertIsNone(v.installed(root/'new-copy/data'))
def fixture(self, root):
base=root/'copy/data';old=base/('installations/'+'a'*32)
private=old/'assets/private';(private/'stock').mkdir(parents=True)
(private/'game.json').write_text('{}');(private/'stock/skater-collections.json').write_text('{}')
(private/'hud').mkdir();(private/'hud/old.txt').write_text('old')
(old/'maps').mkdir();(old/'maps/University.skate').write_bytes(b'unchanged map')
(old/'settings').mkdir();(old/'settings/default-map.json').write_text('"custom.skate"')
(old/'maps.json').write_text('[{"name":"University"}]')
current={group:'new' for group in v.GROUPS}
marker={'version':1,'directory':'installations/'+'a'*32,'pipelines':{**current,'hud':'old'}}
(base/'installation.json').write_text(json.dumps(marker))
source=root/'disc'
for name in ('default.xex','data/big/miscload.big','data/big/miscboot.big','data/big/db.big',
'data/content/createacharacter.big','data/content/worldDIST_University.big'):
p=source/name;p.parent.mkdir(parents=True,exist_ok=True);p.touch()
return base,old,source,current,marker
def test_refresh_preserves_maps_and_publishes_only_after_validation(self):
with tempfile.TemporaryDirectory() as temp:
base,old,source,current,marker=self.fixture(Path(temp))
calls=[]
def run(args,*unused):
calls.append([str(a) for a in args])
with patch.object(v,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run',side_effect=run):
new=install(source,base,Path('unused.exe'),lambda _:None,refresh=True)
self.assertEqual((new/'maps/University.skate').read_bytes(),b'unchanged map')
self.assertEqual((new/'settings/default-map.json').read_text(),'"custom.skate"')
self.assertTrue((old/'assets/private/hud/old.txt').is_file())
self.assertFalse((new/'assets/private/hud/old.txt').exists())
self.assertEqual(v.installed(base)[1]['pipelines'],current)
self.assertTrue(any(any('prepare_runtime_huds.py' in a for a in args) for args in calls))
self.assertFalse(any(any('map_job.py' in a for a in args) for args in calls))
def test_failed_refresh_keeps_previous_record(self):
with tempfile.TemporaryDirectory() as temp:
base,old,source,current,marker=self.fixture(Path(temp))
with patch.object(v,'fingerprints',return_value=current), patch('tools.asset_pipeline.install.run',side_effect=RuntimeError('failed')):
with self.assertRaises(RuntimeError):install(source,base,Path('unused.exe'),lambda _:None,refresh=True)
self.assertEqual(v.installed(base)[1],marker)
self.assertFalse((base/'setup.lock').exists())
if __name__=='__main__':unittest.main()
+69
View File
@@ -0,0 +1,69 @@
"""Content identities for independently refreshable owned-disc asset groups."""
import argparse
import hashlib
import json
from pathlib import Path
import re
GROUPS = ('core', 'hud', 'character', 'environment', 'maps')
COMMON = ('owned_game/**/*.py', 'asset_pipeline/fast_refpack.py', 'asset_pipeline/install.py',
'asset_pipeline/versions.py', 'asset_pipeline/refpack_native.rs')
PARSERS = ('vendor/utt/**/*.py', 'vendor/university/**/*.py', 'vendor/utt/**/*.json', 'vendor/university/**/*.json')
SOURCES = {
'core': ('asset_pipeline/vlt.py', 'asset_pipeline/names.txt', 'asset_pipeline/physics_skeleton.py'),
'hud': ('prepare_hud.py', 'prepare_runtime_huds.py', 'install_prepared_hud.py',
'extract_session_marker.py', 'vendor/skate3_ui/**/*.py', 'vendor/skate3_ui/**/*.json'),
'character': PARSERS + ('extract_default_skater.py', 'default_skater_retail_manifest.json',
'asset_pipeline/character*.py'),
'environment': PARSERS + ('asset_pipeline/sky.py', 'asset_pipeline/backdrop.py',
'asset_pipeline/render_parameters.py', 'asset_pipeline/teleports.py',
'asset_pipeline/environment.py', 'asset_pipeline/map_writer.py',
'asset_pipeline/retail_material.py', 'asset_pipeline/irradiance.py'),
'maps': PARSERS + ('asset_pipeline/map*.py', 'asset_pipeline/dynamic_props.py',
'asset_pipeline/environment.py', 'asset_pipeline/irradiance.py',
'asset_pipeline/retail_material.py', 'asset_pipeline/backdrop.py', 'asset_pipeline/sky.py'),
}
def fingerprints(tools=None):
tools = tools or Path(__file__).resolve().parents[1]
result = {}
for group, patterns in SOURCES.items():
paths = {p for pattern in COMMON + patterns for p in tools.glob(pattern)
if p.is_file() and not p.name.startswith('test_') and '__pycache__' not in p.parts}
digest = hashlib.sha256()
for path in sorted(paths, key=lambda p: p.relative_to(tools).as_posix()):
digest.update(path.relative_to(tools).as_posix().encode() + b'\0')
digest.update(hashlib.sha256(path.read_bytes()).digest())
result[group] = digest.hexdigest()
return result
def changed_groups(previous, current):
changed = {name for name in GROUPS if previous.get(name) != current[name]}
# Other exports consume the core VLT/skeleton/animation data.
if 'core' in changed:
changed.update(GROUPS)
return changed
def installed(base):
marker = base/'installation.json'
if not marker.is_file():
return None
value = json.loads(marker.read_text(encoding='utf-8-sig'))
directory = value.get('directory', '')
if value.get('version') != 1 or not re.fullmatch(r'installations/[0-9a-f]{32}', directory):
raise ValueError('Invalid installation record')
root = (base/directory).resolve()
if not root.is_relative_to(base.resolve()):
raise ValueError('Installation escapes its package data directory')
if not (root/'assets/private/game.json').is_file():
return None
return root, value
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--tools', type=Path)
print(json.dumps(fingerprints(parser.parse_args().tools)))
+12 -6
View File
@@ -18,33 +18,39 @@ def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base',type=Path,required=True)
parser.add_argument('--game-exe',type=Path,required=True)
parser.add_argument('--refresh',action='store_true')
args=parser.parse_args()
import tkinter as tk
from tkinter import filedialog,messagebox,ttk
from tools.asset_pipeline.install import install
from tools.asset_pipeline.versions import installed, fingerprints, changed_groups
previous=installed(args.base) if args.refresh else None
changed=changed_groups(previous[1].get('pipelines',{}),fingerprints()) if previous else set()
updating=previous is not None
window=tk.Tk()
window.title('Skate 3 Rust Engine setup')
window.geometry('700x420');window.resizable(False,False)
icon=ROOT/'docs/images/skating-crab.ico'
if icon.is_file():window.iconbitmap(str(icon))
frame=ttk.Frame(window,padding=24);frame.pack(fill='both',expand=True)
ttk.Label(frame,text='Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16))
ttk.Label(frame,text='Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.',
ttk.Label(frame,text='Update game assets' if updating else 'Set up Skate 3 Rust Engine',font=('Segoe UI',20)).pack(anchor='w',pady=(0,16))
ttk.Label(frame,text=('Changed asset groups: '+', '.join(sorted(changed))+'.\nOnly these groups will be extracted again.\nYour previous installation is kept until the update succeeds.\nUse Update game assets to reuse your selected Xbox source,\nor choose its new location if it has moved.') if updating else 'Select your Skate 3 Xbox 360 ISO, or default.xex inside an\nextracted game folder. Keep the game data beside default.xex.\nSetup prepares the skater, animations and all disc maps.\nNo other apps need installing.\n\nISO extraction needs internet access. Allow free disk space\nand time for the first conversion.',
font=('Segoe UI',11),justify='left').pack(anchor='w')
status=tk.StringVar(value='Choose your game to begin.')
ttk.Label(frame,textvariable=status,wraplength=600).pack(anchor='w',pady=(18,8))
progress=ttk.Progressbar(frame,mode='indeterminate');progress.pack(fill='x')
messages=queue.Queue();running=False;success=False
def start(folder=False):
def start(folder=False,reuse=False):
nonlocal running
iso=(filedialog.askdirectory(parent=window,title='Select the Skate 3 folder containing default.xex') if folder else
saved=previous[1].get('source') if previous else None
iso=saved if reuse and saved and Path(saved).exists() else (filedialog.askdirectory(parent=window,title='Select the Skate 3 folder containing default.xex') if folder else
filedialog.askopenfilename(parent=window,title='Select your Skate 3 Xbox 360 game',
filetypes=[('Xbox 360 game','*.iso *.xex'),('Xbox 360 ISO','*.iso'),('Xbox executable','*.xex')]))
if not iso:return
button.config(state='disabled');folder_button.config(state='disabled');running=True;progress.start()
def work():
try:
install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)))
install(Path(iso),args.base,args.game_exe,lambda text:messages.put(('progress',text)),refresh=updating)
messages.put(('done','Ready'))
except Exception as error:
args.base.mkdir(parents=True,exist_ok=True)
@@ -57,7 +63,7 @@ def main():
else:window.destroy()
buttons=ttk.Frame(frame);buttons.pack(anchor='e',pady=18)
folder_button=ttk.Button(buttons,text='Select extracted folder',command=lambda:start(True));folder_button.pack(side='left',padx=(0,8))
button=ttk.Button(buttons,text='Select ISO or default.xex',command=start);button.pack(side='left')
button=ttk.Button(buttons,text='Update game assets' if updating else 'Select ISO or default.xex',command=lambda:start(reuse=updating));button.pack(side='left')
def poll():
nonlocal running,success
while not messages.empty():