From b3b1517a5c9994b7ed1a6a3205edc70c061ab7bd Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:34:55 +0530 Subject: [PATCH 01/31] feat(web): add batch sizing and output naming Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/archive.mjs | 47 ++++++++++++++++++++++++--------- demo/web/tests/archive.test.mjs | 12 ++++++++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/demo/web/archive.mjs b/demo/web/archive.mjs index f1a30d2e42..8ea97a0c6e 100644 --- a/demo/web/archive.mjs +++ b/demo/web/archive.mjs @@ -208,6 +208,23 @@ export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entrie return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024); } +export function batchMemoryEstimate(summaries, target, libraryBytes) { + if (target !== "linux" && target !== "windows") throw new Error(`Unsupported conversion target: ${target}`); + return summaries.reduce((total, summary) => safeSize(total + memoryEstimate( + summary.totalBytes, + target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes, + libraryBytes, + summary.files + )), 0); +} + +export function batchOutputDirectory(name, index) { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid app index"); + const stem = String(name).replace(/\.zip$/iu, "").normalize("NFKC") + .replace(/[^A-Za-z0-9._-]+/gu, "-").replace(/^[.-]+/u, "").slice(0, 80).replace(/[. -]+$/u, ""); + return `${String(index + 1).padStart(2, "0")}-${stem || "app"}`; +} + async function appendEntry(writer, name, entry, report) { const stream = new TransformStream(); let produced = 0; @@ -282,13 +299,19 @@ class DirectoryWriter { async close() {} } -export async function packageArchive(archive, converted, outputName, libraryFile, destination, report, directory = false) { +export async function packageArchive(archive, converted, outputName, libraryFile, destination, report, directory = false, options = {}) { const libraries = new ZipReader(new BoundedBlobReader(libraryFile)); - const writer = directory ? new DirectoryWriter(destination) : new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true }); + const ownsWriter = !options.writer; + const writer = options.writer ?? (directory ? new DirectoryWriter(destination) : new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true })); + const outputPath = name => { + const path = options.prefix ? `${options.prefix}${name}` : name; + validPath(path); + return path; + }; const names = new Set(); const claim = name => { - validPath(name); - const key = name.replace(/\/$/u, "").toLowerCase(); + const path = outputPath(name); + const key = path.replace(/\/$/u, "").toLowerCase(); if (names.has(key)) throw new Error(`Output path collision: ${name}`); names.add(key); }; @@ -309,18 +332,18 @@ export async function packageArchive(archive, converted, outputName, libraryFile claim(module.name); } const binary = { versionMadeBy: 0x31e, externalFileAttributes: (0o100755 << 16) >>> 0 }; - await writer.add(outputName, new Uint8ArrayReader(converted.executable), binary); - if (converted.diagnostics) await writer.add("windows-diagnostics-imports.txt", new Uint8ArrayReader(new TextEncoder().encode(converted.diagnostics))); - for (const module of converted.modules) await writer.add(module.name, new Uint8ArrayReader(module.bytes), binary); + await writer.add(outputPath(outputName), new Uint8ArrayReader(converted.executable), binary); + if (converted.diagnostics) await writer.add(outputPath("windows-diagnostics-imports.txt"), new Uint8ArrayReader(new TextEncoder().encode(converted.diagnostics))); + for (const module of converted.modules) await writer.add(outputPath(module.name), new Uint8ArrayReader(module.bytes), binary); for (const entry of libraryEntries) { - if (entry.directory) await writer.add(entry.filename, undefined, { directory: true }); - else await appendEntry(writer, entry.filename, entry, report); + if (entry.directory) await writer.add(outputPath(entry.filename), undefined, { directory: true }); + else await appendEntry(writer, outputPath(entry.filename), entry, report); } for (const { entry, name } of originalFiles) { - if (entry.directory) await writer.add("app0/" + name, undefined, { directory: true }); - else await appendEntry(writer, "app0/" + name, entry, report); + if (entry.directory) await writer.add(outputPath("app0/" + name), undefined, { directory: true }); + else await appendEntry(writer, outputPath("app0/" + name), entry, report); } - await writer.close(); + if (ownsWriter) await writer.close(); } finally { await libraries.close(); } diff --git a/demo/web/tests/archive.test.mjs b/demo/web/tests/archive.test.mjs index c176ff2d38..4d375b2114 100644 --- a/demo/web/tests/archive.test.mjs +++ b/demo/web/tests/archive.test.mjs @@ -7,7 +7,7 @@ import { register } from "node:module"; register("./cdn-loader.mjs", import.meta.url, { data: { url: process.env.ANYPS5_TEST_ZIP_URL, path: process.env.ANYPS5_TEST_ZIP_PATH } }); const site = name => pathToFileURL(path.join(process.argv[2], name)).href; -const { Archive, memoryEstimate, MEMORY_LIMIT, ICON_LIMIT, packageArchive } = await import(site("archive.mjs")); +const { Archive, batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT, ICON_LIMIT, packageArchive } = await import(site("archive.mjs")); const { BlobReader, BlobWriter, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter } = await import(process.env.ANYPS5_TEST_ZIP_URL); await import("./archive-reserved.test.mjs"); const { default: createRelinker } = await import(site("native/relinker_wasm.mjs")); @@ -134,6 +134,16 @@ assert.equal(selfModuleSummary.warnings.length, 2); assert.match(selfModuleSummary.warnings.join(" "), /SELF container/u); await selfModule.close(); assert(memoryEstimate(200 * 1024 ** 3, 1024, 1024, 100) > MEMORY_LIMIT); +assert(batchMemoryEstimate([ + { totalBytes: 1024, selectedBytes: 128, windowsSelectedBytes: 256, files: 2 }, + { totalBytes: 2048, selectedBytes: 256, windowsSelectedBytes: 512, files: 3 } +], "linux", 1024) > memoryEstimate(2048, 256, 1024, 3)); +assert.equal(batchOutputDirectory("First App.zip", 0), "01-First-App"); +assert.equal(batchOutputDirectory("First App.zip", 1), "02-First-App"); +assert.equal(batchOutputDirectory("CON.zip", 0), "01-CON"); +assert.equal(batchOutputDirectory("...zip", 0), "01-app"); +assert.throws(() => batchOutputDirectory("app.zip", -1), /Invalid app index/u); +assert.throws(() => batchMemoryEstimate([], "macos", 0), /Unsupported conversion target/u); const invalid = native.Convert({ target: "unsupported" }, () => {}); assert.equal(invalid.ok, false); console.log("Archive validation, missing-folder warnings and large-download budget verified"); From c64fa3d580246cb9fe3264aec60b12b17ed18792 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:36:20 +0530 Subject: [PATCH 02/31] test(web): cover app bundles in a shared archive Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/tests/archive.test.mjs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/demo/web/tests/archive.test.mjs b/demo/web/tests/archive.test.mjs index 4d375b2114..c8801c40d5 100644 --- a/demo/web/tests/archive.test.mjs +++ b/demo/web/tests/archive.test.mjs @@ -111,6 +111,29 @@ for (const target of ["windows", "linux"]) { } } +const bundleArchive = new Archive(source); +await bundleArchive.inspect(() => {}); +const bundleInput = await bundleArchive.conversionInput("linux", false, () => {}); +const bundleConversion = native.Convert(bundleInput, () => {}); +assert.equal(bundleConversion.ok, true, bundleConversion.error); +const bundleLibraries = await zip([["libs/system.prx", new Uint8Array([1, 2, 3])]]); +const bundleWriter = new ZipWriter(new BlobWriter("application/zip"), { zip64: true, level: 9, useWebWorkers: false, useCompressionStream: false }); +await packageArchive(bundleArchive, bundleConversion, bundleInput.outputName, bundleLibraries, null, () => {}, false, { prefix: "01-first/", writer: bundleWriter }); +await packageArchive(bundleArchive, bundleConversion, bundleInput.outputName, bundleLibraries, null, () => {}, false, { prefix: "02-second/", writer: bundleWriter }); +const bundle = await bundleWriter.close(); +const bundleReader = new ZipReader(new BlobReader(bundle)); +const bundleEntries = await bundleReader.getEntries(); +const bundleNames = new Set(bundleEntries.map(entry => entry.filename)); +assert(bundleNames.has(`01-first/${bundleInput.outputName}`)); +assert(bundleNames.has("01-first/libs/system.prx")); +assert(bundleNames.has("01-first/app0/content/resource.bin")); +assert(bundleNames.has(`02-second/${bundleInput.outputName}`)); +assert(bundleNames.has("02-second/libs/system.prx")); +assert(bundleNames.has("02-second/app0/content/resource.bin")); +await bundleReader.close(); +await bundleArchive.close(); +console.log("Multiple app packages append under separate roots in a single ZIP"); + for (const [entries, error] of [ [[["one", executable], ["two", executable]], /exactly one/u], [[["readme", new Uint8Array([1])]], /exactly one/u], From 55a188fcb886b87da04121fc8025cfe1493abfc3 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:37:56 +0530 Subject: [PATCH 03/31] feat(web): stream multiple app packages in one ZIP Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/archive.mjs | 6 ++- demo/web/worker.mjs | 90 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/demo/web/archive.mjs b/demo/web/archive.mjs index 8ea97a0c6e..a479f53bdd 100644 --- a/demo/web/archive.mjs +++ b/demo/web/archive.mjs @@ -225,6 +225,10 @@ export function batchOutputDirectory(name, index) { return `${String(index + 1).padStart(2, "0")}-${stem || "app"}`; } +export function createPackageWriter(destination) { + return new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true }); +} + async function appendEntry(writer, name, entry, report) { const stream = new TransformStream(); let produced = 0; @@ -302,7 +306,7 @@ class DirectoryWriter { export async function packageArchive(archive, converted, outputName, libraryFile, destination, report, directory = false, options = {}) { const libraries = new ZipReader(new BoundedBlobReader(libraryFile)); const ownsWriter = !options.writer; - const writer = options.writer ?? (directory ? new DirectoryWriter(destination) : new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true })); + const writer = options.writer ?? (directory ? new DirectoryWriter(destination) : createPackageWriter(destination)); const outputPath = name => { const path = options.prefix ? `${options.prefix}${name}` : name; validPath(path); diff --git a/demo/web/worker.mjs b/demo/web/worker.mjs index d2cbdb4377..59232fffa1 100644 --- a/demo/web/worker.mjs +++ b/demo/web/worker.mjs @@ -1,6 +1,8 @@ -import { Archive, CONVERSION_LIMIT, MEMORY_LIMIT, memoryEstimate, packageArchive } from "./archive.mjs"; +import { Archive, batchMemoryEstimate, batchOutputDirectory, CONVERSION_LIMIT, MEMORY_LIMIT, createPackageWriter, packageArchive } from "./archive.mjs"; let archive; +let inputs = []; +let summaries = []; let busy = false; let pendingWrite; let lastReport = 0; @@ -15,6 +17,10 @@ function report(event) { self.postMessage({ type: "progress", ...event }); } +function reportFor(item, index) { + return event => report({ ...event, appIndex: index, appCount: inputs.length, appName: item.name }); +} + export async function downloadLibraries(target) { report({ phase: "Downloading system libraries" }); const response = await fetch("./release.json", { cache: "no-cache" }); @@ -40,6 +46,25 @@ export async function downloadLibraries(target) { return { file: new Blob([bytes]), release, expandedBytes: asset.expandedBytes }; } +async function convertInput(item, index, native, libraries, data, destination, directory, writer) { + const current = archive ?? new Archive(item.files ?? item.file); + archive = null; + try { + const itemReport = reportFor(item, index); + if (inputs.length > 1) await current.inspect(itemReport); + const input = await current.conversionInput(data.target, data.toIntel, itemReport); + const converted = native.Convert(input, itemReport); + if (!converted.ok) throw new Error(converted.error); + const options = inputs.length > 1 ? { prefix: `${batchOutputDirectory(item.name, index)}/`, writer } : {}; + await packageArchive(current, converted, input.outputName, libraries.file, destination, itemReport, directory, options); + return input.outputName; + } catch (error) { + throw new Error(`${item.name}: ${error instanceof Error ? error.message : String(error)}`); + } finally { + await current.close(); + } +} + self.onmessage = async ({ data }) => { if (data.type === "written") { if (!pendingWrite) return; @@ -53,27 +78,47 @@ self.onmessage = async ({ data }) => { try { if (data.type === "inspect") { if (archive) await archive.close(); - archive = new Archive(data.files ?? data.file); - const summary = await archive.inspect(report); - self.postMessage({ type: "inspected", summary }); + archive = null; + inputs = data.inputs ?? [{ name: data.file?.name ?? "selected app", file: data.file, files: data.files }]; + if (!inputs.length) throw new Error("Choose at least one ZIP archive."); + summaries = []; + for (let index = 0; index < inputs.length; index++) { + const item = inputs[index]; + const current = new Archive(item.files ?? item.file); + let keepOpen = false; + try { + const summary = await current.inspect(reportFor(item, index)); + summaries.push(summary); + if (inputs.length === 1) { + archive = current; + keepOpen = true; + } + self.postMessage({ type: "inspected", index, count: inputs.length, name: item.name, summary }); + } finally { + if (!keepOpen) await current.close(); + } + } + self.postMessage({ type: "inspection-complete", count: inputs.length }); } else if (data.type === "convert") { + if (data.target !== "linux" && data.target !== "windows") throw new Error(`Unsupported conversion target: ${data.target}`); + if (!["memory", "zip", "disk"].includes(data.mode)) throw new Error("Unknown save mode"); if (data.target === "windows" && data.mode === "disk") throw new Error("Windows folder output is unavailable because browsers block DLL creation. Choose a ZIP save mode."); - if (!archive) throw new Error("Select a ZIP or folder first"); - const selectedBytes = archive.selectedBytesForTarget(data.target); - if (selectedBytes > CONVERSION_LIMIT) throw new Error(`Selected ${data.target} conversion input exceeds the 256 MiB conversion limit. Use the native relinker for this app.`); + if (!inputs.length || summaries.length !== inputs.length) throw new Error("Inspect the selected apps before converting."); + if (inputs.length > 1 && data.mode === "disk") throw new Error("Batch conversion requires a ZIP save mode."); + for (const summary of summaries) { + const selectedBytes = data.target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes; + if (data.target === "windows" && summary.windowsIconTooLarge) throw new Error("A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker."); + if (selectedBytes > CONVERSION_LIMIT) throw new Error(`Selected ${data.target} conversion input exceeds the 256 MiB conversion limit. Use the native relinker for this app.`); + } const libraries = await downloadLibraries(data.target); - const estimate = memoryEstimate(archive.totalBytes, selectedBytes, libraries.expandedBytes, archive.entries.length); + const estimate = batchMemoryEstimate(summaries, data.target, libraries.expandedBytes); if (data.mode === "memory" && estimate > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save ZIP to disk in desktop Chrome or Edge."); report({ phase: "Loading converter" }); const { default: createRelinker } = await import("./native/relinker_wasm.mjs"); const native = await createRelinker(); - let input = await archive.conversionInput(data.target, data.toIntel, report); - const outputName = input.outputName; - const converted = native.Convert(input, report); - input = null; - if (!converted.ok) throw new Error(converted.error); const chunks = []; let size = 0; + let outputName = "anyps5-apps"; const send = (event, transfer = []) => new Promise(resolve => { pendingWrite = resolve; self.postMessage(event, transfer); @@ -93,8 +138,8 @@ self.onmessage = async ({ data }) => { }); } }; - await packageArchive(archive, converted, outputName, libraries.file, destination, report, true); - } else if (data.mode === "memory" || data.mode === "zip") { + outputName = await convertInput(inputs[0], 0, native, libraries, data, destination, true); + } else { const destination = new WritableStream({ async write(chunk) { size += chunk.byteLength; @@ -107,12 +152,17 @@ self.onmessage = async ({ data }) => { chunks.push(chunk.slice()); } }); - await packageArchive(archive, converted, outputName, libraries.file, destination, report); + const writer = inputs.length > 1 ? createPackageWriter(destination) : null; + for (let index = 0; index < inputs.length; index++) { + const name = await convertInput(inputs[index], index, native, libraries, data, destination, false, writer); + if (inputs.length === 1) outputName = name; + } + if (writer) await writer.close(); await destination.close(); - } else throw new Error("Unknown save mode"); - self.postMessage({ type: "complete", mode: data.mode, blob: data.mode === "memory" ? new Blob(chunks, { type: "application/zip" }) : null, outputName, size, release: libraries.release.tag }); - await archive.close(); - archive = null; + } + self.postMessage({ type: "complete", mode: data.mode, blob: data.mode === "memory" ? new Blob(chunks, { type: "application/zip" }) : null, outputName, size, release: libraries.release.tag, apps: inputs.length }); + inputs = []; + summaries = []; } else throw new Error("Unknown worker request"); } catch (error) { self.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) }); From cd3781c1cfe6d61aea85cd27f3b4a0d18a0ad3a5 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:38:22 +0530 Subject: [PATCH 04/31] test(web): exercise batch inspection and conversion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/tests/archive.test.mjs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/demo/web/tests/archive.test.mjs b/demo/web/tests/archive.test.mjs index c8801c40d5..0bc401d74b 100644 --- a/demo/web/tests/archive.test.mjs +++ b/demo/web/tests/archive.test.mjs @@ -243,6 +243,28 @@ try { await assert.rejects(worker.downloadLibraries("windows"), /Cannot verify the latest release \(403\)/u); latestResponse = new Response(JSON.stringify({ tag_name: "v2" })); await assert.rejects(worker.downloadLibraries("windows"), /A new release is being published/u); + + latestResponse = new Response(null, { status: 429 }); + await self.onmessage({ data: { type: "inspect", inputs: [ + { name: "First App.zip", file: source }, + { name: "First App.zip", file: source } + ] } }); + assert.equal(workerMessages.filter(message => message.type === "inspected").length, 2); + assert(workerMessages.some(message => message.type === "inspection-complete" && message.count === 2)); + await self.onmessage({ data: { type: "convert", target: "windows", toIntel: false, mode: "memory" } }); + const batchComplete = workerMessages.find(message => message.type === "complete"); + assert(batchComplete, workerMessages.find(message => message.type === "error")?.message); + assert.equal(batchComplete.apps, 2); + const batchReader = new ZipReader(new BlobReader(batchComplete.blob)); + const batchNames = new Set((await batchReader.getEntries()).map(entry => entry.filename)); + assert(batchNames.has("01-First-App/program.exe")); + assert(batchNames.has("01-First-App/libs/system.prx")); + assert(batchNames.has("01-First-App/app0/content/resource.bin")); + assert(batchNames.has("02-First-App/program.exe")); + assert(batchNames.has("02-First-App/libs/system.prx")); + assert(batchNames.has("02-First-App/app0/content/resource.bin")); + await batchReader.close(); + workerMessages.length = 0; } finally { globalThis.fetch = originalFetch; workerMessages.length = 0; @@ -251,6 +273,15 @@ await self.onmessage({ data: { type: "convert", target: "windows", mode: "disk" assert.equal(workerMessages.length, 1); assert.equal(workerMessages[0].type, "error"); assert.match(workerMessages[0].message, /browsers block DLL creation/u); +workerMessages.length = 0; +await self.onmessage({ data: { type: "inspect", inputs: [ + { name: "valid.zip", file: source }, + { name: "invalid.zip", file: new Blob(["not a ZIP"]) } +] } }); +assert.equal(workerMessages.at(-1).type, "error"); +assert.match(workerMessages.at(-1).message, /invalid\.zip/u); +assert.equal(workerMessages.some(message => message.type === "inspection-complete"), false); delete globalThis.self; console.log("Rate-limited release checks use the verified site manifest; other release-check failures remain errors"); console.log("Windows folder output rejected before conversion or file writes"); +console.log("Multiple ZIP inputs stream into separate app roots; invalid inputs name the failing archive"); From 297b0dfacfeef5bc13872d114c4785cace37b51c Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:39:30 +0530 Subject: [PATCH 05/31] test(web): make batch limits independently testable Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/web-tests.yml | 5 +++-- demo/web/archive.mjs | 24 ++---------------------- demo/web/batch.mjs | 27 +++++++++++++++++++++++++++ demo/web/tests/archive.test.mjs | 12 +----------- demo/web/tests/batch.test.mjs | 30 ++++++++++++++++++++++++++++++ demo/web/worker.mjs | 3 ++- 6 files changed, 65 insertions(+), 36 deletions(-) create mode 100644 demo/web/batch.mjs create mode 100644 demo/web/tests/batch.test.mjs diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index e2b5aa9186..ff75830066 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -25,7 +25,8 @@ jobs: - name: Check browser module syntax run: | node --check demo/web/app.mjs + node --check demo/web/batch.mjs node --check demo/web/output-directory.mjs node --check demo/web/worker.mjs - - name: Test folder output recovery - run: node --test demo/web/tests/output-directory.test.mjs + - name: Test browser converter helpers + run: node --test demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs diff --git a/demo/web/archive.mjs b/demo/web/archive.mjs index a479f53bdd..fdcb24d89a 100644 --- a/demo/web/archive.mjs +++ b/demo/web/archive.mjs @@ -1,8 +1,9 @@ import { BlobReader, Uint8ArrayReader, ZipReader, ZipWriter, configure } from "https://cdn.jsdelivr.net/npm/@zip.js/zip.js@2.7.57/lib/zip-no-worker.js/+esm"; +import { batchMemoryEstimate, batchOutputDirectory, MEMORY_LIMIT, memoryEstimate } from "./batch.mjs"; configure({ useWebWorkers: false, useCompressionStream: false, chunkSize: 65536 }); -export const MEMORY_LIMIT = 256 * 1024 * 1024; +export { batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT }; export const CONVERSION_LIMIT = 256 * 1024 * 1024; export const ICON_LIMIT = 16 * 1024 * 1024; const MAX_ENTRIES = 100000; @@ -204,27 +205,6 @@ export class Archive { async close() { await this.reader.close(); } } -export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entries) { - return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024); -} - -export function batchMemoryEstimate(summaries, target, libraryBytes) { - if (target !== "linux" && target !== "windows") throw new Error(`Unsupported conversion target: ${target}`); - return summaries.reduce((total, summary) => safeSize(total + memoryEstimate( - summary.totalBytes, - target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes, - libraryBytes, - summary.files - )), 0); -} - -export function batchOutputDirectory(name, index) { - if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid app index"); - const stem = String(name).replace(/\.zip$/iu, "").normalize("NFKC") - .replace(/[^A-Za-z0-9._-]+/gu, "-").replace(/^[.-]+/u, "").slice(0, 80).replace(/[. -]+$/u, ""); - return `${String(index + 1).padStart(2, "0")}-${stem || "app"}`; -} - export function createPackageWriter(destination) { return new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true }); } diff --git a/demo/web/batch.mjs b/demo/web/batch.mjs new file mode 100644 index 0000000000..5058dbea4f --- /dev/null +++ b/demo/web/batch.mjs @@ -0,0 +1,27 @@ +export const MEMORY_LIMIT = 256 * 1024 * 1024; + +function safeSize(value) { + if (!Number.isSafeInteger(value) || value < 0) throw new Error("ZIP contains an unsupported file size"); + return value; +} + +export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entries) { + return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024); +} + +export function batchMemoryEstimate(summaries, target, libraryBytes) { + if (target !== "linux" && target !== "windows") throw new Error(`Unsupported conversion target: ${target}`); + return summaries.reduce((total, summary) => safeSize(total + memoryEstimate( + summary.totalBytes, + target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes, + libraryBytes, + summary.files + )), 0); +} + +export function batchOutputDirectory(name, index) { + if (!Number.isSafeInteger(index) || index < 0) throw new Error("Invalid app index"); + const stem = String(name).replace(/\.zip$/iu, "").normalize("NFKC") + .replace(/[^A-Za-z0-9._-]+/gu, "-").replace(/^[.-]+/u, "").slice(0, 80).replace(/[. -]+$/u, ""); + return `${String(index + 1).padStart(2, "0")}-${stem || "app"}`; +} diff --git a/demo/web/tests/archive.test.mjs b/demo/web/tests/archive.test.mjs index 0bc401d74b..d0c2ca7c77 100644 --- a/demo/web/tests/archive.test.mjs +++ b/demo/web/tests/archive.test.mjs @@ -7,7 +7,7 @@ import { register } from "node:module"; register("./cdn-loader.mjs", import.meta.url, { data: { url: process.env.ANYPS5_TEST_ZIP_URL, path: process.env.ANYPS5_TEST_ZIP_PATH } }); const site = name => pathToFileURL(path.join(process.argv[2], name)).href; -const { Archive, batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT, ICON_LIMIT, packageArchive } = await import(site("archive.mjs")); +const { Archive, memoryEstimate, MEMORY_LIMIT, ICON_LIMIT, packageArchive } = await import(site("archive.mjs")); const { BlobReader, BlobWriter, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter } = await import(process.env.ANYPS5_TEST_ZIP_URL); await import("./archive-reserved.test.mjs"); const { default: createRelinker } = await import(site("native/relinker_wasm.mjs")); @@ -157,16 +157,6 @@ assert.equal(selfModuleSummary.warnings.length, 2); assert.match(selfModuleSummary.warnings.join(" "), /SELF container/u); await selfModule.close(); assert(memoryEstimate(200 * 1024 ** 3, 1024, 1024, 100) > MEMORY_LIMIT); -assert(batchMemoryEstimate([ - { totalBytes: 1024, selectedBytes: 128, windowsSelectedBytes: 256, files: 2 }, - { totalBytes: 2048, selectedBytes: 256, windowsSelectedBytes: 512, files: 3 } -], "linux", 1024) > memoryEstimate(2048, 256, 1024, 3)); -assert.equal(batchOutputDirectory("First App.zip", 0), "01-First-App"); -assert.equal(batchOutputDirectory("First App.zip", 1), "02-First-App"); -assert.equal(batchOutputDirectory("CON.zip", 0), "01-CON"); -assert.equal(batchOutputDirectory("...zip", 0), "01-app"); -assert.throws(() => batchOutputDirectory("app.zip", -1), /Invalid app index/u); -assert.throws(() => batchMemoryEstimate([], "macos", 0), /Unsupported conversion target/u); const invalid = native.Convert({ target: "unsupported" }, () => {}); assert.equal(invalid.ok, false); console.log("Archive validation, missing-folder warnings and large-download budget verified"); diff --git a/demo/web/tests/batch.test.mjs b/demo/web/tests/batch.test.mjs new file mode 100644 index 0000000000..b19827d72b --- /dev/null +++ b/demo/web/tests/batch.test.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { batchMemoryEstimate, batchOutputDirectory, memoryEstimate, MEMORY_LIMIT } from "../batch.mjs"; + +test("estimates all ZIP outputs against one shared memory limit", () => { + const summaries = [ + { totalBytes: 1024, selectedBytes: 128, windowsSelectedBytes: 256, files: 2 }, + { totalBytes: 2048, selectedBytes: 256, windowsSelectedBytes: 512, files: 3 } + ]; + + assert.equal(batchMemoryEstimate(summaries, "linux", 1024), summaries.reduce( + (total, summary) => total + memoryEstimate(summary.totalBytes, summary.selectedBytes, 1024, summary.files), + 0 + )); + assert.equal(batchMemoryEstimate(summaries, "windows", 1024), summaries.reduce( + (total, summary) => total + memoryEstimate(summary.totalBytes, summary.windowsSelectedBytes, 1024, summary.files), + 0 + )); + assert.throws(() => batchMemoryEstimate([], "macos", 0), /Unsupported conversion target/u); + assert.throws(() => batchMemoryEstimate([{ totalBytes: Number.MAX_SAFE_INTEGER, selectedBytes: 0, windowsSelectedBytes: 0, files: 0 }], "linux", 0), /unsupported file size/u); + assert.equal(MEMORY_LIMIT, 256 * 1024 * 1024); +}); + +test("creates safe and distinct ZIP folder names for duplicate app filenames", () => { + assert.equal(batchOutputDirectory("First App.zip", 0), "01-First-App"); + assert.equal(batchOutputDirectory("First App.zip", 1), "02-First-App"); + assert.equal(batchOutputDirectory("CON.zip", 0), "01-CON"); + assert.equal(batchOutputDirectory("...zip", 0), "01-app"); + assert.throws(() => batchOutputDirectory("app.zip", -1), /Invalid app index/u); +}); diff --git a/demo/web/worker.mjs b/demo/web/worker.mjs index 59232fffa1..735533fafe 100644 --- a/demo/web/worker.mjs +++ b/demo/web/worker.mjs @@ -1,4 +1,5 @@ -import { Archive, batchMemoryEstimate, batchOutputDirectory, CONVERSION_LIMIT, MEMORY_LIMIT, createPackageWriter, packageArchive } from "./archive.mjs"; +import { batchMemoryEstimate, batchOutputDirectory, MEMORY_LIMIT } from "./batch.mjs"; +import { Archive, CONVERSION_LIMIT, createPackageWriter, packageArchive } from "./archive.mjs"; let archive; let inputs = []; From 919af8c6dc0309b83b93d46b2d1b6ec24588fcb5 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:42:57 +0530 Subject: [PATCH 06/31] feat(web): select multiple app archives Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/app.css | 4 ++ demo/web/app.mjs | 151 +++++++++++++++++++++++++++++++++----------- demo/web/index.html | 8 +-- demo/web/worker.mjs | 13 +++- 4 files changed, 131 insertions(+), 45 deletions(-) diff --git a/demo/web/app.css b/demo/web/app.css index c5258a1102..b7c45cab14 100644 --- a/demo/web/app.css +++ b/demo/web/app.css @@ -4,3 +4,7 @@ *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif}button,input,select{font:inherit}button,a,select,input{touch-action:manipulation}button{cursor:pointer}button:disabled{cursor:not-allowed;opacity:.45}a{color:var(--accent);text-underline-offset:3px}button:focus-visible,a:focus-visible,select:focus-visible,input:focus-visible{outline:2px solid var(--accent);outline-offset:4px}code,pre,.eyebrow,.fine{font-family:ui-monospace,"Cascadia Mono",Consolas,monospace}code{font-size:.9em}h1,h2,h3,p{margin-top:0}[hidden]{display:none!important}header{border-bottom:1px solid var(--line);background:var(--surface)}.bar{max-width:1120px;margin:auto;min-height:68px;padding:12px 24px;display:flex;align-items:center;justify-content:space-between;gap:24px}.brand{display:flex;align-items:center;gap:10px;text-decoration:none;color:var(--text);font-weight:700;font-size:17px;white-space:nowrap}.brand>span:last-child{font-weight:400;color:var(--muted);font-size:14px}.mark{display:flex;gap:3px;align-items:center;margin-right:3px}.mark i{display:block;width:4px;height:20px;background:var(--text)}.mark i:last-child{opacity:.3}nav{display:flex;align-items:center;gap:24px;font-size:13px}nav a{color:var(--muted);text-decoration:none}nav a:hover{color:var(--text)}main{max-width:1024px;margin:auto;padding:58px 24px 36px}.intro{max-width:660px;margin-bottom:30px}.eyebrow{color:var(--muted);letter-spacing:.12em;font-size:11px;margin-bottom:12px}h1{font-size:clamp(29px,4vw,43px);line-height:1.15;letter-spacing:-.035em;font-weight:650;margin-bottom:18px}.converter{border:1px solid var(--line);border-radius:10px;background:var(--surface);overflow:hidden}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:16px;border-bottom:1px solid var(--line);padding:19px 26px}.panel-heading h2{font-size:15px;margin:0;font-weight:600}.private{font-size:12px;color:var(--muted)}.private b{color:var(--good);font-size:9px;margin-right:5px}.drop{margin:24px;border:1px dashed var(--line);border-radius:6px;text-align:center;padding:28px 16px 17px;background:var(--bg)}.drop.dragging{border-color:var(--accent);background:var(--sunken)}.archive-icon{display:inline-flex;align-items:center;justify-content:center;width:40px;height:45px;border:1px solid var(--line);border-radius:4px;font-size:25px;color:var(--muted);margin-bottom:15px}.drop h3{font-size:19px;margin-bottom:8px;font-weight:600}.drop>p{color:var(--muted);font-size:13px;margin-bottom:20px}.button{display:inline-flex;align-items:center;justify-content:center;gap:18px;border:1px solid transparent;border-radius:6px;background:var(--accent);color:var(--on-accent);text-decoration:none;font-weight:600;font-size:13px;padding:11px 20px;white-space:nowrap}.button:hover:not(:disabled){filter:brightness(.94)}.drop .fine{margin:15px 0 0;font-size:10px}.fine{font-size:11px;line-height:1.6;color:var(--muted);margin-bottom:0}.plain{border:0;background:none;color:var(--muted);padding:8px;font-size:13px}.plain:hover{color:var(--text)}#theme{font-size:19px;padding:3px}.settings{display:grid;grid-template-columns:1fr 1fr;gap:36px;padding:0 26px 20px}fieldset{border:0;margin:0;padding:0;min-width:0}legend,.save-settings>label{display:block;font-size:12px;font-weight:600;margin-bottom:10px}.choices{display:flex;gap:10px}.choices label{flex:1;display:flex;gap:9px;align-items:center;border:1px solid var(--line);border-radius:6px;padding:10px;font-size:13px;cursor:pointer}.choices label:has(input:checked){border-color:var(--accent)}.choices span{font-size:11px;color:var(--muted);margin-left:auto}input{accent-color:var(--accent)}input[type=radio],input[type=checkbox]{margin:0;width:15px;height:15px;flex-shrink:0}select{width:100%;padding:10px;border:1px solid var(--line);border-radius:6px;background:var(--surface);color:var(--text);font-size:13px}.save-settings .fine{margin-top:8px}.options{padding:0 26px 23px;font-size:12px;color:var(--muted)}.options label{display:flex;gap:9px;align-items:center}.action-row{border-top:1px solid var(--line);padding:17px 26px;display:flex;gap:12px;align-items:center}.action-row>p{margin:0 auto 0 0;font-size:12px}.muted{color:var(--muted)}.selection{display:flex;justify-content:space-between;align-items:center;padding:25px 26px;gap:16px}.selection h3{font-size:18px;margin-bottom:6px;overflow-wrap:anywhere}.selection p{margin-bottom:0;font-size:12px}.selection .eyebrow{font-size:10px;margin-bottom:6px}.warnings,.notice{padding:16px 20px;background:var(--warning-bg);color:var(--warning);font-size:13px;border:1px solid var(--line);border-radius:6px}.warnings{margin:0 26px 24px}.warnings ul{padding-left:18px;margin:0 0 12px}.warnings label{display:flex;gap:8px;align-items:center}.notice{margin-bottom:20px}.status{border-top:1px solid var(--line);padding:22px 26px}.status-line{display:flex;justify-content:space-between;font-size:13px;gap:16px}progress{appearance:none;width:100%;height:5px;margin:14px 0 8px;border:0;border-radius:3px;overflow:hidden;background:var(--line)}progress::-webkit-progress-bar{background:var(--line)}progress::-webkit-progress-value{background:var(--accent)}progress::-moz-progress-bar{background:var(--accent)}#current-file{overflow-wrap:anywhere;min-height:18px}details{margin-top:18px;font-size:11px;color:var(--muted)}summary{cursor:pointer}pre{max-height:180px;overflow:auto;white-space:pre-wrap;word-break:break-word;font-size:11px}.error{margin:0 26px 24px;padding:15px;background:var(--error-bg);color:var(--error);border-radius:6px;white-space:pre-wrap;overflow-wrap:anywhere}.done{padding:24px 26px;border-top:1px solid var(--line)}.done h3{font-size:20px;color:var(--good);margin-bottom:8px}.done p{font-size:13px;color:var(--muted)}.done .fine{margin:15px 0 0}.compatibility{margin:12px 0 0;color:var(--muted);font-size:11px;line-height:1.8}.limits{border-top:1px solid var(--line);padding-top:22px;margin:30px 0 0;color:var(--muted);font-size:11px;line-height:1.8}footer{max-width:1024px;margin:auto;border-top:1px solid var(--line);padding:22px 24px;display:flex;justify-content:space-between;font-size:11px;color:var(--muted)}footer div{display:flex;gap:20px}footer a{color:var(--muted);text-decoration:none}@media(max-width:680px){.bar{flex-wrap:wrap;gap:12px;padding:14px 20px}nav{gap:17px;font-size:12px}main{padding:34px 16px 26px}.panel-heading{padding:16px}.private{font-size:10px}.drop{margin:16px}.settings{grid-template-columns:1fr;padding:0 16px 18px;gap:22px}.options{padding:0 16px 20px}.action-row{padding:16px;flex-wrap:wrap}.action-row>p{width:100%}.selection{padding:20px 16px}.warnings{margin:0 16px 20px}.status,.done{padding:20px 16px}.error{margin:0 16px 20px}footer{flex-wrap:wrap;gap:12px}footer div{flex-wrap:wrap;gap:14px}} .input-actions{display:flex;justify-content:center;flex-wrap:wrap;gap:12px} +.app-list{display:grid;gap:8px;max-height:220px;overflow:auto;padding-left:22px;margin:14px 0 0;font-size:12px;color:var(--muted)} +.app-list li{padding:8px 0;border-top:1px solid var(--line)} +.app-list strong,.app-list span{display:block;overflow-wrap:anywhere} +.app-list strong{color:var(--text)} diff --git a/demo/web/app.mjs b/demo/web/app.mjs index 79fc9f901d..1b7847598e 100644 --- a/demo/web/app.mjs +++ b/demo/web/app.mjs @@ -1,10 +1,12 @@ import { abortAndDiscardOutput, OutputDirectoryTransaction } from "./output-directory.mjs"; +import { batchMemoryEstimate } from "./batch.mjs"; const $ = id => document.getElementById(id); const stages = ["Converting executable", "Converting PRX modules", "Scanning syscalls", "Scanning instructions", "Converting instructions", "Analysing imports", "Building output image", "Writing output"]; let worker; -let file; -let summary; +let selectedInputs = []; +let summaries = []; +let inspectionComplete = false; let working = false; let failing = false; let sink; @@ -26,27 +28,55 @@ function bytes(value) { return `${(value / 1024 ** unit).toFixed(1)} ${["B", "KiB", "MiB", "GiB"][unit]}`; } +function sum(values) { + let total = 0; + for (const value of values) { + if (!Number.isSafeInteger(value) || value < 0 || total > Number.MAX_SAFE_INTEGER - value) return null; + total += value; + } + return total; +} + function update() { const target = document.querySelector('input[name="target"]:checked').value; - const folderBlocked = target === "windows" && $("save").value === "disk"; + const multi = selectedInputs.length > 1; + const inspected = inspectionComplete && summaries.length === selectedInputs.length; + const folderBlocked = $("save").value === "disk" && (target === "windows" || multi); const folderOption = $("save").querySelector('[value="disk"]'); - folderOption.disabled = !canStream || target === "windows"; - folderOption.textContent = target === "windows" ? "Save to folder · unavailable for Windows (DLL restrictions)" : "Save to folder · uncompressed"; + folderOption.disabled = !canStream || target === "windows" || multi; + folderOption.textContent = target === "windows" ? "Save to folder · unavailable for Windows (DLL restrictions)" : multi ? "Save to folder · unavailable for batch output" : "Save to folder · uncompressed"; const libraryBytes = release?.platforms[target]?.expandedBytes; - const selectedBytes = summary ? target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes : null; - const conversionTooLarge = selectedBytes !== null && selectedBytes > 256 * 1024 * 1024; - const iconTooLarge = target === "windows" && Boolean(summary?.windowsIconTooLarge); + const selectedBytes = inspected ? sum(summaries.map(summary => target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes)) : null; + const conversionTooLarge = inspected && summaries.some(summary => (target === "windows" ? summary.windowsSelectedBytes : summary.selectedBytes) > 256 * 1024 * 1024); + const iconTooLarge = target === "windows" && inspected && summaries.some(summary => summary.windowsIconTooLarge); const conversionBlocked = conversionTooLarge || iconTooLarge; - const estimate = summary && libraryBytes !== undefined ? Math.ceil(summary.totalBytes * 1.02) + selectedBytes * 4 + libraryBytes + summary.files * 1024 + 8 * 1024 * 1024 : null; - const tooLarge = estimate !== null && estimate > 256 * 1024 * 1024; + let estimate = null; + let estimateOverflow = false; + if (inspected && libraryBytes !== undefined) { + try { estimate = batchMemoryEstimate(summaries, target, libraryBytes); } + catch (error) { + if (!(error instanceof Error) || error.message !== "ZIP contains an unsupported file size") throw error; + estimateOverflow = true; + } + } + const tooLarge = estimateOverflow || (estimate !== null && estimate > 256 * 1024 * 1024); $("save").querySelector('[value="memory"]').disabled = Boolean(tooLarge); const memoryBlocked = $("save").value === "memory" && tooLarge; - if (summary) $("archive-info").textContent = `${summary.files} files · ${bytes(summary.totalBytes)} expanded · ${summary.modules} modules · ${bytes(selectedBytes)} ${target} conversion input` + (summary.root ? ` · Root: ${summary.root}` : ""); - $("save-note").textContent = $("save").value === "disk" ? "Creates a subfolder with the executable, libs and app0. No ZIP compression. Incomplete output folders are removed after cancellation or failure. Browsers may block DLL and other restricted files; use ZIP for those apps." : $("save").value === "zip" ? "Writes a compressed ZIP directly to disk without holding the whole archive in memory." : "Up to 256 MiB including expanded files and output overhead."; + if (inspected) { + const totalFiles = sum(summaries.map(summary => summary.files)); + const totalBytes = sum(summaries.map(summary => summary.totalBytes)); + const modules = sum(summaries.map(summary => summary.modules)); + const sizeText = value => value === null ? "beyond the supported size range" : bytes(value); + $("archive-info").textContent = multi + ? `${selectedInputs.length} apps · ${sizeText(totalFiles)} files · ${sizeText(totalBytes)} expanded · ${sizeText(modules)} modules · ${sizeText(selectedBytes)} ${target} conversion input` + : `${summaries[0].files} files · ${bytes(summaries[0].totalBytes)} expanded · ${summaries[0].modules} modules · ${bytes(target === "windows" ? summaries[0].windowsSelectedBytes : summaries[0].selectedBytes)} ${target} conversion input` + (summaries[0].root ? ` · Root: ${summaries[0].root}` : ""); + } + $("save-note").textContent = multi ? $("save").value === "memory" ? "Creates one ZIP with each app in its own folder. The complete bundle must fit in the 256 MiB memory-download budget." : "Writes one compressed ZIP containing each app in its own folder, without holding the whole bundle in memory." : $("save").value === "disk" ? "Creates a subfolder with the executable, libs and app0. No ZIP compression. Incomplete output folders are removed after cancellation or failure. Browsers may block DLL and other restricted files; use ZIP for those apps." : $("save").value === "zip" ? "Writes a compressed ZIP directly to disk without holding the whole archive in memory." : "Up to 256 MiB including expanded files and output overhead."; $("browser-note").hidden = supported && canStream && canSaveZip && !memoryBlocked && !folderBlocked && !conversionBlocked; - $("browser-note").textContent = !supported ? "This browser or connection does not support the converter. Use a current browser over HTTPS, or use the native relinker." : folderBlocked ? "Browsers block direct DLL creation. Windows output requires ZIP. Choose Save ZIP to disk or Download in memory." : iconTooLarge ? "The Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `This ${target} conversion input exceeds 256 MiB. Use the native relinker for this app.` : memoryBlocked ? "This archive exceeds the memory-download limit. Choose Save ZIP to disk. If that option is unavailable, open this page in desktop Chrome or Edge." : "Some direct disk saving options are unavailable in this browser or context. Small apps can still be downloaded in memory. For larger apps, use desktop Chrome or Edge over HTTPS."; - $("convert").disabled = !supported || working || !summary || !release || conversionBlocked || memoryBlocked || folderBlocked || (summary.warnings.length > 0 && !$("acknowledge").checked); - $("ready-note").textContent = summary ? (folderBlocked ? "Choose a ZIP save mode for Windows." : iconTooLarge ? "The Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `The ${target} conversion input exceeds 256 MiB. Use the native relinker.` : memoryBlocked ? "A direct disk save is required for this archive." : release ? `Libraries: ${release.tag} · ${$("save").value === "disk" ? "Uncompressed folder" : "ZIP compression level 9"}` : "Loading release information…") : "Choose a ZIP or folder to get started."; + $("browser-note").textContent = !supported ? "This browser or connection does not support the converter. Use a current browser over HTTPS, or use the native relinker." : folderBlocked && multi ? "Batch conversion creates one ZIP bundle. Choose Save ZIP to disk or Download in memory." : folderBlocked ? "Browsers block direct DLL creation. Windows output requires ZIP. Choose Save ZIP to disk or Download in memory." : iconTooLarge ? "A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `A ${target} conversion input exceeds 256 MiB. Use the native relinker for that app.` : estimateOverflow ? "The combined output exceeds the browser's supported size range. Choose Save ZIP to disk." : memoryBlocked ? "This output exceeds the memory-download limit. Choose Save ZIP to disk. If that option is unavailable, open this page in desktop Chrome or Edge." : "Some direct disk saving options are unavailable in this browser or context. Small apps can still be downloaded in memory. For larger apps, use desktop Chrome or Edge over HTTPS."; + const hasWarnings = inspected && summaries.some(summary => summary.warnings.length > 0); + $("convert").disabled = !supported || working || !inspected || !release || conversionBlocked || memoryBlocked || folderBlocked || (hasWarnings && !$("acknowledge").checked); + $("ready-note").textContent = inspected ? (folderBlocked && multi ? "Choose a ZIP save mode for the app bundle." : folderBlocked ? "Choose a ZIP save mode for Windows." : iconTooLarge ? "A Windows icon exceeds 16 MiB. Choose Linux or use the native relinker." : conversionTooLarge ? `A ${target} conversion input exceeds 256 MiB. Use the native relinker for that app.` : memoryBlocked ? "A direct disk save is required for this output." : release ? `Libraries: ${release.tag} · ${multi ? "ZIP bundle" : $("save").value === "disk" ? "Uncompressed folder" : "ZIP compression level 9"}` : "Loading release information…") : "Choose one folder or one or more ZIP files to get started."; } function lock(value) { @@ -59,7 +89,9 @@ async function fail(message) { worker?.terminate(); worker = null; working = true; - summary = null; + selectedInputs = []; + summaries = []; + inspectionComplete = false; const transaction = outputTransaction; outputTransaction = null; outputDirectory = null; @@ -75,6 +107,9 @@ async function fail(message) { $("status").hidden = true; $("cancel").hidden = true; $("drop").hidden = false; + $("selection").hidden = true; + $("warnings").hidden = true; + $("app-list").replaceChildren(); $("file").value = ""; working = false; failing = false; @@ -86,13 +121,14 @@ async function onMessage({ data }) { try { if (data.type === "progress") { if (data.message) { - logs.push(data.message.trim()); + logs.push(`${data.appName ? `${data.appName}: ` : ""}${data.message.trim()}`); logs = logs.slice(-150); $("log").textContent = logs.join("\n"); return; } - $("phase").textContent = data.phase ?? stages[data.stage]; - $("current-file").textContent = data.input ?? ""; + const phase = data.phase ?? stages[data.stage]; + $("phase").textContent = data.appCount > 1 ? `App ${data.appIndex + 1}/${data.appCount} · ${phase}` : phase; + $("current-file").textContent = [data.appName, data.input].filter(Boolean).join(" · "); if (Number.isFinite(data.total) && data.total > 0) { const percent = Math.min(100, Math.floor(data.completed / data.total * 100)); $("progress").value = percent; @@ -102,13 +138,34 @@ async function onMessage({ data }) { $("percent").textContent = ""; } } else if (data.type === "inspected") { - summary = data.summary; + summaries[data.index] = data.summary; + } else if (data.type === "inspection-complete") { + inspectionComplete = true; working = false; $("selection").hidden = false; $("drop").hidden = true; - $("archive-name").textContent = file.name; - $("warning-list").replaceChildren(...summary.warnings.map(message => { const item = document.createElement("li"); item.textContent = message; return item; })); - $("warnings").hidden = !summary.warnings.length; + $("archive-name").textContent = selectedInputs.length > 1 ? `${selectedInputs.length} app ZIPs` : selectedInputs[0].name; + $("app-list").replaceChildren(...selectedInputs.map((input, index) => { + const item = document.createElement("li"); + const name = document.createElement("strong"); + name.textContent = input.name; + const details = document.createElement("span"); + const summary = summaries[index]; + details.textContent = `${summary.files} files · ${bytes(summary.totalBytes)} expanded · ${summary.modules} modules`; + item.append(name, details); + return item; + })); + $("app-list").hidden = selectedInputs.length < 2; + const warningItems = []; + for (let index = 0; index < summaries.length; index++) { + for (const message of summaries[index].warnings) { + const item = document.createElement("li"); + item.textContent = selectedInputs.length > 1 ? `${selectedInputs[index].name}: ${message}` : message; + warningItems.push(item); + } + } + $("warning-list").replaceChildren(...warningItems); + $("warnings").hidden = warningItems.length === 0; $("status").hidden = true; $("cancel").hidden = true; lock(false); @@ -144,7 +201,7 @@ async function onMessage({ data }) { $("done").hidden = false; $("status").hidden = true; $("cancel").hidden = true; - $("result-info").textContent = `${bytes(data.size)} ${data.mode === "disk" ? "files" : "ZIP"} · ${data.release} system libraries · ${data.outputName}`; + $("result-info").textContent = `${data.apps > 1 ? `${data.apps} apps · ` : ""}${bytes(data.size)} ${data.mode === "disk" ? "files" : "ZIP"} · ${data.release} system libraries · ${data.apps > 1 ? "combined app bundle" : data.outputName}`; if (data.blob) { downloadUrl = URL.createObjectURL(data.blob); $("download").href = downloadUrl; @@ -156,11 +213,14 @@ async function onMessage({ data }) { $("result-info").textContent += ` · Saved in ${outputTransaction.name}/`; if (document.querySelector('input[name="target"]:checked').value === "linux") $("result-info").textContent += ` · On Linux, run chmod +x "${data.outputName}" before launching.`; } + $("result-note").textContent = data.apps > 1 ? "Extract the ZIP; each app is in its own folder with its executable, libs and app0 files. On Linux, run chmod +x */*.elf before launching." : "Extract the entire ZIP before launching the executable."; $("result-note").hidden = data.mode === "disk"; outputDirectory = null; outputTransaction?.commit(); outputTransaction = null; - summary = null; + selectedInputs = []; + summaries = []; + inspectionComplete = false; worker.terminate(); worker = null; lock(false); @@ -169,13 +229,17 @@ async function onMessage({ data }) { } catch (error) { await fail(error.message); } } -async function selectFile(selected, files) { - if (working || !supported || !selected) return; - if (!files && !selected.name.toLowerCase().endsWith(".zip")) { await fail("Choose a ZIP archive."); return; } +async function selectInputs(inputs) { + if (working || !supported || !inputs.length) return; + if (inputs.some(input => !input.files && !input.name.toLowerCase().endsWith(".zip")) || (inputs.length > 1 && inputs.some(input => input.files))) { + await fail("Batch conversion accepts ZIP archives only. Choose one folder or select ZIP files."); + return; + } worker?.terminate(); if (downloadUrl) { URL.revokeObjectURL(downloadUrl); downloadUrl = null; } - summary = null; - file = selected; + selectedInputs = inputs; + summaries = []; + inspectionComplete = false; working = true; logs = []; $("log").textContent = ""; @@ -184,7 +248,7 @@ async function selectFile(selected, files) { $("status").hidden = false; $("cancel").hidden = false; $("phase").textContent = "Inspecting files"; - $("current-file").textContent = file.name; + $("current-file").textContent = inputs.length > 1 ? `Inspecting ${inputs.length} ZIP archives` : inputs[0].name; $("percent").textContent = ""; $("progress").removeAttribute("value"); lock(true); @@ -192,7 +256,7 @@ async function selectFile(selected, files) { worker = new Worker(new URL("./worker.mjs", import.meta.url), { type: "module" }); worker.onmessage = onMessage; worker.onerror = event => fail(event.message || "The converter worker stopped. Try a smaller app or use the native relinker."); - worker.postMessage(files ? { type: "inspect", files } : { type: "inspect", file }); + worker.postMessage({ type: "inspect", inputs }); } $("choose").onclick = () => $("file").click(); @@ -200,12 +264,21 @@ $("change").onclick = () => { $("drop").hidden = false; }; $("choose-folder").onclick = () => { $("folder").value = ""; $("folder").click(); }; $("folder").onchange = () => { const files = [...$("folder").files].map(file => ({ path: file.webkitRelativePath, file })); - if (files.length) selectFile({ name: files[0].path.split("/")[0] }, files); + if (files.length) selectInputs([{ name: files[0].path.split("/")[0], files }]); +}; +$("file").onchange = () => { + const inputs = [...$("file").files].map(file => ({ name: file.name, file })); + $("file").value = ""; + selectInputs(inputs); }; -$("file").onchange = () => selectFile($("file").files[0]); for (const type of ["dragenter", "dragover"]) $("drop").addEventListener(type, event => { event.preventDefault(); if (!working) $("drop").classList.add("dragging"); }); $("drop").ondragleave = () => $("drop").classList.remove("dragging"); -$("drop").ondrop = event => { event.preventDefault(); $("drop").classList.remove("dragging"); if (working) return; if (event.dataTransfer.files.length !== 1) { fail("Drop one ZIP archive at a time."); return; } selectFile(event.dataTransfer.files[0]); }; +$("drop").ondrop = event => { + event.preventDefault(); + $("drop").classList.remove("dragging"); + if (working) return; + selectInputs([...event.dataTransfer.files].map(file => ({ name: file.name, file }))); +}; for (const id of ["save", "target", "acknowledge"]) $(id).addEventListener("change", update); $("cancel").onclick = () => fail("Cancelled. Choose the ZIP or folder again to restart."); $("convert").onclick = async () => { @@ -214,19 +287,21 @@ $("convert").onclick = async () => { lock(true); update(); try { + const target = document.querySelector('input[name="target"]:checked').value; if ($("save").value === "zip") { - const handle = await window.showSaveFilePicker({ suggestedName: file.name.replace(/\.zip$/iu, "") + "-" + document.querySelector('input[name="target"]:checked').value + ".zip", types: [{ description: "ZIP archive", accept: { "application/zip": [".zip"] } }] }); + const name = selectedInputs.length > 1 ? `anyps5-${target}-apps.zip` : `${selectedInputs[0].name.replace(/\.zip$/iu, "")}-${target}.zip`; + const handle = await window.showSaveFilePicker({ suggestedName: name, types: [{ description: "ZIP archive", accept: { "application/zip": [".zip"] } }] }); sink = await handle.createWritable(); } else if ($("save").value === "disk") { const parent = await window.showDirectoryPicker({ mode: "readwrite" }); - const name = file.name.replace(/\.zip$/iu, "") + "-" + document.querySelector('input[name="target"]:checked').value; + const name = selectedInputs[0].name.replace(/\.zip$/iu, "") + "-" + target; outputTransaction = await OutputDirectoryTransaction.create(parent, name); outputDirectory = outputTransaction.directory; } $("status").hidden = false; $("cancel").hidden = false; $("error").hidden = true; - worker.postMessage({ type: "convert", target: document.querySelector('input[name="target"]:checked').value, toIntel: $("intel").checked, mode: $("save").value }); + worker.postMessage({ type: "convert", target, toIntel: $("intel").checked, mode: $("save").value }); } catch (error) { if (error.name === "AbortError") { working = false; lock(false); update(); } else await fail(error.message); diff --git a/demo/web/index.html b/demo/web/index.html index 793792ab62..caefa1c459 100644 --- a/demo/web/index.html +++ b/demo/web/index.html @@ -11,10 +11,10 @@

LOCAL CONVERSION

PS5 apps to Windows/Linux converter

-

Convert an app

Files stay on your device
-

Choose a folder or drop a ZIP

One executable ELF, sce_sys/ and sce_module/.
Other app files can stay in the archive.

ZIP & ZIP64 · Decrypted ELF files only

- - +

Convert apps

Files stay on your device
+

Choose a folder or drop ZIP files

Each app needs one executable ELF, sce_sys/ and sce_module/.
Other app files can stay in the archive.

ZIP & ZIP64 · Decrypted ELF files only · Multiple ZIPs become one bundle

+ +
Convert for

Choose a ZIP or folder to get started.

diff --git a/demo/web/worker.mjs b/demo/web/worker.mjs index 735533fafe..1844ef2cd3 100644 --- a/demo/web/worker.mjs +++ b/demo/web/worker.mjs @@ -19,7 +19,15 @@ function report(event) { } function reportFor(item, index) { - return event => report({ ...event, appIndex: index, appCount: inputs.length, appName: item.name }); + let started = false; + return event => { + if (!started) { + lastReport = 0; + lastPhase = ""; + started = true; + } + report({ ...event, appIndex: index, appCount: inputs.length, appName: item.name }); + }; } export async function downloadLibraries(target) { @@ -112,8 +120,7 @@ self.onmessage = async ({ data }) => { if (selectedBytes > CONVERSION_LIMIT) throw new Error(`Selected ${data.target} conversion input exceeds the 256 MiB conversion limit. Use the native relinker for this app.`); } const libraries = await downloadLibraries(data.target); - const estimate = batchMemoryEstimate(summaries, data.target, libraries.expandedBytes); - if (data.mode === "memory" && estimate > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save ZIP to disk in desktop Chrome or Edge."); + if (data.mode === "memory" && batchMemoryEstimate(summaries, data.target, libraries.expandedBytes) > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save ZIP to disk in desktop Chrome or Edge."); report({ phase: "Loading converter" }); const { default: createRelinker } = await import("./native/relinker_wasm.mjs"); const native = await createRelinker(); From e78695baf510ed3b3dc871d0d3005576e8460a27 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:46:07 +0530 Subject: [PATCH 07/31] docs(web): explain batch conversion bundles Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/user/USAGE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index b448949fe2..de139ff6e8 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -6,6 +6,10 @@ The online converter can save Linux output as an uncompressed folder in supported browsers. It does not overwrite an existing destination folder. If conversion fails or is cancelled, it aborts the active file and attempts to remove the newly created output folder. If the browser prevents cleanup, the error names the folder that may contain incomplete files. +### Browser batch conversion + +Select or drop multiple app ZIP archives to convert them into one ZIP bundle. The converter inspects each app separately and puts its executable, libraries and `app0/` files in a numbered folder, so archives with the same filename remain separate. Batch output uses ZIP mode; saving to an uncompressed folder remains available for one Linux app at a time. The 256 MiB memory-download limit applies to the whole bundle, so use **Save ZIP to disk** for larger batches. If a batch is cancelled or any app fails, the converter aborts the incomplete bundle; any cleanup failure is reported instead of offering it as a completed result. + ## Input and conversion Use a clean ELF executable. Place its bundled ELF modules in `sce_module/`, `sce_modules/`, or `prx/` beside the input executable, or under the parent directory selected with `--sce-module-path `. The option accepts absolute paths and paths relative to the current working directory; do not include the `sce_module` directory itself. Only files directly inside the module directories are processed. SELF/fSELF containers are skipped with a warning; unwrap them before relinking if they need to be included. `tools/unwrap_self.py ` does that for a decrypted container: it places the segments at the offsets of the SELF segment records and restores `p_offset % p_align == p_vaddr % p_align`, and it refuses a container whose load segments all look encrypted. `prx/` can coexist with either `sce_module/` or `sce_modules/`. Both `sce_module/` and `sce_modules/` present, or all three absent, is an error. From 1bb9f49ac452dbabb8859628ec44216095a20d02 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:46:14 +0530 Subject: [PATCH 08/31] docs(web): clarify batch output size limits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/web/index.html b/demo/web/index.html index caefa1c459..e0d468febc 100644 --- a/demo/web/index.html +++ b/demo/web/index.html @@ -22,7 +22,7 @@
-

Memory downloads have a 256 MiB budget, checked against expanded files and output overhead. Larger results require desktop Chrome or Edge with direct disk saving. Executable and modules are limited to 256 MiB combined; the icon is also limited to 16 MiB and counts toward the Windows conversion limit. Use the native relinker for larger conversion inputs.

+

Memory downloads have a 256 MiB budget, checked against expanded files and output overhead. For multiple ZIPs, the limit applies to the combined bundle; larger batches require Save ZIP to disk in desktop Chrome or Edge. Executable and modules are limited to 256 MiB per app; the icon is also limited to 16 MiB and counts toward the Windows conversion limit. Use the native relinker for larger conversion inputs.

Conversion does not guarantee that an app runs. Check the community compatibility list for known blockers and reports.

From 339c644b289628a8dd04cd2a289fe3058d545a7a Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:46:21 +0530 Subject: [PATCH 09/31] test(web): run batch helper checks in CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/web-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml index ff75830066..b6765cf5d2 100644 --- a/.github/workflows/web-tests.yml +++ b/.github/workflows/web-tests.yml @@ -25,6 +25,7 @@ jobs: - name: Check browser module syntax run: | node --check demo/web/app.mjs + node --check demo/web/archive.mjs node --check demo/web/batch.mjs node --check demo/web/output-directory.mjs node --check demo/web/worker.mjs From 09afe38730d7b9d40d57b8aa9e13777012d170a3 Mon Sep 17 00:00:00 2001 From: Avijit Singh Date: Thu, 15 Oct 2026 06:50:12 +0530 Subject: [PATCH 10/31] fix(web): identify invalid batch archives Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- demo/web/worker.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/web/worker.mjs b/demo/web/worker.mjs index 1844ef2cd3..b172e24b25 100644 --- a/demo/web/worker.mjs +++ b/demo/web/worker.mjs @@ -103,6 +103,8 @@ self.onmessage = async ({ data }) => { keepOpen = true; } self.postMessage({ type: "inspected", index, count: inputs.length, name: item.name, summary }); + } catch (error) { + throw new Error(`${item.name}: ${error instanceof Error ? error.message : String(error)}`); } finally { if (!keepOpen) await current.close(); } From f95dc97134a6f1f0030e28c7e783fab8610bacae Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Thu, 15 Oct 2026 00:32:54 +0000 Subject: [PATCH 11/31] perf(agc): release unused registered shader resource plans --- .../src/Driver/Shaders/ShaderRegistry.cpp | 7 ++++ .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 16 ++++++++ core/shader/recompiler/CMakeLists.txt | 1 + .../include/Optimization/ResourceProgram.hpp | 1 + core/shader/recompiler/Recompiler.cpp | 41 +++++++++++++++---- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index c85ab3ee3e..617e72387d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -725,6 +725,7 @@ void BuildRegisteredAbiKey(const QueueState& state, const VulkanDevice& device, struct RegisteredPreparation { std::size_t codeOffset = 0; + bool registration = false; ShaderRecompiler::SwappcInfo swappc{}; std::vector userData; std::array memory{}; @@ -770,6 +771,7 @@ std::unique_ptr PlanRegistered(const ShaderSnapshot& snap auto plan = std::make_unique(); plan->codeOffset = codeOffset; ShaderRecompiler::RdnaProgram decoded; + plan->registration = registration; try { decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); } catch (const ShaderRecompiler::UnsupportedInstructionError& error) { @@ -878,6 +880,11 @@ std::vector PrepareRegistered(RegisteredPreparation& pla request.layout.pushConstantSizeBytes = 0; append(); } + if (plan.registration) { + for (const auto& entry : entries) { + if (entry.handle != nullptr) ShaderRecompiler::ReleaseUndrawnResourcePlan(*entry.handle); + } + } return entries; } diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 6d79855198..40fd7a4a00 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1860,6 +1860,22 @@ void verifyPlanRoundTrip() { std::cout << "plan round trip: " << fixture.name << ", " << plan->valueStorage.Size() << " values, " << ShaderDiskCache::EncodePlanEntry(key, *plan).size() << " bytes per entry, " << succeeded << " of 1000 runtimes materialized\n"; ++index; } + auto releaseFixture = fixtures.front(); + releaseFixture.waveSize = 32u; + auto registeredRequest = releaseFixture.Request(true); + const auto registeredHandle = PrepareShader(registeredRequest); + const auto registeredPlan = GetResourcePlan(registeredRequest); + ShaderDiskCache::Flush(); + ReleaseUndrawnResourcePlan(*registeredHandle); + const auto beforeReload = ShaderDiskCache::Totals(); + const auto reloadedPlan = GetResourcePlan(registeredRequest); + require(ShaderDiskCache::Totals().planHits == beforeReload.planHits + 1u, "a released registration plan was not reloaded from the disk cache"); + require(samePlan(*registeredPlan, *reloadedPlan), "a released registration plan changed when reloaded"); + RequestMemoryView registeredMemory(registeredRequest.context.memory); + const auto runtimeCapture = CaptureResources(registeredRequest, registeredMemory.MakeRuntime(registeredRequest.context.userData, registeredRequest.shader.codeAddress), *registeredHandle); + const auto beforeKeep = ShaderDiskCache::Totals(); + ReleaseUndrawnResourcePlan(*registeredHandle); + require(GetResourcePlan(registeredRequest) == runtimeCapture->plan && ShaderDiskCache::Totals().planHits == beforeKeep.planHits, "a runtime-captured plan was released"); const auto covers = [&](const std::function& predicate) { return std::any_of(plans.begin(), plans.end(), [&](const std::shared_ptr& plan) { return predicate(*plan); }); }; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 18cea782dc..aa023f0561 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -248,6 +248,7 @@ function(add_shader_recompiler target runtimeTarget) ResolveSource ResourceMaterializer::EvaluateNanoseconds ResourceMaterializer::MaterializeNanoseconds + ReleaseUndrawnResourcePlan ResourceMaterializer::SpecializationNanoseconds RuntimeAbi::RequireVersion RuntimeAbi::HeapCapacity diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 77328b8672..835513ad1e 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -80,6 +80,7 @@ void BuildPreparedShaderKey(const RecompileRequest& request, std::vector RegistrationResult(const RecompileRequest& request, const SourceHandle& handle); [[nodiscard]] std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle); [[nodiscard]] std::shared_ptr ResolveSource(const RecompileRequest& request); +void ReleaseUndrawnResourcePlan(const SourceHandle& handle); [[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle); } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 326d926eca..ba73b3f46a 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -326,6 +326,7 @@ struct SourceEntry : std::enable_shared_from_this { std::vector code; std::vector linkedCallFallbacks; std::shared_ptr plan; + bool runtimeCaptured = false; // A plan build that threw (an unsupported resource chain or control flow) is remembered and // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the // title issues every frame (0x1048947300 at the intro video). APS5_NO_FAILURE_MEMO=1 rebuilds. @@ -466,6 +467,25 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr return source; } +std::shared_ptr acquirePlan(const RecompileRequest& request, const std::shared_ptr& source, bool runtimeCapture) { + for (;;) { + { + std::lock_guard lock(source->mutex); + if (source->plan != nullptr) { + if (runtimeCapture) source->runtimeCaptured = true; + return source->plan; + } + } + if (getSource(request) != source) throw std::runtime_error("ShaderRecompiler: resource plan source changed while reloading"); + } +} + +void releaseUndrawnPlan(const SourceHandle& handle) { + if (handle.source == nullptr || !diskPlans()) return; + std::lock_guard lock(handle.source->mutex); + if (!handle.source->runtimeCaptured) handle.source->plan.reset(); +} + std::array partialThreads(const RecompileRequest& request) { return request.context.compute ? request.context.compute->partialThreads : std::array{}; } @@ -1104,7 +1124,7 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { return materializeResult(variant, request, snapshot); } const auto source = getSource(request); - materializer.Materialize(*source->plan, runtime, snapshot); + materializer.Materialize(*acquirePlan(request, source, true), runtime, snapshot); return materializeVariant(*source, request, snapshot); } @@ -1312,10 +1332,17 @@ auto recompileReporting(const RecompileRequest& request, Impl&& impl) -> decltyp std::shared_ptr GetResourcePlan(const RecompileRequest& request) { static_cast(RequestInputInfo(request)); - if (request.useCache) return getSource(request)->plan; + if (request.useCache) { + const auto source = getSource(request); + return acquirePlan(request, source, false); + } return makeResourcePlan(request); } +void ReleaseUndrawnResourcePlan(const SourceHandle& handle) { + releaseUndrawnPlan(handle); +} + namespace { // The capture's materialization; with pure flat slots in the plan the walk's read addresses are @@ -1344,7 +1371,7 @@ std::shared_ptr CaptureResources(const RecompileRequest& auto capture = std::make_shared(); if (request.useCache) { capture->source = getSource(request); - capture->plan = capture->source->plan; + capture->plan = acquirePlan(request, capture->source, true); } else { capture->plan = makeResourcePlan(request); } @@ -1428,13 +1455,13 @@ std::optional PreparedShaderInvocation::TryCreate(cons std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { auto capture = std::make_shared(); capture->source = handle->source; - capture->plan = handle->source->plan; + capture->plan = acquirePlan(request, handle->source, true); materializeCapture(*capture, runtime); return capture; } std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { - if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (capture.source != handle->source) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); result->cacheHit = true; @@ -1462,7 +1489,7 @@ std::shared_ptr RegistrationResult(const RecompileRequest std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle) { if (!MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); - if (capture.source != handle.source || capture.plan != handle.source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (capture.source != handle.source) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle.source, handle.artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle.artifact, request, capture.snapshot)); result->cacheHit = true; @@ -1478,7 +1505,7 @@ std::shared_ptr CaptureResources(const RecompileRequest& if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); auto capture = std::make_shared(); capture->source = handle.source; - capture->plan = handle.source->plan; + capture->plan = acquirePlan(request, handle.source, true); if (profile) capture->sourceNanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); materializeCapture(*capture, runtime); return capture; From c00b0dfb9f9ae02663344876b12d4e04fe3e388b Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Thu, 15 Oct 2026 09:53:44 +0000 Subject: [PATCH 12/31] fix(shader): avoid retaining IR after plan cache misses --- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 22 +++++++++++++++++++ core/shader/recompiler/Recompiler.cpp | 6 ++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 40fd7a4a00..3086748af2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1871,6 +1871,28 @@ void verifyPlanRoundTrip() { const auto reloadedPlan = GetResourcePlan(registeredRequest); require(ShaderDiskCache::Totals().planHits == beforeReload.planHits + 1u, "a released registration plan was not reloaded from the disk cache"); require(samePlan(*registeredPlan, *reloadedPlan), "a released registration plan changed when reloaded"); + + auto reloadFixture = fixtures.front(); + reloadFixture.waveSize = 32u; + reloadFixture.code.insert(reloadFixture.code.begin(), 0xbf800004u); + const auto reloadRequest = reloadFixture.Request(true); + const auto reloadHandle = PrepareShader(reloadRequest); + static_cast(GetResourcePlan(reloadRequest)); + ShaderDiskCache::Flush(); + std::vector reloadKey; + ShaderDiskCache::BuildPlanKey(reloadRequest, 32u, reloadKey); + const auto reloadPath = ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(reloadKey); + require(std::filesystem::is_regular_file(reloadPath), "the reload regression plan was not persisted"); + const auto preservedPath = reloadPath.string() + ".reload-preserved"; + std::filesystem::rename(reloadPath, preservedPath); + ReleaseUndrawnResourcePlan(*reloadHandle); + static_cast(GetResourcePlan(reloadRequest)); + auto secondLayout = reloadRequest; + secondLayout.layout.pushConstantSizeBytes = 0u; + ShaderPreparationContext preparation; + static_cast(PrepareShader(secondLayout, &preparation)); + require(preparation.ResourcePrograms() == 1u, "a plan reload retained full IR after an artifact for that layout was already available"); + RequestMemoryView registeredMemory(registeredRequest.context.memory); const auto runtimeCapture = CaptureResources(registeredRequest, registeredMemory.MakeRuntime(registeredRequest.context.userData, registeredRequest.shader.codeAddress), *registeredHandle); const auto beforeKeep = ShaderDiskCache::Totals(); diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index ba73b3f46a..6d16e1c46d 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -397,6 +397,8 @@ bool diskPlans() { return enabled && ShaderDiskCache::Enabled() && !DebugProbeActive(); } +bool sameLayout(const BindingLayout& left, const BindingLayout& right); + std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true) { static PrimedSharedMutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. @@ -455,7 +457,9 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr try { ResourceProgram resource(request, preparation, reclaim); source->plan = std::move(resource.plan); - source->program = std::move(resource.program); + if (std::ranges::none_of(source->variants, [&](const auto& variant) { return sameLayout(variant->layout, request.layout); })) { + source->program = std::move(resource.program); + } } catch (...) { if (FailureMemo()) source->planFailure = std::current_exception(); throw; From 72471f5c68d5cdd18eceed230b36f6ea46d291ad Mon Sep 17 00:00:00 2001 From: xXPBSXx <20614131+xXPBSXx@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:57:25 +0200 Subject: [PATCH 13/31] test(libSceNetCtl): cover GetStateV6, GetResult and GetNatInfo in guest_net_ctl sceNetCtlGetStateV6, sceNetCtlGetResult and sceNetCtlGetNatInfo are exported and called by titles but no test calls them. The checks pin what the code does today: a null output is refused with the invalid-address error, GetStateV6 reports disconnected or IP obtained, GetResult writes 0, and GetNatInfo reports not connected. Co-Authored-By: Claude Sonnet 5.5 --- core/libs/tests/GuestNetCtl.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/libs/tests/GuestNetCtl.cpp b/core/libs/tests/GuestNetCtl.cpp index f0196de5a7..51e7010a62 100644 --- a/core/libs/tests/GuestNetCtl.cpp +++ b/core/libs/tests/GuestNetCtl.cpp @@ -21,6 +21,9 @@ extern "C" { int APS5_VABI sceNetCtlGetInfo(int, NetCtlInfo*); int APS5_VABI sceNetCtlGetState(int*); +int APS5_VABI sceNetCtlGetStateV6(int*); +int APS5_VABI sceNetCtlGetResult(int, int*); +int APS5_VABI sceNetCtlGetNatInfo(NetCtlNatInfo*); } namespace { @@ -89,6 +92,15 @@ int main() { WSADATA data{}; Require(WSAStartup(MAKEWORD(2, 2), &data) == 0); #endif + Require(sceNetCtlGetStateV6(nullptr) == kErrInvalidAddr); + int stateV6 = -1; + Require(sceNetCtlGetStateV6(&stateV6) == 0); + Require(stateV6 == 0 || stateV6 == kStateIpObtained); + Require(sceNetCtlGetResult(0, nullptr) == kErrInvalidAddr); + int errorCode = -1; + Require(sceNetCtlGetResult(0, &errorCode) == 0 && errorCode == 0); + NetCtlNatInfo natInfo{}; + Require(sceNetCtlGetNatInfo(&natInfo) == kErrNotConnected); NetCtlInfo info{}; Require(sceNetCtlGetInfo(0, &info) == kErrInvalidCode); Require(sceNetCtlGetInfo(22, &info) == kErrInvalidCode); From d8911150d9e36c732c741b24caaffc2c07191ed3 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:31:35 -0400 Subject: [PATCH 14/31] test(shader): cover plan reloads behind a relocated push layout A prepared artifact also serves draws whose push constants start at another offset. When such a capture reloads a released plan that is missing from the disk cache, the rebuilt resource program must not stay on the source entry. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 3086748af2..903926bcd7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1893,6 +1893,31 @@ void verifyPlanRoundTrip() { static_cast(PrepareShader(secondLayout, &preparation)); require(preparation.ResourcePrograms() == 1u, "a plan reload retained full IR after an artifact for that layout was already available"); + auto relocatedFixture = fixtures.front(); + relocatedFixture.waveSize = 32u; + relocatedFixture.code.insert(relocatedFixture.code.begin(), 0xbf800005u); + const auto relocatedRequest = relocatedFixture.Request(true); + const auto relocatedHandle = PrepareShader(relocatedRequest); + static_cast(GetResourcePlan(relocatedRequest)); + ShaderDiskCache::Flush(); + std::vector relocatedKey; + ShaderDiskCache::BuildPlanKey(relocatedRequest, 32u, relocatedKey); + const auto relocatedPath = ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(relocatedKey); + std::filesystem::rename(relocatedPath, relocatedPath.string() + ".reload-preserved"); + ReleaseUndrawnResourcePlan(*relocatedHandle); + auto drawn = relocatedRequest; + drawn.layout.pushConstantOffsetBytes = 16u; + drawn.layout.pushConstantSizeBytes = 112u; + const auto relocatedInvocation = PreparedShaderInvocation::TryCreate(drawn, relocatedHandle); + require(relocatedInvocation.has_value(), "the prepared artifact refused a relocated push layout"); + RequestMemoryView drawnMemory(drawn.context.memory); + static_cast(relocatedInvocation->Materialize(*relocatedInvocation->Capture(drawnMemory.MakeRuntime(drawn.context.userData, drawn.shader.codeAddress)))); + auto relocatedProbe = relocatedRequest; + relocatedProbe.layout.pushConstantSizeBytes = 0u; + ShaderPreparationContext relocatedPreparation; + static_cast(PrepareShader(relocatedProbe, &relocatedPreparation)); + require(relocatedPreparation.ResourcePrograms() == 1u, "a plan reload retained full IR after the prepared artifact served a relocated push layout"); + RequestMemoryView registeredMemory(registeredRequest.context.memory); const auto runtimeCapture = CaptureResources(registeredRequest, registeredMemory.MakeRuntime(registeredRequest.context.userData, registeredRequest.shader.codeAddress), *registeredHandle); const auto beforeKeep = ShaderDiskCache::Totals(); From b9a329613899c0cc96f3522a9baa19be41e4e886 Mon Sep 17 00:00:00 2001 From: fdfawef324 <287205748+fdfawef324@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:31:35 -0400 Subject: [PATCH 15/31] fix(shader): drop rebuilt IR when a prepared artifact serves the reload Prepared captures pass their source handle to the plan reload. When the handle matches the request, its artifact serves the draw, so the rebuilt resource program is not kept even if the push layout differs from the artifact's. A handle that does not match the source is rejected before the entry changes. Same change as f1b42dae4 on top of c00b0dfb9. Co-Authored-By: Claude Opus 5.5 --- core/shader/recompiler/Recompiler.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 6d16e1c46d..1a173644ed 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -399,7 +399,7 @@ bool diskPlans() { bool sameLayout(const BindingLayout& left, const BindingLayout& right); -std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true) { +std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true, const SourceHandle* preparedHandle = nullptr) { static PrimedSharedMutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. static std::unordered_map, std::vector>, SourceKeyHash> sources; @@ -442,6 +442,8 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr { std::lock_guard lock(source->mutex); if (source->plan == nullptr) { + const bool hasPreparedArtifact = preparedHandle != nullptr && preparedHandle->artifact != nullptr; + if (hasPreparedArtifact && (preparedHandle->source != source || !MatchesPreparedShader(request, *preparedHandle))) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the resource plan source"); if (FailureMemo() && source->planFailure) std::rethrow_exception(source->planFailure); const bool disk = diskPlans(); std::vector planKey; @@ -457,7 +459,7 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr try { ResourceProgram resource(request, preparation, reclaim); source->plan = std::move(resource.plan); - if (std::ranges::none_of(source->variants, [&](const auto& variant) { return sameLayout(variant->layout, request.layout); })) { + if (!hasPreparedArtifact && std::ranges::none_of(source->variants, [&](const auto& variant) { return sameLayout(variant->layout, request.layout); })) { source->program = std::move(resource.program); } } catch (...) { @@ -471,7 +473,7 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr return source; } -std::shared_ptr acquirePlan(const RecompileRequest& request, const std::shared_ptr& source, bool runtimeCapture) { +std::shared_ptr acquirePlan(const RecompileRequest& request, const std::shared_ptr& source, bool runtimeCapture, const SourceHandle* preparedHandle = nullptr) { for (;;) { { std::lock_guard lock(source->mutex); @@ -480,7 +482,7 @@ std::shared_ptr acquirePlan(const RecompileRequest& reques return source->plan; } } - if (getSource(request) != source) throw std::runtime_error("ShaderRecompiler: resource plan source changed while reloading"); + if (getSource(request, nullptr, true, preparedHandle) != source) throw std::runtime_error("ShaderRecompiler: resource plan source changed while reloading"); } } @@ -1459,7 +1461,7 @@ std::optional PreparedShaderInvocation::TryCreate(cons std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { auto capture = std::make_shared(); capture->source = handle->source; - capture->plan = acquirePlan(request, handle->source, true); + capture->plan = acquirePlan(request, handle->source, true, handle.get()); materializeCapture(*capture, runtime); return capture; } @@ -1509,7 +1511,7 @@ std::shared_ptr CaptureResources(const RecompileRequest& if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); auto capture = std::make_shared(); capture->source = handle.source; - capture->plan = acquirePlan(request, handle.source, true); + capture->plan = acquirePlan(request, handle.source, true, &handle); if (profile) capture->sourceNanoseconds = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); materializeCapture(*capture, runtime); return capture; From 739e451332fbe284b5395993f9189354774be326 Mon Sep 17 00:00:00 2001 From: Ondra97 <47064087+Ondra97@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:24:30 +0200 Subject: [PATCH 16/31] fix(libkernel): preserve directory identity across Windows renames --- core/libs/prx/libc/src/General.cpp | 18 +++- .../File/include/DirectoryDescriptor.hpp | 1 + core/libs/prx/libkernel/File/include/File.hpp | 5 + .../libkernel/File/src/DescriptorControl.cpp | 2 +- .../File/src/DirectoryDescriptor.cpp | 52 +++++++--- .../prx/libkernel/File/src/NativeStat.cpp | 2 - core/libs/prx/libkernel/File/src/Open.cpp | 8 +- core/libs/prx/libkernel/File/src/Stdio.cpp | 28 ++++-- core/libs/tests/GuestKernelSys.cpp | 97 +++++++++++++++++++ docs/dev/debt/libkernel.md | 4 +- 10 files changed, 184 insertions(+), 33 deletions(-) diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index 0f18419dfd..efe7f31d7d 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -319,9 +319,25 @@ int SetWorkingDirectory(WorkingDirectory& state, const std::filesystem::path& re const auto canonical = std::filesystem::canonical(resolved, error); if (error) return DirectoryFailure(error); if (!std::filesystem::is_directory(canonical, error)) return error ? DirectoryFailure(error) : 20; - const auto relative = canonical.lexically_relative(state.root); + auto relative = canonical.lexically_relative(state.root); +#ifdef _WIN32 + if (relative.empty() || *relative.begin() == "..") { + auto ancestor = canonical; + while (!std::filesystem::equivalent(ancestor, state.root, error)) { + if (error) return DirectoryFailure(error); + const auto parent = ancestor.parent_path(); + if (parent == ancestor) return 45; + ancestor = parent; + } + relative = canonical.lexically_relative(ancestor); + } +#endif if (relative.empty() || *relative.begin() == "..") return 45; +#ifdef _WIN32 + state.current = relative == "." ? state.root : state.root / relative; +#else state.current = canonical; +#endif return 0; } } diff --git a/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp b/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp index a740c9ac0a..f6ab25f51a 100644 --- a/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp +++ b/core/libs/prx/libkernel/File/include/DirectoryDescriptor.hpp @@ -11,6 +11,7 @@ extern "C" int OpenInDirectory_nid_no_patch(int directory, const char* path, int namespace File { int OpenDirectoryDescriptor(const std::filesystem::path& path); +bool IsDirectoryDescriptor(int fd); std::optional DirectoryDescriptorPath(int fd); void ForgetDirectoryDescriptor(int fd); void DuplicateDirectoryDescriptor(int from, int to); diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index d759d1e3f9..6c8fbfadc2 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" @@ -27,6 +29,9 @@ std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_ bool DescriptorIsOpen_nid_no_patch(int descriptor); namespace File { +#ifdef _WIN32 +std::optional WindowsDescriptorPath_nid_no_patch(int descriptor); +#endif int HostStandardError(); [[nodiscard]] std::unique_lock LockPositionedTransfer(int descriptor); void SharePositionedTransfer(int from, int to); diff --git a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp index 174f0571b1..9ac6cb3980 100644 --- a/core/libs/prx/libkernel/File/src/DescriptorControl.cpp +++ b/core/libs/prx/libkernel/File/src/DescriptorControl.cpp @@ -141,7 +141,7 @@ const char* KindName(Kind kind) { #ifdef _WIN32 std::optional Classify(int fd, const char* function) { if (fd < 0 || !DescriptorIsOpen_nid_no_patch(fd)) return std::nullopt; - if (File::DirectoryDescriptorPath(fd)) return Kind::Directory; + if (File::IsDirectoryDescriptor(fd)) return Kind::Directory; if (File::IsRandomDevice(fd)) return Kind::RandomDevice; if (File::IsArchivedFile(fd)) return Kind::ArchivedFile; switch (::GetFileType(WindowsDescriptorHandle_nid_no_patch(fd))) { diff --git a/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp b/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp index 05868357b8..aedc308062 100644 --- a/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp +++ b/core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp @@ -17,10 +17,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -32,7 +34,6 @@ constexpr std::uint8_t GuestRegularType = 8; struct DirectoryState { std::mutex mutex; - std::filesystem::path path; std::vector> entries; std::size_t cursor = 0; bool loaded = false; @@ -48,13 +49,15 @@ std::shared_ptr Find(int fd) { return found == g_directories.end() ? nullptr : found->second; } -void Load(DirectoryState& state) { +void Load(DirectoryState& state, int fd) { if (state.loaded) return; + const auto path = File::WindowsDescriptorPath_nid_no_patch(fd); + if (!path) throw std::system_error(errno, std::generic_category(), "directory descriptor path"); state.loaded = true; state.entries.emplace_back(".", GuestDirectoryType); state.entries.emplace_back("..", GuestDirectoryType); std::error_code error; - for (const auto& entry : std::filesystem::directory_iterator(state.path, error)) { + for (const auto& entry : std::filesystem::directory_iterator(*path, error)) { std::error_code typeError; state.entries.emplace_back(entry.path().filename().string(), entry.is_directory(typeError) ? GuestDirectoryType : GuestRegularType); } @@ -65,20 +68,37 @@ void Load(DirectoryState& state) { namespace File { int OpenDirectoryDescriptor(const std::filesystem::path& path) { - const int fd = ::_open("NUL", _O_RDONLY | _O_BINARY); - if (fd < 0) return -1; - std::lock_guard lock(g_mutex); const auto state = std::make_shared(); - state->path = path; - g_directories[fd] = state; - g_directoryCount = g_directories.size(); + SECURITY_ATTRIBUTES security{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; + const auto handle = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) { + errno = WindowsOpenError_nid_no_patch(GetLastError()); + return -1; + } + const int fd = ::_open_osfhandle(reinterpret_cast(handle), _O_RDONLY | _O_BINARY); + if (fd < 0) { + CloseHandle(handle); + return -1; + } + try { + std::lock_guard lock(g_mutex); + g_directories[fd] = state; + g_directoryCount = g_directories.size(); + } catch (...) { + ::_close(fd); + throw; + } return fd; } +bool IsDirectoryDescriptor(int fd) { + return g_directoryCount != 0 && Find(fd) != nullptr; +} + std::optional DirectoryDescriptorPath(int fd) { - const auto state = Find(fd); - if (!state) return std::nullopt; - return state->path; + if (!IsDirectoryDescriptor(fd)) return std::nullopt; + return WindowsDescriptorPath_nid_no_patch(fd); } void ForgetDirectoryDescriptor(int fd) { @@ -123,7 +143,7 @@ std::optional SeekDirectoryDescriptor(int fd, std::int64_t offset, state.cursor = 0; return std::int64_t{0}; } - Load(state); + Load(state, fd); state.cursor = static_cast(target); return target; } @@ -133,7 +153,7 @@ int ReadDirectoryDescriptor(int fd, char* buf, int nbytes) { if (!found) return SCE_KERNEL_ERROR_ENOTDIR; auto& state = *found; std::lock_guard lock(state.mutex); - Load(state); + Load(state, fd); std::size_t used = 0; while (state.cursor < state.entries.size()) { const auto& [name, type] = state.entries[state.cursor]; @@ -188,7 +208,11 @@ extern "C" const char* GetDirectoryDescriptorPath_nid_no_patch(int fd, int* erro } const auto directory = File::DirectoryDescriptorPath(fd); if (!directory) { +#ifdef _WIN32 + *error = File::IsDirectoryDescriptor(fd) ? errno : ENOTDIR; +#else *error = ENOTDIR; +#endif return nullptr; } struct PathTag {}; diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index b110e980df..c09c0e6606 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -1,5 +1,4 @@ #include "prx/libkernel/File/include/NativeStat.hpp" -#include "prx/libkernel/File/include/DirectoryDescriptor.hpp" #include "prx/libkernel/File/include/ArchivedFile.hpp" #include "prx/libc/include/General.hpp" @@ -43,7 +42,6 @@ static int DoStat(const std::filesystem::path& p, NativeStat* st) { return result; } static int DoFstat(int fd, NativeStat* st) { - if (const auto directory = File::DirectoryDescriptorPath(fd)) return DoStat(*directory, st); if (_fstat64(fd, st) != 0) return -1; return ReadHandleInfo(reinterpret_cast(::_get_osfhandle(fd)), st); } diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 23fd92fc59..824398bb0a 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -58,13 +58,13 @@ static int NativeRead(int fd, void* buf, std::size_t n) { if (n > static_cast(std::numeric_limits::max())) { throw std::runtime_error("sceKernelRead: nbytes exceeds platform limit"); } - const auto positioned = File::LockPositionedTransfer(fd); - const int result = ::_read(fd, buf, static_cast(n)); - if (result < 0 && _doserrno == ERROR_NO_DATA) errno = EAGAIN; - if (result == 0 && File::DirectoryDescriptorPath(fd)) { + if (File::IsDirectoryDescriptor(fd)) { errno = EISDIR; return -1; } + const auto positioned = File::LockPositionedTransfer(fd); + const int result = ::_read(fd, buf, static_cast(n)); + if (result < 0 && _doserrno == ERROR_NO_DATA) errno = EAGAIN; return result; } static int NativeWrite(int fd, const void* buf, std::size_t n) { diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 684d0b121a..95facb5935 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -87,8 +87,7 @@ static int NativeChmod(const std::filesystem::path& path, int mode) { RecordWrittenPath_nid_no_patch(path); return ::_wchmod(path.wstring().c_str(), mode); } -static std::optional NativeDescriptorPath(int descriptor) { - if (auto directory = File::DirectoryDescriptorPath(descriptor)) return directory; +std::optional File::WindowsDescriptorPath_nid_no_patch(int descriptor) { const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); if (handle == INVALID_HANDLE_VALUE) { errno = EBADF; @@ -100,20 +99,27 @@ static std::optional NativeDescriptorPath(int descriptor) path.resize(length); length = ::GetFinalPathNameByHandleW(handle, path.data(), length, FILE_NAME_NORMALIZED); } - if (length == 0 || length >= path.size()) { + if (length == 0) { + errno = WindowsOpenError_nid_no_patch(GetLastError()); + return std::nullopt; + } + if (length >= path.size()) { errno = EINVAL; return std::nullopt; } path.resize(length); + if (path.starts_with(LR"(\\?\UNC\)")) path.replace(0, 8, LR"(\\)"); + else if (path.starts_with(LR"(\\?\)") && path.size() > 6 && path[5] == L':' && path[6] == L'\\') path.erase(0, 4); return path; } static int NativeFchmod(int descriptor, int mode) { - const auto path = NativeDescriptorPath(descriptor); + const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor); return path ? NativeChmod(*path, mode) : -1; } static int NativeFtruncate(int descriptor, std::int64_t length) { const int result = File::TruncateSharedMemory_nid_no_patch(descriptor, length); if (result != -2) return result; + if (File::IsDirectoryDescriptor(descriptor)) return EINVAL; const auto handle = reinterpret_cast(WithoutParameterHandler([descriptor] { return ::_get_osfhandle(descriptor); })); if (handle == INVALID_HANDLE_VALUE) return EBADF; if (::GetFileType(handle) != FILE_TYPE_DISK) return EINVAL; @@ -156,7 +162,7 @@ static int NativeFutimes(int descriptor, const KernelTimeval* times) { const int result = SetTimes(handle, times); CloseHandle(handle); if (result == 0) { - if (const auto path = NativeDescriptorPath(descriptor)) RecordWrittenPath_nid_no_patch(*path); + if (const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor)) RecordWrittenPath_nid_no_patch(*path); } return result; } @@ -215,9 +221,13 @@ static std::int64_t NativePositionedTransfer(int descriptor, void* buf, std::siz errno = EBADF; return -1; } + if (File::IsDirectoryDescriptor(descriptor)) { + errno = write ? EBADF : EISDIR; + return -1; + } LARGE_INTEGER position{}; if (::GetFileType(handle) != FILE_TYPE_DISK || !::SetFilePointerEx(handle, LARGE_INTEGER{}, &position, FILE_CURRENT)) { - errno = !File::DirectoryDescriptorPath(descriptor) ? ESPIPE : write ? EBADF : EISDIR; + errno = ESPIPE; return -1; } OVERLAPPED overlapped{}; @@ -430,7 +440,7 @@ static std::int64_t PosixResult(std::int64_t result) { #ifdef _WIN32 static int NativeFstatfs(int descriptor, KernelStatfs* fs) { - if (!File::DirectoryDescriptorPath(descriptor)) { + if (!File::IsDirectoryDescriptor(descriptor)) { const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); if (handle == INVALID_HANDLE_VALUE) return GUEST_EBADF; const DWORD type = ::GetFileType(handle); @@ -438,7 +448,7 @@ static int NativeFstatfs(int descriptor, KernelStatfs* fs) { if (type != FILE_TYPE_DISK) throw std::runtime_error("_fstatfs: descriptor " + std::to_string(descriptor) + " is not a file on a Windows volume"); } - const auto path = NativeDescriptorPath(descriptor); + const auto path = File::WindowsDescriptorPath_nid_no_patch(descriptor); if (!path) throw std::runtime_error("_fstatfs: no path for descriptor " + std::to_string(descriptor) + ", errno=" + std::to_string(errno)); std::wstring root(path->native().size() + 2, L'\0'); if (!::GetVolumePathNameW(path->c_str(), root.data(), static_cast(root.size()))) @@ -1152,7 +1162,7 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co const auto result = offset != nullptr ? NativePositionedTransfer(d, &none, 0, *offset, write) : NativeTransferBuffer(d, &none, 0, write); return result < 0 ? SceErrorFromErrno(errno) : 0; } - if (!write && offset == nullptr && File::DirectoryDescriptorPath(d)) return SceErrorFromErrno(GUEST_EISDIR); + if (!write && offset == nullptr && File::IsDirectoryDescriptor(d)) return SceErrorFromErrno(GUEST_EISDIR); if (write && offset == nullptr && iovcnt > 1 && total <= static_cast(std::numeric_limits::max()) && File::Appends(d)) { std::vector gathered; gathered.reserve(total); diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index d16ff6789a..c596ee9243 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -1,5 +1,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libkernel/File/include/FileFlags.hpp" +#include "SceTypes.hpp" +#include #include #include #include @@ -9,6 +11,7 @@ #include #include #include +#include #include #ifdef _WIN32 #include @@ -34,6 +37,13 @@ int APS5_VABI chroot_nid_postfix(const char*); char* APS5_VABI getcwd_nid_postfix(char*, std::size_t); int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); int APS5_VABI sceKernelClose(int); +int APS5_VABI sceKernelRename(const char*, const char*); +int APS5_VABI sceKernelFstat(int, FileStat*); +int APS5_VABI sceKernelGetdents(int, char*, int); +std::int64_t APS5_VABI sceKernelLseek(int, std::int64_t, int); +std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); +int APS5_VABI dup_nid_postfix(int); +int APS5_VABI dup2_nid_postfix(int, int); int APS5_VABI setsid_nid_postfix(void); int APS5_VABI link_nid_postfix(const char*, const char*); int APS5_VABI symlink_nid_postfix(const char*, const char*); @@ -110,7 +120,93 @@ void CheckPathconf(const std::filesystem::path& root, const std::string& file) { Require(pathconf_nid_postfix(longPath.c_str(), 4) == -1 && Error() == 63); } +void DirectoryRenameTests(const std::filesystem::path& parent) { + const auto fixture = parent / "directory-identity"; + const auto original = fixture / "original"; + const auto moved = fixture / "moved"; + const auto twice = fixture / "twice"; + Require(std::filesystem::create_directory(fixture)); + Require(std::filesystem::create_directory(original)); + { std::ofstream(original / "marker").put('o'); } + { std::ofstream(original / "old-only").put('a'); } + const auto readMarker = [](int directory) { + const int file = openat_nid_postfix(directory, "marker", SCE_KERNEL_O_RDONLY); + Require(file >= 0); + char marker = 0; + Require(sceKernelRead(file, &marker, 1) == 1); + Require(sceKernelClose(file) == 0); + return marker; + }; + const auto names = [](int directory) { + std::set result; + std::array buffer{}; + for (;;) { + const int bytes = sceKernelGetdents(directory, buffer.data(), buffer.size()); + Require(bytes >= 0); + if (bytes == 0) return result; + for (int offset = 0; offset < bytes;) { + std::uint16_t length = 0; + std::memcpy(&length, buffer.data() + offset + 4, sizeof(length)); + Require(length >= 12 && offset + length <= bytes); + result.emplace(buffer.data() + offset + 8); + offset += length; + } + } + }; + const int first = sceKernelOpen(original.string().c_str(), SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); + Require(first >= 0 && readMarker(first) == 'o'); + FileStat before{}; + Require(sceKernelFstat(first, &before) == 0 && (before.st_mode & 0170000) == 0040000); + const int duplicate = dup_nid_postfix(first); + const int target = sceKernelOpen((original / "marker").string().c_str(), SCE_KERNEL_O_RDONLY, 0); + Require(duplicate >= 0 && target >= 0); + Require(dup2_nid_postfix(first, target) == target); + Require(sceKernelClose(first) == 0); + Require(sceKernelRename(original.string().c_str(), moved.string().c_str()) == 0); + Require(std::filesystem::create_directory(original)); + { std::ofstream(original / "marker").put('r'); } + { std::ofstream(original / "new-only").put('b'); } + const char marker = readMarker(duplicate); + std::fprintf(stderr, "directory marker after rename/replacement: %c, expected o\n", marker); + Require(marker == 'o' && readMarker(target) == 'o'); + const std::set expected{".", "..", "marker", "old-only"}; + Require(names(duplicate) == expected); + Require(names(target).empty()); + FileStat after{}; + Require(sceKernelFstat(target, &after) == 0 && after.st_dev == before.st_dev && after.st_ino == before.st_ino); + Require(sceKernelLseek(target, 0, 0) == 0); + char directoryByte = 0; + Require(sceKernelRead(target, &directoryByte, 1) == static_cast(0x80020015u)); + Require(sceKernelPread(target, &directoryByte, 1, 0) == static_cast(0x80020015u)); + char previous[4096]; + Require(getcwd_nid_postfix(previous, sizeof(previous)) == previous); + Require(fchdir_nid_postfix(target) == 0 && readMarker(-100) == 'o'); + Require(chdir_nid_postfix(previous) == 0); + Require(mkdirat_nid_postfix(target, "created", 0755) == 0); + Require(std::filesystem::is_directory(moved / "created") && !std::filesystem::exists(original / "created")); + Require(unlinkat_nid_postfix(duplicate, "created", 0x800) == 0); + Require(sceKernelRename(moved.string().c_str(), twice.string().c_str()) == 0); + Require(std::filesystem::create_directory(moved)); + { std::ofstream(moved / "different").put('c'); } + Require(sceKernelLseek(target, 0, 0) == 0 && names(target) == expected); + Require(sceKernelClose(duplicate) == 0); + Require(readMarker(target) == 'o'); + Require(unlinkat_nid_postfix(target, "marker", 0) == 0); + Require(unlinkat_nid_postfix(target, "old-only", 0) == 0); + Require(std::filesystem::remove(twice)); + Require(sceKernelFstat(target, &after) == 0 && after.st_dev == before.st_dev && after.st_ino == before.st_ino && (after.st_mode & 0170000) == 0040000); + Require(sceKernelClose(target) == 0); + Require(std::filesystem::remove(original / "marker") && std::filesystem::remove(original / "new-only")); + Require(std::filesystem::remove(moved / "different")); + Require(std::filesystem::remove(original) && std::filesystem::remove(moved) && std::filesystem::remove(fixture)); +} + int main() { +#ifdef _WIN32 + auto hostDirectorySpelling = std::filesystem::current_path().native(); + CharUpperBuffW(hostDirectorySpelling.data(), static_cast(hostDirectorySpelling.size())); + Require(SetCurrentDirectoryW(hostDirectorySpelling.c_str()) != 0); +#endif Require(setsid_nid_postfix() == -1 && Error() == 1); Require(umask_nid_postfix(0) == 022); Require(umask_nid_postfix(07777) == 0); @@ -118,6 +214,7 @@ int main() { const auto root = std::filesystem::path("anyps5-kernel-sys-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); Require(std::filesystem::create_directory(root)); + DirectoryRenameTests(root); const auto file = (root / "file.txt").string(); std::ofstream(file).put('x'); CheckPathconf(root, file); diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 2fb9d30137..a4bc1338c2 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -113,7 +113,7 @@ - [EVFILT_VNODE, sceKernelAddFileEvent and sceKernelDeleteFileEvent](../../../core/libs/prx/libkernel/Equeue/src/FileEvent.cpp) (libkernel) watch the host file or directory behind the descriptor and turn host changes into FreeBSD 11 vnode hints, which [`filt_vfsvnode`](https://github.com/freebsd/freebsd-src/blob/34bd34750187bae02a0086402c78c031c59dd8b3/sys/kern/vfs_subr.c#L5032) adds whole when they share a bit with the requested `fflags`: a write that grows a file is `NOTE_WRITE | NOTE_EXTEND`, another write `NOTE_WRITE`, a shrinking truncate or an attribute change `NOTE_ATTRIB`, a higher link count `NOTE_LINK`, a lower one or a pending delete `NOTE_DELETE`, a new path `NOTE_RENAME`; in a directory, an entry created, removed or renamed is `NOTE_WRITE`, with `NOTE_LINK` for a subdirectory, and on Linux an entry moved from or to another directory adds `NOTE_EXTEND`; inotify queues the two halves of a rename one after the other, so a watcher woken by the first can read it alone, and a half read without the other is classified only after a `getdents64` on the watched directory, which takes the directory lock that `rename` holds until both halves are queued ([`do_renameat2`](https://github.com/torvalds/linux/blob/v6.8/fs/namei.c#L4975) around `vfs_rename` and its `fsnotify_move`, [`iterate_dir`](https://github.com/torvalds/linux/blob/v6.8/fs/readdir.c#L103)), an ordering taken from the kernel source, not from inotify(7), which promises none; a directory registered with `NOTE_EXTEND` on Windows is accepted, as `vfs_kqfilter` does not check it, but never receives it, because the directory handle cannot tell a move between directories from a removal plus an addition. FreeBSD's hints depend on the filesystem and the console's filesystems are not known: a growing `ftruncate` reports `NOTE_WRITE | NOTE_EXTEND` where `vop_setattr_post` reports `NOTE_ATTRIB`, and on Windows `utimes` on a file reports `NOTE_WRITE`. The three Windows notifications are told apart in the error, which names the `parent`, `entries` or `subdirectories` watch, the path it was given and `GetLastError`, but a `FindFirstChangeNotification` that fails still throws rather than returning an error to the guest: the EVFILT_VNODE attach path is [`vfs_kqfilter`](https://github.com/freebsd/freebsd-src/blob/34bd34750187bae02a0086402c78c031c59dd8b3/sys/kern/vfs_subr.c#L4936), which returns `EINVAL`, `ENOMEM` or 0 after `fget` has turned a descriptor that is not open into `EBADF`, and consults no directory permission, so no FreeBSD errno stands for a host that refuses to notify, and what the console does on such a host is not measured. Only the `parent` watch can fail this way in practice: `entries` and `subdirectories` need the `FILE_LIST_DIRECTORY` that opening the watched directory already took. The parent handle produces no hint of its own, as `RunHostWatch` only calls `FindNextChangeNotification` on it, so a later change may leave the watch on its 50 ms poll instead of throwing; that is not done here. Windows uses `FindFirstChangeNotification` on the parent directory (and on a watched directory itself) and compares `GetFileInformationByHandleEx` snapshots at least every 50 ms, because a write through an open handle is reported only when the handle is flushed or closed, so file hints can arrive 50 ms late, and a file deleted while watched, which the POSIX delete of Windows 10 1903 moves into `$Extend\$Deleted` until its last handle closes, reports `NOTE_DELETE` without `NOTE_RENAME` and keeps its handle without a parent directory watch, as that directory cannot be watched; Linux uses inotify on the file and is not run. `NOTE_REVOKE`, `NOTE_OPEN`, `NOTE_CLOSE`, `NOTE_CLOSE_WRITE`, `NOTE_READ`, unknown bits and descriptors other than files, directories, pipes and sockets throw. `sceKernelAddFileEvent(eq, fd, watch, udata)` and `sceKernelDeleteFileEvent(eq, fd)` take the [OpenOrbis](https://github.com/OpenOrbis/OpenOrbis-PS4-Toolchain/blob/0a1aaf9d/include/orbis/libkernel.h#L35) and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0/src/SharpProspero/Interop/Kernel/KernelEqueue.cs#L164) signatures; `watch` is taken as the `fflags` and the event is added with `EV_CLEAR`, neither confirmed on a title (feat(libkernel): implement kevent EV_DISABLE, EV_DISPATCH and EV_FORCEONESHOT) (fix(libkernel): follow kqueue_register for EV_FORCEONESHOT and modify flags) - [sem_timedwait](../../../core/libs/prx/libkernel/Pthread/Posix/Sem.cpp) (libkernel) consumes an available count before checking the timeout and waits indefinitely for a null timeout, following FreeBSD 11's `sem_timedwait`. Console behavior for null and invalid timeouts when a count is available is unverified. - [tcsendbreak, tcdrain, tcflush, tcflow, tcgetpgrp, tcsetpgrp](../../../core/libs/prx/libkernel/File/src/Terminal.cpp) (libkernel) follow FreeBSD 11.4's [`termios.c`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/lib/libc/gen/termios.c): `tcflush` answers `EINVAL` for a queue other than `TCIFLUSH`, `TCOFLUSH` or `TCIOFLUSH` and `tcflow` for an action outside `TCOOFF`..`TCION` before the descriptor is looked at, and every other call fails as its terminal ioctl does on a descriptor that is not a terminal, with the errors `tcgetattr` gives, so `tcsendbreak` never reaches its 0.4 s pause. FreeBSD's `pipe_ioctl` answers `TIOCGPGRP` and `TIOCSPGRP` on a pipe or FIFO with the descriptor's SIGIO owner, which is not modelled, so `tcgetpgrp` and `tcsetpgrp` throw on a host pipe or FIFO above descriptor 2; descriptors 0 to 2 are taken as non-terminals that are not pipes, whatever the host attaches to them, so a pipe the guest creates on a freed descriptor 0 to 2 gets `ENOTTY` there. The PS5 `libkernel` is assumed to use this FreeBSD code, and `tcdrain` is not a cancellation point here -- [Windows file timestamps](../../../core/libs/prx/libkernel/File/include/WindowsFileTime.hpp) use the host filesystem's timestamp range and resolution. `utimes`, `futimes`, and `lutimes` reject timestamps outside nonnegative signed 64-bit Windows file-time ticks with `EINVAL`, including exactly 1601-01-01 00:00:00 UTC, whose zero value means "leave unchanged" to `SetFileTime`. Guest directory descriptors still resolve through their recorded path. Filesystems with coarser timestamp resolution cannot retain every guest microsecond +- [Windows file timestamps](../../../core/libs/prx/libkernel/File/include/WindowsFileTime.hpp) use the host filesystem's timestamp range and resolution. `utimes`, `futimes`, and `lutimes` reject timestamps outside nonnegative signed 64-bit Windows file-time ticks with `EINVAL`, including exactly 1601-01-01 00:00:00 UTC, whose zero value means "leave unchanged" to `SetFileTime`. Directory timestamp updates resolve the current path of the opened handle. Filesystems with coarser timestamp resolution cannot retain every guest microsecond - [EVFILT_VNODE on macOS](../../../core/libs/prx/libkernel/Equeue/src/FileEvent.cpp) (libkernel) registers `EVFILT_VNODE` on a duplicate of the descriptor in a host kqueue, whose notes have FreeBSD's values, and drops the `NOTE_LINK` that XNU adds to `NOTE_DELETE` when the watched file or directory is removed, since FreeBSD's `vop_remove_post` and `vop_rmdir_post` report `NOTE_DELETE` alone. As on Windows, a directory registered with `NOTE_EXTEND` is accepted but never receives it: XNU reports a move between directories as `NOTE_WRITE` alone, with `NOTE_LINK` for a subdirectory - [Windows file identity](../../../core/libs/prx/libkernel/File/src/NativeStat.cpp) (libkernel) - `stat` and `fstat` report the volume serial number as `st_dev` and the 64-bit file index folded into the guest's 32-bit `st_ino` (never 0), so two different files can in principle share an `st_ino`; the fold is not taken from a reference. `st_nlink` is clamped to 65535, the width of the guest field. The call keeps the `_wstat64` values when `GetFileInformationByHandle` fails - [lstat of a Windows symbolic link](../../../core/libs/prx/libkernel/File/src/NativeStat.cpp) (libkernel) reports a link whose target is relative, as `symlink` creates them, with the link's own times, `st_dev` and `st_ino`, mode `S_IFLNK | 0777` and the length of the target `readlink` returns as `st_size`. FreeBSD gives a new link `0777` less the creating process's umask, which the host does not keep, so the permission bits are not the console's. A link with an absolute target, a junction and any other reparse point throw, as for `readlink` @@ -121,7 +121,7 @@ - On Linux, [direct memory](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) lives in one sparse memfd at file offset = physical address, so the [GPU imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) can share one udmabuf buffer object per 64 MiB of it. Released direct memory is not punched out of the file, because a live import of its chunk pins the pages and the GPU would keep reading the old ones: its pages stay committed (the process keeps the high-water mark of the direct memory it used) and are zeroed when the range is allocated again. - [system](../../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) runs no command. It follows FreeBSD 11's `lib/libc/stdlib/system.c` for a sandbox without `/bin/sh`: a null command returns 1 unconditionally, as that version does, and any command returns the wait status of a child that could not exec the shell and called `_exit(127)` (`0x7f00`). Whether the console's application sandbox has no shell, and whether `vfork` succeeds there (FreeBSD returns -1 when it fails), is unverified - [sceKernelWriteThrottlingStatus](../../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed; returns `EFAULT` (`0x8002000e`) when passed a null status buffer pointer. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown -- [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws, where the NUL descriptor used to return 0, because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies +- [Windows directory descriptors](../../../core/libs/prx/libkernel/File/src/DirectoryDescriptor.cpp) (libkernel) take one snapshot of the directory, on the first `getdents`, `getdirentries` or non-zero `lseek` (`SEEK_SET` or `SEEK_CUR`), and keep it until it is discarded; `lseek` to offset 0 only discards the snapshot, and the next read lists the directory again, so every rewind costs O(entries) on the next read. `lseek` takes the offset as an entry index and `SEEK_CUR` with offset 0 only reports the index. `SEEK_END` throws because FreeBSD defines it relative to the directory size, which is not modelled. A negative result returns `EINVAL` and a `SEEK_CUR` offset that overflows returns `EOVERFLOW`, as FreeBSD `vn_seek` reports them. Names whose UTF-8 encoding exceeds the one-byte `d_namlen` limit are not representable in this guest directory-record layout: `getdents` returns `ENAMETOOLONG` without consuming the entry. Repeated calls therefore report the same error and entries after that name cannot be reached through this descriptor; FreeBSD's `getdents` behavior for host filenames that exceed its `MAXNAMLEN` is not verified. `getdirentries` still reports 0 through `basep` instead of the position of the block it read, so only 0 and values returned by `lseek` are valid offsets; FreeBSD offsets are filesystem cookies. Descriptors retain the opened directory's native handle across renames. Enumeration reloads and `*at` operations resolve its current path, so a concurrent rename between resolution and the operation is not atomic. Enumeration of a removed directory and network-share path queries have not been verified - [/dev/random and /dev/urandom](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) (libkernel) opened for reading return a descriptor backed by the host null device whose reads are filled with host OS entropy (`getentropy`, `rand_s` on Windows); both paths behave the same, never block, and positioned reads ignore the offset (FreeBSD `random(4)`). Only the exact paths are recognised, `fstat`, `lseek` and `mmap` see the null device, and opening them for writing throws - [pthread_mutexattr_getpshared and pthread_mutexattr_setpshared](../../../core/libs/prx/libkernel/Pthread/Posix/Mutex.cpp) (libkernel) store and report private or shared attributes, following FreeBSD 11 libthr [thr_mutexattr.c](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libthr/thread/thr_mutexattr.c). Invalid values return `EINVAL` without changing the attribute. Guest threads execute in one host process; sharing mutexes across processes is not modeled. Console behavior and invalid-attribute error precedence are unverified. - [pthread_condattr_setpshared](../../../core/libs/prx/libkernel/Pthread/Posix/Cond.cpp) (libkernel) stores a private or shared attribute, following FreeBSD 11 libthr [thr_condattr.c](https://github.com/freebsd/freebsd-src/blob/stable/11/lib/libthr/thread/thr_condattr.c): a null or uninitialized attribute and values other than `PTHREAD_PROCESS_PRIVATE` (0) and `PTHREAD_PROCESS_SHARED` (1) return `EINVAL` without changing it. Guest threads execute in one host process; sharing condition variables across processes is not modeled. PSFlyCast (PPSA99247) imports it and fails to load without it. Console behavior is unverified. From bb86b5f357b4b7ecfd58e11a70747afee9febb1d Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:56:26 +0200 Subject: [PATCH 17/31] test(agc): isolate the depth clear cases per recorder and cover plane fills against a UAV clear Each case re-activates the test device's recorder, and the DCB submit re-activates the recorder of the second device it creates, so the cases no longer depend on the order they run in. PlaneFillCase checks that a plane fill before a UAV clear is overwritten by it and a plane fill after it wins. --- .../tests/execution/DepthClearHle.cpp | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp index 93f7886211..f3c8208133 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthClearHle.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -98,12 +99,26 @@ std::vector Pm4Packet(std::uint32_t opcode, std::initializer_list return words; } +Graphics::Recorder* testRecorder = nullptr; +Graphics::Recorder* driverRecorder = nullptr; + +void UseTestDevice(AgcDriver::VulkanDevice& device) { + testRecorder = AgcDriver::VulkanDeviceTestAccess::ContextOf(device).recorder; + Require(testRecorder != nullptr, "depth clear: the test device has no recorder"); + testRecorder->Activate(); +} + void Submit(std::vector> packets) { std::vector words; for (const auto& packet : packets) words.insert(words.end(), packet.begin(), packet.end()); Packet dcb{words.data(), static_cast(words.size()), 0, {}}; + if (driverRecorder != nullptr) driverRecorder->Activate(); Require(sceAgcDriverSubmitDcb(&dcb) == 0, "depth clear: DCB submission failed"); AgcDriverWaitIdle_nid_postfix(); + if (driverRecorder == nullptr) { + driverRecorder = Graphics::Recorder::Active(); + Require(driverRecorder != nullptr && driverRecorder != testRecorder, "depth clear: the driver's device did not activate its own recorder"); + } } std::array TextureDescriptor(std::uint64_t address, std::uint32_t format, std::uint32_t tile = TileLinear, std::uint32_t type = Type2DArray) { @@ -284,6 +299,7 @@ void RequireDepthTexels(AgcDriver::VulkanDevice& device, const GuestBlock& plane } Graphics::DepthClearStatistics ClearSequence(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "mode " + std::to_string(mode) + ": "; @@ -341,6 +357,7 @@ Graphics::DepthClearStatistics ClearSequence(AgcDriver::VulkanDevice& device, co } void IntegerViewCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "integer view, mode " + std::to_string(mode) + ": "; @@ -362,6 +379,7 @@ void IntegerViewCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, in } void DriverMatchCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "driver submit, mode " + std::to_string(mode) + ": "; @@ -379,6 +397,7 @@ void DriverMatchCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, in } void SixteenBitCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "16-bit plane, mode " + std::to_string(mode) + ": "; @@ -443,6 +462,7 @@ void SixteenBitCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int } void StencilCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); Graphics::SetDepthPlaneClearForTest(mode != 0); const auto before = Graphics::DepthClearCounts(); const std::string label = "depth and stencil, mode " + std::to_string(mode) + ": "; @@ -464,6 +484,40 @@ void StencilCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mo else Require(difference.clears == 1u, label + "the clear was not taken (" + std::to_string(difference.clears) + " clears, " + std::to_string(Declined(difference)) + " declined)"); } +void PlaneFillCase(AgcDriver::VulkanDevice& device, const Shaders& shaders, int mode) { + UseTestDevice(device); + Graphics::SetDepthPlaneClearForTest(mode != 0); + const auto before = Graphics::DepthClearCounts(); + const std::string label = "plane fill, mode " + std::to_string(mode) + ": "; + auto& code = NewBlock(); + std::memcpy(code.Data(), ClearKernel.data(), sizeof(ClearKernel)); + auto& plane = NewBlock(); + auto& color = NewBlock(); + const Surface surface{plane.Address(), 0, 0, false, VK_FORMAT_D32_SFLOAT}; + const auto planeBytes = static_cast(Graphics::DepthSliceBytes({Width, Height}, 4u)); + const auto drawLess = [&] { return Draw(device, shaders, surface, {.depthCompare = VK_COMPARE_OP_LESS}); }; + const auto drawGreater = [&] { return Draw(device, shaders, surface, {.depthCompare = VK_COMPARE_OP_GREATER}); }; + Require(Draw(device, shaders, surface, {.depthWrite = true, .depthCompare = VK_COMPARE_OP_ALWAYS}) == Covered, label + "the producer did not cover the plane"); + Require(drawLess() == 0u, label + "the plane does not hold the 0.5 the producer wrote"); + + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(0.125f)); + Require(drawGreater() == Covered, label + "a plane fill of 0.125 did not clear the depth"); + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(1.0f)); + SetColor(color, 0.25f); + RunClear(device, code, plane, color, Format32Float, 8, 4); + Require(drawLess() == 0u, label + "a clear to 0.25 after a plane fill of 1.0 left the fill in the depth"); + Require(drawGreater() == Covered, label + "a clear to 0.25 after a plane fill of 1.0 did not leave 0.25 everywhere"); + + SetColor(color, 1.0f); + RunClear(device, code, plane, color, Format32Float, 8, 4); + Graphics::NoteDepthMetadataFill(plane.Address(), planeBytes, std::bit_cast(0.125f)); + Require(drawLess() == 0u, label + "a plane fill of 0.125 after a clear to 1.0 left the clear in the depth"); + Require(drawGreater() == Covered, label + "a plane fill of 0.125 after a clear to 1.0 did not leave 0.125 everywhere"); + const auto difference = Difference(Graphics::DepthClearCounts(), before); + if (mode == 0) Require(difference.matched == 0u && difference.clears == 0u, label + "the lever counted with the switch off"); + else Require(difference.clears == 2u, label + "the lever took " + std::to_string(difference.clears) + " clears, expected 2"); +} + } int main() { @@ -502,6 +556,7 @@ int main() { for (const int mode : {0, 1}) run(mode == 0 ? "integer view, switch off" : "integer view, lever", [&] { IntegerViewCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "16-bit plane, switch off" : "16-bit plane, lever", [&] { SixteenBitCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "depth and stencil, switch off" : "depth and stencil, lever", [&] { StencilCase(*device, shaders, mode); }); + for (const int mode : {0, 1}) run(mode == 0 ? "plane fill, switch off" : "plane fill, lever", [&] { PlaneFillCase(*device, shaders, mode); }); for (const int mode : {0, 1}) run(mode == 0 ? "driver submit, switch off" : "driver submit, lever", [&] { DriverMatchCase(*device, shaders, mode); }); Graphics::SetDepthPlaneClearForTest(true); LibcRunShutdown_nid_postfix(); From 53af7be7ee88d35ea76b53f85690bf80489c97df Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:48:46 +0200 Subject: [PATCH 18/31] feat(libScePad): toggle mouse capture with F7 beside the middle button --- .../prx/libScePad/include/InputMapping.hpp | 1 + .../prx/libSceVideoOut/tests/InputMapping.cpp | 20 +++++++++++++++++++ .../libSceVideoOut/tests/KeyboardMouse.cpp | 2 ++ docs/user/INPUT_MAPPING.md | 4 ++-- 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libScePad/include/InputMapping.hpp b/core/libs/prx/libScePad/include/InputMapping.hpp index a8914303d5..62bc9f2fa2 100644 --- a/core/libs/prx/libScePad/include/InputMapping.hpp +++ b/core/libs/prx/libScePad/include/InputMapping.hpp @@ -62,6 +62,7 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_G, MouseButton::None, InputControl::RightStickDown}, InputBinding{SDL_SCANCODE_BACKSPACE, MouseButton::None, InputControl::TouchLeft}, InputBinding{SDL_SCANCODE_TAB, MouseButton::None, InputControl::TouchRight}, + InputBinding{SDL_SCANCODE_F7, MouseButton::None, InputControl::ToggleMouse}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Middle, InputControl::ToggleMouse}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Left, InputControl::Button, PadButton::Square}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Right, InputControl::Button, PadButton::R2}, diff --git a/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp b/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp index 1fe0f4b30b..f501f2aa77 100644 --- a/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp +++ b/core/libs/prx/libSceVideoOut/tests/InputMapping.cpp @@ -147,6 +147,25 @@ void nonAsciiPaths() { require(message.find(utf8Name) != std::string::npos, "a missing file names the path in UTF-8"); } +std::size_t toggleMouseBindings(const Pad::InputConfiguration& configuration, SDL_Scancode key, Pad::MouseButton button) { + std::size_t count = 0; + for (const auto& binding : configuration.bindings) { + if (binding.control == Pad::InputControl::ToggleMouse && binding.key == key && binding.mouseButton == button) ++count; + } + return count; +} + +void toggleMouseDefaults() { + useConfig("Cross = KEY:F\n"); + auto configuration = Pad::LoadInputMapping(); + require(toggleMouseBindings(configuration, SDL_SCANCODE_F7, Pad::MouseButton::None) == 1, "F7 toggles the mouse by default"); + require(toggleMouseBindings(configuration, SDL_SCANCODE_UNKNOWN, Pad::MouseButton::Middle) == 1, "the middle button toggles the mouse by default"); + useConfig("ToggleMouse = MOUSE:Right\n"); + configuration = Pad::LoadInputMapping(); + require(toggleMouseBindings(configuration, SDL_SCANCODE_F7, Pad::MouseButton::None) == 0, "a ToggleMouse line replaces F7"); + require(toggleMouseBindings(configuration, SDL_SCANCODE_UNKNOWN, Pad::MouseButton::Right) == 1, "a ToggleMouse line adds its own binding"); +} + void deadzoneOffMatchesRawSticks() { for (const std::int16_t axis : {std::int16_t{-32768}, std::int16_t{-20000}, std::int16_t{-1}, std::int16_t{0}, std::int16_t{1}, std::int16_t{1234}, std::int16_t{32767}}) { const auto stick = Pad::StickWithDeadzone(axis, axis, 0); @@ -183,6 +202,7 @@ int main() try { mouseLookDefault(); mouseLookParse(); mouseLookReject(); + toggleMouseDefaults(); nonAsciiPaths(); deadzoneOffMatchesRawSticks(); deadzoneShapesSticks(); diff --git a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp index fa9b6f7e3a..7810dc4d65 100644 --- a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp @@ -283,6 +283,8 @@ void testToggleRequests() { const auto now = Pad::KeyboardMouse::Clock::now(); check(keyboardMouse.HandleEvent(PadSimulation::ButtonEvent(SDL_BUTTON_MIDDLE, true), now).toggleMouse, "middle click asks to toggle the mouse"); check(!keyboardMouse.HandleEvent(PadSimulation::ButtonEvent(SDL_BUTTON_MIDDLE, false), now).toggleMouse, "releasing it asks for nothing"); + check(keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F7, true), now).toggleMouse, "F7 asks to toggle the mouse"); + check(!keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F7, false), now).toggleMouse, "releasing F7 asks for nothing"); check(keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F11, true), now).toggleFullscreen, "F11 asks to toggle fullscreen"); } diff --git a/docs/user/INPUT_MAPPING.md b/docs/user/INPUT_MAPPING.md index 882ce0a000..68344cd645 100644 --- a/docs/user/INPUT_MAPPING.md +++ b/docs/user/INPUT_MAPPING.md @@ -38,7 +38,7 @@ Without an `anyps5-input.ini`, or for actions the file does not mention, these b | `LeftStickLeft`, `LeftStickRight`, `LeftStickUp`, `LeftStickDown` | A, D, W, S | | `RightStickLeft`, `RightStickRight`, `RightStickUp`, `RightStickDown` | F, H, T, G | | `TouchLeft`, `TouchRight` | Backspace, Tab | -| `ToggleMouse` | Middle mouse button | +| `ToggleMouse` | Middle mouse button, F7 | | `ToggleFullscreen` | F11 | `L2` has no built-in keyboard or mouse binding. @@ -78,7 +78,7 @@ MouseSensitivity = 2.5 ## Mouse as the right stick -`ToggleMouse` (the middle mouse button by default) captures the mouse and drives the right stick from how fast the mouse moves. Capture uses raw relative input, so the pointer speed and acceleration set in the operating system don't apply. Press it again to release the mouse. Capture also ends, and every held key is let go, when the window loses focus. +`ToggleMouse` (the middle mouse button or F7 by default; a mouse without a middle button can use F7) captures the mouse and drives the right stick from how fast the mouse moves. Capture uses raw relative input, so the pointer speed and acceleration set in the operating system don't apply. Press it again to release the mouse. Capture also ends, and every held key is let go, when the window loses focus. The stick tilts in the direction the mouse moves, and further the faster it moves. Speed is measured in counts per second over the time between updates, so the same hand speed gives the same tilt at any frame rate. Any movement tilts the stick at least `MouseDeadzoneOffset`, so slow movement isn't lost inside the game's own deadzone. When the mouse stops, the stick eases back and is centered `MouseDecayMs` after the last movement. From 367d46e480bcaefe59b27fdef04b7ee8b2979e37 Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:49:11 +0200 Subject: [PATCH 19/31] feat(libScePad): bind L2 to R and the first mouse side button --- .../prx/libScePad/include/InputMapping.hpp | 2 ++ .../libSceVideoOut/tests/KeyboardMouse.cpp | 19 +++++++++++++++++++ docs/user/INPUT_MAPPING.md | 3 +-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libScePad/include/InputMapping.hpp b/core/libs/prx/libScePad/include/InputMapping.hpp index a8914303d5..eab287c508 100644 --- a/core/libs/prx/libScePad/include/InputMapping.hpp +++ b/core/libs/prx/libScePad/include/InputMapping.hpp @@ -42,6 +42,7 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_C, MouseButton::None, InputControl::Button, PadButton::Circle}, InputBinding{SDL_SCANCODE_Q, MouseButton::None, InputControl::Button, PadButton::L1}, InputBinding{SDL_SCANCODE_E, MouseButton::None, InputControl::Button, PadButton::R1}, + InputBinding{SDL_SCANCODE_R, MouseButton::None, InputControl::Button, PadButton::L2}, InputBinding{SDL_SCANCODE_LALT, MouseButton::None, InputControl::Button, PadButton::R1}, InputBinding{SDL_SCANCODE_RALT, MouseButton::None, InputControl::Button, PadButton::R1}, InputBinding{SDL_SCANCODE_LSHIFT, MouseButton::None, InputControl::Button, PadButton::L3}, @@ -65,6 +66,7 @@ inline constexpr std::array InputMapping{ InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Middle, InputControl::ToggleMouse}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Left, InputControl::Button, PadButton::Square}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::Right, InputControl::Button, PadButton::R2}, + InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::X1, InputControl::Button, PadButton::L2}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::None, InputControl::Button, PadButton::Up, 1}, InputBinding{SDL_SCANCODE_UNKNOWN, MouseButton::None, InputControl::Button, PadButton::Down, -1} }; diff --git a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp index fa9b6f7e3a..50895e00aa 100644 --- a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -286,6 +287,23 @@ void testToggleRequests() { check(keyboardMouse.HandleEvent(KeyEvent(SDL_SCANCODE_F11, true), now).toggleFullscreen, "F11 asks to toggle fullscreen"); } +void testDefaultL2() { + const auto l2 = static_cast(Pad::PadButton::L2); + const auto r2 = static_cast(Pad::PadButton::R2); + const auto now = Pad::KeyboardMouse::Clock::now(); + for (const auto& [name, press, release] : { + std::tuple{"R", KeyEvent(SDL_SCANCODE_R, true), KeyEvent(SDL_SCANCODE_R, false)}, + std::tuple{"mouse button 4", PadSimulation::ButtonEvent(SDL_BUTTON_X1, true), PadSimulation::ButtonEvent(SDL_BUTTON_X1, false)}}) { + Pad::KeyboardMouse keyboardMouse(defaults()); + keyboardMouse.HandleEvent(press, now); + const auto held = keyboardMouse.Compose(PadInputState{}); + check((held.buttons & l2) != 0 && held.analogButtonsL2 == 255, std::string(name) + " presses L2 fully"); + check((held.buttons & r2) == 0 && held.analogButtonsR2 == 0, std::string(name) + " leaves R2 alone"); + keyboardMouse.HandleEvent(release, now); + const auto released = keyboardMouse.Compose(PadInputState{}); + check((released.buttons & l2) == 0 && released.analogButtonsL2 == 0, std::string(name) + " released lets go of L2"); + } +} void testTitleReadsMouse() { const auto square = static_cast(Pad::PadButton::Square); @@ -349,6 +367,7 @@ int main() { {"wheel and keys on focus loss", testWheelFocusLoss}, {"focus loss releases the mouse", testFocusLossReleasesMouse}, {"toggle requests", testToggleRequests}, + {"default L2", testDefaultL2}, {"title reads the mouse", testTitleReadsMouse}, }; for (const auto& [name, test] : tests) { diff --git a/docs/user/INPUT_MAPPING.md b/docs/user/INPUT_MAPPING.md index 882ce0a000..f4d867dd73 100644 --- a/docs/user/INPUT_MAPPING.md +++ b/docs/user/INPUT_MAPPING.md @@ -30,6 +30,7 @@ Without an `anyps5-input.ini`, or for actions the file does not mention, these b | `Square` | Left mouse button | | `L1` | Q | | `R1` | E, Left Alt, Right Alt | +| `L2` | R, Mouse button 4 (X1) | | `R2` | Right mouse button | | `L3` | Left Shift, Right Shift | | `R3` | Left Ctrl, Right Ctrl | @@ -41,8 +42,6 @@ Without an `anyps5-input.ini`, or for actions the file does not mention, these b | `ToggleMouse` | Middle mouse button | | `ToggleFullscreen` | F11 | -`L2` has no built-in keyboard or mouse binding. - For example, this changes Cross to F or Space, moves the left stick to IJKL, uses the mouse buttons for Square and R2, and keeps all other built-in bindings: ```ini From 06ca6a066bc3c1f5189991becb9b3b5eb356b658 Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:50:00 +0200 Subject: [PATCH 20/31] fix(libScePad): smooth a slow-reporting mouse over its measured report interval --- .../prx/libScePad/include/KeyboardMouse.hpp | 14 +++++ .../prx/libSceVideoOut/src/KeyboardMouse.cpp | 35 ++++++++++- .../libSceVideoOut/tests/KeyboardMouse.cpp | 58 +++++++++++++++++++ docs/user/INPUT_MAPPING.md | 2 +- 4 files changed, 107 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libScePad/include/KeyboardMouse.hpp b/core/libs/prx/libScePad/include/KeyboardMouse.hpp index 53336f6a47..c302c3f3b4 100644 --- a/core/libs/prx/libScePad/include/KeyboardMouse.hpp +++ b/core/libs/prx/libScePad/include/KeyboardMouse.hpp @@ -14,6 +14,10 @@ namespace Pad { inline constexpr double MouseFullTiltCountsPerSecond = 3000.0; inline constexpr double MouseMinimumStepSeconds = 0.0005; +inline constexpr double MouseRateWindowSeconds = 0.25; +inline constexpr double MouseRateGapSeconds = 0.05; +inline constexpr double MouseSmoothingReports = 3.0; +inline constexpr double MouseSmoothingLimitMs = 30.0; class MouseStick { public: @@ -22,16 +26,26 @@ public: void Advance(double seconds); void Reset(); double Speed() const; + double ReportRate() const; + double SmoothingMs() const; std::array Deflection() const; private: + void measureReportRate(double step, bool moved); + InputSettings settings; double pendingX = 0.0; double pendingY = 0.0; double pendingSeconds = 0.0; + double pendingCounts = 0.0; + int pendingReports = 0; double speedX = 0.0; double speedY = 0.0; double idleSeconds = 0.0; + double rateSeconds = 0.0; + double rateCounts = 0.0; + int rateReports = 0; + double reportRate = 0.0; }; struct InputRequests { diff --git a/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp index 9b58c374de..75052bfb4f 100644 --- a/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/src/KeyboardMouse.cpp @@ -9,6 +9,32 @@ Pad::MouseStick::MouseStick(const InputSettings& settings) : settings(settings) void Pad::MouseStick::Move(int deltaX, int deltaY) { pendingX += deltaX; pendingY += deltaY; + pendingCounts += std::hypot(deltaX, deltaY); + ++pendingReports; +} + +void Pad::MouseStick::measureReportRate(double step, bool moved) { + if (moved) { + rateSeconds += std::min(idleSeconds + step, MouseRateGapSeconds); + rateCounts += pendingCounts; + rateReports += pendingReports; + } + pendingCounts = 0.0; + pendingReports = 0; + if (rateSeconds < MouseRateWindowSeconds) return; + if (rateReports >= 10 && rateCounts >= 2.0 * rateReports) reportRate = rateReports / rateSeconds; + rateSeconds = 0.0; + rateCounts = 0.0; + rateReports = 0; +} + +double Pad::MouseStick::ReportRate() const { + return reportRate; +} + +double Pad::MouseStick::SmoothingMs() const { + const double reports = reportRate > 0.0 ? std::min(MouseSmoothingReports * 1000.0 / reportRate, MouseSmoothingLimitMs) : 0.0; + return settings.mouseSmoothingMs > 0.0 ? std::max(settings.mouseSmoothingMs, reports) : 0.0; } void Pad::MouseStick::Advance(double seconds) { @@ -23,13 +49,15 @@ void Pad::MouseStick::Advance(double seconds) { const double targetY = pendingY * scale * settings.mouseVerticalScale; pendingX = 0.0; pendingY = 0.0; + measureReportRate(step, moved); idleSeconds = moved ? 0.0 : idleSeconds + step; if (idleSeconds * 1000.0 >= settings.mouseDecayMs) { speedX = 0.0; speedY = 0.0; return; } - const double keep = settings.mouseSmoothingMs > 0.0 ? std::exp(-step * 1000.0 / settings.mouseSmoothingMs) : 0.0; + const double smoothing = SmoothingMs(); + const double keep = smoothing > 0.0 ? std::exp(-step * 1000.0 / smoothing) : 0.0; speedX = targetX + (speedX - targetX) * keep; speedY = targetY + (speedY - targetY) * keep; const double speed = std::hypot(speedX, speedY); @@ -43,6 +71,11 @@ void Pad::MouseStick::Reset() { pendingX = 0.0; pendingY = 0.0; pendingSeconds = 0.0; + pendingCounts = 0.0; + pendingReports = 0; + rateSeconds = 0.0; + rateCounts = 0.0; + rateReports = 0; speedX = 0.0; speedY = 0.0; idleSeconds = 0.0; diff --git a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp index fa9b6f7e3a..ad8464174e 100644 --- a/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp +++ b/core/libs/prx/libSceVideoOut/tests/KeyboardMouse.cpp @@ -182,6 +182,62 @@ void testSmoothing() { check(early > 0.5 * 0.55 && early < 0.5 * 0.72, "smoothing reaches about 63 percent after one time constant"); } +void testSlowReportingMouse() { + const double speed = 0.5 * Pad::MouseFullTiltCountsPerSecond; + const double expected = expectedDeflection(defaults().settings, 0.5); + for (const double reportHz : {125.0, 250.0, 1000.0}) { + std::vector events{PadSimulation::MouseModeAt(0.0, true)}; + PadSimulation::AppendMotion(events, 0.0, 2000.0, speed, 0.0, reportHz); + for (const double poll : {16.7, 8.3, 6.9, 1.0}) { + const auto samples = Simulate(defaults(), events, poll, 2000.0); + double low = 1.0; + double high = 0.0; + for (const auto& sample : samples) { + if (sample.ms < 1000.0) continue; + low = std::min(low, axis(sample.state.sticks[2])); + high = std::max(high, axis(sample.state.sticks[2])); + } + check(high - low < 0.125, "a " + std::to_string(reportHz) + " Hz mouse at " + std::to_string(poll) + " ms polls wavers by " + std::to_string(high - low)); + check(std::abs((high + low) / 2.0 - expected) < 0.05, "a " + std::to_string(reportHz) + " Hz mouse at " + std::to_string(poll) + " ms polls stays around the expected tilt"); + } + } +} + +void testReportRateSmoothing() { + for (const double reportHz : {125.0, 250.0, 1000.0}) { + Pad::MouseStick mouse(defaults().settings); + check(mouse.SmoothingMs() == defaults().settings.mouseSmoothingMs, "smoothing starts at the setting"); + const double periodSeconds = 1.0 / reportHz; + for (int report = 0; report < static_cast(reportHz); ++report) { + mouse.Move(6, 0); + mouse.Advance(periodSeconds); + } + check(std::abs(mouse.ReportRate() - reportHz) < 0.1 * reportHz, "measured " + std::to_string(mouse.ReportRate()) + " reports per second for a " + std::to_string(reportHz) + " Hz mouse"); + const double smoothing = mouse.SmoothingMs(); + check(std::abs(smoothing - std::max(10.0, 3000.0 / reportHz)) < 0.15 * smoothing, "smoothing " + std::to_string(smoothing) + " ms for a " + std::to_string(reportHz) + " Hz mouse"); + } + Pad::MouseStick slow(defaults().settings); + for (int step = 0; step < 40; ++step) { + slow.Move(30, 0); + slow.Advance(0.025); + } + check(std::abs(slow.ReportRate() - 40.0) < 2.0 && slow.SmoothingMs() == Pad::MouseSmoothingLimitMs, "smoothing for very slow reports stops at 30 ms"); + auto off = defaults().settings; + off.mouseSmoothingMs = 0.0; + Pad::MouseStick unsmoothed(off); + for (int report = 0; report < 125; ++report) { + unsmoothed.Move(6, 0); + unsmoothed.Advance(0.008); + } + check(unsmoothed.SmoothingMs() == 0.0, "a smoothing of 0 stays off for a slow mouse"); + Pad::MouseStick single(defaults().settings); + for (int report = 0; report < 125; ++report) { + single.Move(1, 0); + single.Advance(0.008); + } + check(single.ReportRate() == 0.0 && single.SmoothingMs() == defaults().settings.mouseSmoothingMs, "single-count reports are not taken for a slow mouse"); +} + std::vector waveMotion(double base, double swing) { constexpr double Pi = 3.14159265358979323846; std::vector events{PadSimulation::MouseModeAt(0.0, true)}; @@ -341,6 +397,8 @@ int main() { {"decay", testDecay}, {"max speed clamp", testMaxSpeedClamp}, {"smoothing", testSmoothing}, + {"slow reporting mouse", testSlowReportingMouse}, + {"report rate smoothing", testReportRateSmoothing}, {"poll rate independence", testPollRateIndependence}, {"vertical scale", testVerticalScale}, {"mouse mode off", testMouseModeOff}, diff --git a/docs/user/INPUT_MAPPING.md b/docs/user/INPUT_MAPPING.md index 882ce0a000..2568fc894d 100644 --- a/docs/user/INPUT_MAPPING.md +++ b/docs/user/INPUT_MAPPING.md @@ -67,7 +67,7 @@ The same file also accepts these settings, each at most once, in the form `Setti | `MouseCurve` | Number from 0.2 to 5, the response exponent. Above 1 gives finer control at low speed; below 1 reaches large tilts sooner. | `1` | | `MouseDeadzoneOffset` | Number from 0 to 0.9, the smallest tilt any movement produces as a fraction of full tilt. Set it just above the game's own deadzone: raise it if slow movement doesn't move the camera, lower it if small movements jump. | `0.2` | | `MouseMaxSpeed` | Number from 0.1 to 10, the highest tracked speed as a multiple of the full-tilt speed. Above 1, a fast flick keeps the stick at full tilt a little longer after the mouse stops. | `1` | -| `MouseSmoothingMs` | Number from 0 to 500, the smoothing time constant in milliseconds. 0 follows every mouse report directly. | `10` | +| `MouseSmoothingMs` | Number from 0 to 500, the smoothing time constant in milliseconds. A mouse that reports fewer than 300 times per second, such as a 125 or 250 Hz mouse, is smoothed over at least three of its report intervals, up to 30 ms; the report rate is measured from the movement itself. 0 follows every mouse report directly and turns this off too. | `10` | | `MouseDecayMs` | Number from 1 to 1000, the milliseconds without movement after which the stick is centered. | `30` | ```ini From 17dd47d71e104c7221da0ab290b3ba47563d0424 Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:41:49 +0200 Subject: [PATCH 21/31] fix(agc): print the revalidate profile per 10 s window --- .../prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 36d2c9aa87..0a7f0d1f50 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -2594,10 +2594,10 @@ void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point s const auto nowMs = std::chrono::duration_cast(now.time_since_epoch()).count(); auto last = profile.lastReport.load(); if (nowMs - last < 10000 || !profile.lastReport.compare_exchange_strong(last, nowMs)) return; - const auto byReason = [](const auto& counts, const auto& names) { + const auto byReason = [](auto& counts, const auto& names) { std::string reasons; for (std::size_t i = 0; i < names.size(); ++i) { - const auto count = counts[i].load(std::memory_order_relaxed); + const auto count = counts[i].exchange(0, std::memory_order_relaxed); if (count == 0) continue; reasons += " " + std::string(names[i]) + " " + std::to_string(count); } @@ -2606,7 +2606,7 @@ void countRevalidate(bool fast, bool ok, std::chrono::steady_clock::time_point s const auto reasons = byReason(profile.fastFails, FastFailNames); const auto fullReasons = byReason(profile.fullByReason, FastFailNames); const auto fallbacks = byReason(profile.ownFallbacks, OwnRefreshFallbackNames); - AgcDriver::ProfilePrint_nid_no_patch("[rescache] revalidate %llu calls %.1f ms: fast %llu, full %llu, failed %llu; fast-fail by reason:%s; full walks: T1 refreshed %llu (%llu storage images, %llu view sources, %llu views served by their own pending source, %llu overlaps left by the refresh accepted), full by reason:%s, T1 fallback by reason:%s, proofs verified %llu; fast-fail pending image: own 0 (the query excepts the own object), foreign %llu, snapshot %llu, T1 eligible %llu; keys proven: %llu sampled, %llu storage; epoch gate: %llu registry scans skipped, %llu import loops skipped\n", static_cast(profile.calls.load()), profile.nanoseconds.load() / 1e6, static_cast(profile.fast.load()), static_cast(profile.full.load()), static_cast(profile.failed.load()), reasons.c_str(), static_cast(profile.ownRefreshed.load()), static_cast(profile.ownStorageRefreshes.load()), static_cast(profile.ownViewRefreshes.load()), static_cast(profile.ownSourceViews.load()), static_cast(profile.refreshedOverlaps.load()), fullReasons.c_str(), fallbacks.c_str(), static_cast(profile.proofsVerified.load()), static_cast(profile.pendingForeign.load()), static_cast(profile.pendingSnapshot.load()), static_cast(profile.pendingT1Eligible.load()), static_cast(profile.keysProven.load()), static_cast(profile.storageKeysProven.load()), static_cast(profile.serialSkips.load()), static_cast(profile.importSkips.load())); + AgcDriver::ProfilePrint_nid_no_patch("[rescache] revalidate (10 s): %llu calls %.1f ms: fast %llu, full %llu, failed %llu; fast-fail by reason:%s; full walks: T1 refreshed %llu (%llu storage images, %llu view sources, %llu views served by their own pending source, %llu overlaps left by the refresh accepted), full by reason:%s, T1 fallback by reason:%s, proofs verified %llu; fast-fail pending image: own 0 (the query excepts the own object), foreign %llu, snapshot %llu, T1 eligible %llu; keys proven: %llu sampled, %llu storage; epoch gate: %llu registry scans skipped, %llu import loops skipped\n", static_cast(profile.calls.exchange(0, std::memory_order_relaxed)), profile.nanoseconds.exchange(0, std::memory_order_relaxed) / 1e6, static_cast(profile.fast.exchange(0, std::memory_order_relaxed)), static_cast(profile.full.exchange(0, std::memory_order_relaxed)), static_cast(profile.failed.exchange(0, std::memory_order_relaxed)), reasons.c_str(), static_cast(profile.ownRefreshed.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownStorageRefreshes.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownViewRefreshes.exchange(0, std::memory_order_relaxed)), static_cast(profile.ownSourceViews.exchange(0, std::memory_order_relaxed)), static_cast(profile.refreshedOverlaps.exchange(0, std::memory_order_relaxed)), fullReasons.c_str(), fallbacks.c_str(), static_cast(profile.proofsVerified.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingForeign.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingSnapshot.exchange(0, std::memory_order_relaxed)), static_cast(profile.pendingT1Eligible.exchange(0, std::memory_order_relaxed)), static_cast(profile.keysProven.exchange(0, std::memory_order_relaxed)), static_cast(profile.storageKeysProven.exchange(0, std::memory_order_relaxed)), static_cast(profile.serialSkips.exchange(0, std::memory_order_relaxed)), static_cast(profile.importSkips.exchange(0, std::memory_order_relaxed))); } // APS5_NO_EPOCH_REVALIDATE=1: the per-element registry, cache and stamp checks and the per-region From cb3d1d1dcee76499ea4cae5c21a0117dd209ef2a Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:29:55 +0200 Subject: [PATCH 22/31] fix(agc): say "cumulative" on the profile lines that count from the start --- .../Execution/src/Driver/Dispatch/IndirectDispatch.cpp | 2 +- .../prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp | 8 ++++---- .../prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp index f567a93e2e..29a81c9231 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp @@ -20,7 +20,7 @@ void Driver::countIndirect(int path, double readMs) { lastReport = now; std::uint64_t cpu = 0; for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i]; - std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), static_cast(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000); + std::fprintf(stderr, "[indirect] (cumulative): gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), static_cast(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000); } int Driver::indirectArgumentsPath(std::span packet, std::uint64_t arguments) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index bdbb94ebe8..9253c1da2c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2337,7 +2337,7 @@ bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::spa const auto now = std::chrono::steady_clock::now(); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; - AgcDriver::ProfilePrint_nid_no_patch("[fill-sync] fills: %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write; phases ms: decide %.0f record %.0f notes %.0f; %llu uniform, %llu pattern (%llu doubling steps in place; pattern buffers reused %llu, made %llu)\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered), decideMs, recordMs, notesMs, static_cast(uniformFills), static_cast(patternFills), static_cast(doublingSteps), static_cast(patternBuffersReused), static_cast(patternBuffersMade)); + AgcDriver::ProfilePrint_nid_no_patch("[fill-sync] fills (cumulative): %llu synced (%llu with a completion label pending) waited %.0f ms, %llu ordered by barrier, %llu with no pending write; phases ms: decide %.0f record %.0f notes %.0f; %llu uniform, %llu pattern (%llu doubling steps in place; pattern buffers reused %llu, made %llu)\n", static_cast(synced), static_cast(labelSynced), syncedMs, static_cast(ordered), static_cast(fills - synced - ordered), decideMs, recordMs, notesMs, static_cast(uniformFills), static_cast(patternFills), static_cast(doublingSteps), static_cast(patternBuffersReused), static_cast(patternBuffersMade)); } } auto phaseStart = fillStart; @@ -3074,7 +3074,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { } if (profile && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) { lastReport = std::chrono::steady_clock::now(); - AgcDriver::ProfilePrint_nid_no_patch("[flip] %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); + AgcDriver::ProfilePrint_nid_no_patch("[flip] (cumulative): %llu presents from the resident image (%llu refreshed first), through guest memory: %llu not pending, %llu unsuitable; %llu GPU frame dumps\n", static_cast(residentPresents), static_cast(refreshedPresents), static_cast(notPending), static_cast(unsuitable), static_cast(gpuDumps)); } CaptureTrace::Log("present dump=%d address=%llx width=%u height=%u resident=%d generation=%llu", dumpFrame ? state->nextDumpIndex : -1, static_cast(buffer.address), buffer.width, buffer.height, resident != nullptr, static_cast(resident ? resident->Generation() : 0)); if (passthrough && resident == nullptr) ensureHdrScaler(DisplayTexelFormat(buffer.pixelFormat)); @@ -4993,8 +4993,8 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } if (profile && std::chrono::steady_clock::now() - d.cacheReport > std::chrono::seconds(10)) { d.cacheReport = std::chrono::steady_clock::now(); - AgcDriver::ProfilePrint_nid_no_patch("[rescache] %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); - AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call: %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped), static_cast(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast(computePipelinesCreatedLocked.load(std::memory_order_relaxed))); + AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast(d.cacheHits), static_cast(d.cacheMisses), static_cast(d.cacheInvalidated), state->resourceCache.Size(), static_cast(d.templateRefreshed), d.templateRevalidateMs, static_cast(d.templateSameWords), static_cast(ResourceCache::Finds()), static_cast(ResourceCache::Touches())); + AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call (cumulative): %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast(d.procLookups), d.profiledDispatches != 0 ? static_cast(d.procLookups) / static_cast(d.profiledDispatches) : 0.0, static_cast(d.preBarriersRecorded), static_cast(d.preBarriersSkipped), static_cast(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast(computePipelinesCreatedLocked.load(std::memory_order_relaxed))); reportRecipes(); reportDispatchMove(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 0a7f0d1f50..2005f9dcdc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -5095,7 +5095,7 @@ void ShaderResources::MarkGpuWrites(Recorder& recorder) { if (nowMs - last < 10000 || !counters.lastReport.compare_exchange_strong(last, nowMs)) return; const auto elements = counters.elements.load(); const auto readOnly = counters.readOnly.load(); - AgcDriver::ProfilePrint_nid_no_patch("[buffers] descriptor elements bound: %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); + AgcDriver::ProfilePrint_nid_no_patch("[buffers] descriptor elements bound (cumulative): %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast(elements), static_cast(elements - readOnly), static_cast(readOnly), static_cast(counters.notesSkipped.load())); } void ShaderResources::CheckFaults() const { From 0d699295a9a579bf5f2a4b2445e737180683f7fc Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:36:32 +0200 Subject: [PATCH 23/31] docs(agc): say which profile lines count per window and which since start --- docs/dev/BUILD.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 71dcccda64..adfaf7cb5c 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -203,6 +203,24 @@ A class range covers a command together with its own barriers. In `VulkanDevice: - `shadow-publish`: copies of unit shadow slabs into the import, with their barriers (`recordCopies` in `UnitShadow.cpp`, reached from `PublishShadow`). Copied bytes. - `template-refresh`: the `vkCmdUpdateBuffer` updates of a cached dispatch's changed data words, as one range from the first update to the last (`ShaderResources::refreshData`, called from `recordDispatch`). Refreshed bytes. +## Driver profile lines + +With `APS5_PROFILE_DRAW` set (any value), the driver prints a profile line on stderr at most every 10 s. A line either counts the last 10 s (its counters are read and cleared when it is printed; the text says `(10 s)` or `over 10 s`) or counts from the start of the process (the text says `(cumulative)`). A rate is the difference of two cumulative lines divided by the time between them. + +| Line | Counts | +|---|---| +| `[rescache] revalidate` | last 10 s | +| `[texture-cache]` | last 10 s | +| `[pipecache]` | last 10 s | +| `[rescache]` (hits, misses, entries) | from the start | +| `[vk]` | from the start | +| `[indirect]` (gpu-side, cpu-side) | from the start | +| `[flip]` | from the start | +| `[fill-sync]` | from the start | +| `[buffers] descriptor elements bound` | from the start | + +`[gputime]` is described above and needs `APS5_PROFILE_GPU`. Other profile lines state their window in their text or are not listed here because their counters were not checked. + ## Driver thread priority Set `APS5_THREAD_PRIORITY=high` to raise the AGC driver's queue workers and the video output presenter above the title's threads. On Linux the threads get nice `APS5_THREAD_NICE` (default -10) through rtkit; on Windows they get `THREAD_PRIORITY_HIGHEST`. `APS5_THREAD_PRIORITY=rt` asks rtkit for real-time priority `APS5_THREAD_RT` (default 10, at most rtkit's `MaxRealtimePriority`) under rtkit's policy, `SCHED_RR` by default; on Windows it sets `THREAD_PRIORITY_TIME_CRITICAL`. rtkit only grants it when the process's hard `RLIMIT_RTTIME` is at most its `RTTimeUSecMax` (200000 us by default), so a hard limit above `APS5_THREAD_RTTIME_US` (default 200000, capped at `RTTimeUSecMax`) is lowered to it and the soft limit to at most half of it. The hard limit cannot be raised again and stays lowered for the whole process even if rtkit then refuses; see [technical debt](TechnicalDebt.md#functional). There is no fallback between the modes: when rtkit or `SetThreadPriority` refuses, a queue worker reports the refusal as a driver failure, which the next submission, wait or shutdown rethrows, and the presenter's start throws it. Each raised thread logs its priority once as `[thread-priority]`. The setting is `off` by default; a value other than `off`, `high` or `rt` is reported the same way as a refusal. From ad11cb1b3ecd4955541bbc282ac9c895e5fa54d1 Mon Sep 17 00:00:00 2001 From: Gustavo Ariel Gamarra Rojas <111788638+GustavoGamarra95@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:49:00 -0300 Subject: [PATCH 24/31] fix(agc): place device memory in host memory when the driver refuses a device-local allocation The budget check in DeviceMemoryType trusts VK_EXT_memory_budget, which lags behind the driver: with the device-local heap nearly full the budget can still report room, the allocation fails with VK_ERROR_OUT_OF_DEVICE_MEMORY, and the title stops. Device buffers (after the pool's reclaim and retry), textures and storage textures now retry in a host memory type allowed by their requirements. --- .../Graphics/include/BufferPool.hpp | 2 +- .../Graphics/include/Context.hpp | 47 ++++++++++++++++--- .../Graphics/src/BufferPool.cpp | 18 ++++++- .../Graphics/src/Resources.cpp | 6 ++- .../libSceAgcDriver/Graphics/src/Texture.cpp | 4 +- .../prx/libSceAgcDriver/tests/BufferPool.cpp | 37 ++++++++++++++- .../tests/DeviceMemoryFallback.cpp | 11 +++++ 7 files changed, 110 insertions(+), 15 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 3101ae4f90..fec2dc20f6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -62,7 +62,7 @@ public: // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties); static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties); - VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation); + VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional hostType = std::nullopt, std::uint32_t* allocatedType = nullptr); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index d5f3cd7fee..f58e1746c8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -304,15 +305,47 @@ struct Context { const auto budget = reported.heapBudget[heap]; const auto usage = reported.heapUsage[heap]; if (budget == 0 || (usage < budget && bytes <= budget - usage)) return type; + const auto host = HostMemoryType(mask); + if (!host) return type; + static std::atomic placements{0}, placedBytes{0}; + const auto count = placements.fetch_add(1) + 1; + const auto total = placedBytes.fetch_add(bytes) + bytes; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u)); + return *host; + } + + std::optional HostMemoryType(std::uint32_t mask) const { for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) { - if ((mask & (1u << i)) == 0 || (memory.memoryHeaps[memory.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) continue; - static std::atomic placements{0}, placedBytes{0}; - const auto count = placements.fetch_add(1) + 1; - const auto total = placedBytes.fetch_add(bytes) + bytes; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), i, static_cast(count), static_cast(total >> 20u)); - return i; + if ((mask & (1u << i)) != 0 && (memory.memoryHeaps[memory.memoryTypes[i].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return i; } - return type; + return std::nullopt; + } + + std::optional PoolHostFallback(std::uint32_t mask, std::uint32_t type) const { + if (type >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[type].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; + return HostMemoryType(mask); + } + + std::optional RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused, VkDeviceSize bytes, const char* what) const { + if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; + const auto host = HostMemoryType(mask); + if (!host) return std::nullopt; + static std::atomic refusals{0}, refusedBytes{0}; + const auto count = refusals.fetch_add(1) + 1; + const auto total = refusedBytes.fetch_add(bytes) + bytes; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local memory type %u is out of device memory for %llu bytes; placed in host memory type %u; %llu such allocations, %llu MiB\n", what, refused, static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u)); + return host; + } + + VkResult AllocateDeviceMemory(VkMemoryAllocateInfo& allocation, std::uint32_t mask, VkDeviceMemory& allocated, const char* what) const { + const auto allocate = Function("vkAllocateMemory"); + auto result = allocate(device, &allocation, nullptr, &allocated); + if (result != VK_ERROR_OUT_OF_DEVICE_MEMORY) return result; + const auto host = RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex, allocation.allocationSize, what); + if (!host) return result; + allocation.memoryTypeIndex = *host; + allocated = VK_NULL_HANDLE; + return allocate(device, &allocation, nullptr, &allocated); } }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 7beb86bef3..5b745d0dee 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include +#include #include #include #include @@ -51,7 +52,8 @@ BufferPool::~BufferPool() { for (auto& [memory, block] : slabBlocks) freeBlock(*block); } -VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation) { +VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional hostType, std::uint32_t* allocatedType) { + if (allocatedType != nullptr) *allocatedType = allocation.memoryTypeIndex; VkDeviceMemory memory = VK_NULL_HANDLE; auto result = allocateMemory(device, &allocation, nullptr, &memory); const auto initialResult = result; @@ -75,6 +77,20 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation ++attempts; } } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && hostType.has_value() && *hostType != allocation.memoryTypeIndex) { + auto host = allocation; + host.memoryTypeIndex = *hostType; + memory = VK_NULL_HANDLE; + result = allocateMemory(device, &host, nullptr, &memory); + ++attempts; + if (result == VK_SUCCESS) { + if (allocatedType != nullptr) *allocatedType = *hostType; + static std::atomic placements{0}; + const auto count = placements.fetch_add(1, std::memory_order_relaxed) + 1; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u; %llu such allocations\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count)); + return memory; + } + } if (result == VK_SUCCESS) { if (attempts > 1) std::fprintf(stderr, "[bufferpool] %s recovered: allocation=%llu reclaimed=%llu released=%llu initialResult=%d attempts=%u\n", operation, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), static_cast(initialResult), attempts); return memory; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index 8e5b642c4e..9d3fa0c1cf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -178,8 +178,10 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa allocation.allocationSize = ClassAllocationBytes(requirements, capacity); allocationBytes = allocation.allocationSize; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, allocation.allocationSize, "device buffer"); - hostPlaced = allocation.memoryTypeIndex != context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer"); + const auto deviceType = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + std::uint32_t allocatedType = allocation.memoryTypeIndex; + memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer", allocation.memoryTypeIndex == deviceType ? context.HostMemoryType(requirements.memoryTypeBits) : std::nullopt, &allocatedType); + hostPlaced = allocatedType != deviceType; Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device"); ready = true; } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 82f7339415..556f5159f4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -347,7 +347,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory texture"); + owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory texture", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); const VkImage uploadImage = image; @@ -1402,7 +1402,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, allocation.allocationSize = requirements.size; memoryBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture"); - memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory storage texture"); + memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory storage texture", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); } catch (...) { release(); diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index 7863a8ce05..e34aebe254 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -175,8 +176,7 @@ VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDev reported->heapUsage[0] = reportedUsage; } -void HostPlacement() { - mock = MockDevice{}; +Context budgetContext() { auto context = mockContext(); context.physical = reinterpret_cast(std::uintptr_t{1}); context.memoryProperties2 = mockMemoryProperties2; @@ -185,6 +185,12 @@ void HostPlacement() { context.memory.memoryHeaps[1] = {16384 * MiB, 0}; context.memory.memoryTypes[0].heapIndex = 0; context.memory.memoryTypes[1].heapIndex = 1; + return context; +} + +void HostPlacement() { + mock = MockDevice{}; + auto context = budgetContext(); constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; reportedBudget = 1620 * MiB; reportedUsage = 1600 * MiB; @@ -207,6 +213,32 @@ void HostPlacement() { Expect(MemoryOf(again.Handle()) == device, "a retained device-local buffer was not reused"); } +void RefusedDevicePlacement() { + mock = MockDevice{}; + auto context = budgetContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY}; + VkDeviceMemory placed = VK_NULL_HANDLE; + { + DeviceBuffer refused(context, 9 * MiB, usage); + placed = MemoryOf(refused.Handle()); + Expect(mock.memoryTypes.at(placed) == 1, "a device buffer whose device-local allocation ran out of device memory within the reported budget was not placed in the host memory type"); + } + Expect(!mock.memoryBytes.contains(placed) && mock.liveBuffers.empty(), "a device buffer placed in host memory after a refused device-local allocation was kept in the device pool after its release"); + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}; + context.memory.memoryHeaps[1].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + bool threw = false; + try { + DeviceBuffer refused(context, 9 * MiB, usage); + } catch (const std::runtime_error&) { + threw = true; + } + Expect(threw, "a refused device-local allocation without a host memory type did not fail"); + Expect(mock.liveBuffers.empty(), "a failed device buffer left its VkBuffer alive"); +} + void SizeClassesAndDirections() { mock = MockDevice{}; auto context = mockContext(); @@ -789,6 +821,7 @@ int main(int argc, char** argv) { PartialSlabDoesNotRetry(); CacheReclaimRecovery(); HostPlacement(); + RefusedDevicePlacement(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp index 2ed8ec1ba9..0b1d8b1d93 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp @@ -53,6 +53,16 @@ void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::u Expect(type == expected, std::string(what) + ": chose memory type " + std::to_string(type) + ", expected " + std::to_string(expected)); } +void RefusedFallbackTests() { + const auto context = mockContext(true); + const std::uint32_t both = (1u << HostType) | (1u << DeviceType); + const auto refused = context.RefusedDeviceMemoryFallback(both, DeviceType, 9 * MiB, "test"); + Expect(refused.has_value() && *refused == HostType, "a device-local type out of device memory did not fall back to the host type"); + Expect(!context.RefusedDeviceMemoryFallback(both, HostType, 9 * MiB, "test").has_value(), "a host type out of memory was given a fallback"); + Expect(!context.RefusedDeviceMemoryFallback(1u << DeviceType, DeviceType, 9 * MiB, "test").has_value(), "a fallback was chosen outside the allowed type mask"); + Expect(!context.RefusedDeviceMemoryFallback(both, 7, 9 * MiB, "test").has_value(), "an unknown memory type was given a fallback"); +} + void BufferPlacementTests() { constexpr auto coherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; Context context = mockContext(true); @@ -106,6 +116,7 @@ int main() { Choose("usage already over the budget", true, 2952 * MiB, 3633 * MiB, both, HostType); Choose("no host type allowed", true, 3000 * MiB, 3000 * MiB, 1u << DeviceType, DeviceType); BufferPlacementTests(); + RefusedFallbackTests(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); return 1; From 79b6f797458fc2f23351b5bf11ff5fb343f2461b Mon Sep 17 00:00:00 2001 From: Gustavo Ariel Gamarra Rojas <111788638+GustavoGamarra95@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:08:47 -0300 Subject: [PATCH 25/31] fix(agc): fall back to host memory for render, depth and copy images refused device memory Depth targets, depth plane copies, multisampled and render targets, block and depth copies and attachment proxies allocated device-local memory without the fallback, so a full device-local heap still stopped the title at them. --- core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp | 4 ++-- .../prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp | 2 +- core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp | 2 +- core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 0ac8d9b104..5885409dc3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -256,7 +256,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory depth target"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "depth target"), "vkAllocateMemory depth target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth"); if (target.samples != 1u) owned = std::make_shared(this->context, image, memory); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -846,7 +846,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory depth plane copy"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "depth plane copy"), "vkAllocateMemory depth plane copy"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth plane copy"); stagingSize = static_cast(sliceBytes()) * layers; staging = std::make_unique(context, stagingSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 4ca7a22552..503116d8c4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -121,7 +121,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &created.memory), "vkAllocateMemory multisampled target"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, created.memory, "multisampled target"), "vkAllocateMemory multisampled target"); Check(context.Function("vkBindImageMemory")(context.device, created.image, created.memory, 0), "vkBindImageMemory multisampled target"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = created.image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index 9d3fa0c1cf..ec204a099b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -294,7 +294,7 @@ RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bo VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory render target"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "render target"), "vkAllocateMemory render target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 556f5159f4..31090998b6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -681,7 +681,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory block copy"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, owned->memory, "block copy"), "vkAllocateMemory block copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory block copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -738,7 +738,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory depth copy"); + owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory depth copy", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -1832,7 +1832,7 @@ VkImageView StorageTexture::AttachmentProxyView() { VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &proxyMemory), "vkAllocateMemory attachment proxy"); + Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, proxyMemory, "attachment proxy"), "vkAllocateMemory attachment proxy"); Check(context.Function("vkBindImageMemory")(context.device, proxyImage, proxyMemory, 0), "vkBindImageMemory attachment proxy"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = proxyImage; From 9b2339e735a172453a1751da0711791e46fa70c1 Mon Sep 17 00:00:00 2001 From: Gustavo Ariel Gamarra Rojas <111788638+GustavoGamarra95@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:38:39 -0300 Subject: [PATCH 26/31] fix(agc): reclaim the buffer pool before placing an attachment in host memory Render, depth and multisampled targets and attachment proxies that the driver refuses device memory now go through the buffer pool's recovery: the pool releases its idle allocations and the attachment is retried in device-local memory. Only a second refusal places it in host memory, and that placement is logged with the bytes reclaimed. --- .../Graphics/include/BufferPool.hpp | 1 + .../Graphics/src/BufferPool.cpp | 6 ++++- .../Graphics/src/DepthSurface.cpp | 3 ++- .../Graphics/src/MultisampleTarget.cpp | 3 ++- .../Graphics/src/Resources.cpp | 2 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 2 +- .../prx/libSceAgcDriver/tests/BufferPool.cpp | 27 +++++++++++++++++++ 7 files changed, 39 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index fec2dc20f6..084ff7fce2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -156,6 +156,7 @@ private: }; std::shared_ptr GetBufferPool(const Context& context); +VkDeviceMemory AllocateAttachmentMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 5b745d0dee..e47cb4f4c4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -87,7 +87,7 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation if (allocatedType != nullptr) *allocatedType = *hostType; static std::atomic placements{0}; const auto count = placements.fetch_add(1, std::memory_order_relaxed) + 1; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u; %llu such allocations\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count)); + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u until it is released; %llu such allocations\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count)); return memory; } } @@ -386,4 +386,8 @@ std::shared_ptr GetBufferPool(const Context& context) { return context.bufferPool; } +VkDeviceMemory AllocateAttachmentMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation) { + return GetBufferPool(context)->AllocateMemory(allocation, operation, context.HostMemoryType(mask)); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 5885409dc3..2e77a19c66 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -256,7 +257,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "depth target"), "vkAllocateMemory depth target"); + memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth"); if (target.samples != 1u) owned = std::make_shared(this->context, image, memory); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 503116d8c4..5f1c042eb5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -121,7 +122,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, created.memory, "multisampled target"), "vkAllocateMemory multisampled target"); + created.memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory multisampled target"); Check(context.Function("vkBindImageMemory")(context.device, created.image, created.memory, 0), "vkBindImageMemory multisampled target"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = created.image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index ec204a099b..d9a4278601 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -294,7 +294,7 @@ RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bo VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "render target"), "vkAllocateMemory render target"); + memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory render target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 31090998b6..e5ac7aad2f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1832,7 +1832,7 @@ VkImageView StorageTexture::AttachmentProxyView() { VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, proxyMemory, "attachment proxy"), "vkAllocateMemory attachment proxy"); + proxyMemory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory attachment proxy"); Check(context.Function("vkBindImageMemory")(context.device, proxyImage, proxyMemory, 0), "vkBindImageMemory attachment proxy"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = proxyImage; diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index e34aebe254..6fdbe1d265 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -239,6 +240,31 @@ void RefusedDevicePlacement() { Expect(mock.liveBuffers.empty(), "a failed device buffer left its VkBuffer alive"); } +void AttachmentReclaim() { + mock = MockDevice{}; + auto context = budgetContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 8 * MiB; + allocation.memoryTypeIndex = 0; + const auto refuse = [&](std::initializer_list results) { + { DeviceBuffer idle(context, 4 * MiB, usage); } + Expect(mock.memoryBytes.size() == 1, "a released device buffer was not kept in the device pool"); + const auto pooled = mock.memoryBytes.begin()->first; + mock.results = results; + const auto memory = AllocateAttachmentMemory(context, allocation, 3, "vkAllocateMemory attachment"); + Expect(!mock.memoryBytes.contains(pooled), "an attachment refused device memory did not reclaim the pool before retrying"); + const auto type = mock.memoryTypes.at(memory); + mockFreeMemory(VK_NULL_HANDLE, memory, nullptr); + return type; + }; + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 0, "an attachment was placed in host memory although reclaiming the pool made room in device memory"); + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 1, "an attachment still refused device memory after reclaiming the pool was not placed in host memory"); + Expect(mock.liveBytes == 0, "an attachment allocation leaked memory"); +} + void SizeClassesAndDirections() { mock = MockDevice{}; auto context = mockContext(); @@ -822,6 +848,7 @@ int main(int argc, char** argv) { CacheReclaimRecovery(); HostPlacement(); RefusedDevicePlacement(); + AttachmentReclaim(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); return 1; From cfde713e10ba76d86797975a9170fa824e8b60e2 Mon Sep 17 00:00:00 2001 From: Gustavo Ariel Gamarra Rojas <111788638+GustavoGamarra95@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:38:27 -0300 Subject: [PATCH 27/31] fix(agc): release the texture caches before placing any image in host memory Textures, storage textures, block, depth and depth plane copies and attachments now share one allocation path: the pool trims its idle tiers, releases the texture caches, retries device-local memory and places the image in host memory only when that retry is refused too. Block and depth plane copies no longer fall back on their first refusal. --- .../Graphics/include/BufferPool.hpp | 2 +- .../Graphics/include/Context.hpp | 26 ++--------------- .../Graphics/src/BufferPool.cpp | 4 +-- .../Graphics/src/DepthSurface.cpp | 4 +-- .../Graphics/src/MultisampleTarget.cpp | 2 +- .../Graphics/src/Resources.cpp | 2 +- .../libSceAgcDriver/Graphics/src/Texture.cpp | 10 +++---- .../prx/libSceAgcDriver/tests/BufferPool.cpp | 28 ++++++++++++++++++- .../tests/DeviceMemoryFallback.cpp | 8 +++--- 9 files changed, 45 insertions(+), 41 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 084ff7fce2..5994cf4860 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -156,7 +156,7 @@ private: }; std::shared_ptr GetBufferPool(const Context& context); -VkDeviceMemory AllocateAttachmentMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation); +VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index f58e1746c8..21a08c9361 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -321,31 +321,9 @@ struct Context { return std::nullopt; } - std::optional PoolHostFallback(std::uint32_t mask, std::uint32_t type) const { - if (type >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[type].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; - return HostMemoryType(mask); - } - - std::optional RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused, VkDeviceSize bytes, const char* what) const { + std::optional RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused) const { if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; - const auto host = HostMemoryType(mask); - if (!host) return std::nullopt; - static std::atomic refusals{0}, refusedBytes{0}; - const auto count = refusals.fetch_add(1) + 1; - const auto total = refusedBytes.fetch_add(bytes) + bytes; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local memory type %u is out of device memory for %llu bytes; placed in host memory type %u; %llu such allocations, %llu MiB\n", what, refused, static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u)); - return host; - } - - VkResult AllocateDeviceMemory(VkMemoryAllocateInfo& allocation, std::uint32_t mask, VkDeviceMemory& allocated, const char* what) const { - const auto allocate = Function("vkAllocateMemory"); - auto result = allocate(device, &allocation, nullptr, &allocated); - if (result != VK_ERROR_OUT_OF_DEVICE_MEMORY) return result; - const auto host = RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex, allocation.allocationSize, what); - if (!host) return result; - allocation.memoryTypeIndex = *host; - allocated = VK_NULL_HANDLE; - return allocate(device, &allocation, nullptr, &allocated); + return HostMemoryType(mask); } }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index e47cb4f4c4..945b9591eb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -386,8 +386,8 @@ std::shared_ptr GetBufferPool(const Context& context) { return context.bufferPool; } -VkDeviceMemory AllocateAttachmentMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation) { - return GetBufferPool(context)->AllocateMemory(allocation, operation, context.HostMemoryType(mask)); +VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation) { + return GetBufferPool(context)->AllocateMemory(allocation, operation, context.RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex)); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 2e77a19c66..ee8c80e43d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -257,7 +257,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth target"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth"); if (target.samples != 1u) owned = std::make_shared(this->context, image, memory); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -847,7 +847,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, memory, "depth plane copy"), "vkAllocateMemory depth plane copy"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth plane copy"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth plane copy"); stagingSize = static_cast(sliceBytes()) * layers; staging = std::make_unique(context, stagingSize, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 5f1c042eb5..14123d9136 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -122,7 +122,7 @@ public: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - created.memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory multisampled target"); + created.memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory multisampled target"); Check(context.Function("vkBindImageMemory")(context.device, created.image, created.memory, 0), "vkBindImageMemory multisampled target"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = created.image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index d9a4278601..da7b171b7d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -294,7 +294,7 @@ RenderTarget::RenderTarget(const Context& context, const ColorTarget& target, bo VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - memory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory render target"); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory render target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = image; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index e5ac7aad2f..b6df085342 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -347,7 +347,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory texture", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); const VkImage uploadImage = image; @@ -681,7 +681,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.AllocateDeviceMemory(allocation, requirements.memoryTypeBits, owned->memory, "block copy"), "vkAllocateMemory block copy"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory block copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory block copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -738,7 +738,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - owned->memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory depth copy", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -1402,7 +1402,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, allocation.allocationSize = requirements.size; memoryBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture"); - memory = GetBufferPool(context)->AllocateMemory(allocation, "vkAllocateMemory storage texture", context.PoolHostFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex)); + memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory storage texture"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); } catch (...) { release(); @@ -1832,7 +1832,7 @@ VkImageView StorageTexture::AttachmentProxyView() { VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - proxyMemory = AllocateAttachmentMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory attachment proxy"); + proxyMemory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory attachment proxy"); Check(context.Function("vkBindImageMemory")(context.device, proxyImage, proxyMemory, 0), "vkBindImageMemory attachment proxy"); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = proxyImage; diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index 6fdbe1d265..50356ef052 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -254,7 +254,7 @@ void AttachmentReclaim() { Expect(mock.memoryBytes.size() == 1, "a released device buffer was not kept in the device pool"); const auto pooled = mock.memoryBytes.begin()->first; mock.results = results; - const auto memory = AllocateAttachmentMemory(context, allocation, 3, "vkAllocateMemory attachment"); + const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory attachment"); Expect(!mock.memoryBytes.contains(pooled), "an attachment refused device memory did not reclaim the pool before retrying"); const auto type = mock.memoryTypes.at(memory); mockFreeMemory(VK_NULL_HANDLE, memory, nullptr); @@ -751,6 +751,31 @@ void CacheReclaimRecovery() { Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "the failed cache release leaked a resource"); } +void ImageCacheReclaim() { + mock = MockDevice{}; + reclaimMock = ReclaimMock{}; + auto context = budgetContext(); + context.cacheReclaim = mockCacheReclaim; + reportedBudget = 1620 * MiB; + reportedUsage = 200 * MiB; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 8 * MiB; + allocation.memoryTypeIndex = 0; + const auto refuse = [&](std::initializer_list results) { + ExternalAllocation(4 * MiB); + mock.results = results; + const auto requests = reclaimMock.requests; + const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory texture"); + Expect(reclaimMock.requests == requests + 1 && reclaimMock.external == VK_NULL_HANDLE, "a texture refused device memory did not release the texture caches before retrying"); + const auto type = mock.memoryTypes.at(memory); + mockFreeMemory(VK_NULL_HANDLE, memory, nullptr); + return type; + }; + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 0, "a texture was placed in host memory although releasing the texture caches made room in device memory"); + Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 1, "a texture still refused device memory after releasing the texture caches was not placed in host memory"); + Expect(mock.liveBytes == 0, "a texture allocation leaked memory"); +} + void SuccessfulAllocationKeepsPool() { mock = MockDevice{}; { @@ -846,6 +871,7 @@ int main(int argc, char** argv) { SuccessfulAllocationKeepsPool(); PartialSlabDoesNotRetry(); CacheReclaimRecovery(); + ImageCacheReclaim(); HostPlacement(); RefusedDevicePlacement(); AttachmentReclaim(); diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp index 0b1d8b1d93..a5ff02af5d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp @@ -56,11 +56,11 @@ void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::u void RefusedFallbackTests() { const auto context = mockContext(true); const std::uint32_t both = (1u << HostType) | (1u << DeviceType); - const auto refused = context.RefusedDeviceMemoryFallback(both, DeviceType, 9 * MiB, "test"); + const auto refused = context.RefusedDeviceMemoryFallback(both, DeviceType); Expect(refused.has_value() && *refused == HostType, "a device-local type out of device memory did not fall back to the host type"); - Expect(!context.RefusedDeviceMemoryFallback(both, HostType, 9 * MiB, "test").has_value(), "a host type out of memory was given a fallback"); - Expect(!context.RefusedDeviceMemoryFallback(1u << DeviceType, DeviceType, 9 * MiB, "test").has_value(), "a fallback was chosen outside the allowed type mask"); - Expect(!context.RefusedDeviceMemoryFallback(both, 7, 9 * MiB, "test").has_value(), "an unknown memory type was given a fallback"); + Expect(!context.RefusedDeviceMemoryFallback(both, HostType).has_value(), "a host type out of memory was given a fallback"); + Expect(!context.RefusedDeviceMemoryFallback(1u << DeviceType, DeviceType).has_value(), "a fallback was chosen outside the allowed type mask"); + Expect(!context.RefusedDeviceMemoryFallback(both, 7).has_value(), "an unknown memory type was given a fallback"); } void BufferPlacementTests() { From 57fc04be0122b5456660bd0a0dcddb41ad5106e0 Mon Sep 17 00:00:00 2001 From: Gustavo Ariel Gamarra Rojas <111788638+GustavoGamarra95@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:15:48 -0300 Subject: [PATCH 28/31] fix(agc): place repacked and stencil copies in host memory after a refused reclaim Repacked copies and depth stencil copies allocated device-local memory directly and stopped the title on a refusal; they now take the same allocation path as the other images. Device buffers choose their host fallback type the same way, the host placement line carries the cumulative MiB, and the debt file records that a placed image stays in host memory for its lifetime. --- core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp | 5 +++-- core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp | 2 +- core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp | 2 +- core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp | 2 +- docs/dev/debt/libSceAgcDriver.md | 1 + 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 945b9591eb..e2f7fc6dc7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -85,9 +85,10 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation ++attempts; if (result == VK_SUCCESS) { if (allocatedType != nullptr) *allocatedType = *hostType; - static std::atomic placements{0}; + static std::atomic placements{0}, placedBytes{0}; const auto count = placements.fetch_add(1, std::memory_order_relaxed) + 1; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u until it is released; %llu such allocations\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count)); + const auto total = placedBytes.fetch_add(allocation.allocationSize, std::memory_order_relaxed) + allocation.allocationSize; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[bufferpool] %s: memory type %u is out of device memory for %llu bytes after reclaiming %llu and releasing %llu; placed in host memory type %u until it is released; %llu such allocations, %llu MiB\n", operation, allocation.memoryTypeIndex, static_cast(allocation.allocationSize), static_cast(reclaimed), static_cast(released), *hostType, static_cast(count), static_cast(total >> 20u)); return memory; } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index ee8c80e43d..4d82b509aa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -699,7 +699,7 @@ private: VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &stencilCopy.memory), "vkAllocateMemory stencil copy"); + stencilCopy.memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory stencil copy"); Check(context.Function("vkBindImageMemory")(context.device, stencilCopy.image, stencilCopy.memory, 0), "vkBindImageMemory stencil copy"); stencilCopy.staging = std::make_unique(context, static_cast(target.extent.width) * target.extent.height, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index da7b171b7d..678f501634 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -180,7 +180,7 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, allocation.allocationSize, "device buffer"); const auto deviceType = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); std::uint32_t allocatedType = allocation.memoryTypeIndex; - memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer", allocation.memoryTypeIndex == deviceType ? context.HostMemoryType(requirements.memoryTypeBits) : std::nullopt, &allocatedType); + memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer", context.RefusedDeviceMemoryFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex), &allocatedType); hostPlaced = allocatedType != deviceType; Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device"); ready = true; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index b6df085342..fc5a7d3152 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -623,7 +623,7 @@ Texture::Texture(const Context& context, const std::shared_ptr& allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory repacked copy"); + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory repacked copy"); Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory repacked copy"); const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 56e47332fd..e8a112d344 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -75,6 +75,7 @@ - [Covered-centroid raster state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) accepts PA_SC_AA_CONFIG.COVERED_CENTROID_IS_CENTER only when the pixel program is skipped or SPI_PS_INPUT_ENA and SPI_PS_INPUT_ADDR have no active perspective or linear centroid input. Bit 29 follows [PS5_Mesa GFX10.3 registers](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/registers/gfx103.json); [RADV](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/vulkan/radv_cmd_buffer.c) sets it on GFX10.3 and later. Active centroid interpolation and unspecified input registers still throw. Console covered-pixel centroid results were not measured. - [Compute LDS beyond the device's shared-memory limit](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords, rounded up to an even count so that 64-bit atomics stay 8-byte aligned, per workgroup (the dword after the LDS is the atomic lock on a device without `shaderBufferInt64Atomics`), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. - [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. +- [Host placement after a device-memory refusal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) (libSceAgcDriver): when the driver refuses a device-local allocation with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool's trim and the texture-cache release, device buffers and images (textures, storage textures, block, repacked, depth, depth plane and stencil copies, render, depth and multisampled targets and attachment proxies) are allocated in a host memory type their requirements allow. A host-placed device buffer leaves the pool on release. A host-placed image keeps that placement until it is destroyed and is never moved back to device memory, so every access to it crosses PCIe for its lifetime. The placements are counted in a rate-limited `[bufferpool]` line with the cumulative MiB, not the resident bytes. The 1x1 denormal clear probe still allocates device memory directly and throws on a refusal. - [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only and never compacted, so a module a later run no longer builds pipelines from stays in the file until it is deleted, and the whole file is read into memory while the replay runs (its modules reference that buffer until the last replayed pipeline was built). Only the first `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines recorded are replayed (a record is never rewritten, so a pipeline past the bound stays past it), and the pipeline store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES` (1024 each): a title with more configurations rebuilds the rest at the draw as before. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, which a dispatch that finds it resident checks against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost, an entry whose build fails is skipped, and both are counted in the `[pipeline-journal]` line rather than stopping the run. A replayed pipeline that the title never asks for again costs its build time and its driver memory (its optimized relink is deferred to its first draw, so a run whose first frames bind many replayed pipelines queues their relinks then); nothing ages recipes out of the file. Two runs sharing the file take turns under its lock file, but each only knows the modules it loaded or wrote itself, so a module the other run appended meanwhile may be appended again (the loader keeps the first copy). Records a run appends after another run's load are replayed by that run only after its next start. The file header's format number is the only guard against a change of the recipe codec, `CanonicalPipelineState` or the store key that keeps the structures' sizes (the recompiler's source version does not cover the driver's sources). - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline - [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV. From b8692a74f2b7e3f41358be4d0f2c76f971a44d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Sun, 18 Oct 2026 19:29:55 +0200 Subject: [PATCH 29/31] fix(agc): route the sRGB expand copy through AllocateImageMemory --- core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index fc5a7d3152..86fc376680 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -265,7 +265,7 @@ ExpandImage CreateExpandImage(const Context& context, const GuestTextureResource VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &result.owned->memory), "vkAllocateMemory sRGB copy"); + result.owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory sRGB copy"); Check(context.Function("vkBindImageMemory")(context.device, result.image, result.owned->memory, 0), "vkBindImageMemory sRGB copy"); result.bytes = requirements.size; return result; From 831ecb05f712cb227066ac7edd85b82b1c2dc14c Mon Sep 17 00:00:00 2001 From: BIGFERRY999 Date: Thu, 15 Oct 2026 01:01:57 +0100 Subject: [PATCH 30/31] feat(libc): implement fabsf and fabsl --- core/libs/prx/libc/src/Math.cpp | 2 ++ core/libs/tests/GuestMath.cpp | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 40587106b1..c0d0e627e2 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -305,6 +305,8 @@ double APS5_VABI acos_nid_postfix(double x) { return std::acos(x); } double APS5_VABI sqrt_nid_postfix(double x) { return std::sqrt(x); } float APS5_VABI sqrtf_nid_postfix(float x) { return std::sqrt(x); } double APS5_VABI fabs_nid_postfix(double x) { return std::fabs(x); } +float APS5_VABI fabsf_nid_postfix(float x) { return std::fabs(x); } +long double APS5_VABI fabsl_nid_postfix(long double x) { return std::fabsl(x); } int APS5_VABI abs_nid_postfix(int x) { return x < 0 ? static_cast(0u - static_cast(x)) : x; } double APS5_VABI trunc_nid_postfix(double x) { return std::trunc(x); } double APS5_VABI ceil_nid_postfix(double x) { return std::ceil(x); } diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 5f15ebd7c2..a849aa00fd 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -71,6 +71,8 @@ float APS5_VABI hypotf_nid_postfix(float, float); double APS5_VABI hypot_nid_postfix(double, double); double APS5_VABI fdim_nid_postfix(double, double); float APS5_VABI fdimf_nid_postfix(float, float); +float APS5_VABI fabsf_nid_postfix(float); +long double APS5_VABI fabsl_nid_postfix(long double); float APS5_VABI tanf_nid_postfix(float); int APS5_VABI rand_nid_postfix(); void APS5_VABI srand_nid_postfix(unsigned int); @@ -186,6 +188,11 @@ static void CheckExactOperations() { Require(fabs_nid_postfix(-3.5) == 3.5 && fabs_nid_postfix(-INFINITY) == INFINITY); Require(std::bit_cast(fabs_nid_postfix(-0.)) == 0); Require(std::isnan(fabs_nid_postfix(-nan)) && !std::signbit(fabs_nid_postfix(-nan))); + Require(fabsf_nid_postfix(-3.5f) == 3.5f && fabsf_nid_postfix(-INFINITY) == INFINITY); + Require(std::bit_cast(fabsf_nid_postfix(-0.f)) == 0); + Require(std::isnan(fabsf_nid_postfix(-static_cast(nan))) && !std::signbit(fabsf_nid_postfix(-static_cast(nan)))); + Require(fabsl_nid_postfix(-3.5L) == 3.5L && fabsl_nid_postfix(-INFINITY) == INFINITY); + Require(std::isnan(fabsl_nid_postfix(-static_cast(nan))) && !std::signbit(fabsl_nid_postfix(-static_cast(nan)))); Require(trunc_nid_postfix(2.7) == 2. && trunc_nid_postfix(-2.7) == -2.); Require(std::bit_cast(trunc_nid_postfix(-0.5)) == 0x8000000000000000ull); Require(trunc_nid_postfix(4503599627370497.) == 4503599627370497. && trunc_nid_postfix(-1e300) == -1e300); From d5eeed0b34665e9c06dd1332c52f4a14b7589627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Sun, 18 Oct 2026 20:02:58 +0200 Subject: [PATCH 31/31] fix(libc): use std::fabs overload for fabsl --- core/libs/prx/libc/src/Math.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index c0d0e627e2..d16e315293 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -306,7 +306,7 @@ double APS5_VABI sqrt_nid_postfix(double x) { return std::sqrt(x); } float APS5_VABI sqrtf_nid_postfix(float x) { return std::sqrt(x); } double APS5_VABI fabs_nid_postfix(double x) { return std::fabs(x); } float APS5_VABI fabsf_nid_postfix(float x) { return std::fabs(x); } -long double APS5_VABI fabsl_nid_postfix(long double x) { return std::fabsl(x); } +long double APS5_VABI fabsl_nid_postfix(long double x) { return std::fabs(x); } int APS5_VABI abs_nid_postfix(int x) { return x < 0 ? static_cast(0u - static_cast(x)) : x; } double APS5_VABI trunc_nid_postfix(double x) { return std::trunc(x); } double APS5_VABI ceil_nid_postfix(double x) { return std::ceil(x); }