diff --git a/core/libs/prx/libSceAgc/tests/Commands.cpp b/core/libs/prx/libSceAgc/tests/Commands.cpp index 8f509542f8..5379206a97 100644 --- a/core/libs/prx/libSceAgc/tests/Commands.cpp +++ b/core/libs/prx/libSceAgc/tests/Commands.cpp @@ -13,8 +13,13 @@ #include "prx/libSceAgc/DcbState/include/Display.hpp" #include "prx/libSceAgc/DcbState/include/ContextState.hpp" #include "prx/libSceAgc/DcbFlow/include/Sync.hpp" +#include "prx/libSceAgc/DcbFlow/include/Memory.hpp" +#include "prx/libSceAgc/Acb/include/Memory.hpp" #include "prx/libSceAgc/DcbFlow/include/Control.hpp" #include "prx/libSceAgc/DcbDraw/include/Instancing.hpp" +#include "prx/libSceAgc/Acb/include/Dispatch.hpp" +#include "prx/libSceAgc/Cb/include/Dispatch.hpp" +#include "prx/libSceAgc/DcbFlow/include/Dispatch.hpp" #include "prx/libSceAgc/DcbDraw/include/IndexBuffer.hpp" #include "prx/libSceAgc/DcbDraw/include/DrawIndexed.hpp" #include "prx/libSceAgc/DcbDraw/include/DrawNonIndexed.hpp" @@ -842,6 +847,32 @@ void testDefaultsTypes() { checkCompactTypes(sceAgcGetRegisterDefaults2Internal(0), 15, 0xf58fea31u, 0x00040400u, 0); } +void testDispatchPackets() { + Storage graphics; + const auto* indirect = sceAgcDcbDispatchIndirect(&graphics.buffer, 0x40, 0x8); + check(indirect == graphics.words.data() && graphics.words[0] == 0xc0011600u && graphics.words[1] == 0x40u && graphics.words[2] == 0x49u, "graphics DISPATCH_INDIRECT packet mismatch"); + Storage compute; + const auto* arguments = reinterpret_cast(0x123456789abcull); + sceAgcAcbDispatchIndirect(&compute.buffer, arguments, 0x2000u); + const std::array computeExpected{0xc0021600u, 0x56789abcu, 0x1234u, 0x2041u}; + check(std::equal(computeExpected.begin(), computeExpected.end(), compute.words.begin()), "compute DISPATCH_INDIRECT packet mismatch"); + Storage direct; + sceAgcCbDispatch(&direct.buffer, 7, 3, 1, 0x8000u); + const std::array directExpected{0xc0031500u, 7, 3, 1, 0x8041u}; + check(std::equal(directExpected.begin(), directExpected.end(), direct.words.begin()), "DISPATCH_DIRECT packet mismatch"); + Storage rejected; + rejected.words.fill(0xa5a5a5a5u); + const auto rejectedWords = rejected.words; + const auto* rejectedCursor = rejected.buffer.cursor_up; + expectFailure([&] { sceAgcCbDispatch(&rejected.buffer, 1, 1, 1, 0x100u); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + expectFailure([&] { sceAgcAcbDispatchIndirect(&rejected.buffer, reinterpret_cast(0x1002ull), 0); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + expectFailure([&] { sceAgcAcbDispatchIndirect(&rejected.buffer, nullptr, 0); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + check(sceAgcDcbDispatchIndirectGetSize() == 12 && sceAgcAcbDispatchIndirectGetSize() == 16 && sceAgcCbDispatchGetSize() == 20, "dispatch packet sizes"); +} + void testEventWrite() { Storage storage; const auto* samplePacket = sceAgcDcbEventWrite(&storage.buffer, 0x38, reinterpret_cast(0x1000)); @@ -883,6 +914,35 @@ void testEventWrite() { check(cacheFlushWithAddress.words[0] == 0xc0004600u && cacheFlushWithAddress.words[1] == 0x16u, "incorrect event 0x16 with address packet"); } +void testAcquireMem() { + Storage storage; + const auto* base = reinterpret_cast(0x1234567890ull); + const auto* packet = sceAgcDcbAcquireMem(&storage.buffer, 0, 3, 0x1234u, base, 0x1000, 400); + const std::array expected{0xc0065800u, 3, 0x11, 0, 0x12345678u, 0, 10, 0x1234u}; + check(packet == storage.words.data() && std::equal(expected.begin(), expected.end(), storage.words.begin()), "acquire packet does not round its range outward to 256 bytes"); + Storage whole; + sceAgcDcbAcquireMem(&whole.buffer, 0, 0, 0, base, 0xffffffffffffffffull, 0); + check(whole.words[0] == 0xc0065800u && whole.words[2] == 0 && whole.words[4] == 0 && whole.words[6] == 0, "an unbounded acquire did not cover the whole address space"); + Storage large; + sceAgcDcbAcquireMem(&large.buffer, 0, 0, 0, base, (1ull << 40u) + 1u, 0); + check(large.words[2] == 0 && large.words[4] == 0, "an acquire over 1 TiB did not cover the whole address space"); + Storage asynchronous; + sceAgcAcbAcquireMem(&asynchronous.buffer, 0x55u, base, 0x100, 80); + const std::array compute{0xc0065800u, 0x80000000u, 2, 0, 0x12345678u, 0, 2, 0x55u}; + check(std::equal(compute.begin(), compute.end(), asynchronous.words.begin()), "the asynchronous compute acquire did not select engine 1 with no CB/DB operation"); + Storage rejected; + rejected.words.fill(0xa5a5a5a5u); + const auto rejectedWords = rejected.words; + const auto* rejectedCursor = rejected.buffer.cursor_up; + expectFailure([&] { sceAgcDcbAcquireMem(&rejected.buffer, 2, 0, 0, base, 0x100, 0); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + expectFailure([&] { sceAgcDcbAcquireMem(&rejected.buffer, 0, 0, 0x80000u, base, 0x100, 0); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + expectFailure([&] { sceAgcDcbAcquireMem(&rejected.buffer, 0, 0, 0, base, 0x100, 40u * 0x10000u); }); + check(rejected.words == rejectedWords && rejected.buffer.cursor_up == rejectedCursor, "a refused packet changed the command buffer"); + check(sceAgcDcbAcquireMemGetSize() == 32 && sceAgcAcbAcquireMemGetSize() == 32, "acquire packet size"); +} + } int main(int argc, char** argv) { @@ -896,7 +956,9 @@ int main(int argc, char** argv) { } testPackets(); testEventWrite(); + testAcquireMem(); testNop(); + testDispatchPackets(); testClearState(); testIndexedIndirectDraws(); testTagHeaders(); diff --git a/core/libs/prx/libSceNet/tests/Ipv6Only.cpp b/core/libs/prx/libSceNet/tests/Ipv6Only.cpp index 9470a5be35..b34ae99196 100644 --- a/core/libs/prx/libSceNet/tests/Ipv6Only.cpp +++ b/core/libs/prx/libSceNet/tests/Ipv6Only.cpp @@ -48,6 +48,9 @@ static void Exercise(int descriptor) { RequireOption(descriptor, 1); std::uint32_t length = sizeof(enabled); int output = 0; + Require(sceNetGetsockopt(descriptor, 41, 27, &output, nullptr) == static_cast(0x80410116u) && *sceNetErrnoLoc() == 22); + Require(sceNetGetsockopt(-1, 41, 27, &output, &length) == static_cast(0x80410109u) && *sceNetErrnoLoc() == 9); + Require(sceNetSetsockopt(-1, 41, 27, &enabled, sizeof(enabled)) == static_cast(0x80410109u) && *sceNetErrnoLoc() == 9); Require(sceNetGetsockopt(descriptor, 41, 28, &output, &length) == static_cast(0x8041012au) && *sceNetErrnoLoc() == 42); Require(sceNetSetsockopt(descriptor, 41, 28, &enabled, sizeof(enabled)) == static_cast(0x8041012au) && *sceNetErrnoLoc() == 42); } diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp index a04a9c3481..604e85b34c 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp @@ -7,7 +7,7 @@ #include "prx/libkernel/KernelErrors.hpp" -static constexpr size_t DIRECT_MEMORY_SIZE = 12288ULL * 1024 * 1024; +static constexpr size_t DIRECT_MEMORY_SIZE = 13824ULL * 1024 * 1024; static constexpr size_t PS5_PAGE_SIZE = 0x4000; #ifdef _WIN32 diff --git a/core/libs/tests/GuestAprRead.cpp b/core/libs/tests/GuestAprRead.cpp index 88c16dffc7..da7ee7df7c 100644 --- a/core/libs/tests/GuestAprRead.cpp +++ b/core/libs/tests/GuestAprRead.cpp @@ -68,7 +68,7 @@ private: class DirectBuffer { public: DirectBuffer() { - Require(sceKernelAllocateDirectMemory(0, 12288ll << 20, MappingBytes, 0x4000, 0, &physical) == 0, "cannot allocate direct memory"); + Require(sceKernelAllocateDirectMemory(0, 13824ll << 20, MappingBytes, 0x4000, 0, &physical) == 0, "cannot allocate direct memory"); if (sceKernelMapDirectMemory(&pointer, MappingBytes, 0x33, 0, physical, 0x4000) != 0) { sceKernelReleaseDirectMemory(physical, MappingBytes); diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 158985c7e5..be7fc86b3d 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -726,10 +726,6 @@ static void CheckReleaseDirectMemoryClearsMappings() { } } -static void CheckDirectMemorySize() { - Require(sceKernelGetDirectMemorySize() == 0x300000000ull); -} - static void CheckReleaseDirectMemoryRejectsInvalidRanges() { constexpr std::size_t page = 0x4000; Require(sceKernelReleaseDirectMemory(-1, page) == SCE_KERNEL_ERROR_EINVAL); @@ -3588,7 +3584,6 @@ int main(int argc, char** argv) { CheckSavedDirectMemorySurvivesCachePressure(); CheckReusedDirectMemorySections(); CheckReleaseDirectMemoryClearsMappings(); - CheckDirectMemorySize(); CheckReleaseDirectMemoryRejectsInvalidRanges(); CheckFixedMappingReplacesPartialOverlap(); CheckFixedMappingOverHoleInsideAllocation(); diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index 61b22528e4..315904e281 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -63,7 +63,7 @@ - [sceKernelGetModuleList2 and sceKernelGetModuleInfo2](../../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the title-module variants follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/kernel/process.cpp): `sceKernelGetModuleList2` lists what `sceKernelGetModuleList` lists without system libraries, and `sceKernelGetModuleInfo2` answers `EPERM` for a system library after the `EFAULT`, `EINVAL` and `ESRCH` checks of `sceKernelGetModuleInfo`, as a payload process on PS5 firmware 13.60 got for libkernel's handle, a size of 0, a null info and a bad handle. A system library here is a loaded `.prx` image that is not a relinked title module (`.guest.prx`); the main program and the title's own modules are listed. Which modules the console lists is in the entry above. PPSA34063 imports both and fails to load without them - [sceKernelGetLibkernelTextLocation](../../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - returns 0 and the main program's first segment, as [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/kernel/ps4_libkernel.pas#L338-L348) does, where a payload process on PS5 firmware 13.60 got 0 with address 0 and size 0, twice, though libkernel's text was at 0x800000000; what a title gets is not measured. A null address or size faulted there and throws here. libkernel.sprx and libkernel_sys.sprx export the function at the same address - [sceKernelReleaseFlexibleMemory](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - on a PS5 (firmware 13.60) the released range stays mapped, with its protection and contents (64 MiB checked after 0.5 s), and a fixed no-overwrite map over it fails with `ENOMEM`, as here. What it does to the flexible budget was not measured, because a payload process has none (`sceKernelAvailableFlexibleMemorySize` did not change on map or release); here the released bytes return to the budget while the pages stay mapped, and a later map over them charges them again. The host pages behind a released range stay committed until `sceKernelMunmap`, as `madvise` keeps them for every advice; freeing them would lose the contents the console still returns. -- [sceKernelAvailableFlexibleMemorySize](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - the segments of the title's executable and of its `.guest.prx` modules count against the configured flexible-memory budget (448 MiB by default) while loaded, for this size and for flexible maps. A title may declare a larger budget through `kernel.flexibleMemorySize` in `param.json`; this does not reduce the 12 GiB direct-memory size. Rejection of zero or a value not aligned to 16 KiB has not been measured on a console. This segment-accounting model follows [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/module.cpp#L165), which maps every segment of a module outside its system module directory as flexible memory on the PS4. [Counted](../../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) are the segments the title's ELF gives permissions, each rounded to 16 KiB pages; not counted are segments without permissions, the relinker's own blocks (on Linux both are the read/write/execute `PT_LOAD`s, plus the header block at file offset 0; on Windows everything outside the `.elf` sections) and the host system libraries. The Linux image carries no marker for the relinker's blocks, so they are told apart by those flags and that offset only: a title segment that is writable and executable, or starts at file offset 0, is not counted. A module whose segments exceed what is left still loads, and the available size then reads 0; shadPS4 fails the segment map instead ([memory.cpp](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/memory.cpp#L519-L524) returns `EINVAL`, [module.cpp](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/module.cpp#L168-L169) asserts). Which segments the PS5 charges, its rounding, whether system modules use the title's budget and when an unload releases the charge are not measured. PPSA15508 subtracts the reported size and a 10 MiB module allowance from its own 448 MiB budget at start-up and stops when the result exceeds its 146 MiB code and data limit ([#2883](https://github.com/boykopovar/AnyPS5/issues/2883)) +- [sceKernelAvailableFlexibleMemorySize](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - the segments of the title's executable and of its `.guest.prx` modules count against the configured flexible-memory budget (448 MiB by default) while loaded, for this size and for flexible maps. A title may declare a larger budget through `kernel.flexibleMemorySize` in `param.json`; this does not reduce the 13.5 GiB direct-memory size. Rejection of zero or a value not aligned to 16 KiB has not been measured on a console. This segment-accounting model follows [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/module.cpp#L165), which maps every segment of a module outside its system module directory as flexible memory on the PS4. [Counted](../../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) are the segments the title's ELF gives permissions, each rounded to 16 KiB pages; not counted are segments without permissions, the relinker's own blocks (on Linux both are the read/write/execute `PT_LOAD`s, plus the header block at file offset 0; on Windows everything outside the `.elf` sections) and the host system libraries. The Linux image carries no marker for the relinker's blocks, so they are told apart by those flags and that offset only: a title segment that is writable and executable, or starts at file offset 0, is not counted. A module whose segments exceed what is left still loads, and the available size then reads 0; shadPS4 fails the segment map instead ([memory.cpp](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/memory.cpp#L519-L524) returns `EINVAL`, [module.cpp](https://github.com/shadps4-emu/shadPS4/blob/3a9f57265d28975658f1ca941daa5c879c7a74bd/src/core/module.cpp#L168-L169) asserts). Which segments the PS5 charges, its rounding, whether system modules use the title's budget and when an unload releases the charge are not measured. PPSA15508 subtracts the reported size and a 10 MiB module allowance from its own 448 MiB budget at start-up and stops when the result exceeds its 146 MiB code and data limit ([#2883](https://github.com/boykopovar/AnyPS5/issues/2883)) - [sceKernelAvailableDirectMemorySize](../../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - its arguments follow a payload process on a PS5 (firmware 13.60) ([probes](https://gist.github.com/orezraey/42a06a7ea0d9fb57ac9c06eaf9e6756c)). One answer is not modelled: in a process that had not yet called `sceKernelGetDirectMemorySize` or otherwise touched direct memory, the console returned `EBADF` (`0x80020009`); here the call always searches the pool. Titles open the direct-memory device during start-up, so they are not expected to see it - [sceKernelReleaseDirectMemory](../../../core/libs/prx/libkernel/DirectMemory/MemoryPool.cpp) (libkernel) - releases the allocated pages of the range and skips pages that are not allocated, returning 0, while `sceKernelCheckedReleaseDirectMemory` returns `ENOENT` for them, as [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/c206f35a546230d9c9808b823c0f36f5850cd4af/src/core/memory.cpp#L295-L313) does; not measured on a console. Absolum (PPSA28311) releases such a range when it disposes of its video logos - [Memory protection bits 0x100 and 0x200](../../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - taken as audio coprocessor (ACP) read and write, as in the [KytyPS5](https://github.com/KytyPS5/KytyPS5) and [etaHEN](https://github.com/etaHEN/etaHEN) headers; the coprocessor work runs on the host CPU, so they grant host read and read/write access like the GPU bits diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 0ad3826de4..057f7551c6 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -244,7 +244,7 @@ For bundled title modules, use the `.guest.prx` files the relinker writes into ` On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.dll` and `libwinpthread-1.dll` from the `mingw64/bin` directory of the toolchain that built the libraries; the release archives already contain them. The libraries are loaded without searching `PATH`, so a copy elsewhere on the system is not used, and a library that needs one of them fails with error 126 (the specified module could not be found) although its `.prx` file is present. With `APS5_MIMALLOC` set, the title loads `mimalloc.dll` and `mimalloc-redirect.dll` from `libs/` and allocates through mimalloc. Copy both from `build/core/libs/libs`; the release archives contain them. Without them the title keeps the C runtime's heap and prints nothing. -On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 12288 MiB per title) is committed when the title maps it, not when the title allocates it or first uses its pages. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover the mapped direct memory together with all other committed memory. Otherwise `sceKernelMapDirectMemory` throws `replace guest placeholder with resident direct memory` with the Windows error (`commit direct memory pages` when a page is mapped at a second address, `keep unmapped direct memory` when it is unmapped); enlarge the page file or close other applications. +On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed when the title maps it, not when the title allocates it or first uses its pages. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover the mapped direct memory together with all other committed memory. Otherwise `sceKernelMapDirectMemory` throws `replace guest placeholder with resident direct memory` with the Windows error (`commit direct memory pages` when a page is mapped at a second address, `keep unmapped direct memory` when it is unmapped); enlarge the page file or close other applications. On Linux, when the Vulkan driver imports dma-buf memory (not the NVIDIA proprietary driver), shared direct memory is imported through `/dev/udmabuf`, and the user who runs the game needs read-write access to it. Many distributions create it as `root:kvm` with mode `0660`: add the user to the `kvm` group and log in again (an ACL such as `setfacl -m u:$USER:rw /dev/udmabuf` lasts until the next reboot). Without access, startup prints `[gpu] open /dev/udmabuf: Permission denied`, these ranges are copied instead of imported, and GPU stores to them through FLAT/GLOBAL addresses fail with `BDA access failed`. A range larger than 64 MiB is imported as one sparse buffer over 64 MiB udmabuf buffers when the device binds sparse memory (RADV, ANV). Without sparse binding, or when the device's sparse block size is larger than 64 KiB, ranges above udmabuf's `size_limit_mb` (64 MiB by default) are copied as well. diff --git a/tools/build_web.py b/tools/build_web.py index 2020aa2cf4..104161f497 100644 --- a/tools/build_web.py +++ b/tools/build_web.py @@ -33,22 +33,55 @@ def request(url): return urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) +def release_metadata(release): + if not isinstance(release, dict): + raise ValueError("release metadata must be a JSON object") + for field in ("tag_name", "html_url"): + if not isinstance(release.get(field), str) or not release[field]: + raise ValueError(f"release metadata {field} must be nonempty text") + assets = release.get("assets") + if not isinstance(assets, list) or any(not isinstance(asset, dict) or not isinstance(asset.get("name"), str) + or not asset["name"] for asset in assets): + raise ValueError("release metadata assets must be a list of named asset objects") + return release + + +def asset_metadata(asset, maximum): + name = asset["name"] + if type(asset.get("size")) is not int or not 0 < asset["size"] <= maximum: + raise ValueError(f"Invalid release asset {name}: archive size must be a positive integer no greater than {maximum}") + if not isinstance(asset.get("browser_download_url"), str) or not asset["browser_download_url"]: + raise ValueError(f"Invalid release asset {name}: browser_download_url must be nonempty text") + if asset.get("digest") is not None and not isinstance(asset["digest"], str): + raise ValueError(f"Invalid release asset {name}: digest must be text or null") + return asset + + +def library_assets(release): + assets = {} + for platform in ("windows", "linux"): + name = f"prx-{platform}-{release['tag_name']}.zip" + matches = [asset for asset in release["assets"] if asset["name"] == name] + if len(matches) != 1: + raise ValueError(f"Missing or ambiguous release asset: {name}") + assets[platform] = asset_metadata(matches[0], 512 * 1024 * 1024) + return assets + + def publish_libraries(output, release=None): if release is None: with request("https://api.github.com/repos/boykopovar/AnyPS5/releases/latest") as response: release = json.load(response) + release = release_metadata(release) tag = release["tag_name"] if not re.fullmatch(r"v[0-9A-Za-z][0-9A-Za-z._-]*", tag): raise ValueError("Invalid release tag") manifest = {"tag": tag, "url": release["html_url"], "platforms": {}} + assets = library_assets(release) destination = output / "libraries" destination.mkdir(exist_ok=True) for platform in ("windows", "linux"): - name = f"prx-{platform}-{tag}.zip" - matches = [asset for asset in release["assets"] if asset["name"] == name] - if len(matches) != 1: - raise ValueError(f"Missing or ambiguous release asset: {name}") - asset = matches[0] + asset = assets[platform] temporary = destination / f"{platform}.download" try: digest = hashlib.sha256() @@ -104,6 +137,7 @@ def publish_libraries(output, release=None): def publish_release(output): with request("https://api.github.com/repos/boykopovar/AnyPS5/releases/latest") as response: release = json.load(response) + release = release_metadata(release) tag = release["tag_name"] if not re.fullmatch(r"v[0-9A-Za-z][0-9A-Za-z._-]*", tag) or tag.endswith("."): raise ValueError("Invalid release tag") @@ -111,9 +145,8 @@ def publish_release(output): matches = [asset for asset in release["assets"] if asset["name"] == name] if len(matches) != 1: raise ValueError(f"Missing or ambiguous release asset: {name}; publish a release with the web converter") - asset = matches[0] - if not 0 < asset["size"] <= 128 * 1024 * 1024: - raise ValueError("Invalid web release archive size") + asset = asset_metadata(matches[0], 128 * 1024 * 1024) + library_assets(release) with request(asset["browser_download_url"]) as response: data = response.read(asset["size"] + 1) if len(data) != asset["size"]: diff --git a/tools/tests/test_build_web.py b/tools/tests/test_build_web.py index 78cdb95d0e..3c0506b29e 100644 --- a/tools/tests/test_build_web.py +++ b/tools/tests/test_build_web.py @@ -233,5 +233,52 @@ class BuildWebCliTests(unittest.TestCase): self.assertEqual(err.errors, "backslashreplace") +class ReleaseMetadataTests(unittest.TestCase): + def test_malformed_api_responses_are_cli_errors_without_site_changes(self): + documents = ({}, [], None, {"tag_name": 1}, {"tag_name": "v1", "html_url": "url", "assets": None}, + {"tag_name": "v1", "html_url": "url", "assets": [None]}, + {"tag_name": "v1", "html_url": "url", "assets": [{"name": 1}]}) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + web = root / "web" + web.mkdir() + preserved = web / "keep.txt" + preserved.write_bytes(b"previous site") + for document in documents: + with self.subTest(document=document): + errors, output = io.StringIO(), io.StringIO() + with patch.object(build_web, "request", return_value=io.BytesIO(json.dumps(document).encode())), contextlib.redirect_stderr(errors), contextlib.redirect_stdout(output): + with self.assertRaises(SystemExit) as raised: + build_web.main(["--build", str(root), "--release"]) + self.assertEqual(raised.exception.code, 2) + self.assertNotIn("Traceback", errors.getvalue()) + self.assertIn("release", errors.getvalue().lower()) + self.assertEqual(output.getvalue(), "") + self.assertEqual(preserved.read_bytes(), b"previous site") + self.assertEqual(list(root.iterdir()), [web]) + self.assertEqual(list(web.iterdir()), [preserved]) + + def test_selected_asset_metadata_is_checked_before_any_download(self): + def asset(platform): + return {"name": f"prx-{platform}-v1.zip", "size": 10, "browser_download_url": "https://example.invalid/asset", "digest": None} + changes = [("size", x) for x in (True, 0, -1, 1.5, "10", 512*1024*1024+1)] + changes += [("browser_download_url", x) for x in (None, "", 1)] + changes += [("digest", x) for x in (False, 1, [])] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + existing = root / "release.json" + existing.write_bytes(b"previous manifest") + for field, value in changes: + with self.subTest(field=field, value=value): + windows, linux = asset("windows"), asset("linux") + linux[field] = value + release = {"tag_name": "v1", "html_url": "https://example.invalid/v1", "assets": [windows, linux]} + with patch.object(build_web, "request", side_effect=AssertionError("download occurred before preflight")): + with self.assertRaisesRegex(ValueError, "release asset"): + build_web.publish_libraries(root, release) + self.assertEqual(existing.read_bytes(), b"previous manifest") + self.assertEqual(list(root.iterdir()), [existing]) + + if __name__ == "__main__": unittest.main()