diff --git a/CMakeLists.txt b/CMakeLists.txt index 6514f2235a..bc59d5e48c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,10 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND}) set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20) + add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND}) + set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180) add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE} diff --git a/core/libs/prx/libSceAgc/DcbFlow/include/Control.hpp b/core/libs/prx/libSceAgc/DcbFlow/include/Control.hpp index 46c70ed15e..6ded533662 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/include/Control.hpp +++ b/core/libs/prx/libSceAgc/DcbFlow/include/Control.hpp @@ -1,4 +1,12 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_CONTROL_HPP #define CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_CONTROL_HPP +#include "SceTypes.hpp" +#include + +extern "C" { +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData); +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData); +} + #endif diff --git a/core/libs/prx/libSceAgc/DcbFlow/include/Memory.hpp b/core/libs/prx/libSceAgc/DcbFlow/include/Memory.hpp index 184131f904..f089c9a83e 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/include/Memory.hpp +++ b/core/libs/prx/libSceAgc/DcbFlow/include/Memory.hpp @@ -2,7 +2,11 @@ #define CORE_LIBS_PRX_LIBSCEAGC_DCBFLOW_INCLUDE_MEMORY_HPP #include "SceTypes.hpp" +#include -extern "C" std::uint32_t* APS5_VABI sceAgcDcbAcquireMem(CommandBuffer* buf, std::uint8_t engine, std::uint32_t cbDbOp, std::uint32_t gcrControl, const volatile void* base, std::uint64_t sizeBytes, std::uint32_t pollCycles); +extern "C" { +std::uint32_t* APS5_VABI sceAgcDcbAcquireMem(CommandBuffer* buf, std::uint8_t engine, std::uint32_t cbDbOp, std::uint32_t gcrControl, const volatile void* base, std::uint64_t sizeBytes, std::uint32_t pollCycles); +std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize(); +} #endif diff --git a/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp b/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp index eefbb3120c..987ef180f3 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp +++ b/core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp @@ -12,10 +12,19 @@ extern "C" { -// unknown signature -void* APS5_VABI sceAgcDcbAtomicGds_0900(void) { - NotImplemented_nid_no_patch(__func__); - return nullptr; +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData) { + const auto operation = (control & 0xffu) | ((control & 0x200u) << 7u) | + ((control & 0x20u) << 13u) | ((control & 0x40u) << 14u) | + ((control & 0x20u) == 0 && mode == 1 ? 0x20000u : 0u); + return Agc::Command::Emit(buf, 0x1du, { + operation, ((gdsOp & 1u) << 8u) | (src & 0x3fu), offset, index, + mask & 0x00ff00ffu, data, static_cast(compareData), + static_cast(compareData >> 32u), static_cast(extraData), + static_cast(extraData >> 32u)}, __func__); +} + +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer* buf, std::uint32_t control, std::uint32_t mode, std::uint32_t gdsOp, std::uint32_t src, std::uint16_t offset, std::uint16_t index, std::uint32_t mask, std::uint32_t data, std::uint64_t compareData, std::uint64_t extraData) { + return sceAgcDcbAtomicGds_0900(buf, control, mode, gdsOp, src, offset, index, mask, data, compareData, extraData); } diff --git a/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp b/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp index 18dd28abf7..21d0a2eeb2 100644 --- a/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp +++ b/core/libs/prx/libSceAgc/DcbFlow/src/Memory.cpp @@ -53,7 +53,7 @@ std::uint32_t APS5_VABI sceAgcDcbAtomicMemGetSize() { } std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize() { - return 36; + return 44; } std::uint32_t* APS5_VABI sceAgcDcbWriteData(CommandBuffer* buf, std::uint8_t dst, std::uint8_t cachePolicy, std::uint64_t address, const void* data, std::uint32_t numDwords, std::uint8_t increment, std::uint8_t writeConfirm) { diff --git a/core/libs/prx/libSceAgc/DcbState/include/ContextState.hpp b/core/libs/prx/libSceAgc/DcbState/include/ContextState.hpp index 409da75f33..99d37d3c6a 100644 --- a/core/libs/prx/libSceAgc/DcbState/include/ContextState.hpp +++ b/core/libs/prx/libSceAgc/DcbState/include/ContextState.hpp @@ -1,4 +1,17 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_CONTEXTSTATE_HPP #define CORE_LIBS_PRX_LIBSCEAGC_DCBSTATE_INCLUDE_CONTEXTSTATE_HPP +#include "SceTypes.hpp" +#include + +struct CommandBuffer; + +extern "C" { + +uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operation); +uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, uint32_t operation); +uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation); + +} + #endif diff --git a/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp b/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp index ffdf58c810..ad1eddc97a 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp @@ -9,10 +9,7 @@ extern "C" { uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operation) { - (void)buf; - (void)operation; - NotImplemented_nid_no_patch(__func__); - return nullptr; + return sceAgcDcbContextStateOp_0100(buf, operation); } uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation) { diff --git a/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp b/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp index 415b81e362..3de0a5eda6 100644 --- a/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp +++ b/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp @@ -8,6 +8,7 @@ #include extern "C" std::uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(std::uint32_t operation); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, std::uint32_t operation); extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation); namespace { @@ -37,6 +38,15 @@ void testSizes() { check(sceAgcDcbContextStateOp_0100(&buffer, operation) == words.data(), "context state op did not start at the cursor"); const auto written = static_cast(buffer.cursor_up - words.data()) * sizeof(std::uint32_t); check(written == sizes[operation], "context state size differs from the written packets"); + + std::array unversionedWords{}; + CommandBuffer unversionedBuffer{unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), + unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), + nullptr, nullptr, 0}; + check(sceAgcDcbContextStateOp(&unversionedBuffer, operation) == unversionedWords.data(), "context state op did not start at the cursor"); + const auto unversionedWritten = static_cast(unversionedBuffer.cursor_up - unversionedWords.data()) * sizeof(std::uint32_t); + check(unversionedWritten == sizes[operation], "context state size differs from the written packets"); + check(words == unversionedWords, "unversioned context state op mismatch with versioned op"); } } diff --git a/core/libs/prx/libSceAgc/tests/EndOfShaderActionSize.cpp b/core/libs/prx/libSceAgc/tests/EndOfShaderActionSize.cpp index df935a0534..cf78ada4a3 100644 --- a/core/libs/prx/libSceAgc/tests/EndOfShaderActionSize.cpp +++ b/core/libs/prx/libSceAgc/tests/EndOfShaderActionSize.cpp @@ -2,6 +2,7 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libc/include/Shutdown.hpp" +#include #include #include #include @@ -11,6 +12,9 @@ extern "C" { std::uint32_t APS5_VABI sceAgcDcbQueueEndOfShaderActionGetSize(); std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize(); std::uint32_t* APS5_VABI sceAgcCbReleaseMem(CommandBuffer*, std::uint8_t, std::uint16_t, std::uint8_t, std::uint8_t, const volatile Label*, std::uint8_t, std::uint64_t, std::uint16_t, std::uint16_t, std::uint8_t, std::uint32_t); +std::uint32_t APS5_VABI sceAgcDcbAtomicGdsGetSize(); +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds_0900(CommandBuffer*, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint16_t, std::uint16_t, std::uint32_t, std::uint32_t, std::uint64_t, std::uint64_t); +std::uint32_t* APS5_VABI sceAgcDcbAtomicGds(CommandBuffer*, std::uint32_t, std::uint32_t, std::uint32_t, std::uint32_t, std::uint16_t, std::uint16_t, std::uint32_t, std::uint32_t, std::uint64_t, std::uint64_t); } namespace { @@ -19,19 +23,43 @@ void check(bool condition, const char* message) { if (!condition) throw std::runtime_error(message); } +void testEndOfShaderActionSize() { + std::array words{}; + CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0}; + const auto* packet = sceAgcCbReleaseMem(&buffer, 0x2f, 0, 0, 0, nullptr, 0, 0, 0, 0, 0, 0); + const auto releaseBytes = static_cast((buffer.cursor_up - packet) * sizeof(std::uint32_t)); + check(((packet[0] >> 8u) & 0xffu) == 0x49u && releaseBytes == 32, "end-of-shader RELEASE_MEM size mismatch"); + check(sceAgcDcbQueueEndOfShaderActionGetSize() == releaseBytes, "DCB end-of-shader action size differs from its RELEASE_MEM"); + check(sceAgcDcbQueueEndOfShaderActionGetSize() == sceAgcAcbQueueEndOfShaderActionGetSize(), "DCB and ACB end-of-shader action sizes differ"); +} + +void testAtomicGds() { + constexpr std::uint32_t expectedBytes = 11u * sizeof(std::uint32_t); + check(sceAgcDcbAtomicGdsGetSize() == expectedBytes, "DCB atomic GDS size does not match 11-dword packet"); + + std::array words{}; + CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0}; + const auto* packet = sceAgcDcbAtomicGds_0900(&buffer, 0x261u, 1, 3, 7, 0x100, 0x200, 0x00ff00ffu, 0x12345678u, 0x1122334455667788ull, 0x99aabbccddeeff00ull); + check(packet != nullptr && packet == words.data(), "DCB atomic GDS returned unexpected packet pointer"); + check(buffer.cursor_up == words.data() + 11, "DCB atomic GDS cursor advance does not match 11 dwords"); + check(packet[0] == 0xc0091d00u, "DCB atomic GDS header opcode mismatch"); + + std::array unversionedWords{}; + CommandBuffer unversionedBuffer{unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), unversionedWords.data(), unversionedWords.data() + unversionedWords.size(), nullptr, nullptr, 0}; + const auto* unversionedPacket = sceAgcDcbAtomicGds(&unversionedBuffer, 0x261u, 1, 3, 7, 0x100, 0x200, 0x00ff00ffu, 0x12345678u, 0x1122334455667788ull, 0x99aabbccddeeff00ull); + check(unversionedPacket != nullptr && unversionedPacket == unversionedWords.data(), "unversioned DCB atomic GDS returned unexpected packet pointer"); + check(unversionedBuffer.cursor_up == unversionedWords.data() + 11, "unversioned DCB atomic GDS cursor advance does not match 11 dwords"); + check(std::equal(words.begin(), words.begin() + 11, unversionedWords.begin()), "versioned and unversioned DCB atomic GDS emitted different packets"); +} + } int main() { try { - std::array words{}; - CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0}; - const auto* packet = sceAgcCbReleaseMem(&buffer, 0x2f, 0, 0, 0, nullptr, 0, 0, 0, 0, 0, 0); - const auto releaseBytes = static_cast((buffer.cursor_up - packet) * sizeof(std::uint32_t)); - check(((packet[0] >> 8u) & 0xffu) == 0x49u && releaseBytes == 32, "end-of-shader RELEASE_MEM size mismatch"); - check(sceAgcDcbQueueEndOfShaderActionGetSize() == releaseBytes, "DCB end-of-shader action size differs from its RELEASE_MEM"); - check(sceAgcDcbQueueEndOfShaderActionGetSize() == sceAgcAcbQueueEndOfShaderActionGetSize(), "DCB and ACB end-of-shader action sizes differ"); + testEndOfShaderActionSize(); + testAtomicGds(); LibcRunShutdown_nid_postfix(); - std::puts("AGC end-of-shader action size tests passed"); + std::puts("AGC end-of-shader action and atomic GDS size tests passed"); return 0; } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 18a4274f92..9aa2098180 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1212,8 +1212,12 @@ configure_windows_unwind(agc_driver_recorder_tests) if(BUILD_TESTING) set_target_properties(agc_driver_recorder_tests PROPERTIES EXCLUDE_FROM_ALL OFF) add_test(NAME agc_recorder_device_selection COMMAND agc_driver_recorder_tests --device-selection-only) + add_test(NAME agc_gpu_timing_complete COMMAND agc_driver_recorder_tests --gpu-timing-complete-only) + add_test(NAME agc_gpu_timing_incomplete COMMAND agc_driver_recorder_tests --gpu-timing-incomplete-only) + set_tests_properties(agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES + ENVIRONMENT "APS5_PROFILE_GPU=1" TIMEOUT 15 SKIP_RETURN_CODE 77) if(WIN32) - set_tests_properties(agc_recorder_device_selection PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_recorder_device_selection agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only) @@ -1285,6 +1289,12 @@ if(BUILD_TESTING) set_tests_properties(agc_storage_baseline_settled PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_read_written_staging COMMAND agc_driver_recorder_tests --read-written-staging-only) + set_tests_properties(agc_read_written_staging PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_read_written_staging PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_storage_written_buffer_order COMMAND agc_driver_recorder_tests --written-buffer-order-only) set_tests_properties(agc_storage_written_buffer_order PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) if(WIN32) @@ -1297,6 +1307,12 @@ if(BUILD_TESTING) set_tests_properties(agc_storage_split_writeback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_storage_write_back COMMAND agc_driver_recorder_tests --storage-write-back-only) + set_tests_properties(agc_storage_write_back PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120 ENVIRONMENT "APS5_PIN_WAIT_MS=3000") + if(WIN32) + set_tests_properties(agc_storage_write_back PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_copied_direct_ordering COMMAND agc_driver_recorder_tests --copied-direct-only) set_tests_properties(agc_copied_direct_ordering PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) if(WIN32) @@ -1589,6 +1605,15 @@ target_link_libraries(agc_driver_graphics_tests PRIVATE libkernel ) +if(BUILD_TESTING) + add_test(NAME agc_command_batch_fence_failure COMMAND agc_driver_graphics_tests --command-batch-fence-failure) + set_tests_properties(agc_command_batch_fence_failure PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(agc_command_batch_fence_failure PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + set_target_properties(agc_driver_graphics_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_graphics_tests) @@ -1707,6 +1732,10 @@ target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $) +target_include_directories(agc_driver_image_table_cache_bounds_tests PRIVATE $) + +target_include_directories(agc_driver_draw_cache_relocation_tests PRIVATE $) + if(BUILD_TESTING) add_test(NAME agc_driver_store_pending_identity COMMAND agc_driver_store_over_untracked_tests --pending-identity) set_tests_properties(agc_driver_store_pending_identity PROPERTIES @@ -1769,6 +1798,8 @@ target_include_directories(agc_driver_color1555_target_tests PRIVATE $) +target_include_directories(agc_driver_depth_range_remap_draw_tests PRIVATE $) + foreach(stage IN ITEMS vert frag) set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage}) set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv) @@ -1870,6 +1901,15 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_srt_read_patching COMMAND agc_shader_memory_tests --srt-read-patching) set_tests_properties(agc_shader_srt_read_patching PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse) + set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) + set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) + set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + if(WIN32) + set_tests_properties(agc_shader_table_snapshot_reuse agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization) set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_frontend_pair COMMAND agc_shader_memory_tests --frontend-pair) @@ -2086,6 +2126,12 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30) + add_test(NAME agc_driver_dword_pattern_fill_translated COMMAND agc_driver_dword_pattern_fill_tests) + set_tests_properties(agc_driver_dword_pattern_fill_translated PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FILL_HLE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_dword_pattern_fill_translated PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_driver_uint8x4_color_target_no_resident COMMAND agc_driver_uint8x4_color_target_tests) set_tests_properties(agc_driver_uint8x4_color_target_no_resident PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_RESIDENT_TARGETS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30) if(WIN32) @@ -2270,9 +2316,12 @@ if(BUILD_TESTING) endif() endforeach() endforeach() - foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync lazy lazy_sync) + foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync) set(preparedTest "agc_driver_prepared_shaders_${preparedMode}") - if(preparedMode STREQUAL "indirect") + if(preparedMode MATCHES "^abi_deferred_") + string(REPLACE "_" "-" preparedArgument ${preparedMode}) + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --${preparedArgument}) + elseif(preparedMode STREQUAL "indirect") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect) elseif(preparedMode STREQUAL "deferred") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred) @@ -2289,7 +2338,9 @@ if(BUILD_TESTING) add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests) endif() set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") - if(preparedMode STREQUAL "uncached") + if(preparedMode MATCHES "^abi_deferred_") + set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_FAILURE_MEMO=1") + elseif(preparedMode STREQUAL "uncached") set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_DISPATCH_CACHE=1") elseif(preparedMode STREQUAL "registered_workgroup_sync") set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_SYNC_SHADER_PREPARE=1") @@ -2840,6 +2891,20 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_depth_range_remap_tests EXCLUDE_FROM_ALL tests/DepthRangeRemap.cpp) +target_link_libraries(agc_depth_range_remap_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_depth_range_remap_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_depth_range_remap_tests) +if(BUILD_TESTING) + set_target_properties(agc_depth_range_remap_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_depth_range_remap COMMAND agc_depth_range_remap_tests) + set_tests_properties(agc_depth_range_remap PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1") + if(WIN32) + set_tests_properties(agc_depth_range_remap PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_position_invariance_tests EXCLUDE_FROM_ALL tests/PositionInvariance.cpp) target_link_libraries(agc_position_invariance_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_position_invariance_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp index 36b218edf9..1e2b68353b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/AsyncLog.hpp @@ -390,9 +390,7 @@ inline void InstallAsyncLog_nid_no_patch() { static_cast(AsyncLog::Get()); } -inline void FlushAsyncLog_nid_no_patch() { - AsyncLog::Get().Flush(); -} +void FlushAsyncLog_nid_no_patch(); inline void FlushAsyncLogAtEnd_nid_no_patch() { AsyncLog::FlushAtEnd(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index b96a4cdf0a..75efb72f56 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -27,6 +27,7 @@ struct DispatchVariant { std::vector> runs; std::vector words; + std::vector shared; std::uint64_t forgetSerial = 0; std::shared_ptr compiled; std::array meshIndexDescriptor{}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp index 23b6231072..ec149e3d6c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp @@ -53,7 +53,7 @@ struct PreparedGraphicsStage { std::shared_ptr registration; }; -std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target); +std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false); struct DrawRecipeRecord { std::vector> stages; @@ -118,6 +118,7 @@ struct StageCapture { std::shared_ptr compiled; std::shared_ptr capture; std::vector regions; + std::vector> supplied; std::uint64_t forgetSerial = 0; std::uint32_t pushOffset = 0; ShaderRecompiler::SrtUserDataUse userDataUse; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 75c47c8e6d..c65e4ef54c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -53,7 +53,8 @@ public: std::uint64_t DispatchesAdoptedAhead() const; void CheckFailure(); void ReportFailure(std::exception_ptr error); - void Submit(const Packet* packet, std::uint32_t queue); + void Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip = false); + void SubmitFlip(const FlipInfo& info); void SuspendPoint(); std::uint32_t ReserveEopFlip(const FlipInfo& info); void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output); @@ -68,6 +69,7 @@ public: void SettleShaderPreparations(std::span stages); static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters); static std::optional ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); + static void tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span> supplied = {}); static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request, std::string_view suffix = {}); DrawEntryCounters DrawCacheCounters(); struct DispatchCacheFootprint { @@ -76,6 +78,7 @@ public: }; DispatchCacheFootprint DispatchCacheSize(); static std::uint64_t DispatchCacheBytes(); + static ShaderMemory::PendingWrite QueryPendingSpan(std::uint64_t address, std::size_t bytes, std::span known); std::shared_ptr RegisteredShaders(); private: @@ -151,15 +154,14 @@ private: static std::uint64_t drawIdentityKey(const QueueState& queue, const ShaderRegistry& registry); static bool drawRelocationEnabled(); static std::uint64_t relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable); - static void tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords); static void applyShaderDataPatches(DispatchVariant& variant); void learnPendingRelocation(); void relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std::shared_ptr& entry, std::shared_ptr& decode, std::vector& patches); void reportDrawRelocation(); static bool sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b); static bool sameDecode(const DrawDecode& a, const DrawDecode& b); - std::shared_ptr decodeDraw(const QueueState& queue, const Submission& submission); - void resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile); + std::shared_ptr decodeDraw(const QueueState& queue, const Submission& submission, const Graphics::Context& context); + void resolveDrawDecode(const QueueState& queue, const Submission& submission, const Graphics::Context& context, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile); void lookupDraw(const Submission& submission, const std::shared_ptr& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector& programs, const std::vector& roles, const std::vector>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr& entry, std::vector>& matched, std::vector>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array& phaseMs, const std::shared_ptr& dataEntry, DrawDataCandidates& dataCandidates, std::vector& stageData); std::shared_ptr materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected); static void recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector& decodeReads); @@ -208,6 +210,7 @@ private: static std::uint64_t hookWaits(); static ShaderMemory::HookWaitCounter hookWaitCounter(); static bool validateSkipEnabled(); + static bool vouchedValidation(); static bool validateLegacy(); static bool gateRetry(); static bool validateSkipVerify(); @@ -242,10 +245,10 @@ private: void traceCapture(const char* what, std::uint64_t program, std::uint32_t queue, std::span regions, double waitedMs); void reportValidation(ValidateCounters& counters); bool captureStable(std::span captured); - bool validateCaptured(std::uint64_t program, std::uint32_t queue, std::span captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped = nullptr, std::optional* sampling = nullptr, const DataMask* data = nullptr); + bool validateCaptured(std::uint64_t program, std::uint32_t queue, std::span captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped = nullptr, std::optional* sampling = nullptr, const DataMask* data = nullptr, std::span vouched = {}); static void appendEntryRegions(const DispatchVariant& variant, std::vector& regions, const std::vector* words = nullptr); static std::shared_ptr patchDataWords(const DispatchVariant& variant, const std::vector& words); - bool syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span regions, std::uint64_t& synced, PendingView& pending, std::optional& sampling); + bool syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span regions, std::uint64_t& synced, PendingView& pending, std::optional& sampling, std::span skipped = {}); EntryOutcome validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional& sampling, std::vector>* live = nullptr); static int waitTimeoutMs(); static bool pollReapAll(); @@ -328,7 +331,7 @@ private: std::condition_variable changed; std::map workers; - std::uint64_t frameSerial = 0; + std::atomic frameSerial{0}; std::shared_ptr pendingFrameTiming; std::atomic flipsCounted{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index b304c65624..203940d7c3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -211,7 +211,7 @@ void AwaitLibraryPrebuilds(); void CloseRegistrationPreparation(); unsigned RegistrationPreparationThreads(); -void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target); +void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false); ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId); ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 1e0f0044b1..06c5c7bf53 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -196,6 +196,7 @@ unsigned long long CodeOffset(const void* address); // calls FlushGpuWrites itself. void SetFlushHook(void (*hook)(std::uint64_t address, std::size_t bytes)); void SetReadGuard(void (*guard)(std::uint64_t address, std::size_t bytes)); +void GuardRead(std::uint64_t address, std::size_t bytes); void FlushGpuWrites(std::uint64_t address, std::size_t bytes); // Moves with every ForgetPages call (the libc invalidator: memory unmapped or re-registered): odd // while the call stores its page states, moved past it after, so a reader that loads an even diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp index 9ba265842a..3ab0d7e78d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Recipe.hpp @@ -82,7 +82,7 @@ struct DrawRecipe { std::array pushBytes{}; VkShaderStageFlags pushStages = 0; std::optional masked; - std::set fragmentOutputs; + std::uint32_t fragmentOutputs = 0; VkPipelineStageFlags shaderStages = 0; bool hasDepth = false; }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 29bcd747f2..dacb784275 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -16,6 +16,8 @@ namespace ShaderRecompiler { struct SourceHandle; class PreparedShaderInvocation; +struct SrtWriteStamp; +struct SrtGuestWords; } namespace AgcDriver { @@ -56,7 +58,7 @@ public: // by the driver: the [capture] line charges the waits made inside a capture to it. using WaitedMsProvider = double (*)(); static void SetWaitedMsProvider(WaitedMsProvider provider); - explicit ShaderMemory(std::span initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr); + explicit ShaderMemory(std::span initial, PendingWriteQuery pendingWrite = nullptr, PendingWriteObserver observe = nullptr, HookWaitCounter hookWaits = nullptr, PendingWriteQuery pendingSpan = nullptr); ~ShaderMemory(); ShaderMemory(const ShaderMemory&) = delete; ShaderMemory& operator=(const ShaderMemory&) = delete; @@ -74,6 +76,16 @@ public: [[nodiscard]] std::vector TakeRecentRegions(); // The driver's source handle memo outcomes, for the [capture] line (APS5_PROFILE_DRAW). static void CountHandleMemo(bool hit); + struct SupplyCounts { + std::uint64_t shared, copied, spilled; + }; + static SupplyCounts Supplies(); + [[nodiscard]] std::vector> SuppliedBlocks() const; + struct VouchCounts { + std::uint64_t calls, blocks, refused, words; + }; + static VouchCounts Vouches(); + static bool Vouched(PendingWriteQuery pending, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp); private: std::shared_ptr capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation); @@ -83,6 +95,10 @@ private: class WordMask { public: [[nodiscard]] bool Test(std::size_t index) const { return (bits[index / 64u] >> (index % 64u) & 1u) != 0u; } + [[nodiscard]] bool TestBlock(std::size_t index) const { + const auto block = ((std::uint64_t{1} << BlockWords) - 1u) << (index % 64u / BlockWords * BlockWords); + return (bits[index / 64u] & block) == block; + } void Set(std::size_t index) { bits[index / 64u] |= std::uint64_t{1} << (index % 64u); } void SetAll() { bits.fill(~std::uint64_t{0}); } void SetBlock(std::size_t index) { bits[index / 64u] |= ((std::uint64_t{1} << BlockWords) - 1u) << (index % 64u / BlockWords * BlockWords); } @@ -107,22 +123,41 @@ private: bool lazy = false; }; + struct Supplied { + std::shared_ptr words; + std::uint64_t begin = 0; + std::uint64_t end = 0; + bool recent = true; + }; + static bool read(void* context, std::uint64_t address, std::uint32_t* value); static bool readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count); static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes); static bool pending(void* context, std::uint64_t address, std::uint64_t bytes); + static ShaderRecompiler::SrtWriteStamp stampWrites(void* context, std::uint64_t address, std::uint64_t bytes); + static bool unchangedSince(void* context, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp); + static bool supplyWords(void* context, const std::shared_ptr& words); Page& page(std::uint64_t base); + [[nodiscard]] std::size_t pageAtOrAfter(std::uint64_t base) const; + Page& addPage(std::size_t position, std::uint64_t base); [[nodiscard]] bool overlapsInitial(std::uint64_t begin, std::uint64_t end) const; + bool place(std::uint64_t address, const std::uint32_t* values, std::uint64_t count, bool overwrite, bool recent); + void spill(std::size_t index); + void spillTouching(std::uint64_t begin, std::uint64_t end); + [[nodiscard]] std::vector collect(bool recentOnly) const; // Regions given at construction (the registered shader's code and header), referenced as given: // the caller keeps them alive for as long as the capture is used. - std::map> initial; + std::vector>> initial; std::vector> pages; std::uint64_t lastBase = ~std::uint64_t{0}; Page* lastPage = nullptr; + std::vector supplied; + std::vector> spilled; PendingWriteQuery pendingWrite = nullptr; PendingWriteObserver observe = nullptr; HookWaitCounter hookWaits = nullptr; + PendingWriteQuery pendingSpan = nullptr; }; // The positions, among a dispatch-cache variant's stored words, of the pure flat-SRT leaves a @@ -140,6 +175,25 @@ struct DataWordPositionCounts { }; DataWordPositionCounts DataWordPositions(std::span> runs, std::span> leaves, std::span otherReads, std::span words, std::span flattenedSrt, std::vector& positions, std::vector& slots); +struct SharedWords { + std::size_t first = 0; + std::size_t count = 0; + std::size_t offset = 0; + std::weak_ptr block; + std::size_t region = 0; + std::size_t regions = 0; +}; +void AppendSharedWords(std::vector& shared, std::size_t region, std::size_t first, std::span bytes, std::span> blocks); +bool SameCapturedWords(std::span left, std::span leftShared, std::span right, std::span rightShared, std::size_t* compared = nullptr); +struct VouchedRegions { + std::size_t begin = 0; + std::size_t end = 0; + std::size_t words = 0; + std::uint64_t low = 0; + std::uint64_t high = 0; +}; +std::size_t VouchSharedRegions(std::span shared, std::size_t regions, ShaderMemory::PendingWriteQuery pending, std::vector& vouched); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp index a628e872fb..2c7443e9a5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp @@ -54,6 +54,7 @@ public: } extern "C" std::uint32_t AgcDriverReserveEopFlip_nid_postfix(const AgcDriver::FlipInfo& info); +extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info); extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr& output); extern "C" void AgcDriverUnregisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr& output); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 0f46ec72bb..0342ca1652 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -332,9 +332,7 @@ public: void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); bool PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span shaders); -private: - VulkanDevice(const PresentationWindow* window, bool compileOnly); - friend struct VulkanDeviceTestAccess; +public: // The device's Graphics::Context: a copy of the one built at setup (its instance functions // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew. Graphics::Context graphicsContext() const; @@ -342,6 +340,10 @@ private: std::uint32_t computeSubgroupSize(const ShaderRecompiler::RecompileResult& shader) const; VkPipelineShaderStageCreateFlags computeStageFlags(const ShaderRecompiler::RecompileResult& shader) const; std::shared_ptr createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress); + +private: + VulkanDevice(const PresentationWindow* window, bool compileOnly); + friend struct VulkanDeviceTestAccess; ShaderRecompiler::SpirvTarget buildTarget() const; Graphics::Context buildContext() const; // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp index ae76ddf5bb..da9dac3de5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp @@ -12,7 +12,7 @@ void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffse std::fflush(stderr); std::abort(); }; - auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); + auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan); auto invocation = InvocationFor(snapshot, codeOffset, request); const auto capture = CaptureInvocation(*verifyMemory, snapshot, codeOffset, request, invocation); const auto regions = verifyMemory->Regions(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 565d951689..4e0c72af59 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -326,7 +326,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); } else if (!fromAhead) { - shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); + shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan); std::uint64_t forgetAtCapture = 0; static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp index 812b138b74..97c5d143ae 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp @@ -60,7 +60,7 @@ Driver::DispatchCacheFootprint Driver::DispatchCacheSize() { } std::uint64_t Driver::variantBytes(const DispatchVariant& variant) { - return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.runs.size() * sizeof(std::pair) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0); + return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.shared.size() * sizeof(SharedWords) + variant.runs.size() * sizeof(std::pair) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0); } void Driver::accountVariant(const DispatchVariant& variant, bool added) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp index 89c1aeadc7..e732e518ee 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp @@ -22,15 +22,23 @@ void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDis fresh->forgetSerial = forgetAtCapture; std::vector srtRegions; + std::size_t capturedWords = 0; for (const auto& region : captured) { const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); }); if (registered) continue; srtRegions.push_back(region); + capturedWords += region.bytes.size() / sizeof(std::uint32_t); + } + const auto supplied = shaderMemory->SuppliedBlocks(); + fresh->runs.reserve(srtRegions.size()); + fresh->words.reserve(capturedWords); + for (const auto& region : srtRegions) { fresh->runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); const auto count = region.bytes.size() / sizeof(std::uint32_t); const auto offset = fresh->words.size(); fresh->words.resize(offset + count); std::memcpy(fresh->words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); + AppendSharedWords(fresh->shared, fresh->runs.size() - 1u, offset, region.bytes, supplied); } DataWordPositionCounts dataCounts; @@ -91,7 +99,7 @@ void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDis dispatchOrder.push_front(key); created->order = dispatchOrder.begin(); dispatchCache.emplace(key, std::move(created)); - } else if (found->second != missedEntry && !found->second->variants.empty() && found->second->variants.front()->runs == fresh->runs && found->second->variants.front()->words == fresh->words) { + } else if (found->second != missedEntry && !found->second->variants.empty() && found->second->variants.front()->runs == fresh->runs && SameCapturedWords(found->second->variants.front()->words, found->second->variants.front()->shared, fresh->words, fresh->shared)) { attachVariant = found->second->variants.front(); } else { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp index ebefe68205..e4136cbebe 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" +#include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include @@ -28,10 +29,15 @@ std::shared_ptr Driver::patchDataWords( return patched; } -bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span regions, std::uint64_t& synced, PendingView& pending, std::optional& sampling) { +bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled, std::span regions, std::uint64_t& synced, PendingView& pending, std::optional& sampling, std::span skipped) { using Policy = ShaderMemory::PendingWrite; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - for (const auto& region : regions) { + for (std::size_t index = 0, range = 0; index < regions.size(); ++index) { + if (range < skipped.size() && index == skipped[range].begin) { + index = skipped[range++].end - 1u; + continue; + } + const auto& region = regions[index]; if (!pending.Overlaps(region.guestAddress, region.bytes.size())) continue; if (!sampling) sampling.emplace(siteSampling() ? evidenceValidations : evidenceReads); std::uint64_t ValidateCounters::*reason = &ValidateCounters::syncedOff; @@ -78,6 +84,11 @@ bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const S return true; } +bool Driver::vouchedValidation() { + static const bool enabled = std::getenv("APS5_NO_VOUCHED_VALIDATION") == nullptr; + return enabled; +} + EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional& sampling, std::vector>* live) { if (live != nullptr) live->clear(); if (DrawPipeline::Active()) { @@ -86,14 +97,34 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, } const bool masked = live != nullptr && !variant.dataPositions.empty(); const DataMask mask{variant.dataPositions, live}; + thread_local std::vector* vouchedSlot = nullptr; + auto& vouched = ShaderRecompiler::ThreadOwned(vouchedSlot); + vouched.clear(); + if (!variant.shared.empty() && vouchedValidation() && !validateLegacy() && variant.runs.size() == regions.size()) VouchSharedRegions(variant.shared, regions.size(), &QueryPendingSpan, vouched); PendingView pending; pending.Load(); - if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling)) return EntryOutcome::Differing; + std::erase_if(vouched, [&](const VouchedRegions& range) { return pending.Overlaps(range.low, static_cast(range.high - range.low)); }); + thread_local std::vector>* checkedSlot = nullptr; + auto& checked = ShaderRecompiler::ThreadOwned(checkedSlot); + checked.clear(); + if (!vouched.empty()) { + for (std::size_t index = 0, range = 0; index < variant.runs.size(); ++index) { + if (range < vouched.size() && index == vouched[range].begin) { + checked.emplace_back(vouched[range].low, vouched[range].high); + index = vouched[range++].end - 1u; + continue; + } + checked.push_back(variant.runs[index]); + } + } + const std::span> runs = vouched.empty() ? std::span>(variant.runs) : std::span>(checked); + if (!syncPendingRuns(program, queue, *variant.compiled, regions, runsSynced, pending, sampling, vouched)) return EntryOutcome::Differing; auto publish = Graphics::Recorder::PublishGeneration(); auto pendingSerial = Graphics::StorageTexture::PendingSerial(); auto forget = GuestMemory::ForgetSerial(); - if (Graphics::StorageTexture::AnyPendingOverlaps(variant.runs) || Graphics::AnyShadowedOverlaps(variant.runs)) { + if (Graphics::StorageTexture::AnyPendingOverlaps(runs) || Graphics::AnyShadowedOverlaps(runs)) { + vouched.clear(); for (const auto& region : regions) { if (Graphics::StorageTexture::FlushPending(region.guestAddress, region.bytes.size(), nullptr, "dispatch-cache entry")) ++imagesFlushed; } @@ -106,7 +137,7 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, } if ((forget & 1) != 0) return EntryOutcome::ForgetMoved; for (const auto& label : deferredLabels().labels) { - for (const auto& [begin, end] : variant.runs) { + for (const auto& [begin, end] : runs) { if (label.address < end && begin < label.address + label.size) return EntryOutcome::QueuedLabel; } } @@ -118,7 +149,7 @@ EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, pending.Load(); bool unmapped = false; - if (!validateCaptured(program, queue, regions, *variant.compiled, true, pending, &unmapped, &sampling, masked ? &mask : nullptr)) return unmapped ? EntryOutcome::Inaccessible : EntryOutcome::Differing; + if (!validateCaptured(program, queue, regions, *variant.compiled, true, pending, &unmapped, &sampling, masked ? &mask : nullptr, vouched)) return unmapped ? EntryOutcome::Inaccessible : EntryOutcome::Differing; if (Graphics::Recorder::PublishGeneration() != publish) return EntryOutcome::PublishMoved; if (Graphics::StorageTexture::PendingSerial() != pendingSerial) return EntryOutcome::PendingMoved; if (GuestMemory::ForgetSerial() != forget) return EntryOutcome::ForgetMoved; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 71066bbe9a..7d11469870 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -8,6 +8,7 @@ #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "Optimization/ResourceProgram.hpp" #include +#include #include #include #include @@ -162,7 +163,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe if (program.merged) InitializeMerged(queue, program, program.mergedPointer, program.mergedPointerRequired, false); } if (verifyDrawDataHits()) { - const auto full = decodeDraw(queue, submission); + const auto full = decodeDraw(queue, submission, localDevice->graphicsContext()); if (!sameDecode(*refreshed, *full)) { std::size_t differing = 0; while (differing < full->programs.size() && differing < refreshed->programs.size() && full->programs[differing].userData == refreshed->programs[differing].userData) ++differing; @@ -173,15 +174,18 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe } decode = std::move(refreshed); } - resolveDrawDecode(queue, submission, decode, registerKey, drawKey, profile); + resolveDrawDecode(queue, submission, localDevice->graphicsContext(), decode, registerKey, drawKey, profile); const auto& graphics = decode->state; auto pixel = decode->pixel; - std::vector programs = decode->programs; - for (const auto& patch : userPatches) programs[patch.program].userData[patch.index] = patch.value; + const bool writesPrograms = graphics.stages.mesh || drawParameters.indirect || !userPatches.empty(); + std::vector writablePrograms; + if (writesPrograms) writablePrograms = decode->programs; + for (const auto& patch : userPatches) writablePrograms[patch.program].userData[patch.index] = patch.value; + const std::vector& programs = writesPrograms ? writablePrograms : decode->programs; if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn(); const auto setMeshIndexBuffer = [&](const Pm4::DrawParameters& parameters) { if (!graphics.stages.mesh) return; - auto& words = programs.front().userData; + auto& words = writablePrograms.front().userData; require(programs.front().firstUserSgpr == 0 && words.size() >= ShaderRecompiler::MeshIndexBufferUserWord + 4, "mesh program lacks the hidden user words"); const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters); std::copy(descriptor.begin(), descriptor.end(), words.begin() + ShaderRecompiler::MeshIndexBufferUserWord); @@ -222,7 +226,9 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe indirect.drawIndexSgpr = sgprOf(indirect.drawIndexLocation); } std::vector memory; + memory.reserve(2 * programs.size() + 32); std::vector linked; + linked.reserve(programs.size()); for (std::size_t i = 0; i < programs.size(); ++i) { const auto& program = programs[i]; for (const auto& region : program.memory) { @@ -241,12 +247,15 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe decodeReads[i].clear(); vertexInfos[i] = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, &decodeReads[i], false, 8u - program.firstUserSgpr); if (vertexInfos[i]) vertexInfos[i]->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, readRegister(queue.context, 0x207), program.tailContinuation.get()) | graphics.layerExports; + if (vertexInfos[i]) { + vertexInfos[i]->depthRemapMode = graphics.depthRemap.mode; + } }; if (!registerKey) { for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i); phaseTiming.Phase(DrawRowDecode); } - auto shaderMemory = std::make_unique(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); + auto shaderMemory = std::make_unique(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter(), &QueryPendingSpan); std::vector results; std::vector> compiledStages; compiledStages.reserve(programs.size()); @@ -321,7 +330,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe const auto freshRuns = [&](std::size_t i, std::vector>& runs, std::vector& words) { std::vector regions(stageCaptures[i].regions.begin(), stageCaptures[i].regions.end()); - for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))}); + for (const auto& read : decodeReads[i]) regions.push_back({read.address, read.Bytes()}); std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; }); for (const auto& region : regions) { runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); @@ -423,7 +432,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe timing.Mark("shader_compile_and_link"); for (const auto& reads : decodeReads) { - for (const auto& read : reads) memory.push_back({read.address, std::as_bytes(std::span(read.bytes))}); + for (const auto& read : reads) memory.push_back({read.address, read.Bytes()}); } if (pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels); @@ -470,6 +479,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe std::vector snapshots; const auto snapshot = [&] { snapshots.clear(); + snapshots.reserve(memory.size()); for (const auto& region : memory) snapshots.push_back({region.guestAddress, region.bytes}); }; snapshot(); @@ -519,7 +529,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe std::set patched; const auto patch = [&](const std::optional>& word, std::uint32_t value) { if (!word) return; - programs[word->first].userData[word->second] = value; + writablePrograms[word->first].userData[word->second] = value; patched.insert(word->first); }; patch(baseVertexWord, indirect.recordBytes == 20 ? arguments.vertexOffset : arguments.firstVertexOrIndex); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp index 948bc83d91..1d76a6a338 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp @@ -140,7 +140,7 @@ void Driver::insertDrawEntry(std::uint64_t key, std::vectorstages[i]; - const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->meshIndexDescriptor == fresh[i]->meshIndexDescriptor && kept->runs == fresh[i]->runs && kept->words == fresh[i]->words; }); + const auto present = std::find_if(variants.begin(), variants.end(), [&](const std::shared_ptr& kept) { return kept->pushOffset == fresh[i]->pushOffset && kept->meshIndexDescriptor == fresh[i]->meshIndexDescriptor && kept->runs == fresh[i]->runs && SameCapturedWords(kept->words, kept->shared, fresh[i]->words, fresh[i]->shared); }); if (present != variants.end()) { ++counters.present; fresh[i] = *present; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index c761433da8..7662c966f0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -23,7 +23,7 @@ std::shared_ptr Driver::materializeDraw program.binary, {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)}, localDevice->Target(), - {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : graphics.stages.tessellation && pushOffset < Graphics::PipelinePushSlotBytes ? ShaderRecompiler::TessellationLevelPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes}, + {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : graphics.stages.tessellation && pushOffset < Graphics::PipelinePushSlotBytes ? ShaderRecompiler::TessellationLevelPushOffsetBytes : graphics.depthRemap.mode != 0u ? ShaderRecompiler::DepthRemapPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; @@ -56,6 +56,7 @@ std::shared_ptr Driver::materializeDraw }(); stageCapture.regions = shaderMemory.TakeRecentRegions(); + stageCapture.supplied = shaderMemory.SuppliedBlocks(); stageCapture.userDataUse = capture->userDataUse; recompiled[i] = true; memory = shaderMemory.Regions(); @@ -120,7 +121,7 @@ void Driver::recordStageDataPositions(DispatchVariant& variant, const ShaderReco for (const auto& [begin, end] : variant.runs) { for (const auto& read : decodeReads) { const auto from = std::max(begin, read.address); - const auto to = std::min(end, read.address + read.bytes.size()); + const auto to = std::min(end, read.address + read.size); if (from >= to) continue; for (auto word = (from - begin) / sizeof(std::uint32_t); word < (to - begin + sizeof(std::uint32_t) - 1) / sizeof(std::uint32_t); ++word) decoded[offset + static_cast(word)] = true; } @@ -159,9 +160,9 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP if (vertexInfos[i]) variant->vertexInfo = std::make_shared(*vertexInfos[i]); std::vector regions(stageCapture.regions.begin(), stageCapture.regions.end()); - for (const auto& read : decodeReads[i]) regions.push_back({read.address, std::as_bytes(std::span(read.bytes))}); + for (const auto& read : decodeReads[i]) regions.push_back({read.address, read.Bytes()}); std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; }); - tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords); + tagVariantRuns(*variant, regions, stageCapture.userDataUse, decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0), programs[i].hiddenUserWords, stageCapture.supplied); if (drawStageDataHits() && stageCapture.capture != nullptr && !stageCapture.capture->readTrace.leaves.empty()) recordStageDataPositions(*variant, *stageCapture.capture, decodeReads[i]); if (traceDrawCache() && entry != nullptr && matched[i] == nullptr && i < entry->stages.size()) { const auto& variants = entry->stages[i]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp index ded3909f43..f7716f0eeb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp @@ -22,6 +22,7 @@ void ReadUserWords(const QueueState& queue, DrawProgram& program, bool staticAbi const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u); require(userCount <= 32, "graphics user SGPR count exceeds the register bank"); program.userData.clear(); + program.userData.reserve(userCount + 8u); for (std::uint32_t i = 0; i < userCount; ++i) { Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, program.userDataBase + i); program.userData.push_back(staticAbi ? ReadGraphicsRegister(queue.shader, program.userDataBase + i) : readUserData(queue.shader, program.userDataBase + i)); @@ -121,9 +122,10 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const } } -std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Submission& submission) { +std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Submission& submission, const Graphics::Context& context) { auto product = std::make_shared(); product->state = Graphics::DecodeState(queue, VertexWorkgroupRequired(queue, *submission.shaders)); + Graphics::ApplyDepthRemap(context, product->state); DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true); product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue)); product->pixel.targetExportPacking = Graphics::ExportPackings(product->state); @@ -149,11 +151,11 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su return product; } -void Driver::resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile) { +void Driver::resolveDrawDecode(const QueueState& queue, const Submission& submission, const Graphics::Context& context, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile) { if (decode == nullptr || verifyDrawRecipe()) { std::vector readLog; const Graphics::RegisterReadLogScope logScope(verifyDrawRecipe() && registerKey ? &readLog : nullptr); - auto fresh = decodeDraw(queue, submission); + auto fresh = decodeDraw(queue, submission, context); if (verifyDrawRecipe() && registerKey) { std::uint64_t facadeMismatches = 0; for (const auto read : readLog) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp index 148fabe834..9bc95508d0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp @@ -214,6 +214,7 @@ void Driver::traceDrawKeyLookup(const std::array& components, bool Driver::sameVertexInfo(const ShaderRecompiler::ShaderVertexStageInfo& a, const ShaderRecompiler::ShaderVertexStageInfo& b) { if (a.resourcesNum != b.resourcesNum || a.fetchAttribReg != b.fetchAttribReg || a.fetchBufferReg != b.fetchBufferReg || a.fetchEmbedded != b.fetchEmbedded || a.paClVsOutCntl != b.paClVsOutCntl) return false; + if (a.depthRemapMode != b.depthRemapMode) return false; for (std::uint32_t i = 0; i < a.resourcesNum && i < a.resources.size(); ++i) { if (a.resources[i].fields != b.resources[i].fields) return false; const auto& x = a.resourcesDst[i]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp index da86f3e422..33665a380c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" #include #include #include @@ -55,7 +56,7 @@ std::uint64_t Driver::relocatedContentKey(const RelocatableUserWord& word, std:: return hash; } -void Driver::tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords) { +void Driver::tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span> supplied) { std::uint64_t relocatable = eligible && drawRelocationEnabled() ? use.asBase & ~use.asValue : 0; if (hiddenWords != 0) relocatable &= hiddenWords >= 64 ? 0 : ~((std::uint64_t {1} << hiddenWords) - 1); std::vector> reads; @@ -83,12 +84,14 @@ void Driver::tagVariantRuns(DispatchVariant& variant, std::span> found->second & 1u) != 0 ? static_cast(found->second) : std::int8_t {-1}; }; variant.relocatableWords = relocatable; - for (const auto& region : regions) { + for (std::size_t index = 0; index < regions.size(); ++index) { + const auto& region = regions[index]; const auto count = region.bytes.size() / sizeof(std::uint32_t); const auto offset = variant.words.size(); variant.words.resize(offset + count); std::memcpy(variant.words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); if (relocatable == 0) { + AppendSharedWords(variant.shared, variant.runs.size(), offset, region.bytes, supplied); variant.runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); continue; } @@ -97,6 +100,7 @@ void Driver::tagVariantRuns(DispatchVariant& variant, std::span HoldAsyncLog() { return AsyncLog::Get().Hold(); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index a70d23d100..46bcd3af2b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -23,6 +23,27 @@ bool matchesBufferPatternFillKernel(std::span code) { return code.size() >= kernel.size() && std::equal(kernel.begin(), kernel.end(), code.begin()); } +bool matchesDwordPatternFillKernel(std::span code) { + static constexpr std::array kernel{ + 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, + 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, + 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, + 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u, + 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, + 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, + 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, + 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, + 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; + static constexpr std::size_t comparesAt = 27; + static constexpr std::array comparesSwapped{0x7d8c02f9u, 0x06068c02u, 0x7d860609u}; + if (code.size() < kernel.size()) return false; + const auto before = std::span(kernel).first(comparesAt); + const auto after = std::span(kernel).subspan(comparesAt + comparesSwapped.size()); + const auto compares = code.subspan(comparesAt, comparesSwapped.size()); + if (!std::equal(before.begin(), before.end(), code.begin()) || !std::equal(after.begin(), after.end(), code.begin() + comparesAt + comparesSwapped.size())) return false; + return std::equal(compares.begin(), compares.end(), kernel.begin() + comparesAt) || std::equal(compares.begin(), compares.end(), comparesSwapped.begin()); +} + } bool Driver::matchesFillKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { @@ -45,17 +66,7 @@ bool Driver::matchesFillKernel(std::span code, const std::v std::optional MatchDwordPatternFill(std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr; if (!enabled || userData.size() != 10 || packet.size() < 5 || !compute.groupIdEnable[0] || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return std::nullopt; - static constexpr std::array kernel{ - 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, - 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, - 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, - 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u, - 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, - 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, - 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, - 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, - 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; - if (code.size() < kernel.size() || !std::equal(kernel.begin(), kernel.end(), code.begin())) return std::nullopt; + if (!matchesDwordPatternFillKernel(code)) return std::nullopt; const auto stride = (userData[1] >> 16u) & 0x3fffu; const bool swizzled = ((userData[1] >> 31u) & 1u) != 0; const bool addTid = ((userData[3] >> 23u) & 1u) != 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp index 2eb194129d..46d3c66047 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/CaptureValidation.cpp @@ -57,13 +57,23 @@ bool Driver::captureStable(std::span captu return stable; } -bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::span captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped, std::optional* sampling, const DataMask* data) { +bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::span captured, const ShaderRecompiler::RecompileResult& compiled, bool inPlace, const PendingView& view, bool* unmapped, std::optional* sampling, const DataMask* data, std::span vouched) { using Policy = ShaderMemory::PendingWrite; thread_local std::vector* pendingSlot = nullptr; auto& pending = ShaderRecompiler::ThreadOwned(pendingSlot); pending.clear(); - for (std::size_t i = 0; i < captured.size(); ++i) { + thread_local std::vector* skippedSlot = nullptr; + auto& skipped = ShaderRecompiler::ThreadOwned(skippedSlot); + skipped.clear(); + for (const auto& range : vouched) { + if (range.begin < range.end && range.end <= captured.size() && (skipped.empty() || skipped.back().end <= range.begin) && !view.Overlaps(range.low, static_cast(range.high - range.low))) skipped.push_back(range); + } + for (std::size_t i = 0, range = 0; i < captured.size(); ++i) { + if (range < skipped.size() && i == skipped[range].begin) { + i = skipped[range++].end - 1u; + continue; + } const auto& region = captured[i]; if (view.Overlaps(region.guestAddress, region.bytes.size())) pending.push_back(i); } @@ -107,7 +117,12 @@ bool Driver::validateCaptured(std::uint64_t program, std::uint32_t queue, std::s observable.clear(); reachedPending = 0; if (data != nullptr) data->live->clear(); - for (std::size_t i = 0; i < captured.size(); ++i) { + for (std::size_t i = 0, range = 0; i < captured.size(); ++i) { + if (range < skipped.size() && i == skipped[range].begin) { + first += skipped[range].words; + i = skipped[range++].end - 1u; + continue; + } const auto& region = captured[i]; const bool isPending = next < pending.size() && pending[next] == i; if (isPending) ++next; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp index fa2ca447d5..596daf0146 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp @@ -215,4 +215,15 @@ ShaderMemory::PendingWrite Driver::queryPendingWrite(std::uint64_t address, std: return Get().classifyPendingWrite(address, bytes, reason, pending, known); } +ShaderMemory::PendingWrite Driver::QueryPendingSpan(std::uint64_t address, std::size_t bytes, std::span known) { + if (DrawPipeline::Active()) { + auto& pipeline = DrawPipeline::Queue0(); + if (pipeline.Busy() && pipeline.Overlaps(address, bytes)) return ShaderMemory::PendingWrite::Sync; + } + std::uint64_t ValidateCounters::*reason = nullptr; + PendingView pending; + pending.Load(); + return Get().classifyPendingWrite(address, bytes, reason, pending, known); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index b1572bfa26..13d45c0f36 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -8,6 +8,7 @@ #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Execution/include/SubmissionCapture.hpp" #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp" +#include "prx/libSceAgcDriver/Eq/include/Event.hpp" #include "ThreadOwned.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include @@ -37,6 +38,11 @@ bool EopRelease(std::span words) { return words.size() >= 8 && (words[0] >> 30u) == 3u && Pm4::PacketWords(words[0]) == 8u && ((words[0] >> 8u) & 0xffu) == 0x49u && ((words[2] >> 24u) & 7u) == 2u; } +bool& submitsQueue0() { + struct SubmitsQueue0Storage {}; + return HostThreadLocal(); +} + } std::deque& Driver::releasedTails() { @@ -316,12 +322,22 @@ void Driver::executeRewindTail(const Submission& stalled) { execute(tail); } -void Driver::Submit(const Packet* packet, std::uint32_t queue) { +void Driver::SubmitFlip(const FlipInfo& info) { + const auto argument = static_cast(info.argument); + std::array words{FlipPacketHeader, info.handle, static_cast(info.index), info.mode, static_cast(argument), static_cast(argument >> 32u)}; + const Packet packet{words.data(), static_cast(words.size()), 0, {}}; + Submit(&packet, 0, true); +} + +void Driver::Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip) { const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; CheckFailure(); require(queue == 0 || (queue >= 0x20 && queue < SubmissionQueueEnd), "unsupported compute queue"); static const bool ring = ReadSubmissionRingSettings(); if (ring) PrepareSubmissionRing(queue); + static const bool flipsInOrder = std::getenv("APS5_CPU_FLIP_IN_ORDER") != nullptr || std::getenv("APS5_SYNC_FLIP") != nullptr; + const bool unorderedFlip = cpuFlip && !flipsInOrder && !APS5_ENABLE_TIMING_LOG && !submitsQueue0(); + if (queue == 0 && !cpuFlip) submitsQueue0() = true; GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet)); const auto descriptor = *packet; require(descriptor.flags == 0, "nonzero submission flags are not implemented"); @@ -351,11 +367,22 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { auto* const capture = SubmissionCapture::Active(); auto captured = capture != nullptr ? SubmissionCapture::Prepare(queue, SubmissionKind::Submit, submission.commands, submission.conditionalEnds) : CapturedSubmission{}; { - std::lock_guard lock(mutex); + std::unique_lock lock(mutex); rethrowFailure(); checkStopping(); require(accepted != std::numeric_limits::max(), "submission serial overflow"); reserveOutputs(submission); + if (unorderedFlip && capture == nullptr && accepted != 0 && flipsAhead.load(std::memory_order_acquire) == 0) { + const auto now = FrameTiming::Clock::now(); + auto frame = std::make_shared(++frameSerial); + frame->IncludeSubmission(accepted, now, now, now, true); + frame->SetFlip(accepted, 0, now, now); + ++flipsCounted; + submission.flips.begin()->second->GpuReady(frame); + lock.unlock(); + AgcDriverDeliverEopInterrupt(0); + return; + } submission.shaders = shaders; submission.serial = accepted + 1; @@ -384,6 +411,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { void Driver::SuspendPoint() { const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; require(!onWorkerThread(), "worker cannot suspend itself"); + submitsQueue0() = true; auto* const capture = SubmissionCapture::Active(); auto captured = capture != nullptr ? SubmissionCapture::Prepare(0, SubmissionKind::Suspend, {}, {}) : CapturedSubmission{}; std::unique_lock lock(mutex); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 22bee51c49..ad0dfa3e8d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -365,6 +365,7 @@ void ShaderPreparationTransaction::Commit() { destination.rectangleProgress.swap(change.prepared.rectangleProgress); destination.fragments.swap(change.prepared.fragments); std::swap(destination.rectangleRequested, change.prepared.rectangleRequested); + destination.deferred = destination.deferred || change.prepared.deferred; } } committed = true; @@ -1025,6 +1026,29 @@ std::vector PrepareRegistered(const ShaderSnapshot& snap return plan != nullptr ? PrepareRegistered(*plan) : std::vector{}; } +void DeferPipelineStage(PreparedShaderState& prepared, const ShaderSnapshot& snapshot, const char* reason) { + if (!prepared.deferred) APS5_LOG_ERR("Shader 0x%llx could not be prepared when the title created its pipeline (%s); preparing it when it is used", static_cast(snapshot.codeAddress), reason); + prepared.deferred = true; +} + +std::optional FailedRegistration(const ShaderSnapshot& snapshot) { + std::unique_lock lock(snapshot.prepared->mutex); + try { + AwaitRegisteredPreparation(*snapshot.prepared, lock); + } catch (const std::exception& error) { + return error.what(); + } + return std::nullopt; +} + +bool DeferFailedRegistration(const ShaderSnapshot& snapshot) { + const auto failure = FailedRegistration(snapshot); + if (!failure) return false; + std::lock_guard lock(snapshot.prepared->mutex); + DeferPipelineStage(*snapshot.prepared, snapshot, failure->c_str()); + return true; +} + #ifdef __GLIBC__ void trimFreedMemory() { static std::atomic last{0}; @@ -1280,7 +1304,7 @@ unsigned RegistrationPreparationThreads() { return RegistrationPreparePool::LiveWorkers(); } -std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) { +std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) { ShaderPreparationTransaction transaction; require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete"); for (const auto& program : decoded.programs) require(program.snapshot != nullptr, "graphics ABI has no registered shader snapshot"); @@ -1307,18 +1331,29 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod std::vector prepared; std::uint32_t pushOffset = 0; const bool fixedSlots = target.fixedPushSlots; - const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : decoded.state.stages.tessellation ? ShaderRecompiler::TessellationLevelPushOffsetBytes : Graphics::PipelinePushSlotBytes; + const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : decoded.state.stages.tessellation ? ShaderRecompiler::TessellationLevelPushOffsetBytes : decoded.state.depthRemap.mode != 0u ? ShaderRecompiler::DepthRemapPushOffsetBytes : Graphics::PipelinePushSlotBytes; for (std::size_t index = 0; index < decoded.programs.size(); ++index) { if (decoded.roles[index] == ShaderRecompiler::ProgramRole::GeometryBack) continue; const auto& program = decoded.programs[index]; const bool fragment = program.binary.stage == ShaderRecompiler::ShaderStage::Fragment; const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize; - std::optional vertex; - if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr); - if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports; pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots); const auto offset = pushOffset; const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes; + if (deferUnprepared && DeferFailedRegistration(*program.snapshot)) { + prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + continue; + } + std::optional vertex; + try { + if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true, 8u - program.firstUserSgpr); + } catch (const std::exception& error) { + if (!deferUnprepared) throw; + DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what()); + prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + continue; + } + if (vertex) vertex->paClVsOutCntl = PositionExportLayout(program.snapshot.get(), program.codeOffset, program.binary.code, decoded.state.paClVsOutCntl, program.tailContinuation.get()) | decoded.state.layerExports; ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}}; std::vector key; ShaderRecompiler::BuildPreparedShaderKey(request, key); @@ -1331,7 +1366,14 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod } if (handle == nullptr) { PerformanceTimer timing("Shader.PrepareArtifact"); - handle = PrepareShaderWithDiagnostics(request); + try { + handle = PrepareShaderWithDiagnostics(request); + } catch (const std::exception& error) { + if (!deferUnprepared) throw; + DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what()); + prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + continue; + } } const auto bytes = handle->artifact->bindings.pushConstantSizeBytes; require(bytes <= room, "prepared graphics stages exceed the push constant block"); @@ -1368,6 +1410,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std QueueState state{}; std::shared_ptr registry; std::shared_ptr owner; + std::vector> selectedStages; { std::lock_guard lock(mutex); registry = shaders; @@ -1377,7 +1420,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader"); const auto selected = RegisteredHeader(*registry, shader); require(selected != nullptr, "graphics ABI refers to a replaced shader header"); - if (owner == nullptr) owner = selected; + selectedStages.push_back(selected); const auto& snapshot = *selected; require(snapshot.registeredState != nullptr, "registered shader state is missing"); const auto registered = DecodeRegisteredState(snapshot, {}); @@ -1395,6 +1438,13 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std for (const auto reg : primitive) state.userConfig.insert_or_assign(reg.offset, reg.value); const auto localDevice = device.Load(); require(localDevice != nullptr, "graphics ABI device is missing"); + for (const auto& selected : selectedStages) { + if (!DeferFailedRegistration(*selected) && owner == nullptr) owner = selected; + } + if (owner == nullptr) { + transaction.Commit(); + return; + } struct GraphicsAbiKeyStorage {}; auto& key = HostThreadLocal, GraphicsAbiKeyStorage>(); BuildRegisteredAbiKey(state, *localDevice, key); @@ -1404,23 +1454,30 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std for (const auto& stage : abi.stages) { const auto snapshot = stage.lock(); require(snapshot != nullptr, "prepared graphics ABI lost a registered stage"); - if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target()); + if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true); } transaction.Commit(); return; } DrawDecode decoded{}; - decoded.state.stages = Graphics::DecodeShaderStages(state, VertexWorkgroupRequired(state, *registry)); - DecodeGraphicsPrograms(decoded, state, *registry, true, false); - auto prepared = PrepareGraphicsStages(decoded, localDevice->Target()); + try { + decoded.state.stages = Graphics::DecodeShaderStages(state, VertexWorkgroupRequired(state, *registry)); + DecodeGraphicsPrograms(decoded, state, *registry, true, false); + } catch (const std::exception& error) { + for (const auto& selected : selectedStages) DeferPipelineStage(transaction.Edit(*selected), *selected, error.what()); + transaction.Commit(); + return; + } + auto prepared = PrepareGraphicsStages(decoded, localDevice->Target(), true); for (auto& stage : prepared) { + if (stage.entry.handle == nullptr) continue; const auto& entries = transaction.Read(*stage.snapshot).entries; const auto duplicate = std::ranges::any_of(entries, [&](const auto& existing) { return existing.codeOffset == stage.entry.codeOffset && existing.handle->artifact == stage.entry.handle->artifact; }); if (!duplicate) transaction.Edit(*stage.snapshot).entries.push_back(std::move(stage.entry)); } for (const auto& stage : prepared) { if (stage.snapshot->type == 1) continue; - ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target()); + ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true); } if (decoded.state.stages.path == Graphics::ShaderPath::Vertex) { for (const auto& stage : prepared) { @@ -1465,14 +1522,24 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span, RegisteredAbiKeyStorage>(); BuildRegisteredAbiKey(state, *localDevice, key); + std::optional deferral; const auto prepare = [&] { - const auto plan = PlanRegistered(*snapshot, *localDevice, state, false); - if (plan == nullptr) return std::vector{}; - plan->prebuildDevice = localDevice; - return PrepareRegistered(*plan); + try { + const auto plan = PlanRegistered(*snapshot, *localDevice, state, false); + if (plan == nullptr) return std::vector{}; + plan->prebuildDevice = localDevice; + return PrepareRegistered(*plan); + } catch (const std::exception& error) { + deferral = error.what(); + return std::vector{}; + } }; const bool committedAbi = CommittedAbi(*snapshot, key); auto entries = committedAbi ? std::vector{} : prepare(); @@ -1487,6 +1554,7 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::spanartifact == entry.handle->artifact; }); if (!duplicate) prepared.entries.push_back(std::move(entry)); } + if (deferral) DeferPipelineStage(prepared, *snapshot, deferral->c_str()); prepared.registeredAbis.push_back(key); transaction.Commit(); } @@ -1512,7 +1580,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std:: PerformanceContext timingContext(FrameTiming::Preparation()); PerformanceTimer timing("Shader.ResolveGraphicsAbi"); CheckFailure(); - if (primitiveType != 0 && primitiveType != 7 && primitiveType != 17) return; + if ((primitiveType != 0 && primitiveType != 7 && primitiveType != 17) || pixel == nullptr) return; ShaderPreparationTransaction transaction; require(vertex != nullptr && pixel != nullptr, "rectangle ABI requires vertex and fragment shaders"); std::shared_ptr front; @@ -1532,7 +1600,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std:: require(front != fragment, "rectangle stages refer to the same shader"); const auto localDevice = device.Load(); require(localDevice != nullptr, "shader registration device is missing"); - ResolvePreparedGraphics(*front, fragment, primitiveType, localDevice->Target()); + ResolvePreparedGraphics(*front, fragment, primitiveType, localDevice->Target(), true); transaction.Commit(); } @@ -1546,15 +1614,19 @@ void Driver::SettleShaderPreparations(std::span stages) { if (auto snapshot = RegisteredHeader(*shaders, shader)) snapshots.push_back(std::move(snapshot)); } } - for (const auto& snapshot : snapshots) { - std::unique_lock lock(snapshot->prepared->mutex); - AwaitRegisteredPreparation(*snapshot->prepared, lock); - } + for (const auto& snapshot : snapshots) static_cast(FailedRegistration(*snapshot)); } -void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target) { +void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) { ShaderPreparationTransaction transaction; require(fragment.get() != &front, "rectangle stages refer to the same shader"); + if (deferUnprepared) { + const bool frontFailed = DeferFailedRegistration(front); + if ((fragment != nullptr && DeferFailedRegistration(*fragment)) || frontFailed) { + transaction.Commit(); + return; + } + } std::vector> fragments; { const auto& current = transaction.Read(front); @@ -1580,27 +1652,37 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr< for (const auto& pixel : fragments) { const auto& current = transaction.Read(front); const auto& pixelPrepared = transaction.Read(*pixel); - require(!current.entries.empty() && !pixelPrepared.entries.empty(), "rectangle ABI has missing stage artifacts"); + if ((current.entries.empty() && current.deferred) || (pixelPrepared.entries.empty() && pixelPrepared.deferred)) continue; + if (current.entries.empty() || pixelPrepared.entries.empty()) { + require(deferUnprepared || pixel != fragment, "rectangle ABI has missing stage artifacts"); + continue; + } const auto progress = std::ranges::find_if(current.rectangleProgress, [&](const auto& entry) { return entry.fragment.lock() == pixel; }); const auto vertexCount = progress == current.rectangleProgress.end() ? 0u : progress->vertexCount; const auto fragmentCount = progress == current.rectangleProgress.end() ? 0u : progress->fragmentCount; if (vertexCount == current.entries.size() && fragmentCount == pixelPrepared.entries.size()) continue; auto& prepared = transaction.Edit(front); std::vector rectangles; - for (std::size_t vertexIndex = 0; vertexIndex < prepared.entries.size(); ++vertexIndex) { - const auto& vertexEntry = prepared.entries[vertexIndex]; - for (std::size_t fragmentIndex = vertexIndex < vertexCount ? fragmentCount : 0u; fragmentIndex < pixelPrepared.entries.size(); ++fragmentIndex) { - const auto& fragmentEntry = pixelPrepared.entries[fragmentIndex]; - const auto& vertexArtifact = ShaderRecompiler::GetPreparedArtifact(*vertexEntry.handle); - const auto& fragmentArtifact = ShaderRecompiler::GetPreparedArtifact(*fragmentEntry.handle); - const auto exists = [&](const auto& entry) { return entry.vertexId == vertexArtifact.variantId && entry.fragmentId == fragmentArtifact.variantId; }; - if (std::ranges::any_of(prepared.rectangles, exists) || std::ranges::any_of(rectangles, exists)) continue; - ShaderRecompiler::RecompileResult vertexResult; - ShaderRecompiler::RecompileResult fragmentResult; - static_cast(vertexResult) = vertexArtifact; - static_cast(fragmentResult) = fragmentArtifact; - rectangles.push_back({vertexArtifact.variantId, fragmentArtifact.variantId, std::make_shared(ShaderRecompiler::BuildRectListShaders(vertexResult, fragmentResult, target))}); + try { + for (std::size_t vertexIndex = 0; vertexIndex < prepared.entries.size(); ++vertexIndex) { + const auto& vertexEntry = prepared.entries[vertexIndex]; + for (std::size_t fragmentIndex = vertexIndex < vertexCount ? fragmentCount : 0u; fragmentIndex < pixelPrepared.entries.size(); ++fragmentIndex) { + const auto& fragmentEntry = pixelPrepared.entries[fragmentIndex]; + const auto& vertexArtifact = ShaderRecompiler::GetPreparedArtifact(*vertexEntry.handle); + const auto& fragmentArtifact = ShaderRecompiler::GetPreparedArtifact(*fragmentEntry.handle); + const auto exists = [&](const auto& entry) { return entry.vertexId == vertexArtifact.variantId && entry.fragmentId == fragmentArtifact.variantId; }; + if (std::ranges::any_of(prepared.rectangles, exists) || std::ranges::any_of(rectangles, exists)) continue; + ShaderRecompiler::RecompileResult vertexResult; + ShaderRecompiler::RecompileResult fragmentResult; + static_cast(vertexResult) = vertexArtifact; + static_cast(fragmentResult) = fragmentArtifact; + rectangles.push_back({vertexArtifact.variantId, fragmentArtifact.variantId, std::make_shared(ShaderRecompiler::BuildRectListShaders(vertexResult, fragmentResult, target))}); + } } + } catch (const std::exception& error) { + if (!deferUnprepared && pixel == fragment) throw; + if (deferUnprepared) DeferPipelineStage(transaction.Edit(*pixel), *pixel, error.what()); + continue; } prepared.rectangles.insert(prepared.rectangles.end(), std::make_move_iterator(rectangles.begin()), std::make_move_iterator(rectangles.end())); std::erase_if(prepared.rectangleProgress, [&](const auto& entry) { return entry.fragment.expired() || entry.fragment.lock() == pixel; }); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 3ce4e76b81..c1801c0d85 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -2234,13 +2234,17 @@ void SetReadGuard(void (*guard)(std::uint64_t address, std::size_t bytes)) { GuestArena::GuestArenaSetHostReadHook_nid_no_patch(&PrepareHostRead); } +void GuardRead(std::uint64_t address, std::size_t bytes) { + if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, bytes); +} + void FlushGpuWrites(std::uint64_t address, std::size_t bytes) { if (const auto hook = flushHook.load(std::memory_order_acquire)) hook(address, bytes); } void Read(std::uint64_t address, std::span destination, std::size_t alignment) { if (destination.empty()) return; - if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size()); + GuardRead(address, destination.size()); FlushGpuWrites(address, destination.size()); const TimedAccess timed(CounterRead, destination.size()); CountReadCaller(__builtin_return_address(0)); @@ -2287,7 +2291,7 @@ void copyCommitted(std::uint64_t address, std::span destination) { void ReadCommitted(std::uint64_t address, std::span destination) { if (destination.empty()) return; - if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size()); + GuardRead(address, destination.size()); FlushGpuWrites(address, destination.size()); const TimedAccess timed(CounterRead, destination.size()); CountReadCaller(__builtin_return_address(0)); @@ -2296,7 +2300,7 @@ void ReadCommitted(std::uint64_t address, std::span destination) { void ReadCommittedUnsynced(std::uint64_t address, std::span destination) { if (destination.empty()) return; - if (const auto guard = readGuard.load(std::memory_order_acquire)) guard(address, destination.size()); + GuardRead(address, destination.size()); const TimedAccess timed(CounterRead, destination.size()); CountReadCaller(__builtin_return_address(0)); copyCommitted(address, destination); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index d658572a98..253d832b7c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -1,10 +1,12 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include "Optimization/RequestMemoryView.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include "ThreadOwned.hpp" #include #include #include @@ -54,6 +56,13 @@ struct CaptureProfile { // The driver's per-shader source handle memo: captures served from it, and resolves it made. std::atomic handleHits{0}; std::atomic handleMisses{0}; + std::atomic suppliedShared{0}; + std::atomic suppliedCopied{0}; + std::atomic suppliedSpilled{0}; + std::atomic vouchCalls{0}; + std::atomic vouchedBlocks{0}; + std::atomic refusedBlocks{0}; + std::atomic vouchedWords{0}; }; constexpr std::uint64_t WordWaitNanoseconds = 50000; @@ -90,6 +99,29 @@ double WaitedMs() { return provider != nullptr ? provider() : 0.0; } +using GuestRuns = std::vector>; + +std::uint64_t RunEnd(const std::pair& run) { + return run.first + static_cast(run.second) * sizeof(std::uint32_t); +} + +bool RunsTouch(const GuestRuns& runs, std::uint64_t begin, std::uint64_t end) { + const auto after = std::upper_bound(runs.begin(), runs.end(), begin, [](std::uint64_t address, const std::pair& run) { return address < run.first; }); + if (after != runs.begin() && RunEnd(*std::prev(after)) > begin) return true; + return after != runs.end() && after->first < end; +} + +bool RunsOverlap(const GuestRuns& left, const GuestRuns& right) { + std::size_t a = 0; + std::size_t b = 0; + while (a < left.size() && b < right.size()) { + if (left[a].first < RunEnd(right[b]) && right[b].first < RunEnd(left[a])) return true; + if (RunEnd(left[a]) <= RunEnd(right[b])) ++a; + else ++b; + } + return false; +} + template std::vector>& pagePool() { struct PagePoolStorage {}; @@ -104,11 +136,11 @@ void ShaderMemory::SetWaitedMsProvider(WaitedMsProvider provider) { waitedMsProvider.store(provider, std::memory_order_release); } -ShaderMemory::ShaderMemory(std::span regions, PendingWriteQuery pendingWrite, PendingWriteObserver observe, HookWaitCounter hookWaits) : pendingWrite(pendingWrite), observe(observe), hookWaits(hookWaits) { +ShaderMemory::ShaderMemory(std::span regions, PendingWriteQuery pendingWrite, PendingWriteObserver observe, HookWaitCounter hookWaits, PendingWriteQuery pendingSpan) : pendingWrite(pendingWrite), observe(observe), hookWaits(hookWaits), pendingSpan(pendingSpan != nullptr ? pendingSpan : pendingWrite) { const ShaderRecompiler::RequestMemoryView validated(regions); - for (const auto& region : regions) { - initial.emplace(region.guestAddress, region.bytes); - } + initial.reserve(regions.size()); + for (const auto& region : regions) initial.emplace_back(region.guestAddress, region.bytes); + std::sort(initial.begin(), initial.end(), [](const auto& left, const auto& right) { return left.first < right.first; }); } ShaderMemory::~ShaderMemory() { @@ -153,15 +185,11 @@ void ShaderMemory::WordMask::ForEachRun(TRun&& run) const { } } -ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { - if (base == lastBase) return *lastPage; - auto position = pages.begin(); - while (position != pages.end() && position->first < base) ++position; - if (position != pages.end() && position->first == base) { - lastBase = base; - lastPage = position->second; - return *lastPage; - } +std::size_t ShaderMemory::pageAtOrAfter(std::uint64_t base) const { + return static_cast(std::partition_point(pages.begin(), pages.end(), [base](const std::pair& entry) { return entry.first < base; }) - pages.begin()); +} + +ShaderMemory::Page& ShaderMemory::addPage(std::size_t position, std::uint64_t base) { auto& pool = pagePool(); std::unique_ptr owned; if (pool.empty()) { @@ -175,10 +203,21 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { owned->recent.Reset(); owned->wordwise = false; owned->lazy = false; - auto& page = *owned; - pages.insert(position, {base, owned.release()}); + pages.insert(pages.begin() + static_cast(position), {base, owned.get()}); lastBase = base; - lastPage = &page; + lastPage = owned.release(); + return *lastPage; +} + +ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { + if (base == lastBase) return *lastPage; + const auto position = pageAtOrAfter(base); + if (position != pages.size() && pages[position].first == base) { + lastBase = base; + lastPage = pages[position].second; + return *lastPage; + } + auto& page = addPage(position, base); if (CaptureProfiled()) ++CaptureTotals().pages; // A page is either mapped whole or read word by word where the guest mapped less than a page, // or where recorded GPU work writes into it (a whole-page read would wait for that work). @@ -189,6 +228,7 @@ ShaderMemory::Page& ShaderMemory::page(std::uint64_t base) { return page; } const auto started = CaptureProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + GuestMemory::GuardRead(base, PageBytes); GuestMemory::FlushGpuWrites(base, PageBytes); if (CaptureProfiled()) CaptureTotals().pageReadNanoseconds += NanosecondsSince(started); page.lazy = true; @@ -203,7 +243,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val } if (CaptureProfiled()) ++CaptureTotals().reads; if (!self.initial.empty() && address + sizeof(*value) > self.initial.begin()->first && address < InitialEnd(self.initial)) { - const auto next = self.initial.upper_bound(address); + const auto next = std::upper_bound(self.initial.begin(), self.initial.end(), address, [](std::uint64_t value, const auto& entry) { return value < entry.first; }); if (next != self.initial.begin()) { const auto previous = std::prev(next); const auto offset = address - previous->first; @@ -215,6 +255,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val } if (next != self.initial.end() && next->first - address < sizeof(*value)) throw std::runtime_error("AGC driver: shader memory read overlaps a snapshot boundary"); } + if (!self.supplied.empty()) self.spillTouching(address, address + sizeof(*value)); auto& page = self.page(address & ~static_cast(PageBytes - 1)); const auto index = static_cast((address % PageBytes) / sizeof(*value)); if (!page.valid.Test(index) && page.lazy) { @@ -249,6 +290,7 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val word = waited; } } else if (policy == PendingWrite::Raw || policy == PendingWrite::VerifyRaw) { + GuestMemory::GuardRead(address, sizeof(word)); std::memcpy(&word, reinterpret_cast(address), sizeof(word)); ++CaptureTotals().wordsRaw; if (policy == PendingWrite::VerifyRaw) { @@ -289,13 +331,14 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val bool ShaderMemory::overlapsInitial(std::uint64_t begin, std::uint64_t end) const { if (initial.empty()) return false; - const auto after = initial.lower_bound(begin); + const auto after = std::lower_bound(initial.begin(), initial.end(), begin, [](const auto& entry, std::uint64_t value) { return entry.first < value; }); if (after != initial.end() && after->first < end) return true; return after != initial.begin() && std::prev(after)->first + std::prev(after)->second.size() > begin; } bool ShaderMemory::readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count) { auto& self = *static_cast(context); + if (!self.supplied.empty() && count != 0) self.spillTouching(address, count > (std::numeric_limits::max() - address) / sizeof(std::uint32_t) ? std::numeric_limits::max() : address + count * sizeof(std::uint32_t)); while (count != 0) { const auto base = address & ~static_cast(PageBytes - 1); const auto index = static_cast((address % PageBytes) / sizeof(std::uint32_t)); @@ -303,7 +346,22 @@ bool ShaderMemory::readWords(void* context, std::uint64_t address, std::uint32_t const auto bytes = static_cast(words) * sizeof(std::uint32_t); Page* page = nullptr; if (address % sizeof(std::uint32_t) == 0 && address <= std::numeric_limits::max() - bytes && !self.overlapsInitial(address, address + bytes)) page = &self.page(base); - if (page != nullptr && page->valid.All()) { + if (page != nullptr && page->lazy) { + const auto end = (index + words + BlockWords - 1u) / BlockWords * BlockWords; + auto first = index / BlockWords * BlockWords; + while (first < end) { + if (page->valid.TestBlock(first)) { + first += BlockWords; + continue; + } + auto last = first + BlockWords; + while (last < end && !page->valid.TestBlock(last)) last += BlockWords; + std::memcpy(&page->words[first], reinterpret_cast(base + first * sizeof(std::uint32_t)), (last - first) * sizeof(std::uint32_t)); + page->valid.SetRun(first, last - first); + first = last; + } + } + if (page != nullptr && (page->lazy || page->valid.All())) { std::memcpy(values, page->words.data() + index, bytes); page->read.SetRun(index, words); page->recent.SetRun(index, words); @@ -326,7 +384,7 @@ bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_ const auto end = address + bytes; while (address < end) { if (!self.initial.empty() && address < InitialEnd(self.initial)) { - const auto next = self.initial.upper_bound(address); + const auto next = std::upper_bound(self.initial.begin(), self.initial.end(), address, [](std::uint64_t value, const auto& entry) { return value < entry.first; }); if (next != self.initial.begin()) { const auto previous = std::prev(next); if (address - previous->first < previous->second.size()) { @@ -351,6 +409,149 @@ bool ShaderMemory::pending(void* context, std::uint64_t address, std::uint64_t b return policy == PendingWrite::Sync || policy == PendingWrite::VerifyRaw || policy == PendingWrite::VerifyKnownValue; } +ShaderRecompiler::SrtWriteStamp ShaderMemory::stampWrites(void* context, std::uint64_t address, std::uint64_t bytes) { + const auto& self = *static_cast(context); + if (bytes == 0 || address > std::numeric_limits::max() - bytes || self.overlapsInitial(address, address + bytes)) return {}; + const auto fetched = self.pageAtOrAfter(address & ~static_cast(PageBytes - 1)); + if (fetched != self.pages.size() && self.pages[fetched].first < address + bytes) return {}; + GuestMemory::FlushGpuWrites(address, static_cast(bytes)); + const auto layout = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + const auto generation = GuestMemory::CollectWrites(address, static_cast(bytes)); + if (generation == 0) return {}; + if (self.pendingSpan != nullptr && self.pendingSpan(address, static_cast(bytes), {}) != PendingWrite::None) return {}; + return {generation, layout}; +} + +bool ShaderMemory::Vouched(PendingWriteQuery pending, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp) { + CaptureTotals().vouchCalls.fetch_add(1, std::memory_order_relaxed); + if (stamp.generation == 0 || bytes == 0 || address > std::numeric_limits::max() - bytes || stamp.layout != GuestAllocations::GuestAllocationsGeneration_nid_postfix()) return false; + if (pending != nullptr && pending(address, static_cast(bytes), {}) != PendingWrite::None) return false; + GuestMemory::FlushGpuWrites(address, static_cast(bytes)); + if (GuestMemory::CollectWrites(address, static_cast(bytes)) == 0) return false; + return GuestMemory::UnchangedSince(address, static_cast(bytes), stamp.generation); +} + +bool ShaderMemory::unchangedSince(void* context, std::uint64_t address, std::uint64_t bytes, const ShaderRecompiler::SrtWriteStamp& stamp) { + return Vouched(static_cast(context)->pendingSpan, address, bytes, stamp); +} + +bool ShaderMemory::place(std::uint64_t address, const std::uint32_t* values, std::uint64_t count, bool overwrite, bool recent) { + while (count != 0) { + const auto base = address & ~static_cast(PageBytes - 1); + const auto index = static_cast((address % PageBytes) / sizeof(std::uint32_t)); + const auto words = static_cast(std::min(count, PageWords - index)); + const auto position = pageAtOrAfter(base); + Page* found = position != pages.size() && pages[position].first == base ? pages[position].second : nullptr; + if (found == nullptr) { + if (!overwrite && !GuestMemory::Accessible(reinterpret_cast(base), PageBytes)) return false; + found = &addPage(position, base); + } + auto& page = *found; + if (page.wordwise && !overwrite) return false; + const auto head = index / BlockWords * BlockWords; + const auto tail = (index + words + BlockWords - 1) / BlockWords * BlockWords; + const bool headOpen = page.lazy && !page.valid.Test(head); + const bool tailOpen = page.lazy && !page.valid.Test(tail - 1); + if (overwrite || page.valid.None()) { + std::memcpy(page.words.data() + index, values, words * sizeof(std::uint32_t)); + } else { + for (std::size_t word = 0; word < words; ++word) { + if (!page.valid.Test(index + word)) page.words[index + word] = values[word]; + } + } + if (headOpen && head < index) std::memcpy(&page.words[head], reinterpret_cast(base + head * sizeof(std::uint32_t)), (index - head) * sizeof(std::uint32_t)); + if (tailOpen && index + words < tail) std::memcpy(&page.words[index + words], reinterpret_cast(base + (index + words) * sizeof(std::uint32_t)), (tail - index - words) * sizeof(std::uint32_t)); + page.valid.SetRun(page.lazy ? head : index, page.lazy ? tail - head : words); + page.read.SetRun(index, words); + if (recent) page.recent.SetRun(index, words); + address += static_cast(words) * sizeof(std::uint32_t); + values += words; + count -= words; + } + return true; +} + +void ShaderMemory::spill(std::size_t index) { + auto entry = std::move(supplied[index]); + supplied.erase(supplied.begin() + static_cast(index)); + std::size_t at = 0; + for (const auto& [address, count] : entry.words->runs) { + static_cast(place(address, entry.words->words.data() + at, count, true, entry.recent)); + at += count; + } + CaptureTotals().suppliedSpilled += entry.words->words.size(); + spilled.push_back(std::move(entry.words)); +} + +void ShaderMemory::spillTouching(std::uint64_t begin, std::uint64_t end) { + for (std::size_t index = supplied.size(); index-- > 0;) { + const auto& entry = supplied[index]; + if (entry.begin < end && begin < entry.end && RunsTouch(entry.words->runs, begin, end)) spill(index); + } +} + +bool ShaderMemory::supplyWords(void* context, const std::shared_ptr& words) { + auto& self = *static_cast(context); + if (words == nullptr || words->runs.empty()) return false; + std::uint64_t total = 0; + std::uint64_t end = 0; + if (words->ordered && words->low == words->runs.front().first && words->low < words->high) { + total = words->words.size(); + end = words->high; + } else { + for (const auto& [address, count] : words->runs) { + const auto bytes = static_cast(count) * sizeof(std::uint32_t); + if (count == 0 || address % sizeof(std::uint32_t) != 0 || address < end || address > std::numeric_limits::max() - bytes) return false; + end = address + bytes; + total += count; + } + } + const auto begin = words->runs.front().first; + if (total != words->words.size() || self.overlapsInitial(begin, end)) return false; + auto& totals = CaptureTotals(); + for (auto& entry : self.supplied) { + if (entry.words != words) continue; + entry.recent = true; + totals.suppliedShared += total; + return true; + } + const auto fetched = self.pageAtOrAfter(begin & ~static_cast(PageBytes - 1)); + const bool paged = fetched != self.pages.size() && self.pages[fetched].first < end; + const bool overlapping = std::any_of(self.supplied.begin(), self.supplied.end(), [&](const Supplied& entry) { return entry.begin < end && begin < entry.end && RunsOverlap(entry.words->runs, words->runs); }); + if (paged || overlapping) { + for (std::size_t index = self.supplied.size(); index-- > 0;) { + const auto& entry = self.supplied[index]; + if (entry.begin < end && begin < entry.end && RunsOverlap(entry.words->runs, words->runs)) self.spill(index); + } + std::size_t at = 0; + for (const auto& [address, count] : words->runs) { + if (!self.place(address, words->words.data() + at, count, false, true)) return false; + at += count; + } + totals.suppliedCopied += total; + return true; + } + const auto spanBegin = begin & ~static_cast(PageBytes - 1); + const auto spanEnd = (end + PageBytes - 1) & ~static_cast(PageBytes - 1); + if (!GuestMemory::Accessible(reinterpret_cast(spanBegin), static_cast(spanEnd - spanBegin))) { + std::uint64_t first = std::numeric_limits::max(); + std::uint64_t past = 0; + for (const auto& [address, count] : words->runs) { + const auto from = address & ~static_cast(PageBytes - 1); + const auto to = (address + static_cast(count) * sizeof(std::uint32_t) + PageBytes - 1) & ~static_cast(PageBytes - 1); + if (from > past || first == std::numeric_limits::max()) { + if (first != std::numeric_limits::max() && !GuestMemory::Accessible(reinterpret_cast(first), static_cast(past - first))) return false; + first = from; + } + past = std::max(past, to); + } + if (!GuestMemory::Accessible(reinterpret_cast(first), static_cast(past - first))) return false; + } + self.supplied.push_back({words, begin, end, true}); + totals.suppliedShared += total; + return true; +} + ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() { auto& totals = CaptureTotals(); return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)}; @@ -361,6 +562,24 @@ void ShaderMemory::CountHandleMemo(bool hit) { (hit ? CaptureTotals().handleHits : CaptureTotals().handleMisses).fetch_add(1, std::memory_order_relaxed); } +ShaderMemory::VouchCounts ShaderMemory::Vouches() { + auto& totals = CaptureTotals(); + return {totals.vouchCalls.load(std::memory_order_relaxed), totals.vouchedBlocks.load(std::memory_order_relaxed), totals.refusedBlocks.load(std::memory_order_relaxed), totals.vouchedWords.load(std::memory_order_relaxed)}; +} + +ShaderMemory::SupplyCounts ShaderMemory::Supplies() { + auto& totals = CaptureTotals(); + return {totals.suppliedShared.load(std::memory_order_relaxed), totals.suppliedCopied.load(std::memory_order_relaxed), totals.suppliedSpilled.load(std::memory_order_relaxed)}; +} + +std::vector> ShaderMemory::SuppliedBlocks() const { + std::vector> blocks; + blocks.reserve(supplied.size() + spilled.size()); + for (const auto& entry : supplied) blocks.push_back(entry.words); + blocks.insert(blocks.end(), spilled.begin(), spilled.end()); + return blocks; +} + std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle) { return capture(request, handle, nullptr); } @@ -386,6 +605,9 @@ std::shared_ptr ShaderMemory::capture(c runtime.accessible = &accessible; runtime.pendingWrite = &pending; runtime.readWords = &readWords; + runtime.stampWrites = &stampWrites; + runtime.unchangedSince = &unchangedSince; + runtime.supplyWords = &supplyWords; auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { totals.captureNanoseconds += NanosecondsSince(started); @@ -414,34 +636,159 @@ std::shared_ptr ShaderMemory::capture(c return capture; } -std::vector ShaderMemory::Regions() const { +std::vector ShaderMemory::collect(bool recentOnly) const { + struct Cursor { + const ShaderRecompiler::SrtGuestWords* words; + std::size_t run; + std::size_t at; + }; + std::vector cursors; + std::size_t runs = 0; + for (const auto& entry : supplied) { + if (recentOnly && !entry.recent) continue; + cursors.push_back({entry.words.get(), 0u, 0u}); + runs += entry.words->runs.size() + entry.words->words.size() / PageWords; + } std::vector result; - result.reserve(initial.size() + pages.size()); - auto next = initial.begin(); - // Both maps are ordered by address and never overlap, so a merge keeps the result sorted. + result.reserve((recentOnly ? 0u : initial.size()) + pages.size() + runs); + auto next = recentOnly ? initial.end() : initial.begin(); + const auto emitBefore = [&](std::uint64_t address) { + while (true) { + Cursor* lowest = nullptr; + for (auto& cursor : cursors) { + if (cursor.run < cursor.words->runs.size() && (lowest == nullptr || cursor.words->runs[cursor.run].first < lowest->words->runs[lowest->run].first)) lowest = &cursor; + } + const auto run = lowest != nullptr ? lowest->words->runs[lowest->run].first : std::numeric_limits::max(); + if (next != initial.end() && next->first < address && next->first < run) { + result.push_back({next->first, next->second}); + ++next; + } else if (lowest != nullptr && run < address) { + const auto count = lowest->words->runs[lowest->run].second; + const auto words = std::span(lowest->words->words).subspan(lowest->at, count); + for (std::size_t done = 0; done < count;) { + const auto at = run + done * sizeof(std::uint32_t); + const auto piece = std::min(count - done, (PageBytes - static_cast(at % PageBytes)) / sizeof(std::uint32_t)); + result.push_back({at, std::as_bytes(words.subspan(done, piece))}); + done += piece; + } + lowest->at += count; + ++lowest->run; + } else { + return; + } + } + }; for (const auto& [base, pagePointer] : pages) { const auto& page = *pagePointer; - while (next != initial.end() && next->first < base) { - result.push_back({next->first, next->second}); - ++next; - } - page.read.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); }); + const auto& marks = recentOnly ? page.recent : page.read; + if (marks.None()) continue; + marks.ForEachRun([&](std::size_t first, std::size_t end) { + const auto address = base + first * sizeof(std::uint32_t); + emitBefore(address); + result.push_back({address, std::as_bytes(std::span(page.words).subspan(first, end - first))}); + }); } - for (; next != initial.end(); ++next) result.push_back({next->first, next->second}); + emitBefore(std::numeric_limits::max()); return result; } +std::vector ShaderMemory::Regions() const { + return collect(false); +} + std::vector ShaderMemory::TakeRecentRegions() { - std::vector result; - for (auto& [base, pagePointer] : pages) { - auto& page = *pagePointer; - if (page.recent.None()) continue; - page.recent.ForEachRun([&](std::size_t first, std::size_t end) { result.push_back({base + first * sizeof(std::uint32_t), std::as_bytes(std::span(page.words).subspan(first, end - first))}); }); - page.recent.Reset(); - } + auto result = collect(true); + for (auto& [base, page] : pages) page->recent.Reset(); + for (auto& entry : supplied) entry.recent = false; return result; } +void AppendSharedWords(std::vector& shared, std::size_t region, std::size_t first, std::span bytes, std::span> blocks) { + const auto address = reinterpret_cast(bytes.data()); + const auto count = bytes.size() / sizeof(std::uint32_t); + if (count == 0) return; + for (const auto& block : blocks) { + if (block == nullptr || block->words.empty()) continue; + const auto begin = reinterpret_cast(block->words.data()); + const auto end = begin + block->words.size() * sizeof(std::uint32_t); + if (address < begin || address >= end || end - address < count * sizeof(std::uint32_t) || (address - begin) % sizeof(std::uint32_t) != 0) continue; + const auto offset = static_cast((address - begin) / sizeof(std::uint32_t)); + if (!shared.empty()) { + auto& last = shared.back(); + if (last.first + last.count == first && last.offset + last.count == offset && last.region + last.regions == region && !last.block.owner_before(block) && !block.owner_before(last.block)) { + last.count += count; + ++last.regions; + return; + } + } + shared.push_back({first, count, offset, block, region, 1u}); + return; + } +} + +bool SameCapturedWords(std::span left, std::span leftShared, std::span right, std::span rightShared, std::size_t* compared) { + if (compared != nullptr) *compared = 0; + if (left.size() != right.size()) return false; + std::size_t checked = 0; + const auto equal = [&](std::size_t from, std::size_t to) { + checked += to - from; + return std::equal(left.begin() + static_cast(from), left.begin() + static_cast(to), right.begin() + static_cast(from)); + }; + std::size_t at = 0; + std::size_t next = 0; + bool same = true; + for (const auto& run : leftShared) { + while (next < rightShared.size() && rightShared[next].first < run.first) ++next; + if (next == rightShared.size()) break; + const auto& other = rightShared[next]; + const bool placed = other.first == run.first && other.count == run.count && other.offset == run.offset && run.first >= at && run.first <= left.size() && run.count <= left.size() - run.first; + if (!placed || run.block.expired() || run.block.owner_before(other.block) || other.block.owner_before(run.block)) continue; + if (!equal(at, run.first)) { + same = false; + break; + } + at = run.first + run.count; + } + same = same && equal(at, left.size()); + if (compared != nullptr) *compared = checked; + return same; +} + +std::size_t VouchSharedRegions(std::span shared, std::size_t regions, ShaderMemory::PendingWriteQuery pending, std::vector& vouched) { + vouched.clear(); + if (shared.empty() || regions == 0) return 0; + thread_local std::vector>* blocksSlot = nullptr; + auto& blocks = ShaderRecompiler::ThreadOwned(blocksSlot); + blocks.clear(); + auto& totals = CaptureTotals(); + std::size_t words = 0; + for (const auto& entry : shared) { + if (entry.regions == 0 || entry.region > regions || entry.regions > regions - entry.region || (!vouched.empty() && entry.region < vouched.back().end)) continue; + const auto block = entry.block.lock(); + if (block == nullptr || block->low >= block->high) continue; + auto known = std::find_if(blocks.begin(), blocks.end(), [&](const auto& seen) { return seen.first == block.get(); }); + if (known == blocks.end()) { + const bool good = ShaderMemory::Vouched(pending, block->low, block->high - block->low, ShaderRecompiler::LoadStamp(*block)); + (good ? totals.vouchedBlocks : totals.refusedBlocks).fetch_add(1, std::memory_order_relaxed); + known = blocks.insert(blocks.end(), {block.get(), good}); + } + if (!known->second) continue; + if (!vouched.empty() && vouched.back().end == entry.region) { + auto& last = vouched.back(); + last.end += entry.regions; + last.words += entry.count; + last.low = std::min(last.low, block->low); + last.high = std::max(last.high, block->high); + } else { + vouched.push_back({entry.region, entry.region + entry.regions, entry.count, block->low, block->high}); + } + words += entry.count; + } + blocks.clear(); + totals.vouchedWords.fetch_add(words, std::memory_order_relaxed); + return words; +} + DataWordPositionCounts DataWordPositions(std::span> runs, std::span> leaves, std::span otherReads, std::span words, std::span flattenedSrt, std::vector& positions, std::vector& slots) { DataWordPositionCounts counts; positions.clear(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 5e45faf5e4..7bce290b72 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2238,10 +2238,10 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::spanfloat64DenormPreserve; target.float16DenormPreserve = state->float16DenormPreserve; target.spannedWaves = state->spannedWaves; + target.depthRangeRemap = !state->depthRangeUnrestricted; target.fixedPushSlots = state->graphicsPipelineLibrary; target.registrationModules = state->graphicsPipelineLibrary; target.robustBufferAccess2 = state->robustBufferAccess2; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp index dcd1f11e27..c9a0628b12 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp @@ -20,6 +20,7 @@ SampleOrderTarget DepthSurfaceOrderTarget(const Context& context, const DepthTar DepthSampleLocations DepthSurfaceSampleLocations(const Context& context, const DepthTarget& target); bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples); std::uint64_t DepthSurfacesSerial(); +std::uint64_t DepthSurfacesReleased(); std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel); std::uint64_t MultisampledDepthBytes_nid_no_patch(VkExtent2D extent, std::uint32_t bytesPerTexel, std::uint32_t samples); void RequireNoResidentDepthRead_nid_no_patch(std::uint64_t address, std::size_t bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index 0b596093b7..553a5427e2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -29,7 +29,7 @@ struct DrawInputCopy { std::uint64_t registryGeneration = 0; }; DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use); -DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes); +DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes, std::size_t stride = 0); void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived); const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment, bool* behindGpuWrite = nullptr); struct InPlaceIndices { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index a85a858374..70d6251395 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -77,6 +77,9 @@ void SetHostImportWindow(std::uint64_t bytes); void SetHostImportRangeWindow(std::uint64_t bytes); void SetHostImportBudget(std::uint64_t bytes); void SetHostImportDeferredSettles(bool enabled); +// The larger of 6 GiB and half the physical memory, at most the largest host heap less the larger of 2 GiB and half +// that heap: the other half stays for the buffers, mirrors and textures that share it. +std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, std::uint64_t physicalBytes); std::uint64_t HostImportLiveBytes(); struct HostImportSettleCounts { std::uint64_t submitted = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 472140cdf0..2b60aebe38 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -73,6 +73,7 @@ private: std::vector owned; VkExtent2D extent; std::uint32_t layers = 1; + std::uint64_t depthReleases = 0; std::shared_ptr framebuffer; }; void release() noexcept; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index f62ba12556..f9ab938548 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -176,6 +176,7 @@ public: static ReadStatistics ReadCounts(); // Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it. bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes) const; // What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest // batch whose noted range overlaps the access (the open batch counts first, with the serial it // will get), whether that batch's fence has already signaled (its GPU work is done, so the @@ -763,7 +764,7 @@ private: void removeLabels(Batch& batch); void release(Batch& batch) noexcept; static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end); - static std::optional readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end); + static std::optional readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end, std::optional only = std::nullopt); static std::vector> notedReads(const Batch& batch); bool signaled(const Batch& batch) const; // Rebuilds the lock-free snapshot of pending writes from open, inFlight and finishing. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp index b53834e9d8..d555d389a5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp @@ -18,7 +18,9 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& // the draw cache validates them by value with the stage's capture (design_cpu_final rule RD). struct DecodeRead { std::uint64_t address; - std::vector bytes; + std::array data{}; + std::uint32_t size = 0; + std::span Bytes() const { return {data.data(), size}; } }; ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads = nullptr, bool staticAbi = false, std::uint32_t hiddenUserWords = 0); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index a0ffcb295a..4fda54fcfa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -68,6 +68,13 @@ struct PipelineSetLayouts { std::span fragmentKey; }; +struct TableDescriptors { + std::uint32_t binding = 0; + std::vector> runs; + std::vector images; + std::shared_ptr>> textures; +}; + class DescriptorCache { public: explicit DescriptorCache(const Context& context); @@ -83,9 +90,13 @@ public: VkDescriptorPool pool = VK_NULL_HANDLE; std::size_t poolIndex = 0; bool updateAfterBind = false; + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + std::vector sizes; + std::optional variableCount; + std::shared_ptr table; }; SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); - void Free(const SetAllocation& allocation) noexcept; + void Free(const SetAllocation& allocation, std::shared_ptr table = {}) noexcept; SetAllocation AllocateRewritten(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); void Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept; // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. @@ -95,6 +106,8 @@ public: std::uint64_t sets = 0; std::uint64_t pools = 0; std::uint64_t dedicatedPools = 0; + std::uint64_t keptSets = 0; + std::uint64_t reusedSets = 0; }; Stats Counters() const; @@ -115,6 +128,7 @@ private: std::vector updateAfterBindPools; std::vector dedicated; std::unordered_map> recycled; + std::vector kept; Stats stats; }; @@ -125,6 +139,9 @@ using ContentKeyDiff = std::array(ContentKeyCompo struct ImageTableReuseCounts { std::uint64_t reused = 0; std::uint64_t resolved = 0; + std::uint64_t replayed = 0; + std::uint64_t writesKept = 0; + std::uint64_t writesMade = 0; }; ImageTableReuseCounts ImageTableReuses(); @@ -181,7 +198,7 @@ public: std::size_t allocation; std::uint64_t address; std::size_t size; - std::vector words; + std::span words; }; std::shared_ptr PrepareDrawBindings(Recorder& recorder, std::span moved = {}, bool movedOnly = false) const; std::optional> MovedReadOnlyBuffers(std::span shaders, Recorder& recorder) const; @@ -405,7 +422,7 @@ private: // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the // same way but is left out of the write set (no write-back, no pending-write note). `atomic` // is its bufferAtomic (see GuestBufferMemory::AddWritable). - std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read); + std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read, std::uint32_t extent); std::uint64_t guestBufferBytes(std::span words) const; std::size_t addDataBuffer(std::span words); // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache @@ -540,6 +557,16 @@ private: bool splitSets = false; std::vector fragmentLayoutKey; SetObjects fragment; + struct TableWrites { + bool recorded = false; + bool written = false; + std::uint32_t binding = 0; + std::size_t firstImage = 0; + std::size_t images = 0; + std::vector> runs; + }; + std::array tableWrites; + std::shared_ptr keptTable(std::size_t set, std::shared_ptr>>& held) noexcept; std::uint32_t setOf(const Binding& binding) const; std::vector allocations; std::vector descriptorBuffers; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp index 76e24b0e23..2ddac45e80 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp @@ -73,7 +73,7 @@ inline VkShaderStageFlags PushConstantStages(std::span sha VkShaderStageFlags result = 0; for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); - if (!shader.program->pushConstants.empty() || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage); + if (!shader.program->pushConstants.empty() || shader.program->readsDepthRemapPush || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage); } return result; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index ccd4b502a4..fa7bd24286 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -125,6 +125,16 @@ struct State { VkViewport viewport; bool negativeOneToOne; bool depthClamp = false; + float depthScale = 1; + float depthOffset = 0; + float depthClampMin = 0; + float depthClampMax = 1; + struct DepthRemap { + std::uint32_t mode = 0; + float scale = 1; + float offset = 0; + bool operator==(const DepthRemap&) const = default; + } depthRemap; bool alphaToCoverage = false; bool ditheredAlphaToCoverage = false; bool perSampleShading = false; @@ -150,6 +160,7 @@ inline constexpr std::uint32_t PositionExportLayoutBits = 0xffffu | (1u << 21u) ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup = false); State DecodeState(const QueueState& queue, bool vertexWorkgroup = false); +void ApplyDepthRemap(const Context& context, State& state); std::array SampleLocations(const State& state); std::array PixelSampleLocations(const State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel); VkExtent2D SampleLocationGrid(const State& state); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 2064dd48c0..ca716b8f20 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -362,6 +362,7 @@ public: // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); static std::uint64_t SinglePassMoves(); + static std::uint64_t SlabRetiles(); static std::uint64_t RefreshesProved(); static std::uint64_t UntrackedCompares(); std::uint64_t GuestBytes() const; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index dff1ec3e51..8622fa2a81 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -475,6 +475,10 @@ public: return std::any_of(textures.begin(), textures.end(), held) || std::any_of(stencilCopy.textures.begin(), stencilCopy.textures.end(), held); } + bool SampledElsewhere() const { + return std::any_of(textures.begin(), textures.end(), [](const auto& entry) { return entry.second.use_count() > 1; }); + } + std::uint64_t written = ++depthWrites; bool retired = false; @@ -1200,6 +1204,25 @@ bool takeClearedHtile(const DepthTarget& target) { const DepthSurface* drawnDepth = nullptr; std::atomic surfacesSerial{0}; +std::atomic surfacesReleased{0}; + +void releaseRetired(const Context& context) { + auto* recorder = Recorder::Active(); + if (recorder != nullptr && recorder->Device() != context.device) return; + auto& list = surfaces(); + for (auto it = list.begin(); it != list.end();) { + const auto& surface = **it; + if (!surface.retired || surface.context.device != context.device || surface.target.samples != 1u || surface.SampledElsewhere()) { + ++it; + continue; + } + if (drawnDepth == &surface) drawnDepth = nullptr; + std::shared_ptr released = std::move(*it); + it = list.erase(it); + surfacesReleased.fetch_add(1, std::memory_order_release); + if (recorder != nullptr) recorder->Keep(std::move(released)); + } +} std::uint32_t expandedHtileWord(std::uint32_t word, bool stencilInHtile, VkImageAspectFlags cleared) { if (!stencilInHtile) return (cleared & VK_IMAGE_ASPECT_DEPTH_BIT) != 0 ? 0xfffc000fu : word; @@ -1263,6 +1286,10 @@ std::uint64_t DepthSurfacesSerial() { return surfacesSerial.load(std::memory_order_acquire); } +std::uint64_t DepthSurfacesReleased() { + return surfacesReleased.load(std::memory_order_acquire); +} + std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) { const std::uint32_t blockWidth = bytesPerTexel == 4 ? 128u : 256u; const std::uint32_t blockHeight = bytesPerTexel == 1 ? 256u : 128u; @@ -1390,6 +1417,7 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target) auto sampling = target.samples == 1u ? DepthSamplingState{} : prepareDepthSampling(context, target); creationLease = target.samples == 1u || !sampling.refusal.empty() ? GuestAllocations::Lease{} : GuestAllocations::GuestAllocationsAcquireAll_nid_postfix(); lock.lock(); + releaseRetired(context); surfacesSerial.fetch_add(1, std::memory_order_release); surfaces().push_back(std::make_unique(context, target, std::move(sampling))); auto& fresh = *surfaces().back(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 6cc9418efb..025662a590 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -15,6 +15,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include "ThreadOwned.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "Optimization/include/Optimization/ShaderStageInputInfo.hpp" @@ -701,12 +702,19 @@ std::map, std::string>& validationFailures() { return failures; } -std::set CachedFragmentOutputs(const Context& context, std::span shaders, const State& state, bool& memoized, bool& hit) { +std::vector& validationKeyScratch() { + thread_local std::vector* keySlot = nullptr; + auto& key = ShaderRecompiler::ThreadOwned(keySlot); + key.clear(); + return key; +} + +std::uint32_t CachedFragmentOutputs(const Context& context, std::span shaders, const State& state, bool& memoized, bool& hit) { memoized = false; hit = false; - std::vector key; + auto& key = validationKeyScratch(); const bool keyed = ValidationKey(context, shaders, state, key); - static std::map, std::set> memo; + static std::map, std::uint32_t> memo; if (keyed) { memoized = true; std::lock_guard lock(validationMutex()); @@ -715,15 +723,18 @@ std::set CachedFragmentOutputs(const Context& context, std::span< return found->second; } } - std::set outputs; + std::uint32_t outputs = 0; try { - outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation, context.shaderTerminateInvocation); + for (const auto location : ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics, context.shaderClipDistance, context.shaderCullDistance, context.viewportIndexLayer, context.runtimeDescriptorArray, context.demoteToHelperInvocation, context.shaderTerminateInvocation)) { + Require(location < 32, "fragment output location exceeds the output mask"); + outputs |= 1u << location; + } } catch (const std::exception& error) { if (keyed) { std::lock_guard lock(validationMutex()); auto& failures = validationFailures(); if (failures.size() >= 1024) failures.clear(); - failures.emplace(std::move(key), error.what()); + failures.emplace(key, error.what()); } throw; } @@ -731,7 +742,7 @@ std::set CachedFragmentOutputs(const Context& context, std::span< std::lock_guard lock(validationMutex()); // A handful of configurations recur; a runaway key space is dropped wholesale. if (memo.size() >= 1024) memo.clear(); - memo.emplace(std::move(key), outputs); + memo.emplace(key, outputs); } return outputs; } @@ -937,14 +948,20 @@ DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uin return copy; } -DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes) { +DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes, std::size_t stride) { Require(validBytes <= bytes, "the valid bytes of a vertex fetch exceed the fetch"); GuestMemory::FlushGpuWrites(address, bytes); DrawInputCopy copy; copy.buffer = std::make_shared(context, bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); const auto span = copy.buffer->Bytes(); - GuestMemory::Read(address, span.subspan(0, validBytes), 1); - std::fill(span.begin() + static_cast(validBytes), span.end(), std::byte{0}); + auto readable = validBytes; + if (stride != 0 && !GuestMemory::Accessible(reinterpret_cast(address), validBytes)) { + const auto ranges = GuestMemory::CommittedRanges(address, validBytes); + const std::size_t prefix = !ranges.empty() && ranges.front().first == address ? static_cast(ranges.front().second - address) : 0; + if (validBytes - prefix < stride) readable = prefix; + } + GuestMemory::Read(address, span.subspan(0, readable), 1); + std::fill(span.begin() + static_cast(readable), span.end(), std::byte{0}); return copy; } @@ -1015,7 +1032,7 @@ struct DrawInputs { std::vector vertexOffsets; std::vector> inPlaceReads; bool vertexInputBehindGpuWrite = false; - std::set fragmentOutputs; + std::uint32_t fragmentOutputs = 0; VkPipelineStageFlags shaderStages = 0; std::uint32_t meshGroups = 0; std::vector restartTable; @@ -1173,6 +1190,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P if (inputs.retainedVertexInput == nullptr) inputs.vertexInput = BuildVertexInputLayout(context, attributes); } inputs.vertexOffsets.assign(attributes.size(), 0); + inputs.vertexHandles.reserve(attributes.size()); // An indexed draw's vertex offset moves every fetch: the copy must reach the last one. if (draw.indexed) { Require(draw.firstVertex <= std::numeric_limits::max() - inputs.maxIndex, "indexed draw vertex range overflow"); @@ -1239,7 +1257,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P for (const auto f : soloFetches) { const auto begin = fetches[f].begin; const auto bytes = static_cast(fetches[f].end - begin); - auto copy = CopyZeroPaddedDrawInput(context, begin, bytes, fetchValid[f]); + auto copy = CopyZeroPaddedDrawInput(context, begin, bytes, fetchValid[f], fetches[f].stride); soloHandles[f] = copy.buffer->Handle(); inputs.vertexBuffers.push_back(std::move(copy.buffer)); } @@ -1618,6 +1636,15 @@ struct RecordedDraw { std::optional depthLocations; }; +// A remapped viewport depth range reads its scale and offset from the draw's push block (the vertex shader's +// RemapPositionDepth), so draws that differ only in them share one pipeline; the draw's own state supplies them. +void patchDepthRemapPush(std::array& block, const State& state) { + if (state.depthRemap.mode == 0u) return; + static_assert(ShaderRecompiler::DepthRemapPushOffsetBytes + ShaderRecompiler::DepthRemapPushBytes <= PipelinePushSlotBytes); + const std::array words{state.depthRemap.scale, state.depthRemap.offset}; + std::memcpy(block.data() + ShaderRecompiler::DepthRemapPushOffsetBytes, words.data(), sizeof(words)); +} + void recordStencilReplay(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, std::span>> replay, std::span shaders, const ShaderResources& resources, const std::array* bytes = nullptr, VkShaderStageFlags stages = 0) { State selected = state; auto block = bytes != nullptr ? *bytes : AssemblePushConstants(shaders); @@ -1625,6 +1652,7 @@ void recordStencilReplay(const Context& context, VkCommandBuffer commands, const resources.PatchPushConstants(block); stages = PushConstantStages(shaders); } + patchDepthRemapPush(block, state); const auto fragment = std::ranges::find_if(shaders, [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Fragment && !shader.program->stencilReferences.empty(); }); Require(fragment != shaders.end(), "stencil replay fragment stage is missing"); const auto referenceOffset = fragment->pushConstantOffset + fragment->program->stencilReferenceDword * sizeof(std::uint32_t); @@ -1662,6 +1690,7 @@ void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const resources.PatchPushConstants(block); stages = PushConstantStages(shaders); } + patchDepthRemapPush(block, state); if (state.stages.tessellation) { for (const auto& shader : shaders) Require(shader.program->pushConstants.empty() || shader.pushConstantOffset >= PipelinePushSlotBytes || shader.pushConstantOffset + shader.program->pushConstants.size() <= ShaderRecompiler::TessellationLevelPushOffsetBytes, "a stage's push constants overlap the tessellation levels"); const std::array levels{state.stages.minTessellationLevel, state.stages.maxTessellationLevel}; @@ -1949,10 +1978,10 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam // The state with the outputs the pixel shader lacks masked: Vulkan leaves attachments a pixel // shader has no output for undefined, so their writes are masked (shaders that only store to // images or buffers keep their targets as they were). Empty when no mask must change. -std::optional maskedState(const State& state, const std::set& fragmentOutputs) { +std::optional maskedState(const State& state, std::uint32_t fragmentOutputs) { std::optional masked; for (std::size_t index = 0; index < state.blends.size(); ++index) { - if (fragmentOutputs.contains(static_cast(index)) || state.blends[index].colorWriteMask == 0) continue; + if ((index < 32 && ((fragmentOutputs >> index) & 1u) != 0) || state.blends[index].colorWriteMask == 0) continue; if (!masked.has_value()) masked = state; masked->blends[index].colorWriteMask = 0; } @@ -1987,7 +2016,7 @@ bool RecordDraws() { } std::optional KnownValidationFailure(const Context& context, std::span shaders, const State& state) { - std::vector key; + auto& key = validationKeyScratch(); if (!ValidationKey(context, shaders, state, key)) return std::nullopt; std::lock_guard lock(validationMutex()); const auto& failures = validationFailures(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 1b77cf8faf..ce65aeeea7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -1259,6 +1259,12 @@ bool wouldRetireImport(const HostImports& state, const ImportRun& run) { return previous.base + previous.bytes > run.base; } +bool sameRange(const ImportFailure& entry, const GuestAllocations::Lease& lease) { + const auto current = entry.ranges.size() == 1 ? leasedRangeOwner(lease, entry.base, entry.base + entry.bytes) : nullptr; + if (current != nullptr && entry.bytes < current->bytes) return !entry.ranges.front().owner_before(current) && !current.owner_before(entry.ranges.front()); + return sameRun(entry, importRun(lease, entry.base)); +} + bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) { if (entry.ranges.size() == 1 && !entry.joined && entry.bytes < entry.ranges.front()->bytes) { const auto current = leasedRangeOwner(lease, entry.base, entry.base + entry.bytes); @@ -1304,9 +1310,9 @@ std::uint64_t physicalMemoryBytes() { std::atomic budgetOverride{0}; -// Pinned imports count against the driver's system memory heap, which imports and type-3 mirrors share; past it -// ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else the larger of 6 GiB and half the physical -// memory, at most the largest host heap less 2 GiB. +// Pinned imports count against the driver's system memory heap, which imports, type-3 mirrors, buffers and textures +// placed in host memory share; past it ordinary host allocations fail. The budget is APS5_HOST_IMPORT_MIB, else +// HostImportBudget. std::uint64_t importBudget(const Context& context) { if (const auto set = budgetOverride.load(std::memory_order_relaxed); set != 0) return set; static const std::uint64_t configured = []() -> std::uint64_t { @@ -1314,13 +1320,7 @@ std::uint64_t importBudget(const Context& context) { return value != nullptr ? std::strtoull(value, nullptr, 10) << 20u : 0u; }(); if (configured != 0) return configured; - std::uint64_t hostHeap = 0; - for (std::uint32_t index = 0; index < context.memory.memoryHeapCount; ++index) { - if ((context.memory.memoryHeaps[index].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) hostHeap = std::max(hostHeap, context.memory.memoryHeaps[index].size); - } - const auto fromMemory = std::max(6144ull << 20u, physicalMemoryBytes() / 2); - constexpr std::uint64_t reserve = 2048ull << 20u; - return hostHeap > reserve ? std::min(fromMemory, hostHeap - reserve) : fromMemory; + return HostImportBudget(context.memory, physicalMemoryBytes()); } std::atomic rangeImportWindow{64ull << 20u}; @@ -1683,7 +1683,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc retireImport(context.device, state, it, lease); it = next; } - for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRun(it->second, importRun(lease, it->first)) ? std::next(it) : state.failed.erase(it); + for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRange(it->second, lease) ? std::next(it) : state.failed.erase(it); for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it); } @@ -2982,6 +2982,16 @@ void SetHostImportDeferredSettles(bool enabled) { #endif } +std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, const std::uint64_t physicalBytes) { + std::uint64_t hostHeap = 0; + for (std::uint32_t index = 0; index < std::min(memory.memoryHeapCount, VK_MAX_MEMORY_HEAPS); ++index) { + if ((memory.memoryHeaps[index].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) hostHeap = std::max(hostHeap, memory.memoryHeaps[index].size); + } + const auto fromMemory = std::max(6144ull << 20u, physicalBytes / 2); + const auto reserve = std::max(2048ull << 20u, hostHeap / 2); + return hostHeap > reserve ? std::min(fromMemory, hostHeap - reserve) : fromMemory; +} + std::uint64_t HostImportLiveBytes() { auto& state = Imports(); std::lock_guard lock(state.mutex); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 99eafc5467..2ffc1dcd98 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -645,18 +645,20 @@ std::shared_ptr Pipeline::AcquireFramebuffer(std::span colorAttachments ? DepthSurfacesReleased() : 0u; if (resident) { // Entries whose views are gone can never match again and go as soon as no recorded draw holds // them (Kept keeps its framebuffer until the batch completes). - std::erase_if(framebuffers, [](const CachedFramebuffer& entry) { + std::erase_if(framebuffers, [&](const CachedFramebuffer& entry) { if (entry.framebuffer.use_count() != 1) return false; + if (entry.depthReleases != depthReleases) return true; for (std::size_t i = 0; i < entry.owners.size(); ++i) { if (entry.owned[i] && entry.owners[i].expired()) return true; } return false; }); for (auto it = framebuffers.begin(); it != framebuffers.end(); ++it) { - if (it->extent.width != extent.width || it->extent.height != extent.height || it->layers != layers || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue; + if (it->extent.width != extent.width || it->extent.height != extent.height || it->layers != layers || it->depthReleases != depthReleases || !std::equal(it->views.begin(), it->views.end(), targets.begin(), targets.end())) continue; // View handles are recycled once a StorageTexture is destroyed, so the owners must be the // very objects the views were made for. bool same = true; @@ -681,6 +683,7 @@ std::shared_ptr Pipeline::AcquireFramebuffer(std::span(); + state.depthRemap.mode = reader.U32(); } void encodeRecipe(Writer& writer, const GraphicsPipelineRecipe& recipe) { @@ -1037,6 +1039,7 @@ State CanonicalPipelineState(const State& state) { canonical.sampleLocationGrid = state.sampleLocationGrid; canonical.perSampleShading = state.perSampleShading; canonical.quadSampleMask = state.quadSampleMask; + canonical.depthRemap.mode = state.depthRemap.mode; return canonical; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index b5a3d9f5f7..c5ed6c45b9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -2856,6 +2856,17 @@ bool Recorder::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const return bytes != 0 && open != nullptr && overlaps(*open, address, address + bytes); } +std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0) return 0; + const auto end = address + bytes; + const auto writes = [&](const Batch& batch) { return overlaps(batch, address, end) || readOverlap(batch, address, end, ReadKind::AddressBased).has_value(); }; + if (open != nullptr && writes(*open)) return OpenSerial(); + for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) { + if (writes(**it)) return (*it)->serial; + } + return 0; +} + bool Recorder::signaled(const Batch& batch) const { return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS; } @@ -2954,8 +2965,9 @@ void Recorder::NotePendingReadList(std::shared_ptr ranges, ReadK open->readLists.push_back({std::move(ranges), open->reads.size(), kind}); } -std::optional Recorder::readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end) { +std::optional Recorder::readOverlap(const Batch& batch, std::uint64_t address, std::uint64_t end, std::optional only) { const auto listOverlaps = [&](const Batch::ReadListNote& note) { + if (only.has_value() && note.kind != *only) return false; const auto& list = *note.ranges; const auto found = std::partition_point(list.begin(), list.end(), [&](const auto& range) { return range.second <= address; }); return found != list.end() && found->first < end; @@ -2966,7 +2978,7 @@ std::optional Recorder::readOverlap(const Batch& batch, std: if (listOverlaps(batch.readLists[next])) return batch.readLists[next].kind; } const auto& read = batch.reads[index]; - if (address < read.end && read.begin < end) return read.kind; + if ((!only.has_value() || read.kind == *only) && address < read.end && read.begin < end) return read.kind; } for (; next < batch.readLists.size(); ++next) { if (listOverlaps(batch.readLists[next])) return batch.readLists[next].kind; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index 678f501634..4f30207def 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -336,7 +336,9 @@ CommandBatch::CommandBatch(const Context& context) : context(context) { allocation.commandBufferCount = 1; Check(context.Function("vkAllocateCommandBuffers")(context.device, &allocation, &commands), "vkAllocateCommandBuffers"); VkFenceCreateInfo info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - Check(context.Function("vkCreateFence")(context.device, &info, nullptr, &fence), "vkCreateFence"); + VkFence created = VK_NULL_HANDLE; + Check(context.Function("vkCreateFence")(context.device, &info, nullptr, &created), "vkCreateFence"); + fence = created; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index 6a83dea572..b1d3d9a0c7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -233,6 +233,14 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span(vertexAttribReg); info.fetchBufferReg = static_cast(vertexBufferReg); + const auto record = [&](std::uint64_t address, std::span bytes) { + if (reads == nullptr) return; + auto& read = reads->emplace_back(); + read.address = address; + std::copy(bytes.begin(), bytes.end(), read.data.begin()); + read.size = static_cast(bytes.size()); + }; + if (reads != nullptr) reads->reserve(reads->size() + 2u * shader.num_input_semantics); for (std::uint32_t i = 0; i < shader.num_input_semantics; ++i) { const auto& semantic = semantics[i]; if (semantic.static_vb_index == 1 || semantic.static_attribute == 1) throw std::runtime_error("AGC graphics: statically bound vertex attributes are not implemented"); @@ -246,7 +254,7 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span attribWordBytes{}; const auto attribWordAddress = attribTableAddr + static_cast(semantic.semantic) * 4u; AgcDriver::GuestMemory::Read(attribWordAddress, attribWordBytes, 4); - if (reads != nullptr) reads->push_back({attribWordAddress, {attribWordBytes.begin(), attribWordBytes.end()}}); + record(attribWordAddress, attribWordBytes); std::uint32_t attribWord; std::memcpy(&attribWord, attribWordBytes.data(), 4); const auto index = attribWord & 0x1fu; @@ -257,7 +265,7 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span sharpBytes{}; const auto sharpAddress = bufferTableAddr + static_cast(index) * 16u; AgcDriver::GuestMemory::Read(sharpAddress, sharpBytes, 4); - if (reads != nullptr) reads->push_back({sharpAddress, {sharpBytes.begin(), sharpBytes.end()}}); + record(sharpAddress, sharpBytes); std::array sharp{}; std::memcpy(sharp.data(), sharpBytes.data(), 16); if (info.resourcesNum >= ShaderRecompiler::ShaderVertexStageInfo::MaxResources) throw std::runtime_error("AGC graphics: vertex resource count exceeds the supported domain"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 926998f941..a4a9f9e379 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -375,7 +375,10 @@ struct TextureCounters { // ShaderResources::refreshOwnObjects). std::atomic ownRefreshes{0}; std::atomic tablesReused{0}; + std::atomic tablesReplayed{0}; std::atomic tablesResolved{0}; + std::atomic tableWritesKept{0}; + std::atomic tableWritesMade{0}; // Snapshot hits over memory outside the write watch: kept by an unchanged UntrackedHint key, or // by a byte compare. std::atomic untrackedKept{0}; @@ -1574,6 +1577,11 @@ bool TableReuseEnabled() { return !disabled; } +bool TableReplay() { + static const bool disabled = std::getenv("APS5_NO_TABLE_REPLAY") != nullptr; + return !disabled; +} + struct TableState { std::uint64_t epoch = 0; std::uint64_t unwatched = 0; @@ -1607,6 +1615,10 @@ struct ReusedTable { std::vector elements; std::vector queries; std::vector> ranges; + std::vector imageSamplers; + std::vector samplers; + std::vector bound; + bool replayable = false; }; constexpr std::size_t ReusedTableCount = 4; @@ -1820,7 +1832,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element]; const bool read = element < binding.bufferRead.size() && binding.bufferRead[element]; - const auto index = addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read); + const std::uint32_t extent = element < binding.bufferExtent.size() ? binding.bufferExtent[element] : 0u; + const auto index = addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read, extent); const auto& push = shader.program->pushConstants; if (!push.empty()) { const auto position = shader.program->memoryOffsetDword * 4u + element; @@ -2142,6 +2155,7 @@ void ShaderResources::buildComplete() { images.reserve(imageCount); auto& writes = descriptorWrites; writes.reserve(bindings.size()); + std::vector> keptWrites; const auto absentImage = [this](std::uint32_t binding) { constexpr auto bindingCount = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); const auto view = context.nullDescriptors ? VK_NULL_HANDLE : context.placeholderImages->View(binding % bindingCount); @@ -2149,6 +2163,13 @@ void ShaderResources::buildComplete() { }; for (const auto& binding : bindings) { if (binding.flags != 0) { + const auto set = setOf(binding); + const bool sampled = binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + const auto& allocation = set == 0 ? cacheAllocation : fragment.cacheAllocation; + const bool recorded = sampled && set < tableWrites.size() && allocation.layout != VK_NULL_HANDLE; + std::vector> runs; + const auto firstWrite = writes.size(); + const auto firstImage = images.size(); for (std::size_t first = 0; first < binding.imageAllocations.size();) { if (binding.imageAllocations[first] == std::numeric_limits::max()) { ++first; @@ -2157,7 +2178,7 @@ void ShaderResources::buildComplete() { auto last = first; while (last < binding.imageAllocations.size() && binding.imageAllocations[last] != std::numeric_limits::max()) ++last; VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; - write.dstSet = setOf(binding) == 0 ? _set : fragment.set; + write.dstSet = set == 0 ? _set : fragment.set; write.dstBinding = binding.layout.binding; write.dstArrayElement = static_cast(first); write.descriptorCount = static_cast(last - first); @@ -2169,9 +2190,20 @@ void ShaderResources::buildComplete() { else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); } + if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount); writes.push_back(write); first = last; } + if (!recorded) continue; + const auto& held = allocation.table; + auto& counters = TextureCounts(); + if (held != nullptr && held->binding == binding.layout.binding && held->runs == runs && std::equal(held->images.begin(), held->images.end(), images.begin() + static_cast(firstImage), images.end(), [](const VkDescriptorImageInfo& a, const VkDescriptorImageInfo& b) { return a.sampler == b.sampler && a.imageView == b.imageView && a.imageLayout == b.imageLayout; })) { + keptWrites.emplace_back(firstWrite, writes.size()); + counters.tableWritesKept.fetch_add(1, std::memory_order_relaxed); + } else { + counters.tableWritesMade.fetch_add(1, std::memory_order_relaxed); + } + tableWrites[set] = {true, false, binding.layout.binding, firstImage, images.size() - firstImage, std::move(runs)}; continue; } VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; @@ -2212,7 +2244,22 @@ void ShaderResources::buildComplete() { } writes.push_back(write); } - context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + if (keptWrites.empty()) { + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + } else { + std::vector made; + made.reserve(writes.size()); + std::size_t next = 0; + for (std::size_t index = 0; index < writes.size(); ++index) { + if (next < keptWrites.size() && index == keptWrites[next].first) { + index = keptWrites[next++].second - 1u; + continue; + } + made.push_back(writes[index]); + } + if (!made.empty()) context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(made.size()), made.data(), 0, nullptr); + } + for (auto& table : tableWrites) table.written = table.recorded; } for (const auto& allocation : allocations) { if (allocation.adjustment == 0) continue; @@ -3688,12 +3735,26 @@ ResourceCache& SharedResourceCache() { return *cache; } -DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function("vkDestroyDescriptorSetLayout")), destroyPool(context.Function("vkDestroyDescriptorPool")), resetPool(context.Function("vkResetDescriptorPool")), freeDescriptorSets(context.individuallyFreeDescriptorSets ? context.Function("vkFreeDescriptorSets") : nullptr) {} +namespace { +std::size_t KeptSets() { + static const std::size_t count = [] { + const char* text = std::getenv("APS5_KEPT_DESCRIPTOR_SETS"); + return text != nullptr ? static_cast(std::strtoull(text, nullptr, 10)) : std::size_t{8}; + }(); + return count; +} + +} + +DescriptorCache::DescriptorCache(const Context& context) : context(context), destroyLayout(context.Function("vkDestroyDescriptorSetLayout")), destroyPool(context.Function("vkDestroyDescriptorPool")), resetPool(context.Function("vkResetDescriptorPool")), freeDescriptorSets(context.individuallyFreeDescriptorSets ? context.Function("vkFreeDescriptorSets") : nullptr) { + kept.reserve(KeptSets() + 1u); +} DescriptorCache::~DescriptorCache() { for (const auto& pool : pools) destroyPool(context.device, pool.handle, nullptr); for (const auto& pool : updateAfterBindPools) destroyPool(context.device, pool.handle, nullptr); for (const auto pool : dedicated) destroyPool(context.device, pool, nullptr); + for (const auto& set : kept) destroyPool(context.device, set.pool, nullptr); for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr); } @@ -3761,6 +3822,17 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l allocation.pSetLayouts = &layout; if (!fits) { dedicated.reserve(dedicated.size() + 1); + const auto same = std::find_if(kept.begin(), kept.end(), [&](const SetAllocation& set) { + return set.layout == layout && set.updateAfterBind == updateAfterBind && set.variableCount == variableCount && std::equal(set.sizes.begin(), set.sizes.end(), sizes.begin(), sizes.end(), [](const VkDescriptorPoolSize& a, const VkDescriptorPoolSize& b) { return a.type == b.type && a.descriptorCount == b.descriptorCount; }); + }); + if (same != kept.end()) { + auto reused = std::move(*same); + kept.erase(same); + dedicated.push_back(reused.pool); + ++stats.reusedSets; + ++stats.sets; + return reused; + } const auto pool = CreateSetPool(context, sizes, updateAfterBind); allocation.descriptorPool = pool; VkDescriptorSet set = VK_NULL_HANDLE; @@ -3772,7 +3844,7 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l dedicated.push_back(pool); ++stats.dedicatedPools; ++stats.sets; - return {set, pool}; + return {set, pool, 0, updateAfterBind, layout, std::vector(sizes.begin(), sizes.end()), variableCount, {}}; } auto& chain = updateAfterBind ? updateAfterBindPools : pools; for (std::size_t index = chain.size(); index != 0; --index) { @@ -3836,12 +3908,22 @@ void DescriptorCache::Recycle(VkDescriptorSetLayout layout, const SetAllocation& Free(allocation); } -void DescriptorCache::Free(const SetAllocation& allocation) noexcept { +void DescriptorCache::Free(const SetAllocation& allocation, std::shared_ptr table) noexcept { if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; std::lock_guard lock(mutex); if (const auto found = std::find(dedicated.begin(), dedicated.end(), allocation.pool); found != dedicated.end()) { dedicated.erase(found); - destroyPool(context.device, allocation.pool, nullptr); + if (KeptSets() == 0u || allocation.layout == VK_NULL_HANDLE) { + destroyPool(context.device, allocation.pool, nullptr); + return; + } + if (kept.size() >= KeptSets()) { + destroyPool(context.device, kept.back().pool, nullptr); + kept.pop_back(); + } + kept.insert(kept.begin(), allocation); + kept.front().table = std::move(table); + ++stats.keptSets; return; } auto& chain = allocation.updateAfterBind ? updateAfterBindPools : pools; @@ -3881,7 +3963,7 @@ std::uint64_t ShaderResources::guestBufferBytes(std::span w return byteSize; } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) { +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read, std::uint32_t extent) { Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1] & ~0x40000000u, words[2], words[3]}}; Require(descriptor.Type() == 0u, "buffer descriptor uses an unsupported type"); @@ -3899,7 +3981,8 @@ std::size_t ShaderResources::addGuestBuffer(std::span words static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr; Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); written = written || allWritten; - const bool swept = read && dispatchThreads != 0 && descriptor.Stride() != 0 && dispatchThreads * descriptor.Stride() * 2u >= byteSize; + const std::uint64_t recordBytes = descriptor.Stride() != 0 ? descriptor.Stride() : extent; + const bool swept = read && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize; if (written) guestMemory.AddWritable(address, size, atomic, swept); else { guestMemory.AddReadable(address, size); @@ -4541,7 +4624,20 @@ void ShaderResources::resolveImageTables() { const auto stageSamplers = std::span>(samplers).subspan(table.firstSampler, table.samplerCount); table.elementTextures.assign(binding.count, std::numeric_limits::max()); const auto capacityMessage = [&] { return "the image table exceeds the device's capacity of " + std::to_string(context.imageTableCapacity) + " images"; }; - const auto bind = [&](std::uint32_t element, std::span words, std::shared_ptr texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) { + std::vector samplerKey; + for (std::uint32_t sampler = 0; sampler < table.samplerCount; ++sampler) { + const auto& source = samplerSources.at(table.firstSampler + sampler); + samplerKey.insert(samplerKey.end(), source.words.begin(), source.words.end()); + samplerKey.push_back((source.compareEnable ? 1u : 0u) | (source.unnormalized ? 2u : 0u) | (source.integerBorder ? 4u : 0u)); + } + const auto place = [&](std::uint32_t element, std::shared_ptr texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) { + textures.push_back(std::move(texture)); + textureFirstLayer.push_back(firstLayer); + describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); + table.elementTextures[element] = textures.size() - 1u; + item->imageAllocations[static_cast(table.elementBase) + element] = textures.size() - 1u; + }; + const auto bind = [&](std::uint32_t element, std::span words, std::shared_ptr texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) -> std::string { try { RequireFilterMinmax(context, texture->ViewFormat(), binding.imageSamplers.at(element), stageSamplers); RequireBorderSwizzle(resource.bcSwizzle, binding.imageSamplers.at(element), stageSamplers); @@ -4558,18 +4654,17 @@ void ShaderResources::resolveImageTables() { } } catch (const std::runtime_error& error) { table.map[faults + element] = rejectTableElement(table, words, error.what()); - return; + return error.what(); } - textures.push_back(std::move(texture)); - textureFirstLayer.push_back(firstLayer); - describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); - table.elementTextures[element] = textures.size() - 1u; - item->imageAllocations[static_cast(table.elementBase) + element] = textures.size() - 1u; + place(element, std::move(texture), resource, guestBytes, firstLayer); + return {}; }; const auto state = CurrentTableState(); std::vector> cached; const bool storageTable = std::find(binding.imageWritten.begin(), binding.imageWritten.end(), true) != binding.imageWritten.end(); - if (auto* reused = storageTable ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { + if (auto* reused = storageTable || item == nullptr ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { + const bool replay = TableReplay() && reused->replayable && reused->imageSamplers == binding.imageSamplers && reused->samplers == samplerKey; + if (!replay) reused->bound.assign(binding.count, {}); auto next = cached.begin(); for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); @@ -4581,13 +4676,26 @@ void ShaderResources::resolveImageTables() { table.map[faults + element] = rejectTableElement(table, words, entry.message); } else { CompletePendingCpuWrites(context, entry.resource.baseAddress, static_cast(entry.guestBytes)); - bind(element, words, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer); + if (!replay) { + reused->bound[element] = bind(element, words, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer); + } else if (!reused->bound[element].empty()) { + ++next; + table.map[faults + element] = rejectTableElement(table, words, reused->bound[element]); + } else { + place(element, std::move(*next++), entry.resource, entry.guestBytes, entry.firstLayer); + } } } - counters.tablesReused.fetch_add(1, std::memory_order_relaxed); + if (!replay) { + reused->imageSamplers = binding.imageSamplers; + reused->samplers = samplerKey; + reused->replayable = static_cast(table.elementBase) + binding.count <= context.imageTableCapacity; + } + (replay ? counters.tablesReplayed : counters.tablesReused).fetch_add(1, std::memory_order_relaxed); continue; } ReusedTable fresh; + fresh.bound.assign(binding.count, {}); bool reusable = state.epoch != 0; for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); @@ -4676,7 +4784,7 @@ void ShaderResources::resolveImageTables() { fresh.ranges.emplace_back(resource.baseAddress, resource.baseAddress + guestBytes); } } - bind(element, words, std::move(texture), resource, guestBytes, firstLayer); + fresh.bound[element] = bind(element, words, std::move(texture), resource, guestBytes, firstLayer); } counters.tablesResolved.fetch_add(1, std::memory_order_relaxed); if (reusable && CurrentTableState() == state) { @@ -4685,6 +4793,9 @@ void ShaderResources::resolveImageTables() { fresh.device = context.device; fresh.state = state; fresh.pendingSerial = serial; + fresh.imageSamplers = binding.imageSamplers; + fresh.samplers = std::move(samplerKey); + fresh.replayable = static_cast(table.elementBase) + binding.count <= context.imageTableCapacity; RememberTable(std::move(fresh), binding); } } @@ -4694,7 +4805,7 @@ void ShaderResources::resolveImageTables() { ImageTableReuseCounts ImageTableReuses() { const auto& counters = TextureCounts(); - return {counters.tablesReused.load(std::memory_order_relaxed), counters.tablesResolved.load(std::memory_order_relaxed)}; + return {counters.tablesReused.load(std::memory_order_relaxed) + counters.tablesReplayed.load(std::memory_order_relaxed), counters.tablesResolved.load(std::memory_order_relaxed), counters.tablesReplayed.load(std::memory_order_relaxed), counters.tableWritesKept.load(std::memory_order_relaxed), counters.tableWritesMade.load(std::memory_order_relaxed)}; } void ShaderResources::finishImageTables() { @@ -4816,12 +4927,29 @@ ShaderResources::~ShaderResources() { release(); } +std::shared_ptr ShaderResources::keptTable(std::size_t set, std::shared_ptr>>& held) noexcept { + const auto& table = tableWrites[set]; + if (!table.written || KeptSets() == 0u || table.firstImage + table.images > descriptorImages.size()) return nullptr; + try { + if (held == nullptr) held = std::make_shared>>(std::move(textures)); + auto kept = std::make_shared(); + kept->binding = table.binding; + kept->runs = table.runs; + kept->images.assign(descriptorImages.begin() + static_cast(table.firstImage), descriptorImages.begin() + static_cast(table.firstImage + table.images)); + kept->textures = held; + return kept; + } catch (...) { + return nullptr; + } +} + void ShaderResources::release() noexcept { // A set from the cache's pool chain goes back to it; a dedicated pool dies with its set. - if (cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(cacheAllocation); + std::shared_ptr>> held; + if (cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(cacheAllocation, keptTable(0, held)); if (pool) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); if (_layout && ownsLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, _layout, nullptr); - if (fragment.cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(fragment.cacheAllocation); + if (fragment.cacheAllocation.pool && context.descriptorCache != nullptr) context.descriptorCache->Free(fragment.cacheAllocation, keptTable(1, held)); if (fragment.pool) context.Function("vkDestroyDescriptorPool")(context.device, fragment.pool, nullptr); if (fragment.layout && fragment.ownsLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, fragment.layout, nullptr); cacheAllocation = {}; @@ -4829,6 +4957,7 @@ void ShaderResources::release() noexcept { _set = VK_NULL_HANDLE; _layout = VK_NULL_HANDLE; fragment = {}; + tableWrites = {}; } std::uint32_t ShaderResources::setOf(const Binding& binding) const { @@ -4878,6 +5007,7 @@ std::optional> ShaderResources::MovedR const bool same = !item.dataWords.empty() ? item.dataWords == binding.guestDescriptor : !patched && std::memcmp(item.buffer->Bytes().data(), binding.guestDescriptor.data(), item.size) == 0; if (same) continue; if (item.dataWords.empty() && patched) return std::nullopt; + if (moved.empty()) moved.reserve(allocations.size()); moved.push_back({index, 0, item.size, binding.guestDescriptor}); continue; } @@ -4903,6 +5033,7 @@ std::optional> ShaderResources::MovedR const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); if (!GuestMemory::Accessible(reinterpret_cast(begin), bytes)) return std::nullopt; if (guestMemory.WritesOverlap(begin, bytes) || recorder.PendingWriteOverlaps(begin, bytes) || PendingStorageOverlaps(begin, bytes, nullptr) || AnyShadowedOverlaps(begin, bytes)) return std::nullopt; + if (moved.empty()) moved.reserve(allocations.size()); moved.push_back({index, address, static_cast(size)}); } } @@ -4994,12 +5125,16 @@ std::shared_ptr ShaderResources::PrepareDrawBindi selected.clear(); std::array stamp{}; bool stamped = false; + const auto keep = [&](DrawBindings::Snapshot snapshot) { + if (result->snapshots.empty()) result->snapshots.reserve(allocations.size()); + result->snapshots.push_back(std::move(snapshot)); + }; for (std::size_t index = 0; index < allocations.size(); ++index) { const auto& item = allocations[index]; const auto override = std::find_if(moved.begin(), moved.end(), [&](const MovedBuffer& entry) { return entry.allocation == index; }); if (override != moved.end() && !override->words.empty()) { constexpr std::size_t MovedWordBufferSlots = 8; - const auto held = std::find_if(movedWordBuffers.begin(), movedWordBuffers.end(), [&](const MovedWordBuffer& entry) { return entry.allocation == index && entry.size == override->size && entry.words == override->words; }); + const auto held = std::find_if(movedWordBuffers.begin(), movedWordBuffers.end(), [&](const MovedWordBuffer& entry) { return entry.allocation == index && entry.size == override->size && std::ranges::equal(entry.words, override->words); }); std::shared_ptr buffer; if (held != movedWordBuffers.end()) { buffer = held->buffer; @@ -5009,12 +5144,12 @@ std::shared_ptr ShaderResources::PrepareDrawBindi for (const auto& patch : dataPatches) { if (patch.allocation == index && patch.byte < override->size) buffer->Bytes()[patch.byte] = static_cast(patch.adjustment); } - MovedWordBuffer entry{index, override->words, override->size, buffer}; + MovedWordBuffer entry{index, {override->words.begin(), override->words.end()}, override->size, buffer}; if (movedWordBuffers.size() < MovedWordBufferSlots) movedWordBuffers.push_back(std::move(entry)); else movedWordBuffers[movedWordBufferNext++ % MovedWordBufferSlots] = std::move(entry); } selected.push_back(index); - result->snapshots.push_back({0, std::move(buffer), 0, override->size}); + keep({0, std::move(buffer), 0, override->size}); continue; } std::uint64_t address = item.address; @@ -5066,7 +5201,7 @@ std::shared_ptr ShaderResources::PrepareDrawBindi else if (!uploaded || stamp[1] != 0) recorder.NoteUnwrittenDrawSnapshot(begin, bytes, stamp, buffer, offset); } selected.push_back(index); - result->snapshots.push_back({begin, std::move(buffer), offset, bytes}); + keep({begin, std::move(buffer), offset, bytes}); totalSnapshotBytes += bytes; CaptureTrace::Log("draw-snapshot batch=%llu address=%llx bytes=%zu", static_cast(recorder.Submissions() + 1), static_cast(begin), bytes); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 76085491b7..98f2cbbc8c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -541,7 +541,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } } } else if (variable.storage == spv::StorageClassPushConstant) { - Require(!push && (!shader.pushConstants.empty() || mesh || (control && !rectListControl)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface"); + Require(!push && (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface"); push = true; Require(type.size() == 3 && module.decorations[typeId].block, "push constant variable must be a Block struct with exactly one member"); const auto offset = module.offsets.find({typeId, 0}); @@ -613,7 +613,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(shader.vertexAttributes.empty(), "vertex attribute metadata is invalid for this stage"); } Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources"); - Require(push == (!shader.pushConstants.empty() || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V"); + Require(push == (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl && !barycentricPatches)), "recompiler push constant metadata disagrees with SPIR-V"); for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable"); if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices"); if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index ac28f510c1..b4c88f6afe 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -1108,6 +1108,32 @@ ReversedBlend TranslateReversedBlend(std::uint32_t blendControl, std::uint8_t co return {blend, routing->code}; } +void ApplyDepthRemap(const Context& context, State& state) { + const auto& viewport = state.viewport; + const bool inRange = viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1; + if (context.depthRangeUnrestricted || inRange) return; + Require(state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "viewport depth outside [0, 1] is remapped on the vertex path only"); + Require(context.depthClamp, "viewport depth outside [0, 1] needs depth clamping to stand in for the console's clamp"); + const float clampMin = std::clamp(state.depthClampMin, 0.0f, 1.0f); + const float clampMax = std::clamp(state.depthClampMax, 0.0f, 1.0f); + const bool floatDepth = state.depth.has_value() && (state.depth->format == VK_FORMAT_D32_SFLOAT || state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT); + if ((clampMin != state.depthClampMin || clampMax != state.depthClampMax) && floatDepth && (state.depthWrite || state.depthTest)) { + static bool reported = false; + if (!reported) { + reported = true; + std::fprintf(stderr, "[gpu] viewport depth clamp bounds [%g, %g] on a float depth target are clipped to [0, 1]: the device has no VK_EXT_depth_range_unrestricted\n", state.depthClampMin, state.depthClampMax); + } + } + const float range = clampMax - clampMin; + const float scale = range > 0.0f ? state.depthScale / range : state.depthScale; + const float offset = range > 0.0f ? (state.depthOffset - clampMin) / range : state.depthOffset; + state.depthRemap = {state.negativeOneToOne ? 2u : 1u, scale, offset}; + state.viewport.minDepth = clampMin; + state.viewport.maxDepth = clampMax; + state.negativeOneToOne = false; + state.depthClamp = true; +} + State DecodeState(const QueueState& queue, bool vertexWorkgroup) { if (auto reason = DepthMaintenanceRejection(queue); !reason.empty()) throw std::runtime_error(reason); const auto& cx = queue.context; @@ -1342,6 +1368,10 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) { throw std::runtime_error(message.str()); } Require(readFloat(cx, 0xb4) <= readFloat(cx, 0xb5), "inverted viewport depth clamp bounds"); + result.depthScale = zs; + result.depthOffset = zo; + result.depthClampMin = readFloat(cx, 0xb4); + result.depthClampMax = readFloat(cx, 0xb5); result.viewport = collapsed ? VkViewport{xo, yo, 1, 1, minDepth, maxDepth} : VkViewport{xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth}; if (const auto renderControl = find(cx, 0x000); result.depth && renderControl != cx.end() && (renderControl->second & 1u) != 0) { result.viewport.minDepth = result.depth->clearDepth; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ca9fd9b468..68daefa998 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -87,7 +87,7 @@ thread_local const char* uploadReason = "first"; // selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead // (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed, // reported only. -std::atomic singlePassMoves{0}; +std::atomic singlePassMoves{0}, slabRetiles{0}; std::atomic partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0}; // FlushPending listings skipped because no pending unit of the image lay inside the range, images // whose selected pending units were dropped because their memory is no longer registered (see @@ -1003,7 +1003,9 @@ Texture::Texture(const Context& context, const std::shared_ptr& Require(source != nullptr && CanCopyFrom(*source, descriptor), "storage image does not match the sampled texture"); const auto vkFormat = SampledTextureFormat(context, descriptor.format); const auto geometry = DescribeSurface(descriptor); +#ifdef APS5_DEBUG_LOG AGC_LOG_DEFERRED(stdout, "Texture address=0x%llx %ux%u mips=%u viewed from storage image (vk=%d)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, static_cast(vkFormat)); +#endif // Storage images stay in the general layout; the view samples them there. layout = VK_IMAGE_LAYOUT_GENERAL; const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u; @@ -1356,8 +1358,10 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, Require(mipLevel < descriptor.mipCount, "storage texture mip level is outside the texture"); const auto vkFormat = StorageFormatFor(context, ResolveTextureFormat(descriptor.format)); storageFormat = vkFormat; +#ifdef APS5_DEBUG_LOG AGC_LOG_DEFERRED(stdout, "StorageTexture address=0x%llx %ux%u mips=%u mip=%u layers=%u base=%u dim=%d tile=%d format=%u vk=%d", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, mipLevel, descriptor.depthOrLastArray, descriptor.baseArray, static_cast(descriptor.dimension), static_cast(descriptor.tileMode), descriptor.format, static_cast(vkFormat)); +#endif geometry = DescribeSurface(descriptor); mips = geometry.mips; arrayLayers = geometry.layers; @@ -1418,28 +1422,48 @@ namespace { class PendingList { public: using iterator = std::vector::iterator; + using Ranges = std::vector>; iterator begin() { return textures.begin(); } iterator end() { return textures.end(); } void push_back(StorageTexture* texture) { textures.push_back(texture); - ++version; + changed(); } iterator erase(iterator it) { - ++version; - return textures.erase(it); + const auto next = textures.erase(it); + changed(); + return next; } void remove(const StorageTexture* texture) { - if (std::erase(textures, texture) != 0) ++version; + if (std::erase(textures, texture) != 0) changed(); } bool MayOverlap(std::uint64_t address, std::size_t bytes) { - if (textures.empty() || bytes == 0) return false; if (indexed != version) rebuild(); + return !textures.empty() && Overlaps(ranges, address, bytes); + } + bool SurelyClear(std::uint64_t address, std::size_t bytes) const { + const auto current = published.load(std::memory_order_acquire); + return current != nullptr && current->version == changes.load(std::memory_order_acquire) && !Overlaps(current->ranges, address, bytes); + } + +private: + struct Snapshot { + std::uint64_t version = 0; + Ranges ranges; + }; + + static bool Overlaps(const Ranges& ranges, std::uint64_t address, std::size_t bytes) { + if (bytes == 0) return false; const auto end = bytes > std::numeric_limits::max() - address ? std::numeric_limits::max() : address + bytes; const auto it = std::upper_bound(ranges.begin(), ranges.end(), address, [](std::uint64_t value, const std::pair& range) { return value < range.second; }); return it != ranges.end() && it->first < end; } -private: + void changed() { + ++version; + changes.store(version, std::memory_order_release); + } + void rebuild() { ranges.clear(); for (const auto* texture : textures) ranges.emplace_back(texture->Descriptor().baseAddress, texture->Descriptor().baseAddress + texture->GuestBytes()); @@ -1451,11 +1475,14 @@ private: } ranges.resize(merged); indexed = version; + published.store(std::make_shared(Snapshot{version, ranges}), std::memory_order_release); } std::vector textures; - std::vector> ranges; + Ranges ranges; std::uint64_t version = 0; std::uint64_t indexed = ~std::uint64_t{0}; + std::atomic changes{0}; + std::atomic> published; }; // Storage images whose results have not reached guest memory yet. @@ -2271,6 +2298,10 @@ std::uint64_t StorageTexture::SinglePassMoves() { return singlePassMoves.load(std::memory_order_relaxed); } +std::uint64_t StorageTexture::SlabRetiles() { + return slabRetiles.load(std::memory_order_relaxed); +} + bool StorageTexture::otherPendingOverlaps() const { auto& pending = Pending(); std::lock_guard lock(pending.mutex); @@ -3117,6 +3148,15 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se std::vector slabPieces; std::vector> importCopies(segments.size()); std::vector seeds; + struct AddedPiece { + std::size_t pieces; + std::size_t copy; + std::uint64_t begin; + std::uint64_t end; + bool slab; + bool seeded; + }; + std::vector added; const auto addPiece = [&](std::uint64_t scratchOffset, std::uint64_t guestBegin, std::uint64_t guestEnd) { while (guestBegin < guestEnd) { std::size_t segment = 0; @@ -3132,9 +3172,11 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se for (const auto& [seedBegin, seedEnd] : destination->seedUnits) { if (std::none_of(seeds.begin(), seeds.end(), [&](const SeedUnit& seed) { return seed.begin == seedBegin; })) seeds.push_back({seedBegin, seedEnd, destination->slab, &import}); } + added.push_back({slabPieces.size() - 1, slabPieces.back().copies.size() - 1, guestBegin - descriptor.baseAddress, pieceEnd - descriptor.baseAddress, true, !destination->seedUnits.empty()}); if (!shadowed.empty() && shadowed.back().end == guestBegin && shadowed.back().slab == destination->slab) shadowed.back().end = pieceEnd; else shadowed.push_back({guestBegin, pieceEnd, destination->slab}); } else { + added.push_back({0, 0, guestBegin - descriptor.baseAddress, pieceEnd - descriptor.baseAddress, false, false}); importCopies[segment].push_back({scratchOffset, HostImportOffset(import, guestBegin), pieceEnd - guestBegin}); if (!imported.empty() && imported.back().second == guestBegin) imported.back().second = pieceEnd; else imported.emplace_back(guestBegin, pieceEnd); @@ -3143,16 +3185,37 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se guestBegin = pieceEnd; } }; + struct SlabRetile { + std::size_t window; + std::size_t first; + std::size_t last; + }; + std::vector slabRetiled; + const auto blockBytes = ThinBlockLayout(descriptor.tileMode, elementBytes)[0]; for (std::size_t i = 0; i < windows.size(); ++i) { const auto& window = windows[i]; // The tail levels follow each other, sharing the previous one's region. if (i != 0 && scratchPositions[i] == scratchPositions[i - 1]) continue; + const auto first = added.size(); + std::size_t kept = 0; + bool whole = false; for (const auto& [from, to] : runs) { const auto begin = std::max(from, window.tiledBegin); const auto end = std::min(to, window.tiledEnd); if (begin >= end) continue; + ++kept; addPiece(scratchPositions[i] + (begin - window.tiledBegin), descriptor.baseAddress + begin, descriptor.baseAddress + end); + whole = begin == window.tiledBegin && end == window.tiledEnd; } + if (save != nullptr || geometry.thick || mips[window.level].tail || std::count(scratchPositions.begin(), scratchPositions.end(), scratchPositions[i]) != 1 || kept != 1 || !whole) continue; + const auto last = added.size(); + const auto parts = std::span(added).subspan(first, last - first); + const auto sliceBegin = window.tiledBegin - window.window.rangeBegin; + const auto intoSlabs = std::all_of(parts.begin(), parts.end(), [&](const AddedPiece& part) { + const auto& copy = slabPieces[part.pieces].copies[part.copy]; + return part.slab && !part.seeded && copy.dstOffset % 4 == 0 && copy.size % 4 == 0 && (part.begin - sliceBegin) % blockBytes == 0 && (part.end - sliceBegin) % blockBytes == 0; + }); + if (intoSlabs) slabRetiled.push_back({i, first, last}); } std::sort(seeds.begin(), seeds.end(), [](const SeedUnit& a, const SeedUnit& b) { return a.begin < b.begin; }); std::vector copied; @@ -3169,7 +3232,88 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se if (save != nullptr) { for (const auto& piece : copied) saves.push_back({piece.scratch, piece.begin, piece.end - piece.begin}); } - const auto padding = paddingSeeds(segments, copied); + auto padding = paddingSeeds(segments, copied); + struct DirectPiece { + VkBuffer slab; + VkDeviceSize offset; + DetileWindow window; + std::uint64_t linearOffset; + }; + std::vector directPieces; + std::vector> directRanges(windows.size(), {0, 0}); + struct HoleCopies { + VkBuffer source; + VkBuffer slab; + std::vector copies; + }; + std::vector holeCopies; + if (!slabRetiled.empty()) { + std::vector> ranges; + for (const auto& retiled : slabRetiled) ranges.emplace_back(windows[retiled.window].tiledBegin, windows[retiled.window].tiledEnd); + const auto holes = uncoveredBytes(ranges); + for (const auto& retiled : slabRetiled) { + const auto& window = windows[retiled.window]; + const auto parts = std::span(added).subspan(retiled.first, retiled.last - retiled.first); + const auto position = scratchPositions[retiled.window]; + const auto length = window.tiledEnd - window.tiledBegin; + const auto inside = [&](const VkBufferCopy& hole) { return hole.size != 0 && hole.dstOffset >= position && hole.dstOffset + hole.size <= position + length; }; + const auto partCopy = [&](const AddedPiece& part) -> VkBufferCopy& { return slabPieces[part.pieces].copies[part.copy]; }; + const auto partSlab = [&](const AddedPiece& part) { return slabPieces[part.pieces].slab->buffer; }; + std::uint64_t holeBytes = 0, sourcedBytes = 0; + for (const auto& [first, last] : holes) { + const auto begin = std::max(first, window.tiledBegin); + const auto end = std::min(last, window.tiledEnd); + if (begin < end) holeBytes += end - begin; + } + bool sameSlabMoved = false; + for (const auto& [source, copies] : padding.copies) { + for (const auto& hole : copies) { + if (!inside(hole)) continue; + sourcedBytes += hole.size; + const auto holeBegin = window.tiledBegin + (hole.dstOffset - position); + for (const auto& part : parts) { + const auto from = std::max(holeBegin, part.begin); + const auto to = std::min(holeBegin + hole.size, part.end); + if (from < to && source == partSlab(part) && hole.srcOffset + (from - holeBegin) != partCopy(part).dstOffset + (from - part.begin)) sameSlabMoved = true; + } + } + } + if (sourcedBytes != holeBytes || sameSlabMoved) continue; + for (auto& [source, copies] : padding.copies) { + for (auto& hole : copies) { + if (!inside(hole)) continue; + const auto holeBegin = window.tiledBegin + (hole.dstOffset - position); + for (const auto& part : parts) { + const auto from = std::max(holeBegin, part.begin); + const auto to = std::min(holeBegin + hole.size, part.end); + if (from >= to || source == partSlab(part)) continue; + const auto slab = partSlab(part); + auto entry = std::find_if(holeCopies.begin(), holeCopies.end(), [&](const HoleCopies& entry) { return entry.source == source && entry.slab == slab; }); + if (entry == holeCopies.end()) entry = holeCopies.insert(holeCopies.end(), {source, slab, {}}); + entry->copies.push_back({hole.srcOffset + (from - holeBegin), partCopy(part).dstOffset + (from - part.begin), to - from}); + } + hole.size = 0; + } + } + directRanges[retiled.window] = {directPieces.size(), directPieces.size() + parts.size()}; + for (const auto& part : parts) { + auto& copy = partCopy(part); + if (parts.size() == 1) { + directPieces.push_back({partSlab(part), copy.dstOffset, window.window, 0}); + } else { + const std::pair run{part.begin, part.end}; + const auto narrowed = sliceWindows(std::span(&run, 1)); + Require(narrowed.size() == 1, "a storage write-back slab piece is not one window"); + directPieces.push_back({partSlab(part), copy.dstOffset, narrowed.front().window, narrowed.front().window.linearBase - window.window.linearBase}); + } + copy.size = 0; + } + slabRetiles.fetch_add(1, std::memory_order_relaxed); + } + for (auto& pieces : slabPieces) std::erase_if(pieces.copies, [](const VkBufferCopy& copy) { return copy.size == 0; }); + for (auto& entry : padding.copies) std::erase_if(entry.second, [](const VkBufferCopy& copy) { return copy.size == 0; }); + std::erase_if(padding.copies, [](const auto& entry) { return entry.second.empty(); }); + } // A queued label or key store inside a seeded unit lands before the seed copies the import. auto flushBegin = firstStored; auto flushEnd = lastStored; @@ -3179,7 +3323,8 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se } const bool direct = singlePass(); auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); - auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const bool scratchUsed = std::any_of(directRanges.begin(), directRanges.end(), [](const auto& range) { return range.first == range.second; }) || std::any_of(slabPieces.begin(), slabPieces.end(), [](const SlabPieces& pieces) { return !pieces.copies.empty(); }) || std::any_of(importCopies.begin(), importCopies.end(), [](const auto& copies) { return !copies.empty(); }) || !saves.empty() || !padding.copies.empty() || !seeds.empty(); + auto tiledScratch = scratchUsed ? std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT) : nullptr; detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -3192,11 +3337,11 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se commands = recorder->Commands(&covered); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); if (linear) recorder->Keep(linear, linear->Size()); - recorder->Keep(tiledScratch, tiledScratch->Size()); + if (tiledScratch) recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) keepStorageTexture(*recorder, std::move(self)); - Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, (direct ? 3 : 4) - (tiledScratch ? 0 : 1)); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; std::vector> seedReads; @@ -3218,7 +3363,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; if (!direct) context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); - if (!seeds.empty() || !padding.copies.empty()) { + if (!seeds.empty() || !padding.copies.empty() || !holeCopies.empty()) { // The seeded units' import bytes (every earlier writer of them, host stores included) // precede the seed copies; the previous command's trailing barrier may have covered that. constexpr VkAccessFlags transferAccess = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; @@ -3237,6 +3382,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se // The padding's current bytes, under the retile that follows (the import-ready barrier // orders the scratch writes). for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); + for (const auto& [source, slab, copies] : holeCopies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, slab, static_cast(copies.size()), copies.data()); } const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; if (direct) { @@ -3249,7 +3395,12 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se const auto& window = windows[i]; auto retile = window.window; retile.pipeBankXor = descriptor.pipeBankXor; - detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); + for (auto k = directRanges[i].first; k < directRanges[i].second; ++k) { + auto piece = directPieces[k].window; + piece.pipeBankXor = descriptor.pipeBankXor; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, directPieces[k].slab, directPieces[k].offset, elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, piece); + } + if (directRanges[i].first == directRanges[i].second) detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); } singlePassMoves.fetch_add(1, std::memory_order_relaxed); } else { @@ -3267,16 +3418,23 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se const auto& window = windows[i]; auto retile = window.window; retile.pipeBankXor = descriptor.pipeBankXor; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + for (auto k = directRanges[i].first; k < directRanges[i].second; ++k) { + auto piece = directPieces[k].window; + piece.pipeBankXor = descriptor.pipeBankXor; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i] + directPieces[k].linearOffset, directPieces[k].slab, directPieces[k].offset, mips[window.level], true, window.layer, false, piece); + } + if (directRanges[i].first == directRanges[i].second) detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); } } - { + if (tiledScratch) { // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in // part) precede the copies into the slabs and the import. const VkMemoryBarrier scratchDone{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT}; context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &scratchDone, 0, nullptr, 0, nullptr); const auto copyBuffer = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); - for (const auto& pieces : slabPieces) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast(pieces.copies.size()), pieces.copies.data()); + for (const auto& pieces : slabPieces) { + if (!pieces.copies.empty()) copyBuffer(commands, tiledScratch->Handle(), pieces.slab->buffer, static_cast(pieces.copies.size()), pieces.copies.data()); + } for (std::size_t segment = 0; segment < segments.size(); ++segment) { if (!importCopies[segment].empty()) copyBuffer(commands, tiledScratch->Handle(), segments[segment].import->buffer, static_cast(importCopies[segment].size()), importCopies[segment].data()); } @@ -3535,6 +3693,20 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons for (const auto& texture : flush) std::fprintf(stderr, "[flush] image 0x%llx+0x%llx for %s 0x%llx+0x%zx\n", static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), reason, static_cast(address), bytes); } std::lock_guard gpu(GuestMemory::GpuMutex()); + if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion() && recorder->PendingWriterSerial(address, bytes) != 0) { + try { + recorder->Reap(); + if (const auto writer = recorder->PendingWriterSerial(address, bytes); writer == recorder->OpenSerial()) { + Recorder::CountSync(recorder->OpenWriteOverlaps(address, bytes) ? 1 : 3); + recorder->Sync(); + } else if (writer != 0) { + recorder->FinishUpTo(writer); + } + } catch (...) { + for (const auto& texture : flush) texture->reconcilePending(); + throw; + } + } std::exception_ptr failure; const auto previousReason = std::exchange(flushReason, reason); for (const auto& texture : flush) { @@ -3884,6 +4056,7 @@ std::vector> StorageTexture::overlappingPending( // FlushPending's listing; the images stay alive across the caller's scan as there. std::vector> overlapping; auto& pending = Pending(); + if (pending.textures.SurelyClear(address, bytes)) return overlapping; std::lock_guard lock(pending.mutex); if (!pending.textures.MayOverlap(address, bytes)) return overlapping; const auto* exempt = refreshing; diff --git a/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp b/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp index 0325795a41..f334f4b036 100644 --- a/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/AsyncLog.cpp @@ -24,6 +24,9 @@ #else #include #endif +#ifdef __linux__ +#include +#endif namespace { @@ -248,6 +251,9 @@ constexpr int ChildLines = 50; #else const rlimit noCore{0, 0}; setrlimit(RLIMIT_CORE, &noCore); +#endif +#ifdef __linux__ + prctl(PR_SET_DUMPABLE, 0); #endif std::thread([] { std::this_thread::sleep_for(8s); diff --git a/core/libs/prx/libSceAgcDriver/tests/DepthRangeRemap.cpp b/core/libs/prx/libSceAgcDriver/tests/DepthRangeRemap.cpp new file mode 100644 index 0000000000..8452a2ae1a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DepthRangeRemap.cpp @@ -0,0 +1,166 @@ +#include "Recompiler.hpp" +#include "PipelineSpecialization.hpp" +#include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct ModuleFacts { + std::map specializationDefaults; + std::uint32_t clipDistanceLength = 0; + bool clipDistanceBuiltIn = false; + std::size_t multiplies = 0; + bool pushConstantVariable = false; + std::set pushDwordLoads; +}; + +ModuleFacts inspect(std::span words) { + ModuleFacts facts; + std::map specIds; + std::map specValues; + std::map constants; + std::map arrayLengths; + std::map pointerTargets; + std::map variableTypes; + std::set clipVariables; + std::set pushVariables; + std::vector pushIndexIds; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + const auto op = static_cast(words[at] & 0xffffu); + const auto count = words[at] >> 16u; + if (op == spv::OpDecorate && count == 4 && words[at + 2] == spv::DecorationSpecId) specIds[words[at + 1]] = words[at + 3]; + if (op == spv::OpDecorate && count == 4 && words[at + 2] == spv::DecorationBuiltIn && words[at + 3] == spv::BuiltInClipDistance) clipVariables.insert(words[at + 1]); + if (op == spv::OpSpecConstant && count == 4) specValues[words[at + 2]] = words[at + 3]; + if (op == spv::OpConstant && count == 4) constants[words[at + 2]] = words[at + 3]; + if (op == spv::OpTypeArray && count == 4) arrayLengths[words[at + 1]] = words[at + 3]; + if (op == spv::OpTypePointer && count == 4) pointerTargets[words[at + 1]] = words[at + 3]; + if (op == spv::OpVariable && count >= 4) variableTypes[words[at + 2]] = words[at + 1]; + if (op == spv::OpVariable && count >= 4 && words[at + 3] == spv::StorageClassPushConstant) pushVariables.insert(words[at + 2]); + if (op == spv::OpAccessChain && count == 6 && pushVariables.contains(words[at + 3])) pushIndexIds.push_back(words[at + 5]); + if (op == spv::OpFMul) ++facts.multiplies; + } + for (const auto& [id, specId] : specIds) { + if (const auto value = specValues.find(id); value != specValues.end()) facts.specializationDefaults[specId] = value->second; + } + facts.pushConstantVariable = !pushVariables.empty(); + for (const auto id : pushIndexIds) { + if (const auto value = constants.find(id); value != constants.end()) facts.pushDwordLoads.insert(value->second); + } + for (const auto variable : clipVariables) { + facts.clipDistanceBuiltIn = true; + const auto pointer = variableTypes.find(variable); + if (pointer == variableTypes.end()) continue; + const auto array = pointerTargets.find(pointer->second); + if (array == pointerTargets.end()) continue; + const auto length = arrayLengths.find(array->second); + if (length == arrayLengths.end()) continue; + const auto constant = constants.find(length->second); + if (constant != constants.end()) facts.clipDistanceLength = constant->second; + } + return facts; +} + +ModuleFacts inspect(const ShaderRecompiler::RecompileResult& result) { + return inspect(result.spirv.Words()); +} + +constexpr std::array PositionExport{0xf80008cfu, 0u, 0xbf810000u}; + +ShaderRecompiler::RecompileRequest baseRequest(bool remap) { + using namespace ShaderRecompiler; + RecompileRequest request{}; + request.shader = {ShaderStage::Vertex, 0x10000u, PositionExport, 0, {}}; + request.context.waveSize = 64; + ShaderVertexStageInfo vertex{}; + vertex.depthRemapMode = 2u; + request.context.vertex = vertex; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + static constexpr std::array capabilities{spv::CapabilityShader, spv::CapabilityClipDistance}; + request.target.supportedCapabilities = capabilities; + request.target.depthRangeRemap = remap; + request.layout.pushConstantSizeBytes = ShaderRecompiler::DepthRemapPushOffsetBytes; + request.useCache = false; + return request; +} + +void require(bool condition, const char* what) { + if (!condition) throw std::runtime_error(what); +} + +void preparedTransformTests() { + using namespace ShaderRecompiler; + auto request = baseRequest(true); + request.useCache = true; + const auto handle = PrepareShader(request); + const auto materialize = [&](std::uint32_t mode) { + auto moved = request; + moved.context.vertex->depthRemapMode = mode; + const auto invocation = PreparedShaderInvocation::TryCreate(moved, handle); + require(invocation.has_value(), "the prepared vertex shader did not match a request of its own mode"); + return invocation->Materialize(*invocation->Capture(SrtRuntime{})); + }; + const auto prepared = inspect(GetPreparedArtifact(*handle).spirv.Words()); + require(prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase) && !prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 1u) && !prepared.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 2u), "only the depth remap mode may be a specialization constant of the prepared module"); + const auto first = materialize(2u); + require(materialize(2u)->PipelineVariantId() == first->PipelineVariantId(), "the same depth remap mode did not reuse its materialized vertex shader"); + auto other = request; + other.context.vertex->depthRemapMode = 1u; + require(!PreparedShaderInvocation::TryCreate(other, handle).has_value(), "a prepared vertex shader matched a request of another depth remap mode"); +} + +void pushedTransformTests() { + using namespace ShaderRecompiler; + const auto remapped = Recompile(baseRequest(true)); + const auto plain = Recompile(baseRequest(false)); + const auto facts = inspect(remapped); + require(!facts.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 1u) && !facts.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 2u), "the depth remap scale or offset is a specialization constant"); + require(facts.pushConstantVariable && facts.pushDwordLoads.contains(DepthRemapPushOffsetBytes / 4u) && facts.pushDwordLoads.contains(DepthRemapPushOffsetBytes / 4u + 1u), "the remapped vertex shader does not read the scale and offset from the push block"); + require(!inspect(plain).pushConstantVariable, "a vertex shader without a depth remap gained a push constant block"); + auto crowded = baseRequest(true); + crowded.layout.pushConstantSizeBytes = DepthRemapPushOffsetBytes + 4u; + bool refused = false; + try { + static_cast(Recompile(crowded)); + } catch (const std::runtime_error&) { + refused = true; + } + require(refused, "a remapped vertex shader with push data over the depth remap words was accepted"); +} + +} + +int main() { + using namespace ShaderRecompiler; + try { + const auto remapped = inspect(Recompile(baseRequest(true))); + const auto plain = inspect(Recompile(baseRequest(false))); + require(remapped.clipDistanceBuiltIn && remapped.clipDistanceLength == plain.clipDistanceLength + 2u, "the remap did not add its two clip distances"); + require(remapped.multiplies >= plain.multiplies + 2u, "the remap does not scale the clip z and offset it by w"); + require(!plain.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase) && !remapped.specializationDefaults.contains(PipelineSpecialization::DepthRemapBase + 3u), "unexpected specialization constants"); + preparedTransformTests(); + pushedTransformTests(); + std::cout << "Depth range remap tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "Depth range remap test failed: " << error.what() << '\n'; + for (const bool remap : {true, false}) { + const auto facts = inspect(Recompile(baseRequest(remap))); + std::cerr << " remap=" << remap << ": spec ids"; + for (const auto& [id, value] : facts.specializationDefaults) std::cerr << ' ' << id << '=' << value; + std::cerr << "; clip builtin " << facts.clipDistanceBuiltIn << " length " << facts.clipDistanceLength << "; fmul " << facts.multiplies << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp index 006e4efd53..e2932dc814 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include #include #include #include @@ -47,6 +48,7 @@ VkImageUsageFlags lastImageUsage = 0; std::uint32_t imagesDestroyed = 0; std::vector> imageReads; std::vector> imageWrites; +std::vector destroyedImages; VKAPI_ATTR VkResult VKAPI_CALL createImage(VkDevice, const VkImageCreateInfo* info, const VkAllocationCallbacks*, VkImage* image) { lastImageFormat = info->format; @@ -108,7 +110,10 @@ VKAPI_ATTR VkResult VKAPI_CALL getFenceStatus(VkDevice, VkFence) { VKAPI_ATTR void VKAPI_CALL freeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t, const VkCommandBuffer*) {} VKAPI_ATTR void VKAPI_CALL destroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) {} VKAPI_ATTR void VKAPI_CALL destroyImageView(VkDevice, VkImageView, const VkAllocationCallbacks*) {} -VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage, const VkAllocationCallbacks*) { ++imagesDestroyed; } +VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage image, const VkAllocationCallbacks*) { + ++imagesDestroyed; + destroyedImages.push_back(image); +} VKAPI_ATTR void VKAPI_CALL freeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) {} VKAPI_ATTR void VKAPI_CALL cmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {} VKAPI_ATTR void VKAPI_CALL cmdClearDepthStencilImage(VkCommandBuffer, VkImage image, VkImageLayout, const VkClearDepthStencilValue*, std::uint32_t, const VkImageSubresourceRange*) { @@ -374,6 +379,44 @@ void TestMultisampleStencilAliases(Context context) { } +void TestRetiredSurfaceRelease(const Context& context) { + ClearDepthSurfaces(context.device); + constexpr VkExtent2D extent{256, 256}; + const auto bytes = DepthSliceBytes(extent, 4u); + const auto destroyed = [](VkImage image) { return std::find(destroyedImages.begin(), destroyedImages.end(), image) != destroyedImages.end(); }; + constexpr std::uint64_t moving = 0x90000000; + std::vector images; + for (std::uint64_t frame = 0; frame < 4; ++frame) { + DepthSurfaceView(context, {moving + frame * 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + images.push_back(lastImage); + RetireDepthSurfaces(context.device, moving + frame * 0x100000u, bytes); + } + const auto released = DepthSurfacesReleased(); + DepthSurfaceView(context, {moving + 4u * 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + Require(std::all_of(images.begin(), images.end(), destroyed), "a retired depth surface must free its image once a new surface is created"); + Require(!destroyed(lastImage), "the new depth surface must keep its image"); + Require(DepthSurfacesReleased() > released, "freeing retired depth surfaces must be counted for the framebuffer cache"); + + constexpr std::uint64_t sampled = 0x98000000; + DepthSurfaceView(context, {sampled, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + const auto sampledImage = lastImage; + auto view = lookup(context, {sampled, FormatR32Float, 256, 256}); + Require(view != nullptr, "an R32F view of a D32 surface's own extent must sample its depth plane"); + RetireDepthSurfaces(context.device, sampled, bytes); + DepthSurfaceView(context, {sampled + 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + Require(!destroyed(sampledImage), "a retired depth surface must keep its image while its sampled view is held"); + view.reset(); + DepthSurfaceView(context, {sampled + 0x200000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + Require(destroyed(sampledImage), "a retired depth surface must free its image once its sampled view is released"); + + constexpr std::uint64_t kept = 0x9c000000; + DepthSurfaceView(context, {kept, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + const auto keptImage = lastImage; + DepthSurfaceView(context, {kept + 0x100000u, 0, extent, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + Require(!destroyed(keptImage), "a depth surface that was not retired must keep its image"); + ClearDepthSurfaces(context.device); +} + void RunDepthSurfaceReuseTests() { static int deviceTag = 0; Context context{}; @@ -480,6 +523,7 @@ void RunDepthSurfaceReuseTests() { Require(clearsAfter(sliceHtile) == std::vector{firstImage}, "an HTILE clear of the first slice must clear that slice alone"); Require(clearsAfter(2u * sliceHtile) == std::vector{firstImage, secondImage}, "an HTILE clear over both slices must clear both"); } + TestRetiredSurfaceRelease(context); ClearDepthSurfaces(context.device); } diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index c5f368fe78..8202fde0bb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -2,6 +2,7 @@ #include "GraphicsTests.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" @@ -10,6 +11,7 @@ #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "ShaderDiskCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" @@ -28,6 +30,11 @@ #include "ImageTableAbi.hpp" #include "SpirvBackend/SpirvSpecialization.hpp" #include +#ifdef _WIN32 +#include +#else +#include +#endif #include #include #include @@ -35,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -3894,6 +3902,57 @@ void quadPixelMaskTests() { expectFailure([&] { AgcDriver::Graphics::ValidateSampleMask(context, varying, storing); }, "differs between the pixels of a quad"); } +void DepthRemapTests() { + auto queue = makeState(); + queue.context[0x204] = 0; + queue.context[0x113] = std::bit_cast(1.0f); + queue.context[0x114] = 0; + AgcDriver::Graphics::Context remapping{}; + remapping.depthClamp = true; + auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.negativeOneToOne && state.viewport.minDepth == -1 && state.viewport.maxDepth == 1 && state.depthRemap.mode == 0, "a negative-one-to-one depth transform decoded with a remap"); + AgcDriver::Graphics::ApplyDepthRemap(remapping, state); + Require(state.depthRemap.mode == 2 && state.depthRemap.scale == 1.0f && state.depthRemap.offset == 0.0f, "the remap did not take the viewport's depth scale and offset"); + Require(state.viewport.minDepth == 0 && state.viewport.maxDepth == 1 && !state.negativeOneToOne && state.depthClamp, "the remapped viewport does not clamp at the console's bounds in zero-to-one clip space"); + AgcDriver::Graphics::Context unrestricted{}; + unrestricted.depthRangeUnrestricted = true; + auto kept = AgcDriver::Graphics::DecodeState(queue); + AgcDriver::Graphics::ApplyDepthRemap(unrestricted, kept); + Require(kept.depthRemap.mode == 0 && kept.viewport.minDepth == -1 && kept.negativeOneToOne && !kept.depthClamp, "an unrestricted device remapped the depth range"); + AgcDriver::Graphics::Context noClamp{}; + auto refused = AgcDriver::Graphics::DecodeState(queue); + expectFailure([&] { AgcDriver::Graphics::ApplyDepthRemap(noClamp, refused); }, "depth clamping"); + queue.context[0x113] = std::bit_cast(0.5f); + queue.context[0x114] = std::bit_cast(0.5f); + auto inRange = AgcDriver::Graphics::DecodeState(queue); + AgcDriver::Graphics::ApplyDepthRemap(remapping, inRange); + Require(inRange.depthRemap.mode == 0 && inRange.negativeOneToOne && !inRange.depthClamp, "an in-range depth transform was remapped"); + queue.context[0x204] = 0x80000u; + queue.context[0x113] = std::bit_cast(2.0f); + queue.context[0x114] = std::bit_cast(-0.5f); + queue.context[0xb4] = std::bit_cast(0.25f); + queue.context[0xb5] = std::bit_cast(0.75f); + auto bounded = AgcDriver::Graphics::DecodeState(queue); + AgcDriver::Graphics::ApplyDepthRemap(remapping, bounded); + Require(bounded.depthRemap.mode == 1 && bounded.depthRemap.scale == 4.0f && bounded.depthRemap.offset == -1.5f && bounded.viewport.minDepth == 0.25f && bounded.viewport.maxDepth == 0.75f, "a zero-to-one remap did not divide its transform by the clamp range it gives the viewport"); + queue.context[0xb4] = std::bit_cast(-1.0f); + queue.context[0xb5] = std::bit_cast(2.0f); + auto clipped = AgcDriver::Graphics::DecodeState(queue); + AgcDriver::Graphics::ApplyDepthRemap(remapping, clipped); + Require(clipped.depthRemap.mode == 1 && clipped.viewport.minDepth == 0 && clipped.viewport.maxDepth == 1, "clamp bounds beyond [0, 1] were not clipped to the representable depth range"); + Require(clipped.depthRemap.scale == 2.0f && clipped.depthRemap.offset == -0.5f, "clamp bounds clipped to [0, 1] changed the depth transform"); + auto unorm = AgcDriver::Graphics::DecodeState(queue); + unorm.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D16_UNORM, 0.0f, 0}; + unorm.depthTest = unorm.depthWrite = true; + AgcDriver::Graphics::ApplyDepthRemap(remapping, unorm); + Require(unorm.viewport.minDepth == 0 && unorm.viewport.maxDepth == 1, "a normalized depth target refused clipped clamp bounds"); + auto floating = AgcDriver::Graphics::DecodeState(queue); + floating.depth = AgcDriver::Graphics::DepthTarget{0x10000, 0, {64, 64}, VK_FORMAT_D32_SFLOAT, 0.0f, 0}; + floating.depthWrite = true; + AgcDriver::Graphics::ApplyDepthRemap(remapping, floating); + Require(floating.depthRemap.mode == 1 && floating.viewport.minDepth == 0 && floating.viewport.maxDepth == 1 && floating.depthClamp, "a written float depth target refused clipped clamp bounds"); +} + void DepthClipTests() { auto queue = makeState(); const auto direct = AgcDriver::Graphics::DecodeState(queue); @@ -4591,6 +4650,117 @@ AgcDriver::Graphics::Context mockContext() { return context; } +struct CommandBatchFailureProbe { + VkResult allocation = VK_SUCCESS; + VkResult creation = VK_SUCCESS; + VkResult begin = VK_SUCCESS; + VkFence failedFence = VK_NULL_HANDLE; + std::uint32_t freed = 0; + std::uint32_t creates = 0; + std::uint32_t destroys = 0; + std::uint32_t invalidDestroys = 0; + std::uint32_t begins = 0; +}; + +CommandBatchFailureProbe batchProbe; + +VKAPI_ATTR VkResult VKAPI_CALL batchAllocateCommands(VkDevice device, const VkCommandBufferAllocateInfo* info, VkCommandBuffer* commands) { + if (batchProbe.allocation != VK_SUCCESS) { + std::fill_n(commands, info->commandBufferCount, VK_NULL_HANDLE); + return batchProbe.allocation; + } + return mockAllocateCommandBuffers(device, info, commands); +} + +VKAPI_ATTR VkResult VKAPI_CALL batchCreateFence(VkDevice device, const VkFenceCreateInfo* info, const VkAllocationCallbacks* allocator, VkFence* fence) { + ++batchProbe.creates; + if (batchProbe.creation != VK_SUCCESS) { + *fence = batchProbe.failedFence; + return batchProbe.creation; + } + return mockCreateFence(device, info, allocator, fence); +} + +VKAPI_ATTR void VKAPI_CALL batchFreeCommands(VkDevice device, VkCommandPool pool, std::uint32_t count, const VkCommandBuffer* commands) { + batchProbe.freed += count; + mockFreeCommandBuffers(device, pool, count, commands); +} + +VKAPI_ATTR void VKAPI_CALL batchDestroyFence(VkDevice device, VkFence fence, const VkAllocationCallbacks* allocator) { + ++batchProbe.destroys; + if (fence != VK_NULL_HANDLE && fence == batchProbe.failedFence) { + ++batchProbe.invalidDestroys; + return; + } + mockDestroyFence(device, fence, allocator); +} + +VKAPI_ATTR VkResult VKAPI_CALL batchBeginCommands(VkCommandBuffer commands, const VkCommandBufferBeginInfo* info) { + ++batchProbe.begins; + if (batchProbe.begin != VK_SUCCESS) return batchProbe.begin; + return mockBeginCommandBuffer(commands, info); +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL batchFailureProc(VkDevice device, const char* name) { + if (std::strcmp(name, "vkAllocateCommandBuffers") == 0) return reinterpret_cast(batchAllocateCommands); + if (std::strcmp(name, "vkCreateFence") == 0) return reinterpret_cast(batchCreateFence); + if (std::strcmp(name, "vkFreeCommandBuffers") == 0) return reinterpret_cast(batchFreeCommands); + if (std::strcmp(name, "vkDestroyFence") == 0) return reinterpret_cast(batchDestroyFence); + if (std::strcmp(name, "vkBeginCommandBuffer") == 0) return reinterpret_cast(batchBeginCommands); + return mockProc(device, name); +} + +void commandBatchFenceFailureTests() { + using namespace AgcDriver::Graphics; + const std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + std::uint32_t invalidDestroys = 0; + for (const auto result : {VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) { + for (const bool nonnull : {false, true}) { + mock = MockVulkan{}; + batchProbe = {}; + batchProbe.creation = result; + batchProbe.failedFence = nonnull ? reinterpret_cast(std::uintptr_t{0xdeadbeef}) : VK_NULL_HANDLE; + auto context = mockContext(); + context.deviceProc = batchFailureProc; + bool failed = false; + try { + CommandBatch batch(context); + } catch (const std::runtime_error& error) { + const std::string message = error.what(); + failed = message.find("vkCreateFence") != std::string::npos && message.find("Vulkan result " + std::to_string(result)) != std::string::npos; + } + Require(failed && batchProbe.creates == 1 && batchProbe.begins == 0, "fence creation failure lost its error or continued construction"); + Require(mock.live == 0 && batchProbe.freed == 1, "fence creation failure leaked the allocated command buffer"); + invalidDestroys += batchProbe.invalidDestroys; + std::printf("command batch fence result=%d nonnull=%d invalid destroys=%u\n", result, nonnull, batchProbe.invalidDestroys); + const auto destroyed = batchProbe.destroys; + const auto invalid = batchProbe.invalidDestroys; + batchProbe.creation = VK_SUCCESS; + { + CommandBatch recovered(context); + Require(recovered.Handle() != VK_NULL_HANDLE, "command batch did not recover after fence creation failed"); + } + Require(mock.live == 0 && batchProbe.freed == 2 && batchProbe.destroys == destroyed + 1 && batchProbe.invalidDestroys == invalid, "recovered command batch mishandled its owned objects"); + } + } + for (const bool allocated : {false, true}) { + mock = MockVulkan{}; + batchProbe = {}; + batchProbe.allocation = allocated ? VK_SUCCESS : VK_ERROR_OUT_OF_HOST_MEMORY; + batchProbe.begin = VK_ERROR_OUT_OF_HOST_MEMORY; + auto context = mockContext(); + context.deviceProc = batchFailureProc; + bool failed = false; + try { + CommandBatch batch(context); + } catch (const std::runtime_error& error) { + failed = std::string(error.what()).find(allocated ? "vkBeginCommandBuffer" : "vkAllocateCommandBuffers") != std::string::npos; + } + Require(failed && mock.live == 0 && batchProbe.freed == (allocated ? 1u : 0u) && batchProbe.destroys == (allocated ? 1u : 0u) && batchProbe.invalidDestroys == 0, "command batch constructor failure mishandled its owned objects"); + } + Require(invalidDestroys == 0, "command batch destroyed undefined failed fence outputs"); +} + void addressSpaceTeardownTests() { #ifdef _WIN32 _putenv_s("APS5_NO_LEASE_MIRROR", "1"); @@ -5067,6 +5237,16 @@ void pushConstantTests() { fragment.pushConstants.clear(); shaders[1].pushConstantOffset = 999; Require(AgcDriver::Graphics::PushConstantStages(shaders) == VK_SHADER_STAGE_VERTEX_BIT, "empty push constants contributed a stage"); + { + ShaderRecompiler::RecompileResult reading; + reading.readsDepthRemapPush = true; + std::vector encoded; + ShaderRecompiler::ShaderDiskCache::EncodeResult(reading, encoded); + ShaderRecompiler::RecompileResult replayed; + Require(ShaderRecompiler::ShaderDiskCache::DecodeResult(encoded, replayed), "a result that reads the depth remap push words does not decode"); + const std::array remapped{{{ShaderRecompiler::ShaderStage::Vertex, &replayed, 0}}}; + Require(replayed.pushConstants.empty() && AgcDriver::Graphics::PushConstantStages(remapped) == VK_SHADER_STAGE_VERTEX_BIT, "a journalled vertex shader that reads the depth remap push words lost its push stage"); + } Require(AgcDriver::Graphics::AssemblePushConstants(shaders)[8] == std::byte{0}, "empty push constants were copied"); shaders[1].program = nullptr; expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "missing compiled shader"); @@ -5095,10 +5275,12 @@ void descriptorSnapshotTests() { const auto original = findWrite(0).buffers; const auto originalThird = findWrite(4).buffers; std::vector moved; + std::array, 2> words; for (std::size_t index = 0; index < 2; ++index) { if ((selected & (1u << index)) == 0) continue; const std::size_t bytes = index == 0 ? 24 : 12; - moved.push_back({index, 0, bytes, std::vector(bytes / 4, static_cast(0x12340000u + index))}); + words[index].assign(bytes / 4, static_cast(0x12340000u + index)); + moved.push_back({index, 0, bytes, words[index]}); } mock.writes.clear(); const auto snapshots = resources.PrepareDrawBindings(recorder, moved); @@ -5517,19 +5699,27 @@ void descriptorCacheTests() { Require(mock.poolSizes.size() == 2 && mock.poolSizes[0].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER && mock.poolSizes[0].descriptorCount == 4097 && mock.poolSizes[1].type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE && mock.poolSizes[1].descriptorCount == 3, "the dedicated pool is not sized to its set"); Require(cache.Counters().pools == 0 && cache.Counters().sets == 1, "an oversized set was counted in the chain pools"); cache.Free(dedicated); - Require(mock.live == live, "freeing an oversized set did not destroy its pool"); + Require(mock.live == live + 1 && cache.Counters().keptSets == 1, "freeing an oversized set did not keep its set and pool"); + const auto again = cache.Allocate(large, oversized); + Require(again.set == dedicated.set && again.pool == dedicated.pool && mock.live == live + 1 && cache.Counters().reusedSets == 1 && cache.Counters().dedicatedPools == 1, "an oversized set of the kept set's layout and sizes did not take the kept set"); + cache.Free(again); + const std::array otherSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4098}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 3}}}; + const auto other = cache.Allocate(large, otherSizes); + Require(other.pool != dedicated.pool && mock.live == live + 2 && cache.Counters().dedicatedPools == 2, "an oversized set of other sizes took a kept set"); + cache.Free(other); + Require(mock.live == live + 2 && cache.Counters().keptSets == 3, "freeing a second oversized set did not keep it"); const std::array fitting{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}}}; const auto chained = cache.Allocate(layout(4096), fitting); Require(chained.set != VK_NULL_HANDLE && mock.poolMaxSets == 1024 && mock.poolFlags == 0 && cache.Counters().pools == 1, "a set the chain pool holds left the chain"); cache.Free(chained); - Require(mock.descriptorPools.at(chained.pool).resets == 1 && mock.live == live + 2, "a chain pool was not reset after its last set"); + Require(mock.descriptorPools.at(chained.pool).resets == 1 && mock.live == live + 4, "a chain pool was not reset after its last set"); const std::array regular{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}}}; const auto first = cache.Allocate(layout(4096), regular); const auto oversizedMixed = cache.Allocate(large, oversized); const auto second = cache.Allocate(layout(4096), regular); Require(first.pool == chained.pool && second.pool == chained.pool && oversizedMixed.pool != chained.pool && mock.descriptorPools.at(chained.pool).resets == 1, "an oversized set disturbed the chain pool"); cache.Free(oversizedMixed); - Require(mock.live == live + 2 && mock.descriptorPools.count(oversizedMixed.pool) == 0 && mock.descriptorPools.at(chained.pool).resets == 1, "freeing an oversized set between chain sets touched the chain pool"); + Require(oversizedMixed.pool == dedicated.pool && mock.live == live + 4 && mock.descriptorPools.count(oversizedMixed.pool) == 1 && mock.descriptorPools.at(chained.pool).resets == 1, "freeing an oversized set between chain sets touched the chain pool"); cache.Free(first); Require(mock.descriptorPools.at(chained.pool).resets == 1, "a chain pool reset while a set was live"); cache.Free(second); @@ -5540,6 +5730,15 @@ void descriptorCacheTests() { expectFailure([&] { cache.Allocate(large, storageImages); }, "descriptor set exceeds the device's per-set descriptor limit"); const std::array uniformBuffers{{{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1}}}; expectFailure([&] { cache.Allocate(large, uniformBuffers); }, "descriptor set uses an unsupported descriptor type 6"); + const auto keptBefore = mock.live; + std::vector spread; + for (std::uint32_t index = 0; index < 10; ++index) { + const std::array sized{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4100 + index}}}; + spread.push_back(cache.Allocate(large, sized)); + } + Require(mock.live == keptBefore + 10, "ten oversized sets of new sizes did not get ten pools"); + for (const auto& set : spread) cache.Free(set); + Require(mock.live == keptBefore - 2 + 8, "freeing more oversized sets than the cache keeps did not destroy the oldest pools"); } Require(mock.live == 0, "the descriptor cache leaked a pool or layout"); mock = MockVulkan{}; @@ -5673,7 +5872,8 @@ void imageTableSnapshotAllocationTests() { const auto state = DecodeState(makeState()); ShaderResources resources(context, vertex, fragment, state.color, 0, 0); const auto originalMap = findWrite(map.binding).buffers.at(0); - const std::array moved{{{0, 0, 16, {11, 22, 33, 44}}}}; + const std::array movedWords{11, 22, 33, 44}; + const std::array moved{{{0, 0, 16, movedWords}}}; const auto sets = cache.Counters().sets; mock.writes.clear(); mock.variableDescriptorCount.reset(); @@ -5809,6 +6009,31 @@ void samplerTableDeviceLimitTests(std::optional selected = std::n } } +void hostImportBudgetTests() { + using AgcDriver::Graphics::HostImportBudget; + constexpr std::uint64_t MiB = 1ull << 20u; + constexpr std::uint64_t GiB = 1ull << 30u; + VkPhysicalDeviceMemoryProperties memory{}; + Require(HostImportBudget(memory, 0) == 6 * GiB, "a device without memory heaps does not keep the 6 GiB import budget"); + Require(HostImportBudget(memory, 32 * GiB) == 16 * GiB, "a device without a host heap does not take half the physical memory"); + memory.memoryHeapCount = 3; + memory.memoryHeaps[0] = {7920 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + memory.memoryHeaps[1] = {8128 * MiB, 0}; + memory.memoryHeaps[2] = {256 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + Require(HostImportBudget(memory, 16 * GiB) == 4064 * MiB, "imports on a 16 GiB machine take more than half its 8128 MiB host heap"); + memory.memoryHeaps[1].size = 16 * GiB; + Require(HostImportBudget(memory, 32 * GiB) == 8 * GiB, "imports on a 32 GiB machine take more than half its 16 GiB host heap"); + memory.memoryHeaps[1].size = 32 * GiB; + Require(HostImportBudget(memory, 64 * GiB) == 16 * GiB, "imports on a 64 GiB machine take more than half its 32 GiB host heap"); + memory.memoryHeaps[1].size = 3 * GiB; + Require(HostImportBudget(memory, 16 * GiB) == 1 * GiB, "a 3 GiB host heap does not keep its 2 GiB reserve"); + memory.memoryHeaps[1].size = 8128 * MiB; + memory.memoryHeaps[0].size = 64 * GiB; + Require(HostImportBudget(memory, 16 * GiB) == 4064 * MiB, "a device-local heap counted as the host heap"); + memory.memoryHeapCount = 1; + Require(HostImportBudget(memory, 16 * GiB) == 8 * GiB, "a host heap past memoryHeapCount limits the import budget"); +} + void textureCacheBudgetTests() { using AgcDriver::Graphics::TextureCacheBudget; constexpr std::uint64_t GiB = 1ull << 30u; @@ -9279,6 +9504,39 @@ void vertexZeroFillTests() { Require(std::all_of(bytes.begin() + static_cast(valid), bytes.end(), [](std::byte value) { return value == std::byte{0}; }), "a vertex fetch past the record count did not read as zeros"); } +void vertexOverhangTests() { + using AgcDriver::Graphics::CopyZeroPaddedDrawInput; + constexpr std::size_t page = 65536; + constexpr std::size_t stride = 44; + constexpr std::size_t overhang = 8; +#ifdef _WIN32 + auto* block = static_cast(VirtualAlloc(nullptr, page * 2, MEM_RESERVE, PAGE_NOACCESS)); + Require(block != nullptr && VirtualAlloc(block, page, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot reserve the guest pages of the overhang test"); +#else + void* mapped = mmap(nullptr, page * 2, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(mapped != MAP_FAILED && mprotect(mapped, page, PROT_READ | PROT_WRITE) == 0, "cannot reserve the guest pages of the overhang test"); + auto* block = static_cast(mapped); +#endif + constexpr std::size_t offset = 0x650; + const auto readable = page - offset; + std::fill(block + offset, block + page, std::byte{0x7a}); + const auto address = reinterpret_cast(block) + offset; + { + const auto copy = CopyZeroPaddedDrawInput(mockContext(), address, readable + overhang, readable + overhang, stride); + const auto bytes = copy.buffer->Bytes(); + Require(bytes.size() == readable + overhang, "an overhanging vertex fetch changed its size"); + Require(std::all_of(bytes.begin(), bytes.begin() + static_cast(readable), [](std::byte value) { return value == std::byte{0x7a}; }), "an overhanging vertex fetch did not copy the readable bytes"); + Require(std::all_of(bytes.begin() + static_cast(readable), bytes.end(), [](std::byte value) { return value == std::byte{0}; }), "an overhanging vertex fetch did not zero its last record"); + } + expectFailure([&] { CopyZeroPaddedDrawInput(mockContext(), address, readable + stride, readable + stride, stride); }, "is not readable"); + expectFailure([&] { CopyZeroPaddedDrawInput(mockContext(), address, readable + overhang, readable + overhang, 0); }, "is not readable"); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + munmap(block, page * 2); +#endif +} + void vertexZeroFillMergeTests() { using AgcDriver::Graphics::PlanVertexCopies; using AgcDriver::Graphics::SoloZeroPaddedFetchIndices; @@ -9733,6 +9991,10 @@ int main(int argc, char** argv) { stencilReferenceUnusedTests(); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--command-batch-fence-failure") { + commandBatchFenceFailureTests(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--pipeline-journal") { pipelineJournalTests(); std::cout << "pipeline journal recipe, record and replay tests passed\n"; @@ -9893,6 +10155,7 @@ int main(int argc, char** argv) { hardwareScreenOffsetTests(); srgb8TargetTests(); DepthClipTests(); + DepthRemapTests(); DepthStencilTests(); OneDimensionalColorTests(); htileDepthClearTests(); @@ -9971,6 +10234,7 @@ int main(int argc, char** argv) { samplerTableDeviceLimitTests(); misalignedShaderDataTests(); debugBranchTests(); + hostImportBudgetTests(); textureCacheBudgetTests(); sampledTextureBudgetTests(); storageTextureBudgetTests(); @@ -9985,6 +10249,7 @@ int main(int argc, char** argv) { storeAtLabelTests(); nullVertexDescriptorTests(); vertexZeroFillTests(); + vertexOverhangTests(); vertexZeroFillMergeTests(); parallelCompareTests(); pixelParameterSlotTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp b/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp index 45c02ae30e..6f127df513 100644 --- a/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp @@ -58,6 +58,8 @@ struct MockDevice { std::map memories; std::vector binds; VkResult bindResult = VK_SUCCESS; + VkResult allocationResult = VK_SUCCESS; + std::uint64_t allocationAttempts = 0; std::uint64_t nextAddress = 0x500000000ull; VkDeviceSize sparseAlignment = Alignment; }; @@ -119,6 +121,8 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu } VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + ++mock.allocationAttempts; + if (mock.allocationResult != VK_SUCCESS) return mock.allocationResult; *memory = reinterpret_cast(mock.next++); mock.memories[*memory] = info->allocationSize; return VK_SUCCESS; @@ -340,6 +344,34 @@ void LargerSparseBlockIsDecidedBeforeImporting() { Expect(mock.memories.size() == memories && mock.binds.empty(), "a chunk was imported or bound for a range that does not take the sparse path"); } +void RetainsFailedWindowAcrossUnrelatedMutation() { + constexpr std::uint64_t window = 4ull << 20u; + SharedRange range(window * 2, 0, true); + const auto context = mockContext(); + SetHostImportRangeWindow(window); + mock.allocationResult = VK_ERROR_INVALID_EXTERNAL_HANDLE; + const auto address = range.Address() + window; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + const auto attempts = mock.allocationAttempts; + Expect(HostImportFor(context, address, Alignment) == nullptr, "the failed import window was accepted"); + const auto failedAttempts = mock.allocationAttempts; + Expect(failedAttempts > attempts, "the import window did not reach allocation"); + Expect(HostImportFor(context, address, Alignment) == nullptr && mock.allocationAttempts == failedAttempts, "an unchanged failed window was retried"); + SharedRange unrelated(Alignment, 0, true); + Expect(HostImportFor(context, address, Alignment) == nullptr && mock.allocationAttempts == failedAttempts, "an unrelated registry change retried a failed import window"); + Expect(HostImportFor(context, range.Address(), Alignment) == nullptr && mock.allocationAttempts > failedAttempts, "a different import window inherited the first window's failure"); + const auto beforeRemap = mock.allocationAttempts; + { + GuestAllocations::Mutation mutation; + mutation.Remove(range.backing.base); + mutation.Add(range.backing.base, static_cast(range.backing.bytes), true, true, true); + } + mock.allocationResult = VK_SUCCESS; + const auto* imported = HostImportFor(context, address, Alignment); + Expect(imported != nullptr && imported->base == address && imported->bytes == window && mock.allocationAttempts > beforeRemap, "a remapped failed import window was not retried successfully"); + SetHostImportRangeWindow(64ull << 20u); +} + void FailureAfterTheDecisionThrows(bool sealed, VkResult bind, const char* step) { const auto bytes = RangeBytes(); SharedRange range(bytes, 0, sealed); @@ -382,6 +414,7 @@ int main() { LargerSparseBlockIsDecidedBeforeImporting(); FailureAfterTheDecisionThrows(false, VK_SUCCESS, "UDMABUF_CREATE"); FailureAfterTheDecisionThrows(true, VK_ERROR_DEVICE_LOST, "vkQueueBindSparse"); + RetainsFailedWindowAcrossUnrelatedMutation(); } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index e4c841603a..c5d09b3e1e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -38,6 +38,7 @@ #include "VertexRange_vert_spv.h" #include "VertexRangeProducer_frag_spv.h" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetTransfer.hpp" +#include "BdaAbi.hpp" #include #ifdef _WIN32 #ifndef NOMINMAX @@ -391,6 +392,14 @@ public: bool NullDescriptors() const { return nullDescriptors; } const Context& GetContext() const { return context; } bool Graphics() const { return graphics; } + bool Timestamps() const { + std::uint32_t count = 0; + const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); + queues(context.physical, &count, nullptr); + std::vector families(count); + queues(context.physical, &count, families.data()); + return families.at(context.queueFamily).timestampValidBits != 0; + } VkPhysicalDeviceProperties Properties() const { VkPhysicalDeviceProperties properties{}; function("vkGetPhysicalDeviceProperties")(context.physical, &properties); @@ -1194,6 +1203,44 @@ void afterRecordedWorkTests(const Device& device, Recorder& recorder) { Require(seen == 1, "an action ran before the completion label recorded ahead of it"); } +int gpuTimingCompletionTests(const Device& device, bool incomplete) { + const auto& context = device.GetContext(); + if (!device.Timestamps()) { + std::cout << "skipped, queue timestamp queries unavailable\n"; + return 77; + } + Require(Recorder::GpuTimingEnabled(), "GPU timing completion test requires APS5_PROFILE_GPU"); + std::unique_lock gpu(GpuMutex()); + int completed = 0; + Recorder recorder(context); + recorder.Activate(); + for (int round = 0; round < 3; ++round) { + const bool missingEnd = incomplete && round == 1; + const auto first = recorder.BeginGpuTiming(0x12340); + Require(first != Recorder::NoTiming, "first GPU timing range was not allocated"); + if (!missingEnd) recorder.EndGpuTiming(first); + const auto second = recorder.BeginGpuTiming(0x12341); + Require(second != Recorder::NoTiming && second != first, "second GPU timing range was not allocated"); + recorder.EndGpuTiming(second); + recorder.OnComplete([&] { ++completed; }); + const auto serial = recorder.Submissions() + 1; + recorder.Submit(); + recorder.Sync(); + Require(completed == round + 1 && recorder.Idle(), "GPU timing prevented batch completion"); + std::size_t missing = 0; + const auto batches = recorder.CompletedBatches(serial - 1, serial, missing); + Require(missing == 0 && batches.size() == 1 && batches.front().serial == serial, "GPU timing batch completion record is missing"); + const auto& batch = batches.front(); + if (missingEnd) { + Require(batch.gpuStartNs == 0 && batch.gpuEndNs == 0, "incomplete query batch published GPU timings"); + } else { + Require(batch.gpuStartNs > 0 && batch.gpuEndNs >= batch.gpuStartNs, "complete query batch lost GPU timings"); + } + } + std::cout << "GPU timing batch completion checked, incomplete=" << incomplete << '\n'; + return 0; +} + void batchStampTests(Recorder& recorder) { Require(recorder.Idle(), "batch stamps: the recorder is busy"); double previousEnd = 0; @@ -2592,11 +2639,11 @@ void resourceReadTests(const Device& device, Recorder& recorder) { HostImportFor(context, address, bytes); } -void readWrittenStagingTests(const Device& device, Recorder& recorder) { +bool readWrittenStagingTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); if (context.hostImportAlignment == 0) { std::cout << "host imports unavailable: read and written staging not tested\n"; - return; + return false; } constexpr std::size_t bytes = 8u << 20u; #ifdef _WIN32 @@ -2613,7 +2660,7 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) { } if (HostImportFor(context, address, bytes) == nullptr) { std::cout << "host import of the test block refused: read and written staging not tested\n"; - return; + return false; } const auto boundInPlace = [&](std::uint64_t element, std::size_t elementBytes, bool read) { GuestBufferMemory memory(context); @@ -2630,10 +2677,10 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) { Require(!boundInPlace(address + 4096, 1u << 20u, false), "a written 1 MiB element was not staged"); Require(boundInPlace(address + 4096, 4u << 20u, false), "a written-only 4 MiB element was staged past the written window"); Require(!boundInPlace(address + 4096, 4u << 20u, true), "a read and written 4 MiB element was bound in place"); - const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read) { + const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read, std::uint32_t stride, std::uint32_t extent) { const auto element = address + 4096; - constexpr std::uint32_t stride = 16; - constexpr std::uint32_t records = (4u << 20u) / stride; + constexpr std::uint32_t elementBytes = 4u << 20u; + const std::uint32_t records = stride != 0 ? elementBytes / stride : elementBytes; ShaderRecompiler::RecompileResult program; ShaderRecompiler::DescriptorBinding binding; binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; @@ -2644,25 +2691,30 @@ void readWrittenStagingTests(const Device& device, Recorder& recorder) { binding.guestDescriptor = {static_cast(element), (static_cast(element >> 32u) & 0xffffu) | (stride << 16u), records, 0x31000000u}; binding.bufferWritten = {true}; binding.bufferRead = {read}; + binding.bufferExtent = {extent}; program.bindings.push_back(binding); const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0}; ShaderResources resources(context, compute, {}, false, dispatchThreads); const auto reads = resources.InPlaceReads(); - const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return range.first <= element && range.second >= element + records * stride; }); + const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return range.first <= element && range.second >= element + elementBytes; }); resources.MarkGpuWrites(recorder); recorder.Sync(); return direct; }; - Require(!buildInPlace((4u << 20u) / 16u, true), "a dispatch sweeping a read and written 4 MiB element bound it in place"); - Require(buildInPlace(1024, true), "a dispatch touching 16 KiB of a read and written 4 MiB element staged all of it"); - Require(buildInPlace(0, true), "a dispatch of unknown size staged a read and written 4 MiB element"); - Require(buildInPlace((4u << 20u) / 16u, false), "a dispatch sweeping a written-only 4 MiB element staged it"); + Require(!buildInPlace((4u << 20u) / 16u, true, 16, 0), "a dispatch sweeping a read and written 4 MiB element bound it in place"); + Require(buildInPlace(1024, true, 16, 0), "a dispatch touching 16 KiB of a read and written 4 MiB element staged all of it"); + Require(buildInPlace(0, true, 16, 0), "a dispatch of unknown size staged a read and written 4 MiB element"); + Require(buildInPlace((4u << 20u) / 16u, false, 16, 0), "a dispatch sweeping a written-only 4 MiB element staged it"); + Require(!buildInPlace((4u << 20u) / 32u, true, 0, 32), "a dispatch sweeping a read and written 4 MiB raw element bound it in place"); + Require(buildInPlace(1024, true, 0, 32), "a dispatch touching 32 KiB of a read and written 4 MiB raw element staged all of it"); + Require(buildInPlace((4u << 20u) / 32u, true, 0, 0), "a raw element without an access extent was staged"); Require(std::all_of(static_cast(block), static_cast(block) + bytes, [](std::byte value) { return value == std::byte{0x11}; }), "a staged copy-back changed the guest bytes"); { GuestAllocations::Mutation mutation; mutation.Remove(block); } HostImportFor(context, address, bytes); + return true; } bool sparseRunImportTests(const Device& device, bool copied) { @@ -3081,7 +3133,8 @@ void drawDescriptorWriteTests(const Device& device, Recorder& recorder) { const auto original = updates.calls.front(); Require(original.buffers.size() == 2 && original.images.size() == 2, "descriptor test lacks mixed bindings and an image array"); for (std::size_t allocation = 0; allocation < 2; ++allocation) { - const ShaderResources::MovedBuffer moved{allocation, 0, 16, {5, 6, 7, 8}}; + const std::array words{5, 6, 7, 8}; + const ShaderResources::MovedBuffer moved{allocation, 0, 16, words}; const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1)); Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot"); Require(updates.calls.size() == allocation + 2 && updates.copies == 0, "draw descriptors copied GPU memory or needed multiple updates"); @@ -3165,7 +3218,8 @@ void drawSnapshotTableSetTests(const Device& device, Recorder& recorder) { const ColorTarget target{}; ShaderResources resources(context, vertex, fragment, target, 0, 0); Require(resources.SplitSets(), "a fragment image table did not get its own descriptor set"); - const ShaderResources::MovedBuffer moved{0, 0, 16, {5, 6, 7, 8}}; + const std::array movedWords{5, 6, 7, 8}; + const ShaderResources::MovedBuffer moved{0, 0, 16, movedWords}; const auto draw = resources.PrepareDrawBindings(snapshots, std::span(&moved, 1)); Require(draw != nullptr && draw->snapshots.size() == 1, "moved shader data did not create a draw snapshot"); const auto& written = updates.calls.back().written; @@ -4806,17 +4860,21 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche std::uint32_t height; std::uint32_t mipCount; std::uint64_t offset; + bool direct = false; }; constexpr Case cases[] = { {"64 KiB R_X 200x150", TextureTileMode::kR64KBX, 200, 150, 1, 0}, {"64 KiB R_X 200x150, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 0}, - {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0}, + {"64 KiB R_X 520x260, 6 mips", TextureTileMode::kR64KBX, 520, 260, 6, 0, true}, {"64 KiB R_X 200x150 off a 64 KiB boundary, 3 mips", TextureTileMode::kR64KBX, 200, 150, 3, 4096}, {"4 KiB S 100x70, 2 mips", TextureTileMode::kStandard4KB, 100, 70, 2, 0}, {"linear 200x150", TextureTileMode::kLinear, 200, 150, 1, 0}, + {"64 KiB R_X 512x512, 3 mips, no padding below the tail", TextureTileMode::kR64KBX, 512, 512, 3, 0, true}, + {"64 KiB R_X 1000x700", TextureTileMode::kR64KBX, 1000, 700, 1, 0, true}, + {"64 KiB R_X 1900x1100, across two 8 MiB slabs", TextureTileMode::kR64KBX, 1900, 1100, 1, 0, true}, }; const auto pattern = [](std::size_t i) { return static_cast(i * 131u + 7u); }; - constexpr std::array red{255, 0, 0, 255}; + constexpr std::array, 2> colours{{{255, 0, 0, 255}, {0, 255, 0, 255}}}; std::string failures; for (const auto& test : cases) { const std::string what = std::string(watched ? "(watched) " : "") + test.name; @@ -4886,34 +4944,40 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche { auto image = StorageTexture::Create(context, detiler, resource, 0); const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, test.mipCount, 0, 1}; - const auto commands = recorder.Commands(); - recorder.Keep(image); - const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}}; - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); - context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); - RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); - image->MarkDirty(); - image->WriteBack(); - recorder.Submit(); - device.WaitQueue(); - recorder.Sync(); - std::vector read(bytes); - if (watched) AgcDriver::GuestMemory::Read(blockAddress, read); - else std::memcpy(read.data(), memory, bytes); - std::size_t texels = 0, garbage = 0, outside = 0; - for (std::size_t i = 0; i < bytes; i += 4) { - const bool inside = i >= test.offset && i < test.offset + surfaceBytes; - bool isTexel = inside, isPattern = true; - for (std::size_t c = 0; c < 4; ++c) { - const auto byte = std::to_integer(read[i + c]); - isTexel = isTexel && byte == red[c]; - isPattern = isPattern && byte == pattern(i + c); + for (std::size_t pass = 0; pass < (test.direct ? colours.size() : 1u); ++pass) { + const auto& colour = colours[pass]; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue value{{colour[0] / 255.0f, colour[1] / 255.0f, colour[2] / 255.0f, colour[3] / 255.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + const auto slabRetilesBefore = StorageTexture::SlabRetiles(); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto slabRetiled = StorageTexture::SlabRetiles() - slabRetilesBefore; + if (test.direct && watched && UnitShadowEnabled() && AgcDriver::GuestMemory::Watched(address, surfaceBytes) && slabRetiled == 0) failures += what + ": no window was retiled straight into its unit-shadow slab\n"; + std::vector read(bytes); + if (watched) AgcDriver::GuestMemory::Read(blockAddress, read); + else std::memcpy(read.data(), memory, bytes); + std::size_t texels = 0, garbage = 0, outside = 0; + for (std::size_t i = 0; i < bytes; i += 4) { + const bool inside = i >= test.offset && i < test.offset + surfaceBytes; + bool isTexel = inside, isPattern = true; + for (std::size_t c = 0; c < 4; ++c) { + const auto byte = std::to_integer(read[i + c]); + isTexel = isTexel && byte == colour[c]; + isPattern = isPattern && byte == pattern(i + c); + } + if (isTexel) ++texels; + else if (!isPattern) ++(inside ? garbage : outside); + } + if (garbage != 0 || outside != 0 || texels != texelCount) { + failures += what + ", write-back " + std::to_string(pass + 1) + ": stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n"; } - if (isTexel) ++texels; - else if (!isPattern) ++(inside ? garbage : outside); - } - if (garbage != 0 || outside != 0 || texels != texelCount) { - failures += what + ": the write-back stored " + std::to_string(texels) + " texels of " + std::to_string(texelCount) + " and changed " + std::to_string(garbage) + " words no texel holds (" + std::to_string(outside) + " past the surface)\n"; } } recorder.Sync(); @@ -7036,6 +7100,31 @@ void multisampleGuestAccessTests(const Device& device, Recorder& recorder) { refused([&] { AgcDriver::GuestMemory::ReadCommitted(address + 256u, texels); }, "a CPU read", "a committed CPU read"); refused([&] { AgcDriver::GuestMemory::ReadCommittedUnsynced(address + 256u, texels); }, "a CPU read", "a committed CPU read without the flush hook"); refused([&] { RequireNoMultisampleTarget(address, 64u, "a DMA copy"); }, "a DMA copy", "a DMA copy"); + const auto captureSrt = [](std::uint64_t srt, AgcDriver::ShaderMemory::PendingWriteQuery pendingWrite) { + static constexpr std::array code{0xf4080100u, 0xfa000000u, 0xbf8cc07fu, 0xe0700000u, 0x80010000u, 0xbf810000u}; + const std::array userData{static_cast(srt), static_cast(srt >> 32u)}; + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Compute, 0x24000u, code, 0, {}}; + request.context.waveSize = 32u; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderRecompiler::ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + request.layout.pushConstantSizeBytes = 128; + AgcDriver::ShaderMemory memory({}, pendingWrite); + static_cast(memory.Capture(request)); + }; + refused([&] { captureSrt(address + 256u, nullptr); }, "a CPU read", "a shader capture"); + captureSrt(address + 4u * 65536u + 256u, nullptr); + refused([&] { captureSrt(address + 256u, nullptr); }, "a CPU read", "a shader capture on a pooled page"); + const auto raw = [](std::uint64_t, std::size_t, std::span) { return AgcDriver::ShaderMemory::PendingWrite::Raw; }; + const auto synced = [](std::uint64_t, std::size_t, std::span) { return AgcDriver::ShaderMemory::PendingWrite::Sync; }; + refused([&] { captureSrt(address + 256u, raw); }, "a CPU read", "a shader capture's raw word read"); + refused([&] { captureSrt(address + 256u, synced); }, "a CPU read", "a shader capture's synced word read"); + captureSrt(address + 4u * 65536u + 256u, raw); ShaderRecompiler::RecompileResult program; ShaderRecompiler::DescriptorBinding binding; binding.kind = ShaderRecompiler::DescriptorKind::SampledImage; @@ -8918,6 +9007,9 @@ int main(int argc, char** argv) { Device& device = *created; if (argc == 2 && std::string_view(argv[1]) == "--host-import-lifetime-only") return RunHostImportLifetimeTests(device.GetContext()); if (argc == 2 && std::string_view(argv[1]) == "--host-import-mutation-race-only") return RunHostImportMutationRaceTests(device.GetContext()); + if (argc == 2 && (std::string_view(argv[1]) == "--gpu-timing-complete-only" || std::string_view(argv[1]) == "--gpu-timing-incomplete-only")) { + return gpuTimingCompletionTests(device, std::string_view(argv[1]) == "--gpu-timing-incomplete-only"); + } if (argc == 2 && std::string_view(argv[1]) == "--import-watch-only") { importWatchTests(device); return 0; @@ -9059,6 +9151,11 @@ int main(int argc, char** argv) { std::cout << "Completion label storage and ordering tests passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--read-written-staging-only") { + if (!readWrittenStagingTests(device, recorder)) return 77; + std::cout << "read and written staging tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--written-buffer-order-only") { if (!writtenBufferOrderTests(device, recorder)) return 77; std::cout << "written buffer order tests passed\n"; @@ -9106,6 +9203,19 @@ int main(int argc, char** argv) { splitWriteBackTests(device, recorder); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--storage-write-back-only") { + if (device.GetContext().hostImportAlignment == 0) { + std::cout << "skipped, host imports are unavailable\n"; + return 77; + } + storageRefreshTests(device, recorder, false); + storageRefreshTests(device, recorder, true); + writeBackPaddingTests(device, recorder, false); + writeBackPaddingTests(device, recorder, true); + singlePassTests(device, recorder, false); + singlePassTests(device, recorder, true); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--cmask-only") { cmaskPassTests(device, recorder); std::cout << "CMASK copied clear, write-back and failure tests passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 687ea44831..f335fb2c69 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -84,7 +84,7 @@ void removeCacheDirectory(const std::filesystem::path& directory) { } bool sameBinding(const DescriptorBinding& left, const DescriptorBinding& right) { - return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.imageShapes == right.imageShapes && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.bufferRead == right.bufferRead && left.samplerUnnormalized == right.samplerUnnormalized && left.samplerIntegerBorder == right.samplerIntegerBorder && left.samplerCoordinatesIgnored == right.samplerCoordinatesIgnored && left.imageUnnormalized == right.imageUnnormalized && left.imageUnnormalizedBaseLevel == right.imageUnnormalizedBaseLevel && left.imageSamplers == right.imageSamplers; + return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.imageShapes == right.imageShapes && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.bufferRead == right.bufferRead && left.bufferExtent == right.bufferExtent && left.samplerUnnormalized == right.samplerUnnormalized && left.samplerIntegerBorder == right.samplerIntegerBorder && left.samplerCoordinatesIgnored == right.samplerCoordinatesIgnored && left.imageUnnormalized == right.imageUnnormalized && left.imageUnnormalizedBaseLevel == right.imageUnnormalizedBaseLevel && left.imageSamplers == right.imageSamplers; } bool sameBindings(const std::vector& left, const std::vector& right) { @@ -125,6 +125,7 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right const std::string prefix = std::string(what) + ": "; require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ"); require(left.pushConstants == right.pushConstants, prefix + "push constants differ"); + require(left.readsDepthRemapPush == right.readsDepthRemapPush, prefix + "depth remap push reads differ"); require(left.specialization == right.specialization, prefix + "specialization constants differ"); require(left.workgroupMemoryDwords == right.workgroupMemoryDwords, prefix + "workgroup memory stride differs"); require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ"); @@ -181,6 +182,7 @@ DescriptorBinding sampleBinding(std::uint32_t seed) { binding.imageUnnormalizedBaseLevel = {false, true, seed % 3 == 0}; binding.imageSamplers = {0x5u, 0u, 0x80000000u}; binding.bufferRead = {true, false, false, true, true, false}; + binding.bufferExtent = {32, 0, 16, 4096, 8, 0}; return binding; } @@ -195,6 +197,7 @@ RecompileResult sampleResult() { result.bindings[1].role = DescriptorRole::GuestBuffers; result.bindings[1].imageShape.reset(); result.pushConstants = {std::byte{1}, std::byte{0xff}, std::byte{0}, std::byte{0x80}, std::byte{7}}; + result.readsDepthRemapPush = true; result.bdaAbiVersion = 3; result.memoryOffsetDword = 7; result.hostSubgroupSize = 32; @@ -2141,6 +2144,10 @@ std::filesystem::path planEntryPath(PlanFixture& fixture) { int runPlanLoadingProcess(std::string_view mode) { if (mode == "--load-plan-switch") setEnvironment("APS5_TRACE_IMAGE_TABLE", "1"); if (mode == "--load-plan-off") setEnvironment("APS5_NO_PERF_DISK_PLANS", "1"); + if (mode == "--load-plan-reuse-off") { + setEnvironment("APS5_NO_TABLE_SNAPSHOT_REUSE", "1"); + setEnvironment("APS5_NO_TABLE_RESOLUTION_REUSE", "1"); + } auto fixtures = planFixtures(); const auto count = fixtures.size(); std::vector> plans; @@ -2156,6 +2163,8 @@ int runPlanLoadingProcess(std::string_view mode) { } if (mode == "--load-plan") { require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "the second process ran the front end" + counts); + } else if (mode == "--load-plan-reuse-off") { + require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "turning the image table reuse off changed the stored plan or variant keys" + counts); } else if (mode == "--load-plan-off") { require(builds == count && totals.planHits == 0u && totals.planMisses == 0u && totals.planWrites == 0u && totals.hits == count && totals.misses == 0u, "APS5_NO_PERF_DISK_PLANS=1 used the stored plans or missed the stored variants" + counts); } else { @@ -2175,6 +2184,8 @@ int runPlanLoadingProcess(std::string_view mode) { const auto after = ShaderDiskCache::Totals(); require(FrontEndBuilds() == 1u && after.planHits == count && after.planMisses == 1u, "a changed code word reused a stored plan"); std::cout << "second process prepared " << count << " shaders from stored plans without running the front end\n"; + } else if (mode == "--load-plan-reuse-off") { + std::cout << "with the image table reuse off, a process loaded all " << count << " stored plans and variants with the same output\n"; } else if (mode == "--load-plan-off") { std::cout << "APS5_NO_PERF_DISK_PLANS=1 ran the front end for all " << count << " shaders with the same output\n"; } else { @@ -2218,6 +2229,7 @@ void verifyPlanAcrossProcesses(const char* self) { }; run("--load-plan"); run("--load-plan-off"); + run("--load-plan-reuse-off"); std::vector entry; const auto damaged = planEntryPath(fixtures.front()); require(ReadWholeFile(damaged, entry), "cannot read a stored plan entry"); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index d7b34bc9b9..7b43184c7e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1,4 +1,8 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "Optimization/DescriptorBindingBuilder.hpp" #include "Optimization/RequestMemoryView.hpp" @@ -37,6 +41,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -1291,7 +1299,7 @@ std::array TableSelected(const ShaderRecompiler::ImageTableSna namespace Abi = ShaderRecompiler::ImageTableAbi; const auto& column = tables.tables.at(table); require(column.records == 1u && column.keys == 1u && column.codes.size() == 1u && (column.codes[0] & Abi::PoisonFlag) == 0u, message); - return tables.words.at(column.codes[0]).dwords; + return column.words.at(column.codes[0]).dwords; } void verifyImageTableKeys() { @@ -1423,6 +1431,827 @@ void verifyImageTableBulkReads() { const auto& cut = cutByRun->snapshot.tables.tables.at(0); require(cut.keys == 4u && (cut.codes[0] & Abi::PoisonFlag) == 0u && cut.codes[1] == unmapped && cut.codes[2] == unmapped && cut.codes[3] == unmapped, "image table bulk reads: a record crossing into an unmapped page was not Unmapped poison"); require(cutByRun->snapshot.tables == cutByWord->snapshot.tables && cutWordReads.wordReads == 12u && cutRunReads.wordReads == 0u && cutRunReads.bulkWords == 12u, "image table bulk reads: a record cut by an unmapped page read other words in runs than word by word"); + + place(1028u, 4u); + CountingMemory insideWordReads{reinterpret_cast(words.data() + 1028u), reinterpret_cast(words.data() + 1060u)}; + const auto insideByWord = capture(request, insideWordReads, false); + AgcDriver::ShaderMemory inside({}); + const auto insideCaptured = inside.Capture(request); + const auto regions = inside.Regions(); + const auto first = static_cast(reinterpret_cast(words.data())); + const auto past = first + words.size() * sizeof(std::uint32_t); + const auto tableFirst = first + 1028u * sizeof(std::uint32_t); + const auto tablePast = first + 1060u * sizeof(std::uint32_t); + const bool exact = std::all_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return region.guestAddress + region.bytes.size() <= first || region.guestAddress >= past || (region.guestAddress >= tableFirst && region.guestAddress + region.bytes.size() <= tablePast); }); + require(insideCaptured->snapshot.tables == insideByWord->snapshot.tables && TableCovered(regions, words.data() + 1028u, 32u * sizeof(std::uint32_t)) && exact, "image table bulk reads: run reads inside 16-word blocks changed the snapshot or captured words outside the records"); +} + +std::uint64_t pendingTableWord = 0; +std::uint32_t pendingTableValue = 0; +bool pendingTableKnown = false; + +AgcDriver::ShaderMemory::PendingWrite PendingTableWrite(std::uint64_t address, std::size_t bytes, std::span known) { + using Pending = AgcDriver::ShaderMemory::PendingWrite; + if (pendingTableWord < address || pendingTableWord - address >= bytes) return Pending::None; + if (!pendingTableKnown) return Pending::Sync; + if (known.size() == sizeof(pendingTableValue)) std::memcpy(known.data(), &pendingTableValue, sizeof(pendingTableValue)); + return Pending::KnownValue; +} + +std::uint64_t deferredTableRecord = 0; +std::array deferredTableWords{}; +bool deferredTableStored = false; + +void StoreDeferredTableRecord(std::uint64_t address, std::size_t bytes) { + if (deferredTableStored || deferredTableRecord < address || deferredTableRecord - address >= bytes) return; + deferredTableStored = true; + AgcDriver::GuestMemory::StoreOwnBytes(deferredTableRecord, std::as_bytes(std::span(deferredTableWords))); +} + +std::uint32_t* AllocateWatchedTable(std::size_t bytes) { + auto* heap = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536)); + require(heap != nullptr, "table snapshot reuse: cannot allocate the table in the guest arena"); + GuestArena::GuestArenaCommit_nid_postfix(heap, bytes, 0x04u, bytes); +#ifndef _WIN32 + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(heap, bytes); +#endif + return heap; +} + +void ProtectTablePage(std::uintptr_t page, std::uint32_t protection) { + GuestArena::GuestArenaSetProtection_nid_postfix(page, 4096u, protection); +#ifdef _WIN32 + DWORD previous = 0; + require(VirtualProtect(reinterpret_cast(page), 4096u, protection, &previous) != 0, "table snapshot reuse: cannot protect a table page"); +#endif + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096u); +} + +void verifyTableSnapshotReuse() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + constexpr std::uint32_t Records = 4096; + constexpr std::size_t Bytes = Records * 32u; + const auto texture = [](std::uint32_t index) { return TableT(reinterpret_cast(static_cast(0x100000000ull + index * 256ull)), TableFormat8888UNorm, TableType2D, 1u + index % 5u); }; + const auto fill = [&](std::uint32_t* table, std::uint32_t records) { + for (std::uint32_t record = 0; record < records; record++) { + const auto words = texture(record % 1024u); + std::copy(words.begin(), words.end(), table + static_cast(record) * 8u); + } + }; + struct Outcome { + ImageTableSnapshot tables; + bool covered = false; + ImageTableReuseCounts reuse; + AgcDriver::ShaderMemory::SupplyCounts supplied{}; + const std::byte* tableBytes = nullptr; + std::size_t tableRegionBytes = 0; + std::vector> tableRegions; + }; + const auto sortedDisjoint = [](const std::vector& regions) { + for (std::size_t index = 1; index < regions.size(); index++) { + if (regions[index - 1].guestAddress + regions[index - 1].bytes.size() > regions[index].guestAddress) return false; + } + return true; + }; + const auto holds = [](const std::vector& regions, const std::uint32_t* table, std::size_t bytes, const std::uint32_t* expected) { + const auto begin = static_cast(reinterpret_cast(table)); + std::size_t found = 0; + for (const auto& region : regions) { + const auto from = std::max(region.guestAddress, begin); + const auto to = std::min(region.guestAddress + region.bytes.size(), begin + bytes); + if (from >= to) continue; + if (std::memcmp(region.bytes.data() + (from - region.guestAddress), reinterpret_cast(expected) + (from - begin), static_cast(to - from)) != 0) return false; + found += static_cast(to - from); + } + return found == bytes; + }; + const auto capture = [&](const RecompileRequest& request, const std::uint32_t* table, std::size_t bytes, AgcDriver::ShaderMemory::PendingWriteQuery pending) { + const auto before = ResourceMaterializer::TableReuse(); + const auto suppliedBefore = AgcDriver::ShaderMemory::Supplies(); + Outcome outcome; + AgcDriver::ShaderMemory memory({}, pending); + outcome.tables = memory.Capture(request)->snapshot.tables; + const auto regions = memory.Regions(); + const auto suppliedAfter = AgcDriver::ShaderMemory::Supplies(); + outcome.supplied = {suppliedAfter.shared - suppliedBefore.shared, suppliedAfter.copied - suppliedBefore.copied, suppliedAfter.spilled - suppliedBefore.spilled}; + for (const auto& region : regions) { + if (region.guestAddress < reinterpret_cast(table) || region.guestAddress >= reinterpret_cast(table) + bytes) continue; + if (outcome.tableBytes == nullptr) outcome.tableBytes = region.bytes.data(); + if (region.bytes.data() == outcome.tableBytes + outcome.tableRegionBytes) outcome.tableRegionBytes += region.bytes.size(); + outcome.tableRegions.emplace_back(region.guestAddress, region.bytes.size()); + } + outcome.covered = TableCovered(regions, table, bytes) && std::all_of(regions.begin(), regions.end(), [&](const MemoryRegion& region) { + return region.guestAddress < reinterpret_cast(table) || region.guestAddress >= reinterpret_cast(table) + bytes || std::memcmp(reinterpret_cast(static_cast(region.guestAddress)), region.bytes.data(), region.bytes.size()) == 0; + }); + const auto after = ResourceMaterializer::TableReuse(); + outcome.reuse = {after.unwritten - before.unwritten, after.compared - before.compared, after.wordsRead - before.wordsRead, after.shared - before.shared, after.columns - before.columns}; + return outcome; + }; + const auto recordWords = [](const Outcome& outcome, std::uint32_t record) { return outcome.tables.tables.at(0).words.at(outcome.tables.tables.at(0).codes.at(record)).dwords; }; + const auto heldRecords = [&](const Outcome& outcome, const std::uint32_t* table, std::uint32_t records) { + for (std::uint32_t record = 0; record < records; record++) { + std::array words{}; + std::copy_n(table + static_cast(record) * 8u, 8u, words.begin()); + if (recordWords(outcome, record) != words) return false; + } + return true; + }; + const auto sharesColumn = [](const Outcome& left, const Outcome& right) { + const auto& a = left.tables.tables.at(0); + const auto& b = right.tables.tables.at(0); + return a.codes.Shares(b.codes) && a.words.Shares(b.words) && a.readRanges.Shares(b.readRanges); + }; + const auto sharedColumnOf = [](const std::shared_ptr& capture, const Outcome& outcome) { + const auto& a = capture->snapshot.tables.tables.at(0); + const auto& b = outcome.tables.tables.at(0); + return a.codes.Shares(b.codes) && a.words.Shares(b.words) && a.readRanges.Shares(b.readRanges); + }; + + auto* heap = AllocateWatchedTable(Bytes); + fill(heap, Records); + const auto address = reinterpret_cast(heap); + const bool watched = AgcDriver::GuestMemory::CollectWrites(address, Bytes) != 0; + alignas(4096) static TableGuest guest; + FillTableSrt(guest, heap, 32u, Records); + const auto userData = TableUserData(guest); + const auto request = TableRequest(TableWholeCode, userData); + const auto expectUnwritten = [&](const Outcome& outcome, const std::string& what) { + require(heldRecords(outcome, heap, Records) && outcome.covered, ("table snapshot reuse: " + what + ": the snapshot or the captured words do not hold the table").c_str()); + if (watched) require(outcome.reuse.unwritten == 1u && outcome.reuse.compared == 0u && outcome.reuse.wordsRead == 0u, ("table snapshot reuse: " + what + ": an unwritten table was read again").c_str()); + else require(outcome.reuse.compared == 1u && outcome.reuse.wordsRead == Records * 8u, ("table snapshot reuse: " + what + ": an unchanged unwatched table was not compared and reused").c_str()); + }; + const auto expectReread = [&](const Outcome& outcome, const std::string& what) { + require(outcome.reuse.unwritten == 0u && outcome.reuse.wordsRead == Records * 8u, ("table snapshot reuse: " + what + ": the table was not read again exactly once").c_str()); + }; + + const auto fresh = capture(request, heap, Bytes, nullptr); + require(fresh.reuse.unwritten == 0u && fresh.reuse.compared == 0u && fresh.reuse.wordsRead == Records * 8u && heldRecords(fresh, heap, Records) && fresh.covered, "table snapshot reuse: the first snapshot is wrong"); + require(fresh.reuse.columns == 1u && fresh.reuse.shared == 0u, "table snapshot reuse: the first snapshot did not build one column"); + const std::vector original(heap, heap + Bytes / sizeof(std::uint32_t)); + const auto unchanged = capture(request, heap, Bytes, nullptr); + expectUnwritten(unchanged, "an unchanged table"); + require(unchanged.tables == fresh.tables, "table snapshot reuse: a reused snapshot differs from the snapshot it reuses"); + const auto reusedAgain = capture(request, heap, Bytes, nullptr); + expectUnwritten(reusedAgain, "an unchanged table reused again"); + require(sharesColumn(unchanged, fresh) && sharesColumn(reusedAgain, fresh) && unchanged.reuse.shared == 1u && reusedAgain.reuse.shared == 1u && unchanged.reuse.columns == 0u && reusedAgain.reuse.columns == 0u, "table snapshot reuse: an unchanged table reused twice did not share one column without a copy"); + if (watched) { + const auto sharedWords = [&](const Outcome& outcome) { return outcome.supplied.shared == Records * 8u && outcome.supplied.copied == 0u && outcome.supplied.spilled == 0u && outcome.tableRegionBytes == Bytes; }; + require(sharedWords(unchanged) && sharedWords(reusedAgain) && unchanged.tableBytes != nullptr && unchanged.tableBytes == reusedAgain.tableBytes && fresh.tableBytes != unchanged.tableBytes && fresh.supplied.shared == 0u, "table snapshot reuse: the captured words of an unchanged table were copied instead of shared"); + } + require(fresh.tableRegions.size() == Bytes / 4096u && unchanged.tableRegions == fresh.tableRegions && reusedAgain.tableRegions == fresh.tableRegions, "table snapshot reuse: a reused table's words are not reported in the regions a read of them reports"); + { + AgcDriver::ShaderMemory stages({}); + const auto suppliedBefore = AgcDriver::ShaderMemory::Supplies(); + const auto vertex = stages.Capture(request); + const auto vertexRecent = stages.TakeRecentRegions(); + const auto pixel = stages.Capture(request); + const auto pixelRecent = stages.TakeRecentRegions(); + const auto both = stages.Regions(); + require(sharedColumnOf(vertex, fresh) && sharedColumnOf(pixel, fresh) && sortedDisjoint(vertexRecent) && sortedDisjoint(pixelRecent) && sortedDisjoint(both) && holds(vertexRecent, heap, Bytes, original.data()) && holds(pixelRecent, heap, Bytes, original.data()) && holds(both, heap, Bytes, original.data()), "table snapshot reuse: two stages of one shader memory did not each capture the shared table once"); + alignas(4096) static TableGuest halfGuest; + FillTableSrt(halfGuest, heap, 32u, Records / 2u); + const auto halfUserData = TableUserData(halfGuest); + const auto half = stages.Capture(TableRequest(TableWholeCode, halfUserData)); + const auto all = stages.Regions(); + const auto suppliedAfter = AgcDriver::ShaderMemory::Supplies(); + require(half->snapshot.tables.tables.at(0).keys == Records / 2u && sortedDisjoint(all) && holds(all, heap, Bytes, original.data()), "table snapshot reuse: a read over supplied words reported them twice or changed them"); + if (watched) require(suppliedAfter.shared - suppliedBefore.shared == 2u * Records * 8u && suppliedAfter.copied == suppliedBefore.copied && suppliedAfter.spilled - suppliedBefore.spilled == Records * 8u, "table snapshot reuse: a second stage copied the shared words, or a read over them did not move them into pages once"); + } + AgcDriver::ShaderMemory heldMemory({}); + const auto heldCapture = heldMemory.Capture(request); + const auto freshColumn = [&](const Outcome& outcome, const std::string& what) { + require(outcome.reuse.columns == 1u && outcome.reuse.shared == 0u && !sharesColumn(outcome, fresh) && !(outcome.tables == fresh.tables) && outcome.supplied.shared == 0u, ("table snapshot reuse: " + what + ": the snapshot shares the old column or holds its words").c_str()); + require(heldRecords(unchanged, original.data(), Records) && heldRecords(reusedAgain, original.data(), Records) && sharesColumn(reusedAgain, fresh), ("table snapshot reuse: " + what + ": the old column changed under the snapshots sharing it").c_str()); + require(sharedColumnOf(heldCapture, fresh) && holds(heldMemory.Regions(), heap, Bytes, original.data()), ("table snapshot reuse: " + what + ": the words a live shader memory captured before it changed").c_str()); + }; + + const auto changeRecord = [&](std::uint32_t record, std::uint32_t replacement, const std::function&)>& store) { + const auto words = texture(replacement); + store(address + static_cast(record) * 32u, words); + }; + changeRecord(77u, 2000u, [](std::uint64_t at, const std::array& words) { std::memcpy(reinterpret_cast(static_cast(at)), words.data(), sizeof(words)); }); + const auto cpuStore = capture(request, heap, Bytes, nullptr); + expectReread(cpuStore, "a CPU store"); + require(heldRecords(cpuStore, heap, Records) && cpuStore.covered && recordWords(cpuStore, 77u) == texture(2000u), "table snapshot reuse: the snapshot after a CPU store misses the stored T#"); + freshColumn(cpuStore, "a CPU store"); + expectUnwritten(capture(request, heap, Bytes, nullptr), "the table after a CPU store"); + + changeRecord(1234u, 2001u, [](std::uint64_t at, const std::array& words) { AgcDriver::GuestMemory::Write(at, std::as_bytes(std::span(words)), 4); }); + const auto driverStore = capture(request, heap, Bytes, nullptr); + expectReread(driverStore, "a driver store (CP write, DMA copy or fill)"); + require(heldRecords(driverStore, heap, Records) && recordWords(driverStore, 1234u) == texture(2001u), "table snapshot reuse: the snapshot after a driver store misses the stored T#"); + freshColumn(driverStore, "a driver store"); + expectUnwritten(capture(request, heap, Bytes, nullptr), "the table after a driver store"); + + changeRecord(2048u, 2002u, [](std::uint64_t at, const std::array& words) { AgcDriver::GuestMemory::StoreOwnBytes(at, std::as_bytes(std::span(words))); }); + const auto gpuStore = capture(request, heap, Bytes, nullptr); + expectReread(gpuStore, "a GPU result stored by the driver"); + require(heldRecords(gpuStore, heap, Records) && recordWords(gpuStore, 2048u) == texture(2002u), "table snapshot reuse: the snapshot after a stored GPU result misses the stored T#"); + freshColumn(gpuStore, "a GPU result stored by the driver"); + expectUnwritten(capture(request, heap, Bytes, nullptr), "the table after a stored GPU result"); + + AgcDriver::GuestMemory::MarkWritten(address + 3000u * 32u, 32u); + const auto marked = capture(request, heap, Bytes, nullptr); + require(marked.reuse.unwritten == 0u && marked.reuse.compared == 1u && heldRecords(marked, heap, Records) && marked.tables == gpuStore.tables, "table snapshot reuse: a GPU write into an import (MarkWritten) did not make the table compared again"); + require(sharesColumn(marked, gpuStore) && marked.reuse.columns == 0u, "table snapshot reuse: a compared table did not share the column it matched"); + + pendingTableWord = address + 500u * 32u; + pendingTableValue = 0x0badf00du; + pendingTableKnown = true; + const auto known = capture(request, heap, Bytes, &PendingTableWrite); + expectReread(known, "a pending GPU write with a known value"); + require(recordWords(known, 500u)[0] == pendingTableValue && recordWords(known, 499u) == recordWords(gpuStore, 499u), "table snapshot reuse: the snapshot under a pending GPU write misses its known value"); + freshColumn(known, "a pending GPU write with a known value"); + require(!sharesColumn(known, gpuStore), "table snapshot reuse: a pending GPU write's snapshot shares the column read before it"); + pendingTableKnown = false; + const auto synced = capture(request, heap, Bytes, &PendingTableWrite); + require(synced.reuse.unwritten == 0u && heldRecords(synced, heap, Records), "table snapshot reuse: a table under a pending GPU write was reused without a read"); + require(!sharesColumn(synced, known) && recordWords(known, 500u)[0] == pendingTableValue, "table snapshot reuse: a synced read shares or changed the column of the known value it replaced"); + pendingTableWord = 0; + const auto settled = capture(request, heap, Bytes, nullptr); + require(settled.reuse.unwritten == 0u && heldRecords(settled, heap, Records), "table snapshot reuse: a snapshot taken under a pending GPU write was reused without a read"); + expectUnwritten(capture(request, heap, Bytes, nullptr), "the table after the pending GPU write"); + + deferredTableRecord = address + 3500u * 32u; + deferredTableWords = texture(2003u); + struct FlushHookReset { + ~FlushHookReset() { AgcDriver::GuestMemory::SetFlushHook(nullptr); } + } flushHookReset; + AgcDriver::GuestMemory::SetFlushHook(&StoreDeferredTableRecord); + const auto deferred = capture(request, heap, Bytes, nullptr); + AgcDriver::GuestMemory::SetFlushHook(nullptr); + require(deferredTableStored && deferred.reuse.unwritten == 0u && heldRecords(deferred, heap, Records) && recordWords(deferred, 3500u) == texture(2003u), "table snapshot reuse: a GPU result a read would store first was not stored and read before the table was reused"); + freshColumn(deferred, "a GPU result stored by the flush hook"); + expectUnwritten(capture(request, heap, Bytes, nullptr), "the table after a GPU result stored by the flush hook"); + + constexpr std::size_t UnrelatedBytes = 65536; + void* unrelated = ::operator new(UnrelatedBytes, std::align_val_t{UnrelatedBytes}); + { + GuestAllocations::Mutation mutation; + mutation.Add(unrelated, UnrelatedBytes, true, true, false); + } + const auto remapped = capture(request, heap, Bytes, nullptr); + { + GuestAllocations::Mutation mutation; + mutation.Remove(unrelated); + } + ::operator delete(unrelated, std::align_val_t{UnrelatedBytes}); + require(remapped.reuse.unwritten == 0u && remapped.reuse.compared == 1u && heldRecords(remapped, heap, Records), "table snapshot reuse: a table was reused without a read after a guest mapping changed"); + + auto& flipped = heap[100u * 8u + 1u]; + const auto keptWord = flipped; + flipped ^= 0x100u; + const auto changed = capture(request, heap, Bytes, nullptr); + flipped = keptWord; + const auto restored = capture(request, heap, Bytes, nullptr); + const auto& remappedColumn = remapped.tables.tables.at(0); + const auto& changedColumn = changed.tables.tables.at(0); + const auto& restoredColumn = restored.tables.tables.at(0); + require(changed.reuse.columns == 1u && !changedColumn.words.Shares(remappedColumn.words) && !(changedColumn.words == remappedColumn.words) && changedColumn.words.Hash() != remappedColumn.words.Hash(), "table snapshot reuse: a changed table matched the column read before the change"); + require(restored.reuse.columns == 1u && !sharesColumn(restored, remapped) && !sharesColumn(restored, changed) && restored.tables == remapped.tables && restoredColumn.codes.Hash() == remappedColumn.codes.Hash() && restoredColumn.words.Hash() == remappedColumn.words.Hash() && !(restored.tables == changed.tables), "table snapshot reuse: a table written back to its old words is not a new column equal to the old one"); + const SharedValues sealed(std::vector{1u, 2u, 3u}); + const auto alias = sealed; + const SharedValues equal(std::vector{1u, 2u, 3u}); + const SharedValues unequal(std::vector{1u, 2u, 4u}); + require(alias.Shares(sealed) && alias == sealed && !equal.Shares(sealed) && equal == sealed && equal.Hash() == sealed.Hash() && !unequal.Shares(sealed) && !(unequal == sealed) && unequal.Hash() != sealed.Hash() && SharedValues{}.Shares(SharedValues(std::vector{})), "table snapshot reuse: shared values compare by identity before their contents"); + + struct CapturedVariant { + std::vector> runs; + std::vector words; + std::vector shared; + }; + const auto captureVariant = [](const RecompileRequest& request) { + CapturedVariant variant; + AgcDriver::ShaderMemory memory({}); + static_cast(memory.Capture(request)); + const auto supplied = memory.SuppliedBlocks(); + for (const auto& region : memory.TakeRecentRegions()) { + variant.runs.emplace_back(region.guestAddress, region.guestAddress + region.bytes.size()); + const auto offset = variant.words.size(); + variant.words.resize(offset + region.bytes.size() / sizeof(std::uint32_t)); + std::memcpy(variant.words.data() + offset, region.bytes.data(), region.bytes.size()); + AgcDriver::AppendSharedWords(variant.shared, variant.runs.size() - 1u, offset, region.bytes, supplied); + } + return variant; + }; + const auto sameVariant = [](const CapturedVariant& left, const CapturedVariant& right, std::size_t& compared) { return left.runs == right.runs && AgcDriver::SameCapturedWords(left.words, left.shared, right.words, right.shared, &compared); }; + const auto sameBlock = [](const CapturedVariant& left, const CapturedVariant& right) { return left.shared.size() == 1u && right.shared.size() == 1u && !left.shared[0].block.owner_before(right.shared[0].block) && !right.shared[0].block.owner_before(left.shared[0].block); }; + const auto tableWords = static_cast(Records) * 8u; + const auto keptVariant = captureVariant(request); + const auto freshVariant = captureVariant(request); + std::size_t comparedWords = 0; + require(sameVariant(keptVariant, freshVariant, comparedWords) && keptVariant.words == freshVariant.words, "draw cache identity: two captures of an unchanged table did not match"); + if (watched) require(sameBlock(keptVariant, freshVariant) && keptVariant.shared[0].count == tableWords && comparedWords == keptVariant.words.size() - tableWords, "draw cache identity: two captures sharing a table column compared its words instead of its identity"); + else require(keptVariant.shared.empty() && comparedWords == keptVariant.words.size(), "draw cache identity: captures of an unwatched table were not compared word by word"); + + alignas(16) static std::array variantBuffer{}; + const auto variantBufferV = TableV(variantBuffer.data(), 0u, static_cast(sizeof(variantBuffer))); + const std::array variantKeptV{guest.srt[12], guest.srt[13], guest.srt[14], guest.srt[15]}; + std::copy(variantBufferV.begin(), variantBufferV.end(), guest.srt.begin() + 12); + const auto otherSrtVariant = captureVariant(request); + std::copy(variantKeptV.begin(), variantKeptV.end(), guest.srt.begin() + 12); + require(keptVariant.runs == otherSrtVariant.runs && !sameVariant(keptVariant, otherSrtVariant, comparedWords), "draw cache identity: a capture with other SRT words matched through the table column it shares"); + if (watched) require(sameBlock(keptVariant, otherSrtVariant), "draw cache identity: a capture with other SRT words did not share the unchanged table column"); + + auto& variantWord = heap[9u * 8u + 1u]; + const auto variantKeptWord = variantWord; + variantWord ^= 0x100u; + const auto writtenVariant = captureVariant(request); + variantWord = variantKeptWord; + require(keptVariant.runs == writtenVariant.runs && !sameVariant(keptVariant, writtenVariant, comparedWords) && comparedWords == keptVariant.words.size(), "draw cache identity: a capture after a CPU store matched the capture before it"); + const auto rereadVariant = captureVariant(request); + const auto rewrittenVariant = captureVariant(request); + require(sameVariant(keptVariant, rereadVariant, comparedWords) && comparedWords == keptVariant.words.size() && sameVariant(keptVariant, rewrittenVariant, comparedWords) && comparedWords == keptVariant.words.size(), "draw cache identity: a table written back to its old words did not match by its words alone"); + if (watched) require(writtenVariant.shared.empty() && rereadVariant.shared.empty() && rewrittenVariant.shared.size() == 1u && !sameBlock(keptVariant, rewrittenVariant) && keptVariant.shared[0].block.expired(), "draw cache identity: a new column with the old words was taken for the replaced column"); + + const auto identityBlock = std::make_shared(SrtGuestWords{{{0x1000u, 4u}}, {1u, 2u, 3u, 4u}}); + const auto twinBlock = std::make_shared(SrtGuestWords{{{0x1000u, 4u}}, {1u, 2u, 3u, 4u}}); + const std::vector> identityBlocks{identityBlock, twinBlock}; + const auto identityBytes = std::as_bytes(std::span(identityBlock->words)); + const std::vector copiedWords(identityBlock->words); + std::vector whole, tail, twin, copied, misaligned, nothing; + AgcDriver::AppendSharedWords(whole, 0u, 1u, identityBytes, identityBlocks); + AgcDriver::AppendSharedWords(tail, 0u, 2u, identityBytes.subspan(4u), identityBlocks); + AgcDriver::AppendSharedWords(twin, 0u, 1u, std::as_bytes(std::span(twinBlock->words)), identityBlocks); + AgcDriver::AppendSharedWords(copied, 0u, 1u, std::as_bytes(std::span(copiedWords)), identityBlocks); + AgcDriver::AppendSharedWords(misaligned, 0u, 1u, identityBytes.subspan(2u, 8u), identityBlocks); + AgcDriver::AppendSharedWords(nothing, 0u, 1u, identityBytes.subspan(0u, 0u), identityBlocks); + require(whole.size() == 1u && whole[0].first == 1u && whole[0].count == 4u && whole[0].offset == 0u && tail.size() == 1u && tail[0].first == 2u && tail[0].count == 3u && tail[0].offset == 1u && twin.size() == 1u && twin[0].block.lock() == twinBlock && copied.empty() && misaligned.empty() && nothing.empty(), "draw cache identity: words were credited to a block they were not copied from"); + std::vector joined, gapped, crossed, skipped; + AgcDriver::AppendSharedWords(joined, 0u, 1u, identityBytes.subspan(0u, 8u), identityBlocks); + AgcDriver::AppendSharedWords(joined, 1u, 3u, identityBytes.subspan(8u), identityBlocks); + AgcDriver::AppendSharedWords(gapped, 0u, 1u, identityBytes.subspan(0u, 4u), identityBlocks); + AgcDriver::AppendSharedWords(gapped, 1u, 3u, identityBytes.subspan(4u, 4u), identityBlocks); + AgcDriver::AppendSharedWords(gapped, 2u, 4u, identityBytes.subspan(12u, 4u), identityBlocks); + AgcDriver::AppendSharedWords(crossed, 0u, 1u, identityBytes.subspan(0u, 4u), identityBlocks); + AgcDriver::AppendSharedWords(crossed, 1u, 2u, std::as_bytes(std::span(twinBlock->words)).subspan(4u, 4u), identityBlocks); + AgcDriver::AppendSharedWords(skipped, 0u, 1u, identityBytes.subspan(0u, 8u), identityBlocks); + AgcDriver::AppendSharedWords(skipped, 2u, 3u, identityBytes.subspan(8u), identityBlocks); + require(skipped.size() == 2u && skipped[0].region == 0u && skipped[0].regions == 1u && skipped[1].region == 2u && skipped[1].regions == 1u, "draw cache identity: words of regions that are not consecutive were joined"); + require(joined.size() == 1u && joined[0].first == 1u && joined[0].count == 4u && joined[0].offset == 0u && joined[0].region == 0u && joined[0].regions == 2u && gapped.size() == 3u && gapped[1].first == 3u && gapped[1].offset == 1u && gapped[2].first == 4u && gapped[2].offset == 3u && crossed.size() == 2u && crossed[1].block.lock() == twinBlock, "draw cache identity: consecutive words of one block were not joined, or words of two places were"); + const std::vector leftWords{9u, 1u, 2u, 3u, 4u}; + auto rightWords = leftWords; + std::size_t identityCompared = 0; + require(AgcDriver::SameCapturedWords(leftWords, whole, rightWords, whole, &identityCompared) && identityCompared == 1u, "draw cache identity: words copied from one block were compared"); + require(AgcDriver::SameCapturedWords(leftWords, whole, rightWords, twin, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: equal words from two blocks did not match by their words"); + rightWords[0] = 8u; + require(!AgcDriver::SameCapturedWords(leftWords, whole, rightWords, whole, &identityCompared) && identityCompared == 1u, "draw cache identity: a shared block hid a difference outside it"); + rightWords = {9u, 1u, 2u, 3u, 5u}; + const std::vector unowned{{1u, 4u, 0u, {}}}; + const std::vector shifted{{1u, 4u, 1u, identityBlock}}; + const std::vector beyond{{3u, 4u, 0u, identityBlock}}; + auto doomed = std::make_shared(SrtGuestWords{{{0x2000u, 4u}}, {1u, 2u, 3u, 4u}}); + std::vector dead; + AgcDriver::AppendSharedWords(dead, 0u, 1u, std::as_bytes(std::span(doomed->words)), std::span>(&doomed, 1u)); + doomed.reset(); + require(!AgcDriver::SameCapturedWords(leftWords, whole, rightWords, twin, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: different words from two blocks matched"); + require(!AgcDriver::SameCapturedWords(leftWords, unowned, rightWords, unowned, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: runs without a block vouched for words"); + require(!AgcDriver::SameCapturedWords(leftWords, whole, rightWords, shifted, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: runs at other offsets of one block vouched for words"); + require(!AgcDriver::SameCapturedWords(leftWords, beyond, rightWords, beyond, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: a run past the words vouched for them"); + require(dead.size() == 1u && dead[0].block.expired() && !AgcDriver::SameCapturedWords(leftWords, dead, rightWords, dead, &identityCompared) && identityCompared == leftWords.size(), "draw cache identity: a released block vouched for words"); + require(!AgcDriver::SameCapturedWords(leftWords, whole, std::vector(leftWords.begin(), leftWords.end() - 1), {}, &identityCompared) && identityCompared == 0u, "draw cache identity: captures of different sizes matched"); + + const auto compiled = CompileTable(request); + const auto& map = TableMap(*compiled.result); + const auto codes = TableCodes(map, 0); + const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); + require(images != nullptr && codes.size() == Records, "table snapshot reuse: the reused table has no image table binding"); + for (const auto key : {0u, 77u, 1234u, 2048u, 3071u, 4095u}) { + const auto element = map.at(codes.at(key) + Abi::ImageRecordElement); + require((codes[key] & Abi::PoisonFlag) == 0u && std::equal(heap + static_cast(key) * 8u, heap + static_cast(key) * 8u + 8u, images->guestDescriptor.begin() + static_cast(element) * 8), ("table snapshot reuse: key " + std::to_string(key) + " selects the wrong record of a reused table").c_str()); + } + + const auto resolvedBefore = ResourceMaterializer::TableReuse(); + alignas(16) static std::array otherOutput{}; + const auto otherOutputV = TableV(otherOutput.data(), 0u, static_cast(sizeof(otherOutput))); + const std::array outputV{guest.srt[12], guest.srt[13], guest.srt[14], guest.srt[15]}; + std::copy(otherOutputV.begin(), otherOutputV.end(), guest.srt.begin() + 12); + const auto otherBuffer = CompileTable(request); + std::copy(outputV.begin(), outputV.end(), guest.srt.begin() + 12); + const auto resolvedShared = ResourceMaterializer::TableReuse(); + require(otherBuffer.result != compiled.result && otherBuffer.result->variantId == compiled.result->variantId && resolvedShared.resolutionsShared - resolvedBefore.resolutionsShared == 1u && resolvedShared.resolutions == resolvedBefore.resolutions && TableMap(*otherBuffer.result) == map && TableBinding(*otherBuffer.result, DescriptorRole::ImageTable)->guestDescriptor == images->guestDescriptor, "table snapshot reuse: a result for another buffer did not reuse the unchanged table's resolution"); + const auto storedWord = heap[5u * 8u]; + heap[5u * 8u] = heap[6u * 8u]; + const auto restoredMap = map; + const auto storedTable = CompileTable(request); + heap[5u * 8u] = storedWord; + const auto resolvedStored = ResourceMaterializer::TableReuse(); + require(resolvedStored.resolutionsShared == resolvedShared.resolutionsShared && resolvedStored.resolutions - resolvedShared.resolutions == 1u && TableMap(*storedTable.result) != restoredMap, "table snapshot reuse: a table changed by a CPU store took the resolution of its old words"); + const auto& storedMap = TableMap(*storedTable.result); + const auto storedCodes = TableCodes(storedMap, 0); + const auto* storedImages = TableBinding(*storedTable.result, DescriptorRole::ImageTable); + const auto storedElement = storedMap.at(storedCodes.at(5u) + Abi::ImageRecordElement); + require(storedImages != nullptr && std::equal(heap + 6u * 8u, heap + 6u * 8u + 1u, storedImages->guestDescriptor.begin() + static_cast(storedElement) * 8) && std::equal(heap + 5u * 8u + 1u, heap + 5u * 8u + 8u, storedImages->guestDescriptor.begin() + static_cast(storedElement) * 8 + 1), "table snapshot reuse: the resolution after a CPU store does not hold the stored record"); + + constexpr std::uint32_t SmallRecords = 64; + std::vector> smallStorage(17, std::vector(SmallRecords * 8u + 1024u)); + std::vector smallGuests(smallStorage.size()); + std::vector> smallUserData(smallStorage.size()); + std::vector smallRequests; + std::vector smallTables; + for (std::size_t index = 0; index < smallStorage.size(); index++) { + auto* table = reinterpret_cast((reinterpret_cast(smallStorage[index].data()) + 4095u) & ~std::uintptr_t{4095}); + fill(table, SmallRecords); + FillTableSrt(smallGuests[index], table, 32u, SmallRecords); + smallTables.push_back(table); + smallUserData[index] = TableUserData(smallGuests[index]); + smallRequests.push_back(TableRequest(TableWholeCode, smallUserData[index])); + } + const auto smallBytes = static_cast(SmallRecords) * 32u; + const auto first = capture(smallRequests[0], smallTables[0], smallBytes, nullptr); + const auto again = capture(smallRequests[0], smallTables[0], smallBytes, nullptr); + require(first.reuse.compared == 0u && again.reuse.unwritten == 0u && again.reuse.compared == 1u && heldRecords(again, smallTables[0], SmallRecords), "table snapshot reuse: an unchanged table outside the write watch was not compared and reused"); + smallTables[0][5u * 8u] ^= 0x10u; + const auto unwatchedStore = capture(smallRequests[0], smallTables[0], smallBytes, nullptr); + require(unwatchedStore.reuse.compared == 0u && heldRecords(unwatchedStore, smallTables[0], SmallRecords), "table snapshot reuse: a store into a table outside the write watch was not seen"); + for (std::size_t index = 1; index < smallStorage.size(); index++) static_cast(capture(smallRequests[index], smallTables[index], smallBytes, nullptr)); + const auto evicted = capture(smallRequests[0], smallTables[0], smallBytes, nullptr); + const auto kept = capture(smallRequests.back(), smallTables.back(), smallBytes, nullptr); + require(evicted.reuse.unwritten == 0u && evicted.reuse.compared == 0u && kept.reuse.compared == 1u, "table snapshot reuse: the memo kept more than 16 columns"); + + constexpr std::uint32_t PagedRecords = 512; + constexpr std::size_t PagedBytes = PagedRecords * 32u; + auto* paged = AllocateWatchedTable(PagedBytes); + fill(paged, PagedRecords); + alignas(4096) static TableGuest pagedGuest; + FillTableSrt(pagedGuest, paged, 32u, PagedRecords); + const auto pagedUserData = TableUserData(pagedGuest); + const auto pagedRequest = TableRequest(TableWholeCode, pagedUserData); + const auto pagedFresh = capture(pagedRequest, paged, PagedBytes, nullptr); + const auto pagedReused = capture(pagedRequest, paged, PagedBytes, nullptr); + require(pagedFresh.reuse.columns == 1u && sharesColumn(pagedReused, pagedFresh) && heldRecords(pagedReused, paged, PagedRecords), "table snapshot reuse: an unchanged paged table did not share its column"); + const auto hidden = reinterpret_cast(paged) + 4096u; + ProtectTablePage(hidden, 0x01u); + const auto pagedUnmapped = capture(pagedRequest, paged, PagedBytes, nullptr); + ProtectTablePage(hidden, 0x04u); + const auto unmappedCode = Abi::PoisonCode(static_cast(Abi::PoisonReason::Unmapped)); + const auto& hiddenCodes = pagedUnmapped.tables.tables.at(0).codes; + require(pagedUnmapped.reuse.unwritten == 0u && pagedUnmapped.reuse.shared == 0u && pagedUnmapped.reuse.columns == 1u && pagedUnmapped.supplied.shared == 0u && hiddenCodes.at(127u) != unmappedCode && hiddenCodes.at(128u) == unmappedCode && hiddenCodes.at(255u) == unmappedCode && hiddenCodes.at(256u) != unmappedCode, "table snapshot reuse: the records of an unmapped page were not read again as Unmapped poison"); + require(heldRecords(pagedReused, paged, PagedRecords) && sharesColumn(pagedReused, pagedFresh), "table snapshot reuse: unmapping a page changed the column read before it"); + const auto pagedBack = capture(pagedRequest, paged, PagedBytes, nullptr); + require(pagedBack.reuse.unwritten == 0u && pagedBack.reuse.compared == 1u && sharesColumn(pagedBack, pagedFresh) && heldRecords(pagedBack, paged, PagedRecords), "table snapshot reuse: the column read before a page was unmapped was not compared and shared once the page came back"); + static_cast(capture(pagedRequest, paged, PagedBytes, nullptr)); + GuestArena::GuestArenaReset_nid_postfix(paged, PagedBytes); + GuestArena::GuestArenaCommit_nid_postfix(paged, PagedBytes, 0x04u, PagedBytes); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(paged), PagedBytes); + const auto pagedRemapped = capture(pagedRequest, paged, PagedBytes, nullptr); + require(pagedRemapped.reuse.unwritten == 0u && pagedRemapped.reuse.shared == 0u && pagedRemapped.reuse.columns == 1u && !sharesColumn(pagedRemapped, pagedFresh) && heldRecords(pagedRemapped, paged, PagedRecords), "table snapshot reuse: a table mapped again with new contents was not read again"); + std::cout << "table snapshot reuse " << (watched ? "with" : "without") << " write watching passed\n"; +} + +std::atomic tableRacePhase{0}; +std::uint64_t tableRaceBegin = 0; +std::uint64_t tableRaceBytes = 0; + +AgcDriver::ShaderMemory::PendingWrite RacingTableWrite(std::uint64_t address, std::size_t bytes, std::span) { + if (tableRaceBytes != 0u && address == tableRaceBegin && bytes >= tableRaceBytes) { + tableRaceBytes = 0; + tableRacePhase.store(1, std::memory_order_release); + while (tableRacePhase.load(std::memory_order_acquire) != 2) std::this_thread::yield(); + } + return AgcDriver::ShaderMemory::PendingWrite::None; +} + +void verifyTableStampRace() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Records = 4096; + constexpr std::uint32_t Stride = 48; + constexpr std::size_t Bytes = static_cast(Records) * Stride; + constexpr std::size_t RecordWords = Stride / 4u; + const auto texture = [](std::uint32_t index) { return TableT(reinterpret_cast(static_cast(0x100000000ull + index * 256ull)), TableFormat8888UNorm, TableType2D, 1u + index % 5u); }; + auto* heap = AllocateWatchedTable(Bytes); + for (std::uint32_t record = 0; record < Records; record++) { + const auto words = texture(record % 1024u); + std::copy(words.begin(), words.end(), heap + record * RecordWords); + std::fill_n(heap + record * RecordWords + 8u, 4u, 0x5a5a5a5au); + } + const auto address = static_cast(reinterpret_cast(heap)); + if (AgcDriver::GuestMemory::CollectWrites(address, Bytes) == 0) { + std::cout << "table stamp race without write watching skipped\n"; + return; + } + alignas(4096) static TableGuest guest; + FillTableSrt(guest, heap, Stride, Records); + const auto userData = TableUserData(guest); + auto wideCode = TableWholeCode; + std::replace(wideCode.begin(), wideCode.end(), 0x8f108510u, 0x9310b010u); + const auto request = TableRequest(wideCode, userData); + const auto capture = [&](AgcDriver::ShaderMemory::PendingWriteQuery pending) { + const auto before = ResourceMaterializer::TableReuse(); + AgcDriver::ShaderMemory memory({}, pending); + static_cast(memory.Capture(request)); + const auto after = ResourceMaterializer::TableReuse(); + return ImageTableReuseCounts{after.unwritten - before.unwritten, after.compared - before.compared, after.wordsRead - before.wordsRead, after.shared - before.shared, after.columns - before.columns}; + }; + tableRacePhase.store(0, std::memory_order_relaxed); + tableRaceBegin = address; + tableRaceBytes = Bytes - 16u; + std::thread writer([&] { + while (tableRacePhase.load(std::memory_order_acquire) != 1) std::this_thread::yield(); + AgcDriver::GuestMemory::MarkWritten(address + 5u * Stride, 32u); + tableRacePhase.store(2, std::memory_order_release); + }); + const auto first = capture(&RacingTableWrite); + writer.join(); + require(tableRacePhase.load(std::memory_order_relaxed) == 2 && first.columns == 1u, "table stamp race: the racing store did not run while the table was stamped"); + const auto next = capture(nullptr); + require(next.unwritten == 0u && next.compared == 1u, "table stamp race: a GPU write stamped while the table was stamped, before it was noted pending, was hidden by the table's stamp"); + std::cout << "table stamp race with write watching passed\n"; +} + +void verifyTableVouch() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Records = 4096; + constexpr std::uint32_t Stride = 48; + constexpr std::size_t Bytes = static_cast(Records) * Stride; + constexpr std::size_t RecordWords = Stride / 4u; + const auto texture = [](std::uint32_t index) { return TableT(reinterpret_cast(static_cast(0x100000000ull + index * 256ull)), TableFormat8888UNorm, TableType2D, 1u + index % 5u); }; + const auto fill = [&](std::uint32_t* heap) { + for (std::uint32_t record = 0; record < Records; record++) { + const auto words = texture(record % 1024u); + std::copy(words.begin(), words.end(), heap + record * RecordWords); + std::fill_n(heap + record * RecordWords + 8u, 4u, 0x5a5a5a5au); + } + }; + struct Captured { + ImageTableReuseCounts reuse; + std::uint64_t calls = 0; + std::vector addresses; + std::vector counts; + std::vector words; + std::vector shared; + std::vector> blocks; + }; + const auto capture = [](const RecompileRequest& request, AgcDriver::ShaderMemory::PendingWriteQuery pending) { + Captured captured; + const auto before = ResourceMaterializer::TableReuse(); + const auto calls = AgcDriver::ShaderMemory::Vouches().calls; + AgcDriver::ShaderMemory memory({}, pending); + static_cast(memory.Capture(request)); + captured.calls = AgcDriver::ShaderMemory::Vouches().calls - calls; + captured.blocks = memory.SuppliedBlocks(); + for (const auto& region : memory.TakeRecentRegions()) { + const auto offset = captured.words.size(); + captured.addresses.push_back(region.guestAddress); + captured.counts.push_back(region.bytes.size() / sizeof(std::uint32_t)); + captured.words.resize(offset + region.bytes.size() / sizeof(std::uint32_t)); + std::memcpy(captured.words.data() + offset, region.bytes.data(), region.bytes.size()); + AgcDriver::AppendSharedWords(captured.shared, captured.addresses.size() - 1u, offset, region.bytes, captured.blocks); + } + const auto after = ResourceMaterializer::TableReuse(); + captured.reuse = {after.unwritten - before.unwritten, after.compared - before.compared, after.wordsRead - before.wordsRead, after.shared - before.shared, after.columns - before.columns}; + return captured; + }; + const auto vouch = [](const Captured& captured, AgcDriver::ShaderMemory::PendingWriteQuery pending, std::vector& vouched) { + std::vector ranges; + const auto words = AgcDriver::VouchSharedRegions(captured.shared, captured.addresses.size(), pending, ranges); + vouched.assign(captured.addresses.size(), 0u); + std::size_t covered = 0; + for (const auto& range : ranges) { + std::fill(vouched.begin() + static_cast(range.begin), vouched.begin() + static_cast(range.end), std::uint8_t{1}); + for (auto index = range.begin; index < range.end; index++) covered += captured.counts[index]; + } + return covered == words ? words : std::size_t{0}; + }; + + auto* heap = AllocateWatchedTable(Bytes); + fill(heap); + const auto address = static_cast(reinterpret_cast(heap)); + const bool watched = AgcDriver::GuestMemory::CollectWrites(address, Bytes) != 0; + alignas(4096) static TableGuest guest; + FillTableSrt(guest, heap, Stride, Records); + const auto userData = TableUserData(guest); + auto wideCode = TableWholeCode; + std::replace(wideCode.begin(), wideCode.end(), 0x8f108510u, 0x9310b010u); + const auto request = TableRequest(wideCode, userData); + const auto tableWords = static_cast(Records) * 8u; + const auto onlyTable = [&](const Captured& captured, const std::vector& vouched) { + if (vouched.size() != captured.addresses.size()) return false; + for (std::size_t index = 0; index < vouched.size(); index++) { + if ((captured.addresses[index] >= address && captured.addresses[index] < address + Bytes) != (vouched[index] != 0u)) return false; + } + return true; + }; + + const auto fresh = capture(request, nullptr); + require(fresh.reuse.columns == 1u && fresh.reuse.wordsRead == tableWords, "table vouch: the first snapshot did not read the strided table once"); + const auto unchanged = capture(request, nullptr); + if (watched) { + require(unchanged.reuse.unwritten == 1u && unchanged.reuse.wordsRead == 0u, "table vouch: an unwritten strided table was read again"); + require(unchanged.calls == 1u, ("table vouch: an unwritten table of " + std::to_string(Records) + " strided records was vouched for in " + std::to_string(unchanged.calls) + " calls instead of one").c_str()); + require(unchanged.blocks.size() == 1u && unchanged.blocks[0]->ordered && unchanged.blocks[0]->runs.size() == Records && unchanged.blocks[0]->low == address && unchanged.blocks[0]->high == address + static_cast(Records - 1u) * Stride + 32u, "table vouch: the strided table's words were not kept as one ordered block over the span of its records"); + } else { + require(unchanged.reuse.compared == 1u && unchanged.calls == 0u, "table vouch: an unchanged unwatched strided table was not compared"); + } + + std::vector vouched; + const auto vouchesBefore = AgcDriver::ShaderMemory::Vouches(); + const auto unchangedWords = vouch(unchanged, nullptr, vouched); + const auto vouchesAfter = AgcDriver::ShaderMemory::Vouches(); + if (watched) { + require(unchangedWords == tableWords && onlyTable(unchanged, vouched), "table vouch: the regions of an unchanged shared table were not vouched for, or other regions were"); + require(vouchesAfter.calls - vouchesBefore.calls == 1u && vouchesAfter.blocks - vouchesBefore.blocks == 1u && vouchesAfter.refused == vouchesBefore.refused && vouchesAfter.words - vouchesBefore.words == tableWords, "table vouch: one shared block was not vouched for once"); + } else { + require(unchangedWords == 0u && unchanged.shared.empty() && std::none_of(vouched.begin(), vouched.end(), [](std::uint8_t flag) { return flag != 0u; }), "table vouch: an unwatched table's regions were vouched for"); + } + if (!watched) { + std::cout << "table vouch without write watching passed\n"; + return; + } + const auto refused = [&](const std::string& what, AgcDriver::ShaderMemory::PendingWriteQuery pending) { + const auto before = AgcDriver::ShaderMemory::Vouches(); + const auto words = vouch(unchanged, pending, vouched); + const auto after = AgcDriver::ShaderMemory::Vouches(); + require(words == 0u && std::none_of(vouched.begin(), vouched.end(), [](std::uint8_t flag) { return flag != 0u; }) && after.refused - before.refused == 1u, ("table vouch: " + what + ": the shared table was still vouched for").c_str()); + }; + + pendingTableKnown = false; + pendingTableWord = address + 100u * Stride; + refused("a pending GPU write into a record", &PendingTableWrite); + pendingTableWord = address + 100u * Stride + 40u; + refused("a pending GPU write between two records", &PendingTableWrite); + pendingTableWord = address + Bytes + 4096u; + require(vouch(unchanged, &PendingTableWrite, vouched) == tableWords, "table vouch: a pending GPU write outside the table refused it"); + pendingTableWord = 0; + + constexpr std::size_t UnrelatedBytes = 65536; + void* unrelated = ::operator new(UnrelatedBytes, std::align_val_t{UnrelatedBytes}); + { + GuestAllocations::Mutation mutation; + mutation.Add(unrelated, UnrelatedBytes, true, true, false); + } + refused("a changed guest mapping", nullptr); + { + GuestAllocations::Mutation mutation; + mutation.Remove(unrelated); + } + ::operator delete(unrelated, std::align_val_t{UnrelatedBytes}); + + const auto rewrite = [&](const std::string& what, const std::function& store, bool gap) { + auto held = capture(request, nullptr); + if (held.reuse.unwritten == 0u) held = capture(request, nullptr); + std::vector heldVouched; + require(held.reuse.unwritten == 1u && vouch(held, nullptr, heldVouched) == tableWords, ("table vouch: " + what + ": the table was not vouched for before the store").c_str()); + store(); + const auto before = AgcDriver::ShaderMemory::Vouches(); + require(vouch(held, nullptr, heldVouched) == 0u && AgcDriver::ShaderMemory::Vouches().refused - before.refused == 1u, ("table vouch: " + what + ": the table captured before the store was still vouched for").c_str()); + const auto after = capture(request, nullptr); + if (gap) { + require(after.reuse.unwritten == 0u && after.reuse.compared == 1u && after.reuse.columns == 0u, ("table vouch: " + what + ": the table was not compared and kept").c_str()); + require(vouch(held, nullptr, heldVouched) == tableWords, ("table vouch: " + what + ": a capture sharing the compared table was not vouched for again").c_str()); + require(capture(request, nullptr).reuse.unwritten == 1u, ("table vouch: " + what + ": the compared table was not reused unwritten after").c_str()); + } else { + require(after.reuse.unwritten == 0u && after.reuse.compared == 0u && after.reuse.columns == 1u, ("table vouch: " + what + ": the table was not read again").c_str()); + require(vouch(held, nullptr, heldVouched) == 0u, ("table vouch: " + what + ": the replaced table was vouched for after it was read again").c_str()); + } + }; + std::uint32_t replacement = 3000u; + const auto record = [&](std::uint32_t index) { return address + static_cast(index) * Stride; }; + rewrite("a CPU store into a record", [&] { const auto words = texture(replacement++); std::memcpy(heap + 7u * RecordWords, words.data(), sizeof(words)); }, false); + rewrite("a driver store (CP write, DMA copy or fill)", [&] { const auto words = texture(replacement++); AgcDriver::GuestMemory::Write(record(1234u), std::as_bytes(std::span(words)), 4); }, false); + rewrite("a GPU result stored by the driver", [&] { const auto words = texture(replacement++); AgcDriver::GuestMemory::StoreOwnBytes(record(2048u), std::as_bytes(std::span(words))); }, false); + rewrite("a CPU store between two records", [&] { heap[300u * RecordWords + 9u] ^= 0x10u; }, true); + rewrite("a GPU write into an import (MarkWritten)", [&] { AgcDriver::GuestMemory::MarkWritten(record(3000u) + 32u, 16u); }, true); + + { + auto held = capture(request, nullptr); + std::vector heldVouched; + require(held.reuse.unwritten == 1u && !held.blocks.empty() && vouch(held, nullptr, heldVouched) == tableWords, "table vouch: the held table was not vouched for"); + held.blocks.clear(); + heap[11u * RecordWords + 2u] ^= 0x10u; + const auto replaced = capture(request, nullptr); + heap[11u * RecordWords + 2u] ^= 0x10u; + const auto before = AgcDriver::ShaderMemory::Vouches(); + require(replaced.reuse.columns == 1u && held.shared.size() == 1u && held.shared[0].block.expired() && vouch(held, nullptr, heldVouched) == 0u && AgcDriver::ShaderMemory::Vouches().calls == before.calls, "table vouch: a released table block was vouched for"); + static_cast(capture(request, nullptr)); + } + + std::cout << "table vouch with write watching passed\n"; +} + +std::uint64_t spanQueryBegin = 0; +std::uint64_t spanQueryEnd = 0; +std::uint64_t spanQueriesToPages = 0; +bool spanQueryQueued = false; + +AgcDriver::ShaderMemory::PendingWrite PageQuery(std::uint64_t address, std::size_t bytes, std::span) { + if (address <= spanQueryBegin && address + bytes >= spanQueryEnd) ++spanQueriesToPages; + return AgcDriver::ShaderMemory::PendingWrite::None; +} + +AgcDriver::ShaderMemory::PendingWrite QueuedSpanQuery(std::uint64_t address, std::size_t bytes, std::span) { + return spanQueryQueued && address < spanQueryEnd && spanQueryBegin < address + bytes ? AgcDriver::ShaderMemory::PendingWrite::Sync : AgcDriver::ShaderMemory::PendingWrite::None; +} + +void verifyTableSpanQuery() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Records = 4096; + constexpr std::uint32_t Stride = 48; + constexpr std::size_t Bytes = static_cast(Records) * Stride; + constexpr std::size_t RecordWords = Stride / 4u; + auto* heap = AllocateWatchedTable(Bytes); + for (std::uint32_t record = 0; record < Records; record++) { + const auto words = TableT(reinterpret_cast(static_cast(0x100000000ull + (record % 1024u) * 256ull)), TableFormat8888UNorm, TableType2D, 1u + record % 5u); + std::copy(words.begin(), words.end(), heap + record * RecordWords); + std::fill_n(heap + record * RecordWords + 8u, 4u, 0x5a5a5a5au); + } + const auto address = static_cast(reinterpret_cast(heap)); + if (AgcDriver::GuestMemory::CollectWrites(address, Bytes) == 0) { + std::cout << "table span query without write watching skipped\n"; + return; + } + alignas(4096) static TableGuest guest; + FillTableSrt(guest, heap, Stride, Records); + auto wideCode = TableWholeCode; + std::replace(wideCode.begin(), wideCode.end(), 0x8f108510u, 0x9310b010u); + const auto userData = TableUserData(guest); + const auto request = TableRequest(wideCode, userData); + spanQueryBegin = address; + spanQueryEnd = address + static_cast(Records - 1u) * Stride + 32u; + spanQueriesToPages = 0; + spanQueryQueued = false; + const auto capture = [&] { + const auto before = ResourceMaterializer::TableReuse(); + AgcDriver::ShaderMemory memory({}, &PageQuery, nullptr, nullptr, &QueuedSpanQuery); + static_cast(memory.Capture(request)); + const auto after = ResourceMaterializer::TableReuse(); + return ImageTableReuseCounts{after.unwritten - before.unwritten, after.compared - before.compared, after.wordsRead - before.wordsRead, after.shared - before.shared, after.columns - before.columns}; + }; + require(capture().columns == 1u, "table span query: the first capture did not read the table"); + require(capture().unwritten == 1u, "table span query: the unwritten table was not reused through the span query"); + spanQueryQueued = true; + const auto queued = capture(); + require(queued.unwritten == 0u && queued.compared == 1u && queued.columns == 0u, "table span query: a write queued over the table's span did not make the capture compare the table"); + spanQueryQueued = false; + require(capture().unwritten == 1u, "table span query: the table was not reused unwritten once the queued write was gone"); + require(spanQueriesToPages == 0u, ("table span query: " + std::to_string(spanQueriesToPages) + " whole-span checks went to the page query, which waits for queued draws").c_str()); + std::cout << "table span query passed\n"; +} + +void verifyTableEdgeStore() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Records = 4096; + constexpr std::uint32_t Stride = 48; + constexpr std::size_t Bytes = static_cast(Records) * Stride; + constexpr std::size_t Tail = 65536; + constexpr std::size_t RecordWords = Stride / 4u; + const auto texture = [](std::uint32_t index) { return TableT(reinterpret_cast(static_cast(0x100000000ull + index * 256ull)), TableFormat8888UNorm, TableType2D, 1u + index % 5u); }; + auto* heap = AllocateWatchedTable(Bytes + Tail); + for (std::uint32_t record = 0; record < Records; record++) { + const auto words = texture(record % 1024u); + std::copy(words.begin(), words.end(), heap + record * RecordWords); + std::fill_n(heap + record * RecordWords + 8u, 4u, 0x5a5a5a5au); + } + const auto address = static_cast(reinterpret_cast(heap)); + if (AgcDriver::GuestMemory::CollectWrites(address, Bytes) == 0) { + std::cout << "table edge store without write watching skipped\n"; + return; + } + AgcDriver::GuestMemory::BumpCollectEpoch(); + AgcDriver::GuestMemory::Unwatch(address + Bytes, Tail); + alignas(4096) static TableGuest guest; + FillTableSrt(guest, heap, Stride, Records); + auto wideCode = TableWholeCode; + std::replace(wideCode.begin(), wideCode.end(), 0x8f108510u, 0x9310b010u); + const auto userData = TableUserData(guest); + const auto request = TableRequest(wideCode, userData); + struct Captured { + ImageTableReuseCounts reuse; + std::vector> blocks; + }; + const auto capture = [&] { + Captured captured; + const auto before = ResourceMaterializer::TableReuse(); + AgcDriver::ShaderMemory memory({}, nullptr); + static_cast(memory.Capture(request)); + captured.blocks = memory.SuppliedBlocks(); + const auto after = ResourceMaterializer::TableReuse(); + captured.reuse = {after.unwritten - before.unwritten, after.compared - before.compared, after.wordsRead - before.wordsRead, after.shared - before.shared, after.columns - before.columns}; + return captured; + }; + const auto vouched = [](const Captured& captured) { + const auto& block = captured.blocks.front(); + return AgcDriver::ShaderMemory::Vouched(nullptr, block->low, block->high - block->low, LoadStamp(*block)); + }; + const auto last = address + static_cast(Records - 1u) * Stride; + std::uint32_t replacement = 3000u; + const auto across = [&](const std::string& what, const std::function)>& store) { + auto held = capture(); + if (held.reuse.unwritten == 0u) held = capture(); + require(held.reuse.unwritten == 1u && held.blocks.size() == 1u && vouched(held), ("table edge store: " + what + ": the table was not vouched for before the store").c_str()); + std::array words{}; + const auto record = texture(replacement++); + std::copy(record.begin(), record.end(), words.begin()); + std::fill(words.begin() + 8, words.end(), 0x77777777u); + store(std::as_bytes(std::span(words))); + require(!vouched(held), ("table edge store: " + what + " across the heap's edge into unwatched memory did not end the heap's epoch").c_str()); + require(capture().reuse.columns == 1u, ("table edge store: " + what + ": the table was not read again").c_str()); + }; + static_cast(capture()); + across("a driver store", [&](std::span bytes) { AgcDriver::GuestMemory::Write(last, bytes, 4); }); + across("a GPU result stored by the driver", [&](std::span bytes) { AgcDriver::GuestMemory::StoreOwnBytes(last, bytes); }); + std::cout << "table edge store passed\n"; } void verifyImageTableScratchReuse() { @@ -1475,7 +2304,7 @@ void verifyImageTableScratchReuse() { const auto warm = resolve(); const auto fresh = std::async(std::launch::async, resolve).get(); require(warm.snapshot == fresh.snapshot && warm.tables.map == fresh.tables.map && warm.tables.faults == fresh.tables.faults && warm.tables.poison.size() == fresh.tables.poison.size(), "image table scratch: a snapshot resolved after another one on the same thread differs from one resolved on a new thread"); - require(warm.snapshot.words.size() == Entries && warm.tables.elements.size() == Entries && warm.tables.map[Abi::HeaderElementCount] == Entries, "image table scratch: the distinct records were not all indexed"); + require(warm.snapshot.tables.at(0).words.size() == Entries && warm.tables.elements.size() == Entries && warm.tables.map[Abi::HeaderElementCount] == Entries, "image table scratch: the distinct records were not all indexed"); const auto codes = TableCodes(warm.tables.map, 0); for (std::uint32_t entry = 0; entry < Entries; entry++) { const auto element = warm.tables.map.at(codes.at(entry) + Abi::ImageRecordElement); @@ -1581,7 +2410,7 @@ void verifyImageTables() { const auto& snapshot = tables.tables[0]; require(snapshot.base == static_cast(reinterpret_cast(heap.data())) && snapshot.size == 128u && snapshot.records == 4u && snapshot.keys == 4u && snapshot.fault == 0u && !snapshot.outside, "image tables: the column snapshot is wrong"); require(snapshot.codes.size() == 4u && snapshot.codes[0] == snapshot.codes[3] && snapshot.codes[0] != snapshot.codes[1] && snapshot.codes[2] == Abi::PoisonCode(static_cast(Abi::PoisonReason::OutsideDomain)), "image tables: the narrowed codes are wrong"); - require(tables.words.at(snapshot.codes[0]).dwords == heap[0] && tables.words.at(snapshot.codes[1]).dwords == heap[1], "image tables: the snapshot words are wrong"); + require(snapshot.words.at(snapshot.codes[0]).dwords == heap[0] && snapshot.words.at(snapshot.codes[1]).dwords == heap[1], "image tables: the snapshot words are wrong"); for (const auto& material : materials) require(TableCovered(compiled.regions, &material[1], sizeof(std::uint32_t)), "image tables: a material key was not captured"); for (const auto entry : {0u, 1u, 3u}) require(TableCovered(compiled.regions, heap[entry].data(), 32u), "image tables: a table entry was not captured"); const auto& result = *compiled.result; @@ -1682,7 +2511,7 @@ void verifyImageTables() { const auto partial = CompileTable(request); const auto& partialTables = partial.capture->snapshot.tables; require(partialTables.tables[0].keys == 4u && partialTables.tables[0].size == 100u, "image tables: a partial table has the wrong record count"); - const auto& tail = partialTables.words.at(partialTables.tables[0].codes.at(3)); + const auto& tail = partialTables.tables[0].words.at(partialTables.tables[0].codes.at(3)); require(tail.dwords[0] == heap[3][0] && std::all_of(tail.dwords.begin() + 1, tail.dwords.end(), [](std::uint32_t word) { return word == 0u; }), "image tables: the straddling record's tail is not zero"); const auto& partialMap = TableMap(*partial.result); require((partialMap.at(TableCodes(partialMap, 0).at(3) + Abi::ImageRecordFlags) & Abi::PoisonFlag) != 0u, "image tables: a straddling record was bound"); @@ -1903,7 +2732,7 @@ void verifySelectedRecordsWithinLimits(bool pairs, bool direct) { require(directTables.tables.size() == 1u, (name + ": the snapshot beside the direct " + (pairs ? "sampler" : "images") + " does not hold one table").c_str()); const auto& directSnapshot = directTables.tables[mergedDirect->table]; for (const auto key : {2u, 2u + reads / 2u, 1u + reads}) { - require(key < directSnapshot.codes.size() && (directSnapshot.codes[key] & Abi::PoisonFlag) == 0u && directTables.words.at(directSnapshot.codes[key]).dwords == heap[key], (name + ": a key beside the direct " + (pairs ? "sampler" : "images") + " does not bind its record").c_str()); + require(key < directSnapshot.codes.size() && (directSnapshot.codes[key] & Abi::PoisonFlag) == 0u && directSnapshot.words.at(directSnapshot.codes[key]).dwords == heap[key], (name + ": a key beside the direct " + (pairs ? "sampler" : "images") + " does not bind its record").c_str()); } require(directSnapshot.codes.at(1) == outside && directSnapshot.codes.at(reads + 2u) == outside, (name + ": a key no read selects was bound beside the direct " + (pairs ? "sampler" : "images")).c_str()); std::cout << name << ": " << reads << " selected reads and " << (pairs ? "1 direct image with a third sampler" : "2 direct images") << ": " << describe(*pastDirect) << ", the reads merged into one table with " << mergedColumn.keyDomains.size() << " key domains\n"; @@ -1923,7 +2752,7 @@ void verifySelectedRecordsWithinLimits(bool pairs, bool direct) { require(tables.tables.size() == 1u, (name + ": the snapshot does not hold one table").c_str()); const auto& snapshot = tables.tables[0]; for (const auto key : {2u, 2u + limit / 2u, 2u + limit}) { - require(key < snapshot.codes.size() && (snapshot.codes[key] & Abi::PoisonFlag) == 0u && tables.words.at(snapshot.codes[key]).dwords == heap[key], (name + ": a key does not bind its record").c_str()); + require(key < snapshot.codes.size() && (snapshot.codes[key] & Abi::PoisonFlag) == 0u && snapshot.words.at(snapshot.codes[key]).dwords == heap[key], (name + ": a key does not bind its record").c_str()); } require(snapshot.codes.at(1) == outside && snapshot.codes.at(limit + 3u) == outside, (name + ": a key no read selects was bound").c_str()); if (pairs) return; @@ -1955,7 +2784,7 @@ void verifyImageTableSrtKey() { const auto compiled = CompileTable(request); const auto& tables = compiled.capture->snapshot.tables; const auto& snapshot = tables.tables.at(0); - require(snapshot.records == 1u && snapshot.keys == 1u && snapshot.codes.size() == 1u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && tables.words.at(snapshot.codes[0]).dwords == heap[2], "image table SRT key: the snapshot does not hold the selected record alone"); + require(snapshot.records == 1u && snapshot.keys == 1u && snapshot.codes.size() == 1u && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && snapshot.words.at(snapshot.codes[0]).dwords == heap[2], "image table SRT key: the snapshot does not hold the selected record alone"); const auto* images = TableBinding(*compiled.result, DescriptorRole::ImageTable); require(images != nullptr && images->count == 1u, "image table SRT key: the binding holds other entries than the selected one"); require(TableCovered(compiled.regions, &guest.srt[16], sizeof(std::uint32_t)) && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), "image table SRT key: the key or the selected record was not captured"); @@ -1963,7 +2792,7 @@ void verifyImageTableSrtKey() { guest.srt[16] = 3u; const auto moved = CompileTable(request); const auto& movedTables = moved.capture->snapshot.tables; - require(movedTables.words.at(movedTables.tables.at(0).codes.at(0)).dwords == heap[3] && TableCovered(moved.regions, &heap[3], sizeof(heap[3])) && !TableCovered(moved.regions, &heap[2], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the selected record alone"); + require(movedTables.tables.at(0).words.at(movedTables.tables.at(0).codes.at(0)).dwords == heap[3] && TableCovered(moved.regions, &heap[3], sizeof(heap[3])) && !TableCovered(moved.regions, &heap[2], sizeof(std::uint32_t)) && moved.result->spirv == compiled.result->spirv && moved.key == compiled.key, "image table SRT key: a changed key did not move the selected record alone"); } void verifyImageTableMaskedOffset() { @@ -2134,7 +2963,7 @@ void verifyPointerImageTable() { const auto& tables = compiled.capture->snapshot.tables; const auto& snapshot = tables.tables.at(0); require(snapshot.base == base && snapshot.size == 0u && snapshot.keys == Entries && snapshot.records == Entries && !snapshot.outside && snapshot.fault == 0u, "pointer image table: the column snapshot is wrong"); - require(snapshot.codes[0] == snapshot.codes[1] && snapshot.codes[2] == snapshot.codes[255] && snapshot.codes[0] != snapshot.codes[2] && tables.words.at(snapshot.codes[2]).dwords[0] == root[(Palette + 2u * 32u) / 4u], "pointer image table: the snapshot words are wrong"); + require(snapshot.codes[0] == snapshot.codes[1] && snapshot.codes[2] == snapshot.codes[255] && snapshot.codes[0] != snapshot.codes[2] && snapshot.words.at(snapshot.codes[2]).dwords[0] == root[(Palette + 2u * 32u) / 4u], "pointer image table: the snapshot words are wrong"); require(tables.ranges == std::vector>{{base + Palette, Entries * 32u}}, "pointer image table: the snapshot range is wrong"); require(snapshot.readRanges == tables.ranges && compiled.result->imageTableReadRanges.size() == 1u && compiled.result->imageTableReadRanges[0] == snapshot.readRanges, "pointer image table: the consumed ranges did not retain table ownership"); require(TableCovered(compiled.regions, root + Palette / 4u, Entries * 32u) && !TableTouched(compiled.regions, root + (Palette + Entries * 32u) / 4u, Committed - Palette - Entries * 32u), "pointer image table: the captured regions are wrong"); @@ -2255,7 +3084,7 @@ void verifyPointerTableOffsetCarry() { const auto compiled = CompileTable(request); const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); require(snapshot.keys == 256u && snapshot.codes[7] == Abi::PoisonCode(static_cast(Abi::PoisonReason::OffsetCarry)), "pointer table offset carry: the record whose second load wraps past 32 bits is not OffsetCarry poison"); - require((snapshot.codes[6] & Abi::PoisonFlag) == 0u && (snapshot.codes[8] & Abi::PoisonFlag) == 0u && compiled.capture->snapshot.tables.words.at(snapshot.codes[6]).dwords[0] == root[0x100000010ull / 4u] && compiled.capture->snapshot.tables.words.at(snapshot.codes[8]).dwords[0] == root[0x50u / 4u] && root[0x100000010ull / 4u] != root[0x50u / 4u], "pointer table offset carry: the records on either side of the carry were not resolved"); + require((snapshot.codes[6] & Abi::PoisonFlag) == 0u && (snapshot.codes[8] & Abi::PoisonFlag) == 0u && snapshot.words.at(snapshot.codes[6]).dwords[0] == root[0x100000010ull / 4u] && snapshot.words.at(snapshot.codes[8]).dwords[0] == root[0x50u / 4u] && root[0x100000010ull / 4u] != root[0x50u / 4u], "pointer table offset carry: the records on either side of the carry were not resolved"); require(!TableTouched(compiled.regions, root + 0x100000030ull / 4u, 0x20u), "pointer table offset carry: the record whose second load wraps was read"); #ifdef _WIN32 VirtualFree(block, 0, MEM_RELEASE); @@ -2534,7 +3363,7 @@ void verifyChainedTableOffset() { require(column.address && column.stride == 32u && column.addend == 0x20u && column.offset == 0x40u && column.maxKey == 0xffu && column.foldedAddend == 0x20u && column.foldedCarry == 16u, "chained table offset: the column is wrong"); const auto compiled = CompileTable(request); const auto& snapshot = compiled.capture->snapshot.tables.tables.at(0); - const auto& words = compiled.capture->snapshot.tables.words; + const auto& words = snapshot.words; require(snapshot.keys == 256u && (snapshot.codes[3] & Abi::PoisonFlag) == 0u && (snapshot.codes[5] & Abi::PoisonFlag) == 0u, "chained table offset: the records were not resolved"); require(words.at(snapshot.codes[3]).dwords[0] == root[(0x60u + 3u * 32u) / 4u] && words.at(snapshot.codes[3]).dwords[7] == root[(0x60u + 3u * 32u) / 4u + 7u] && words.at(snapshot.codes[5]).dwords[0] == root[(0x60u + 5u * 32u) / 4u] && snapshot.codes[3] != snapshot.codes[5], "chained table offset: a record was read from the wrong offset"); } @@ -2591,7 +3420,7 @@ void verifyPointerChainImageTable() { const auto& tables = compiled.capture->snapshot.tables; const auto& snapshot = tables.tables.at(0); require(snapshot.keys == Entries && snapshot.codes[0] == snapshot.codes[1] && snapshot.codes[0] != snapshot.codes[2] && (snapshot.codes[0] & Abi::PoisonFlag) == 0u && (snapshot.codes[2] & Abi::PoisonFlag) == 0u, "pointer chain image table: the records behind two pointers do not resolve to one element per distinct T#"); - require(tables.words.at(snapshot.codes[0]).dwords[0] == descriptor0[0] && tables.words.at(snapshot.codes[2]).dwords[0] == descriptor1[0] && tables.words.at(snapshot.codes[2]).dwords[1] == descriptor1[1], "pointer chain image table: the snapshot words are wrong"); + require(snapshot.words.at(snapshot.codes[0]).dwords[0] == descriptor0[0] && snapshot.words.at(snapshot.codes[2]).dwords[0] == descriptor1[0] && snapshot.words.at(snapshot.codes[2]).dwords[1] == descriptor1[1], "pointer chain image table: the snapshot words are wrong"); const auto unmapped = Abi::PoisonCode(static_cast(Abi::PoisonReason::Unmapped)); require(snapshot.codes[3] == unmapped && snapshot.codes[4] == unmapped && snapshot.codes[255] == unmapped, "pointer chain image table: a record whose chain reaches a null pointer is not Unmapped poison"); for (std::uint32_t record = 0; record < 3u; record++) { @@ -3902,7 +4731,7 @@ void verifyPixelRequestSerialization() { minimal.context.pixel->layerExported = true; minimal.context.pixel->ditheredAlphaToCoverage = true; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQSMAAAA=", "new requests did not use serialization version 35"); + require(requestPrefix(encoded, 8u) == "NVNQQSQAAAA=", "new requests did not use serialization version 36"); const std::array fallbacks{0x40u, 0x1ac4u}; auto withFallbacks = minimal; withFallbacks.context.linkedCallFallbacks = fallbacks; @@ -4020,7 +4849,7 @@ void verifyPixelRequestSerialization() { for (std::size_t bytes = 0; bytes < 45u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export, dither or front-face flag, quad pixel mask or quad sample mask was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQSQAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQSUAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ @@ -4165,12 +4994,13 @@ void verifyVertexInfoVersion19() { const auto orderedBytes = decodeBase64(serializer.Serialize(ordered)); require(orderedBytes.size() == plainPixel.size() && std::equal(plainPixel.begin(), plainPixel.begin() + static_cast(orderOffset), orderedBytes.begin()) && orderedBytes[orderOffset] == 0x10u && orderedBytes[orderOffset + 11u] == 0x76u && orderedBytes[orderOffset + 12u] == 0x01u && std::equal(plainPixel.begin() + static_cast(orderOffset + sampleOrderBytes), plainPixel.end(), orderedBytes.begin() + static_cast(orderOffset + sampleOrderBytes)), "the sample order is not the four words the version 35 pixel info adds after the pixel iteration fields"); auto orderedVersion34 = orderedBytes; + orderedVersion34.pop_back(); orderedVersion34.erase(orderedVersion34.begin() + static_cast(orderOffset), orderedVersion34.begin() + static_cast(orderOffset + sampleOrderBytes)); orderedVersion34[4] = 34u; const auto orderedReplay34 = serializer.Deserialize(encodeBase64(orderedVersion34)); require(orderedReplay34.request.context.pixel.has_value() && orderedReplay34.request.context.pixel->sampleOrder == std::array{} && serializer.Serialize(orderedReplay34.request) == encodedPixel, "a version 34 request read a sample order, or misread the fields after its pixel iteration fields"); auto orderedVersion30 = orderedBytes; - orderedVersion30.pop_back(); + orderedVersion30.resize(orderedVersion30.size() - 2u); orderedVersion30.erase(orderedVersion30.begin() + static_cast(blendOffset + 4u), orderedVersion30.begin() + static_cast(orderOffset + sampleOrderBytes)); orderedVersion30[4] = 30u; const auto orderedReplay30 = serializer.Deserialize(encodeBase64(orderedVersion30)); @@ -4178,7 +5008,8 @@ void verifyVertexInfoVersion19() { auto orderedSpanned = ordered; orderedSpanned.target.spannedWaves = true; auto orderedVersion32 = decodeBase64(serializer.Serialize(orderedSpanned)); - require(orderedVersion32.back() == 1u, "the spanned-waves flag is not the last request byte"); + require(orderedVersion32.size() > 1u && orderedVersion32.back() == 0u && orderedVersion32[orderedVersion32.size() - 2u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag"); + orderedVersion32.pop_back(); orderedVersion32.erase(orderedVersion32.begin() + static_cast(blendOffset + 4u), orderedVersion32.begin() + static_cast(orderOffset + sampleOrderBytes)); orderedVersion32[4] = 32u; const auto orderedReplay32 = serializer.Deserialize(encodeBase64(orderedVersion32)); @@ -4197,6 +5028,7 @@ void verifyVertexInfoVersion19() { require(std::equal(maskedBytes.begin(), maskedBytes.begin() + static_cast(maskOffset + 1u), sampledBytes.begin()) && sampledBytes[maskOffset + 1u] == 0x1u && sampledBytes[maskOffset + 5u] == 0x2u && sampledBytes[maskOffset + 9u] == 0x4u && sampledBytes[maskOffset + 13u] == 0x8u, "the quad sample masks are not the four words after the quad pixel mask"); auto version30 = sampledBytes; version30.pop_back(); + version30.pop_back(); version30.erase(version30.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u), version30.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); version30[4] = 30u; const auto replay30 = serializer.Deserialize(encodeBase64(version30)); @@ -4204,31 +5036,36 @@ void verifyVertexInfoVersion19() { auto spanned = sampledPixel; spanned.target.spannedWaves = true; auto version32 = decodeBase64(serializer.Serialize(spanned)); - require(version32.back() == 1u, "the spanned-waves flag is not the last request byte"); + require(version32.size() > 1u && version32.back() == 0u && version32[version32.size() - 2u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag"); + version32.pop_back(); version32.erase(version32.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u), version32.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); version32[4] = 32u; const auto replay32 = serializer.Deserialize(encodeBase64(version32)); require(replay32.request.context.pixel.has_value() && !replay32.request.context.pixel->iteratesSamples && replay32.request.context.pixel->sampleLocations == std::array{} && replay32.request.target.spannedWaves && serializer.Serialize(replay32.request) == serializer.Serialize(spanned), "a version 32 request read pixel iteration or sample locations, or lost its spanned-waves flag"); auto version29 = sampledBytes; version29.pop_back(); + version29.pop_back(); version29.erase(version29.begin() + static_cast(maskOffset + 1u + sampleMaskBytes), version29.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); version29[4] = 29u; const auto replay29 = serializer.Deserialize(encodeBase64(version29)); require(replay29.request.context.pixel.has_value() && replay29.request.context.pixel->quadSampleMask == std::array{0x1u, 0x2u, 0x4u, 0x8u} && replay29.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay29.request)) == sampledBytes, "a version 29 request read a reversed-order blend code or misread the fields after the quad sample masks"); auto version28 = sampledBytes; version28.pop_back(); + version28.pop_back(); version28.erase(version28.begin() + static_cast(maskOffset + 1u), version28.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); version28[4] = 28u; const auto replay28 = serializer.Deserialize(encodeBase64(version28)); require(replay28.request.context.pixel.has_value() && replay28.request.context.pixel->quadPixelMask == 0x2u && replay28.request.context.pixel->quadSampleMask == std::array{0xffffu, 0xffffu, 0xffffu, 0xffffu} && replay28.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay28.request)) == maskedBytes, "a version 28 request read quad sample masks or a reversed-order blend code, or misread the fields after them"); auto version27 = maskedBytes; version27.pop_back(); + version27.pop_back(); version27.erase(version27.begin() + static_cast(maskOffset), version27.begin() + static_cast(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); version27[4] = 27u; const auto replay27 = serializer.Deserialize(encodeBase64(version27)); require(replay27.request.context.pixel.has_value() && replay27.request.context.pixel->quadPixelMask == 0xfu && replay27.request.context.pixel->reversedBlend == 0u && serializer.Serialize(replay27.request) == encodedPixel, "a version 27 request read a quad pixel mask or reversed-order blend code, or misread the fields after it"); auto legacyPixel = plainPixel; legacyPixel.pop_back(); + legacyPixel.pop_back(); legacyPixel.erase(legacyPixel.begin() + static_cast(allBitsOffset), legacyPixel.begin() + static_cast(allBitsOffset + 2u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes)); legacyPixel[4] = 26u; const auto replay26 = serializer.Deserialize(encodeBase64(legacyPixel)); @@ -4259,6 +5096,56 @@ void verifyVertexInfoVersion19() { } } +void verifyDepthRemapRequests() { + using namespace ShaderRecompiler; + const std::array code{0xbf810000u}; + const RequestSerializer serializer; + RecompileRequest vertex{}; + vertex.shader = {ShaderStage::Vertex, 0x10000u, code, 0, {}}; + vertex.context.waveSize = 64; + vertex.context.vertex = ShaderVertexStageInfo{}; + vertex.context.vertex->paClVsOutCntl = 0x89abcdefu; + const auto plain = serializer.Serialize(vertex); + vertex.target.depthRangeRemap = true; + vertex.context.vertex->depthRemapMode = 2u; + const auto remapped = serializer.Serialize(vertex); + const auto replay = serializer.Deserialize(remapped); + require(replay.request.target.depthRangeRemap && replay.request.context.vertex.has_value() && replay.request.context.vertex->paClVsOutCntl == 0x89abcdefu, "the depth remap target flag or the vertex info did not round-trip"); + require(replay.request.context.vertex->depthRemapMode == 2u, "the depth remap mode did not round-trip"); + require(serializer.Serialize(replay.request) == remapped, "a version 35 depth remap request changed on re-serialization"); + const auto plainBytes = decodeBase64(plain); + const auto remappedBytes = decodeBase64(remapped); + require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 5u && std::equal(plainBytes.begin(), plainBytes.end() - 5, remappedBytes.begin()), "the depth remap flag and mode are not the last 5 bytes of a vertex request"); + for (const std::uint8_t legacyVersion : {32u, 34u}) { + auto legacyBytes = remappedBytes; + legacyBytes[4] = legacyVersion; + legacyBytes.resize(legacyBytes.size() - 5u); + const auto legacy = serializer.Deserialize(encodeBase64(legacyBytes)); + require(!legacy.request.target.depthRangeRemap && legacy.request.context.vertex.has_value() && legacy.request.context.vertex->depthRemapMode == 0u, ("a version " + std::to_string(legacyVersion) + " vertex request read depth remap fields").c_str()); + require(serializer.Serialize(legacy.request) == plain, ("a version " + std::to_string(legacyVersion) + " vertex request misread the fields before the depth remap").c_str()); + } + auto truncated = remappedBytes; + truncated.pop_back(); + expectFailure([&] { static_cast(serializer.Deserialize(encodeBase64(truncated))); }, "truncated data", "a truncated depth remap mode was accepted"); + auto trailing = remappedBytes; + trailing.push_back(0u); + expectFailure([&] { static_cast(serializer.Deserialize(encodeBase64(trailing))); }, "unexpected trailing data", "a byte after the depth remap fields was accepted"); + RecompileRequest pixel{}; + pixel.shader = {ShaderStage::Fragment, 0x20000u, code, 0, {}}; + pixel.context.waveSize = 64; + pixel.context.pixel = ShaderPixelStageInfo{}; + pixel.target.depthRangeRemap = true; + const auto pixelReplay = serializer.Deserialize(serializer.Serialize(pixel)); + require(pixelReplay.request.target.depthRangeRemap && !pixelReplay.request.context.vertex.has_value(), "a pixel request lost the depth remap target flag or gained vertex info"); + std::vector key; + std::vector other; + RecompileCacheKey::Build(vertex, key); + auto zeroToOne = vertex; + zeroToOne.context.vertex->depthRemapMode = 1u; + RecompileCacheKey::Build(zeroToOne, other); + require(key != other, "the depth remap mode did not change the shader cache key"); +} + void verifyLegacyPixelRequests() { using namespace ShaderRecompiler; static constexpr std::array legacyPixelRequests{ @@ -6206,13 +7093,13 @@ void verifyVolumeBiasTableRejection() { auto& column = snapshot.tables.tables.front(); column.records = 2u; column.keys = 2u; - column.codes = {0u, 1u}; + column.codes = SharedValues(std::vector{0u, 1u}); auto plane = volume; plane.dwords[3] = (TableType2D << 28u) | 0xfacu; for (const auto [word, bits] : {std::pair{4u, 1u}, std::pair{3u, 1u << 12u}, std::pair{3u, 1u << 16u}, std::pair{5u, 1u << 4u}}) { auto invalid = volume; invalid.dwords[word] |= bits; - snapshot.tables.words = {plane, invalid}; + column.words = SharedValues(std::vector{plane, invalid}); const auto resolved = ResourceMaterializer::ResolveImageTables(info, snapshot); const auto mapStart = resolved.map.at(Abi::TableHeader(0u) + Abi::TableMapStart); const auto good = resolved.map.at(mapStart); @@ -6319,6 +7206,14 @@ int main(int argc, char** argv) { std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { + verifyTableSnapshotReuse(); + verifyTableStampRace(); + verifyTableVouch(); + verifyTableSpanQuery(); + verifyTableEdgeStore(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--selected-records-image-limit") { verifySelectedRecordsWithinLimits(false, false); std::cout << "Selected image-table records past the image limit merge back into one table per heap\n"; @@ -6383,6 +7278,7 @@ int main(int argc, char** argv) { verifyCanonicalVersion20And21Requests(); verifyPixelRequestSerialization(); verifyVertexInfoVersion19(); + verifyDepthRemapRequests(); verifyPositionExportComponents(); verifyLegacyPixelRequests(); verifyPixelExportReplay(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthRangeRemapDraw.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthRangeRemapDraw.cpp new file mode 100644 index 0000000000..831ec0b751 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthRangeRemapDraw.cpp @@ -0,0 +1,257 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "Recompiler.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "DepthFastClearHarness.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace Harness = DepthFastClearHarness; +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +struct Transform { + bool negativeOneToOne; + float scale; + float offset; + float clampMin; + float clampMax; +}; + +struct Case { + Transform transform; + float z; + bool drawn; + float probe; + bool covered; + bool remapOnly; + const char* what; +}; + +constexpr Transform Plain{false, 1.0f, 0.0f, 0.0f, 1.0f}; + +alignas(4096) std::array, 32> Depths{}; +alignas(256) std::array, 3>, 128> VertexSets{}; +std::size_t vertexSets = 0; + +// A vertex shader that reads no user data: the fullscreen triangle comes from the vertex index (v5), and the +// clip z is the literal at index 9. Its program carries no push data of its own. +alignas(256) std::array FetchlessCode{ + 0x36020a81u, 0x34020282u, 0x4a0202c1u, 0x7e020b01u, 0x2c040a81u, 0x34040482u, 0x4a0404c1u, 0x7e040b02u, + 0x7e0602ffu, 0u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u, +}; + +std::uint32_t Draw(AgcDriver::VulkanDevice& device, std::uint64_t depth, const Transform& transform, bool remap, float z, bool write, bool fetchless = false, bool targetRemap = false, bool prepared = false) { + Require(vertexSets < VertexSets.size(), "the test ran out of vertex sets"); + auto& vertices = VertexSets[vertexSets++]; + vertices = {{{-1.0f, -1.0f, z, 1.0f}, {3.0f, -1.0f, z, 1.0f}, {-1.0f, 3.0f, z, 1.0f}}}; + Harness::Pixels.fill(std::byte{0}); + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Harness::Pixels.data()), {Harness::Width, Harness::Height}, VK_FORMAT_R8G8B8A8_UNORM, Harness::Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.depth = AgcDriver::Graphics::DepthTarget{depth, 0, {Harness::Width, Harness::Height}, VK_FORMAT_D32_SFLOAT, 1.0f, 0}; + state.depthTest = true; + state.depthWrite = write; + state.depthCompare = write ? VK_COMPARE_OP_ALWAYS : VK_COMPARE_OP_LESS; + state.renderExtent = {Harness::Width, Harness::Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + const float minDepth = transform.negativeOneToOne ? transform.offset - transform.scale : transform.offset; + state.viewport = {0, static_cast(Harness::Height), static_cast(Harness::Width), -static_cast(Harness::Height), minDepth, transform.offset + transform.scale}; + state.negativeOneToOne = transform.negativeOneToOne; + state.depthScale = transform.scale; + state.depthOffset = transform.offset; + state.depthClampMin = transform.clampMin; + state.depthClampMax = transform.clampMax; + state.scissor = {{0, 0}, {Harness::Width, Harness::Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + auto context = device.graphicsContext(); + context.depthRangeUnrestricted = !remap; + AgcDriver::Graphics::ApplyDepthRemap(context, state); + const bool inRange = minDepth >= 0.0f && transform.offset + transform.scale <= 1.0f; + Require((state.depthRemap.mode != 0u) == (remap && !inRange), "the draw did not take the depth path the test asked for"); + + auto target = device.Target(); + target.depthRangeRemap = state.depthRemap.mode != 0u || targetRemap; + ShaderRecompiler::ShaderVertexStageInfo vertexInfo{}; + vertexInfo.depthRemapMode = state.depthRemap.mode; + std::vector vertexUserData(4, 0u); + if (fetchless) { + FetchlessCode[9] = std::bit_cast(z); + } else { + const auto vertexBuffer = Harness::BufferDescriptor(vertices.data(), 16u, static_cast(vertices.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + } + const std::span vertexCode = fetchless ? std::span(FetchlessCode) : std::span(Harness::VertexCode); + const std::array vertexMemory{{{reinterpret_cast(vertexCode.data()), std::as_bytes(vertexCode)}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(vertexCode.data()), vertexCode, 0, {}}, + {32, 0, vertexUserData, std::nullopt, std::nullopt, vertexInfo, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = prepared; + std::shared_ptr vertexHolder; + if (prepared) { + const auto handle = ShaderRecompiler::PrepareShader(vertex); + const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(vertex, handle); + Require(invocation.has_value(), "the prepared vertex shader did not match its own request"); + vertexHolder = invocation->Materialize(*invocation->Capture(ShaderRecompiler::SrtRuntime{std::span(vertexUserData)})); + } else { + vertexHolder = std::make_shared(ShaderRecompiler::Recompile(vertex)); + } + const auto& vertexResult = *vertexHolder; + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + if (fetchless) Require(vertexPush == 0u, "the fetchless vertex shader carries push data of its own"); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = true; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + std::vector pixelUserData(4, 0u); + const std::array pixelMemory{{{reinterpret_cast(Harness::PixelCode.data()), std::as_bytes(std::span(Harness::PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(Harness::PixelCode.data()), Harness::PixelCode, 0, {}}, + {32, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(vertices.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); + std::uint32_t covered = 0; + for (std::size_t index = 0; index < Harness::Pixels.size() / 4u; ++index) { + if (std::to_integer(Harness::Pixels[index * 4u]) == 255u) ++covered; + } + return covered; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto context = device->graphicsContext(); + if (!context.depthClamp) { + std::puts("skipped, the device has no depth clamping"); + return VulkanTestSkipped; + } + constexpr float Step = 1.0f / 64.0f; + constexpr Transform minecraft{true, 1.0f, 0.0f, -1.0f, 1.0f}; + constexpr Transform widened{false, 2.0f, -0.5f, 0.0f, 1.0f}; + constexpr Transform bounded{false, 2.0f, -0.5f, 0.25f, 0.75f}; + constexpr std::array cases{{ + {minecraft, 0.5f - Step, true, 0.5f, false, false, "negative-one-to-one depth just below 0.5"}, + {minecraft, 0.5f + Step, true, 0.5f, true, false, "negative-one-to-one depth just above 0.5"}, + {minecraft, -1.5f, false, 0.5f, true, false, "negative-one-to-one clip z in front of the near plane"}, + {minecraft, 1.5f, false, 0.5f, true, false, "negative-one-to-one clip z behind the far plane"}, + {widened, 0.5f - Step / 2.0f, true, 0.5f, false, false, "scaled zero-to-one depth just below 0.5"}, + {widened, 0.5f + Step / 2.0f, true, 0.5f, true, false, "scaled zero-to-one depth just above 0.5"}, + {widened, -0.125f, false, 0.5f, true, false, "zero-to-one clip z in front of the near plane"}, + {widened, 1.125f, false, 0.5f, true, false, "zero-to-one clip z behind the far plane"}, + {bounded, 0.4f, true, 0.35f, false, false, "depth 0.3 inside clamp bounds [0.25, 0.75], probed at 0.35"}, + {bounded, 0.4f, true, 0.25f, true, false, "depth 0.3 inside clamp bounds [0.25, 0.75], probed at 0.25"}, + {bounded, 0.6f, true, 0.65f, true, false, "depth 0.7 inside clamp bounds [0.25, 0.75], probed at 0.65"}, + {bounded, 0.2f, true, 0.2f, true, true, "depth -0.1 clamped to 0.25, probed at 0.2"}, + {bounded, 0.8f, true, 0.8f, false, true, "depth 1.1 clamped to 0.75, probed at 0.8"}, + }}; + std::size_t surfaces = 0; + std::size_t ran = 0; + for (const bool remap : {true, false}) { + if (!remap && !context.depthRangeUnrestricted) continue; + for (const auto& test : cases) { + if (!remap && (test.remapOnly || (test.transform.negativeOneToOne && !context.depthClipControl))) continue; + Require(surfaces < Depths.size(), "the test ran out of depth surfaces"); + const auto depth = static_cast(reinterpret_cast(Depths[surfaces++].data())); + const auto drawn = Draw(*device, depth, test.transform, remap, test.z, true); + const auto passed = Draw(*device, depth, Plain, false, test.probe, false); + const auto path = remap ? "remapped" : "unrestricted"; + if (drawn != (test.drawn ? Harness::Covered : 0u) || passed != (test.covered ? Harness::Covered : 0u)) { + char detail[256]; + std::snprintf(detail, sizeof(detail), "%s (%s path): the draw covered %u of %u pixels (expected %s) and the probe passed %u (expected %s)", test.what, path, drawn, Harness::Covered, test.drawn ? "all" : "none", passed, test.covered ? "all" : "none"); + throw std::runtime_error(detail); + } + ++ran; + } + } + // Scale and offset reach the vertex shader through the draw's push block, so two draws whose depth remap + // differs only in them build one pipeline between them: the second finds it in the store, and both + // store the depth they should. + { + constexpr Transform widerScale{false, 4.0f, -1.5f, 0.0f, 1.0f}; + Require(surfaces + 2u <= Depths.size(), "the test ran out of depth surfaces"); + const auto depthA = static_cast(reinterpret_cast(Depths[surfaces++].data())); + const auto depthB = static_cast(reinterpret_cast(Depths[surfaces++].data())); + const auto first = Draw(*device, depthA, widened, true, 0.5f + Step / 2.0f, true); + const auto before = AgcDriver::Graphics::PipelineStoreTotals(); + const auto second = Draw(*device, depthB, widerScale, true, 0.5f + Step / 4.0f, true); + const auto after = AgcDriver::Graphics::PipelineStoreTotals(); + Require(first == Harness::Covered && second == Harness::Covered, "a draw with a remapped depth did not cover the target"); + Require(after.misses == before.misses && after.hits > before.hits, "a draw that differs only in the depth scale and offset built another pipeline"); + for (const auto depth : {depthA, depthB}) { + Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered, "a depth remapped by a shared pipeline was stored below 0.5 + 1/64"); + Require(Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a depth remapped by a shared pipeline was stored above 0.5 + 1/64"); + } + ++ran; + } + // A remapped vertex shader with no push data of its own, and a fragment shader with none: the pipeline's + // push stages must still include the vertex stage (a non-GPL layout names them), or the scale and offset + // reach no one. + { + Require(surfaces < Depths.size(), "the test ran out of depth surfaces"); + const auto depth = static_cast(reinterpret_cast(Depths[surfaces++].data())); + const auto drawn = Draw(*device, depth, widened, true, 0.5f + Step / 2.0f, true, true); + Require(drawn == Harness::Covered, "a fetchless remapped vertex shader did not cover the target"); + Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered, "a fetchless remapped vertex shader stored a depth below 0.5 + 1/64, so its scale and offset did not arrive"); + Require(Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a fetchless remapped vertex shader stored a depth above 0.5 + 1/64"); + ++ran; + } + // On a device without VK_EXT_depth_range_unrestricted the target remaps every vertex shader, so an ordinary + // in-range draw (mode 0) with no push data of its own still carries the remap's push block: it must validate + // and draw, from a direct build and from a prepared one. + for (const bool prepared : {false, true}) { + Require(surfaces < Depths.size(), "the test ran out of depth surfaces"); + const auto depth = static_cast(reinterpret_cast(Depths[surfaces++].data())); + const auto drawn = Draw(*device, depth, Plain, false, 0.5f + Step / 2.0f, true, true, true, prepared); + Require(drawn == Harness::Covered, prepared ? "a fetchless mode 0 vertex shader on a remapping target, prepared, did not cover the target" : "a fetchless mode 0 vertex shader on a remapping target did not cover the target"); + Require(Draw(*device, depth, Plain, false, 0.5f, false) == Harness::Covered && Draw(*device, depth, Plain, false, 0.5f + 2.0f * Step, false) == 0u, "a fetchless mode 0 vertex shader on a remapping target stored the wrong depth"); + ++ran; + } + std::printf("depth range remap draw tests passed (%zu cases, unrestricted path %s)\n", ran, context.depthRangeUnrestricted ? "compared" : "absent"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp index b5650189c7..a82e34a36a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheRelocation.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "Optimization/SrtWalker.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "VulkanTestDevice.hpp" #include @@ -11,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -131,6 +135,27 @@ void MovedDataHitsTheCache(bool libraries) { Require(again.lookups == 10 && again.hits == 5, mode + ": a draw at an address seen before did not reuse the cached draw (" + std::to_string(again.hits) + " hits)"); } +void SplitRunsNameTheirSharedWords() { + alignas(16) static std::array data{1u, 2u, 3u, 4u}; + auto table = std::make_shared(); + table->words = {5u, 6u, 7u, 8u}; + const auto dataAddress = static_cast(reinterpret_cast(data.data())); + const auto tableAddress = dataAddress + 0x1000u; + const std::array regions{{{dataAddress, std::as_bytes(std::span(data))}, {tableAddress, std::as_bytes(std::span(table->words))}}}; + ShaderRecompiler::SrtUserDataUse use; + use.asBase = 0x3u; + use.reads = {{dataAddress, 0}, {dataAddress + 8u, 1}}; + const std::array, 1> supplied{table}; + AgcDriver::DriverDetail::DispatchVariant variant; + AgcDriver::DriverDetail::Driver::tagVariantRuns(variant, regions, use, true, 0, supplied); + Require(variant.runs.size() > regions.size(), "two user-data bases into one region did not split it into several runs"); + Require(variant.shared.size() == 1u && variant.shared.front().regions == 1u, "the table region's words were not recorded as one shared run"); + const auto& shared = variant.shared.front(); + std::size_t first = 0; + for (std::size_t run = 0; run < shared.region && run < variant.runs.size(); ++run) first += static_cast((variant.runs[run].second - variant.runs[run].first) / sizeof(std::uint32_t)); + Require(shared.region < variant.runs.size() && variant.runs[shared.region].first == tableAddress && first == shared.first, "the shared words of a variant whose runs were split for relocation name run " + std::to_string(shared.region) + " of " + std::to_string(variant.runs.size()) + ", not the table's run " + std::to_string(variant.runs.size() - 1u)); +} + void UnrelocatedAliasesStayOut(bool libraries) { Registered(SecondVertexHeader, SecondVertexCode.data(), sizeof(SecondVertexCode), 2); Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1); @@ -205,6 +230,7 @@ void MovedForwardedWordReachesStageDataHit(bool libraries) { int main() { try { + SplitRunsNameTheirSharedWords(); bool libraries = false; { const auto device = OpenVulkanTestDevice(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp index 1629b16458..f7a18fada0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp @@ -26,6 +26,13 @@ alignas(256) constexpr std::array Code{ 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; +constexpr std::size_t ComparesAt = 27; +constexpr std::array WithCompares(std::array compares) { + auto code = Code; + for (std::size_t word = 0; word < compares.size(); ++word) code[ComparesAt + word] = compares[word]; + return code; +} +alignas(256) constexpr std::array SwappedCode = WithCompares({0x7d8c02f9u, 0x06068c02u, 0x7d860609u}); constexpr std::array Pattern{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; constexpr std::uint32_t Untouched = 0xcafebabeu; alignas(256) std::array Output{}; @@ -35,6 +42,35 @@ std::vector UserData(void* output, std::uint32_t records, std::ui return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), records, format, Pattern[0], Pattern[1], Pattern[2], Pattern[3], count, period}; } +void SwappedComparesTests() { + const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1}; + const std::array packet{0xc0031500u, 4, 1, 1, 0x41u}; + for (const auto period : {1u, 2u, 4u}) { + const auto data = UserData(Output.data(), 1024, 256, period); + const auto listed = MatchDwordPatternFill(packet, Code, data, compute); + const auto swapped = MatchDwordPatternFill(packet, SwappedCode, data, compute); + Require(listed && swapped && swapped->base == listed->base && swapped->bytes == listed->bytes && swapped->pattern == listed->pattern, "the kernel with its two compares in the other order was not recognized as the same fill"); + } + const auto data = UserData(Output.data(), 1024, 256, 4); + for (std::size_t word = 0; word < SwappedCode.size(); ++word) { + auto altered = SwappedCode; + altered[word] ^= 1u; + Require(!MatchDwordPatternFill(packet, altered, data, compute), "a different shader matched the dword pattern-fill kernel with swapped compares"); + } + Require(!MatchDwordPatternFill(packet, std::span(SwappedCode).first(68), data, compute), "a truncated shader with swapped compares matched"); + constexpr std::array, 5> otherOrders{{ + {0x7d8c02f9u, 0x7d860609u, 0x06068c02u}, + {0x06068c02u, 0x7d8c02f9u, 0x7d860609u}, + {0x7d860609u, 0x06068c02u, 0x7d8c02f9u}, + {0x7d860609u, 0x7d860609u, 0x7d8c02f9u}, + {0x7d8c02f9u, 0x06068c02u, 0x7d8c02f9u}, + }}; + for (const auto& order : otherOrders) { + const auto code = WithCompares(order); + Require(!MatchDwordPatternFill(packet, code, data, compute), "a kernel whose compare words are not one of the two orders matched"); + } +} + void MatchTests() { const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1}; const std::array packet{0xc0031500u, 4, 1, 1, 0x41u}; @@ -74,7 +110,7 @@ void MatchTests() { Require(clamped && clamped->bytes == 256, "the fill exceeded the descriptor's records"); } -std::vector Commands(const std::vector& data, std::uint32_t groups, bool partial, std::uint32_t threads) { +std::vector Commands(std::span code, const std::vector& data, std::uint32_t groups, bool partial, std::uint32_t threads) { std::vector words; const auto registers = [&](std::uint32_t first, std::span values) { words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); @@ -83,7 +119,7 @@ std::vector Commands(const std::vector& data, std: }; const std::array shape{64, 1, 1}; registers(0x207, shape); - const auto address = reinterpret_cast(Code.data()); + const auto address = reinterpret_cast(code.data()); const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; registers(0x20c, program); const std::array resource{(static_cast(data.size()) << 1u) | (1u << 7u)}; @@ -111,18 +147,20 @@ void ExecuteTests() { {4, 1024, 256, 5, false, 0}, {0, 1024, 256, 4, true, 70}, }}; - for (const auto& item : cases) { - Output.fill(Untouched); - const auto words = Commands(UserData(Output.data(), item.records, item.count, item.period), item.groups, item.partial, item.threads); - Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; - AgcDriver::Submit(&packet, 0); - AgcDriverWaitIdle_nid_postfix(); - AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); - const auto launched = item.partial ? item.threads : item.groups * 64u; - const auto filled = std::min({launched, item.records, item.count}); - for (std::uint32_t i = 0; i < Output.size(); ++i) { - const auto expected = i < filled ? Pattern[std::min(i % item.period, 3u)] : Untouched; - Require(Output[i] == expected, "dword pattern fill mismatch at " + std::to_string(i) + " (period " + std::to_string(item.period) + ", count " + std::to_string(item.count) + ", records " + std::to_string(item.records) + "): got " + std::to_string(Output[i]) + ", expected " + std::to_string(expected)); + for (const std::span code : {std::span(Code), std::span(SwappedCode)}) { + for (const auto& item : cases) { + Output.fill(Untouched); + const auto words = Commands(code, UserData(Output.data(), item.records, item.count, item.period), item.groups, item.partial, item.threads); + Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + const auto launched = item.partial ? item.threads : item.groups * 64u; + const auto filled = std::min({launched, item.records, item.count}); + for (std::uint32_t i = 0; i < Output.size(); ++i) { + const auto expected = i < filled ? Pattern[std::min(i % item.period, 3u)] : Untouched; + Require(Output[i] == expected, "dword pattern fill mismatch at " + std::to_string(i) + " (" + (code.data() == SwappedCode.data() ? "swapped" : "listed") + " compares, period " + std::to_string(item.period) + ", count " + std::to_string(item.count) + ", records " + std::to_string(item.records) + "): got " + std::to_string(Output[i]) + ", expected " + std::to_string(expected)); + } } } } @@ -131,7 +169,10 @@ void ExecuteTests() { int main() { try { - if (std::getenv("APS5_NO_FILL_HLE") == nullptr) MatchTests(); + if (std::getenv("APS5_NO_FILL_HLE") == nullptr) { + MatchTests(); + SwappedComparesTests(); + } { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp index d95da537b3..2c65e4d7a8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp @@ -1,5 +1,10 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/SrtWalker.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -16,6 +21,8 @@ #endif #include #include +#include +#include #include #include #include @@ -23,7 +30,10 @@ #include #include #include +#include #include +#include +#include #include namespace { @@ -47,6 +57,12 @@ constexpr std::size_t HeapOffset = OutputOffset + 4096; constexpr std::size_t TexturesOffset = HeapOffset + static_cast(Records) * Stride; constexpr std::size_t BlockBytes = (TexturesOffset + static_cast(Surfaces) * TextureBytes + 65535) & ~static_cast(65535); +constexpr std::uint32_t WideStride = 48; +constexpr std::uint32_t WideRecords = 32768; +constexpr std::size_t WideKernelOffset = 2048; +constexpr std::size_t WideHeapOffset = 65536; +constexpr std::array WideSlots{SrtSlots - 1u, SrtSlots - 2u, SrtSlots - 3u}; +constexpr std::array WideTableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x9310b010u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; constexpr std::array TableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; class GuestBlock { @@ -97,10 +113,9 @@ std::vector Pm4(std::uint32_t opcode, std::initializer_list words = {}) { + const auto kernel = block.At(kernelOffset); const auto srt = block.At(SrtOffset + slot * SrtBytes); - std::vector words; for (const auto& packet : {Pm4(0x76, {0x20c, static_cast(kernel >> 8u)}), Pm4(0x76, {0x20d, static_cast(kernel >> 40u) & 0xffu}), Pm4(0x76, {0x207, 64}), Pm4(0x76, {0x208, 1}), Pm4(0x76, {0x209, 1}), Pm4(0x76, {0x213, 2u << 1u}), Pm4(0x76, {0x240, static_cast(srt)}), Pm4(0x76, {0x241, static_cast(srt >> 32u)}), Pm4(0x15, {1, 1, 1, 0x41})}) { words.insert(words.end(), packet.begin(), packet.end()); } @@ -150,6 +165,173 @@ void SrtMovingPerDispatch(const GuestBlock& block) { Require(most <= Driver::DispatchCacheBytes(), "image table cache bounds: dispatches reading a 2 MiB table through " + std::to_string(SrtSlots) + " SRTs kept " + std::to_string(most >> 20u) + " MiB of captured words in the dispatch cache, over its budget of " + std::to_string(Driver::DispatchCacheBytes() >> 20u) + " MiB"); } +std::array TextureWords(const GuestBlock& block, std::uint32_t element) { + const auto address = block.At(TexturesOffset + static_cast(element / Aliases) * TextureBytes); + return {static_cast(address >> 8u), static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 7u << 14u, 0xfacu | (Type2D << 28u), 0u, 0u, element % Aliases, 0u}; +} + +std::uint64_t WideRecord(const GuestBlock& block, std::uint32_t record) { + return block.At(WideHeapOffset + static_cast(record) * WideStride); +} + +void SetWideRecord(const GuestBlock& block, std::uint32_t record, std::uint32_t element) { + const auto words = TextureWords(block, element); + std::memcpy(block.Bytes(WideHeapOffset + static_cast(record) * WideStride), words.data(), sizeof(words)); +} + +void VouchedLookups(const GuestBlock& block) { + using AgcDriver::ShaderMemory; + using ShaderRecompiler::ResourceMaterializer; + std::memcpy(block.Bytes(WideKernelOffset), WideTableCode.data(), sizeof(WideTableCode)); + for (std::uint32_t record = 0; record < WideRecords; ++record) { + SetWideRecord(block, record, (1u + record % 15u) * Aliases); + std::memset(block.Bytes(WideHeapOffset + static_cast(record) * WideStride + 32u), 0x5a, WideStride - 32u); + } + for (const auto slot : WideSlots) { + auto* srt = reinterpret_cast(block.Bytes(SrtOffset + slot * SrtBytes)); + std::fill(srt, srt + SrtBytes / 4u, 0u); + const auto heap = V(block.At(WideHeapOffset), WideStride, WideRecords); + const auto output = V(block.At(OutputOffset), 0u, 4096u); + std::copy(heap.begin(), heap.end(), srt); + std::copy(output.begin(), output.end(), srt + 12); + } + struct Step { + std::uint64_t vouched = 0; + std::uint64_t refused = 0; + std::uint64_t columns = 0; + std::uint64_t reused = 0; + }; + const auto dispatch = [&](std::size_t slot, std::vector before = {}) { + const auto vouches = ShaderMemory::Vouches(); + const auto reuse = ResourceMaterializer::TableReuse(); + Dispatch(block, slot, WideKernelOffset, std::move(before)); + const auto after = ShaderMemory::Vouches(); + const auto reused = ResourceMaterializer::TableReuse(); + return Step{after.blocks - vouches.blocks, after.refused - vouches.refused, reused.columns - reuse.columns, (reused.unwritten - reuse.unwritten) + (reused.compared - reuse.compared)}; + }; + const auto text = [](const Step& step) { return " (" + std::to_string(step.vouched) + " vouched, " + std::to_string(step.refused) + " refused, " + std::to_string(step.columns) + " columns read, " + std::to_string(step.reused) + " reused)"; }; + const auto read = dispatch(WideSlots[0]); + Require(read.columns == 1u, "image table vouched lookups: the first dispatch did not read the strided table" + text(read)); + const auto shared = dispatch(WideSlots[1]); + Require(shared.columns == 0u && shared.reused == 1u, "image table vouched lookups: a second dispatch over the unwritten table did not reuse its column" + text(shared)); + const auto hit = dispatch(WideSlots[1]); + Require(hit.vouched >= 1u && hit.refused == 0u && hit.columns == 0u && hit.reused == 0u, "image table vouched lookups: a cached dispatch over an unwritten table was not validated by its table's vouch" + text(hit)); + + std::size_t vouchedSlot = WideSlots[1]; + std::size_t otherSlot = WideSlots[0]; + const auto recapture = [&](const std::string& what, std::vector before = {}) { + const auto changed = dispatch(vouchedSlot, std::move(before)); + Require(changed.refused >= 1u && changed.columns == 1u, "image table vouched lookups: " + what + " was hidden by the cached dispatch's vouch" + text(changed)); + const auto other = dispatch(otherSlot); + Require(other.columns == 0u && other.reused == 1u, "image table vouched lookups: the other dispatch after " + what + " did not reuse the column read again" + text(other)); + const auto again = dispatch(otherSlot); + Require(again.vouched >= 1u && again.refused == 0u && again.columns == 0u && again.reused == 0u, "image table vouched lookups: the dispatch after " + what + " was not vouched for again" + text(again)); + std::swap(vouchedSlot, otherSlot); + }; + SetWideRecord(block, 0u, 9u * Aliases); + recapture("a CPU store into a record"); + + const auto words = TextureWords(block, 10u * Aliases); + const auto target = WideRecord(block, 0u); + std::vector cpWrite{0xc0000000u | ((3u + static_cast(words.size()) - 1u) << 16u) | (0x37u << 8u), 0x00100200u, static_cast(target), static_cast(target >> 32u)}; + cpWrite.insert(cpWrite.end(), words.begin(), words.end()); + recapture("a GPU write into a record pending ahead of the dispatch", cpWrite); + Require(std::memcmp(block.Bytes(WideHeapOffset), words.data(), sizeof(words)) == 0, "image table vouched lookups: the GPU write did not land in the record"); + + const auto driverWords = TextureWords(block, 11u * Aliases); + AgcDriver::GuestMemory::Write(WideRecord(block, 0u), std::as_bytes(std::span(driverWords)), 4); + recapture("a driver store into a record"); + + *reinterpret_cast(block.Bytes(WideHeapOffset + 36u)) ^= 0x10u; + const auto gap = dispatch(vouchedSlot); + Require(gap.refused >= 1u && gap.columns == 0u && gap.reused == 0u, "image table vouched lookups: a store between two records did not refuse the vouch, or the table was captured again" + text(gap)); + const auto compared = dispatch(WideSlots[2]); + Require(compared.columns == 0u && compared.reused == 1u, "image table vouched lookups: a new dispatch after a store between two records did not compare and keep the column" + text(compared)); + const auto restamped = dispatch(vouchedSlot); + Require(restamped.vouched >= 1u && restamped.refused == 0u && restamped.columns == 0u, "image table vouched lookups: a dispatch sharing a column compared after a store between two records was not vouched for again" + text(restamped)); + std::puts("image table vouched lookups passed"); +} + +struct QueuedDraw { + std::atomic release{false}; + + void Enqueue(std::uint64_t begin, std::uint64_t end) { + AgcDriver::DriverDetail::DrawPipeline::Queue0().Enqueue([this] { + while (!release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + }, {{begin, end}}); + } +}; + +void VouchUnderQueuedDraws(const GuestBlock& block) { + using AgcDriver::DriverDetail::DrawPipeline; + using AgcDriver::DriverDetail::Driver; + Require(DrawPipeline::Depth() >= 3u, "image table vouch under queued draws: the draw pipeline is off"); + for (std::uint32_t record = 0; record < WideRecords; ++record) SetWideRecord(block, record, (1u + record % 15u) * Aliases); + const auto low = WideRecord(block, 0u); + const auto bytes = static_cast(WideRecord(block, WideRecords - 1u) + 32u - low); + static_cast(Driver::Get()); + const ShaderRecompiler::SrtWriteStamp stamp{AgcDriver::GuestMemory::CollectWrites(low, bytes), GuestAllocations::GuestAllocationsGeneration_nid_postfix()}; + Require(stamp.generation != 0u, "image table vouch under queued draws: the table is not write-watched"); + auto& pipeline = DrawPipeline::Queue0(); + QueuedDraw elsewhere; + QueuedDraw between; + QueuedDraw into; + const auto releaseAll = [&] { + elsewhere.release = true; + between.release = true; + into.release = true; + }; + DrawPipeline::Active() = true; + const auto vouch = [&](const std::string& what, bool expected) { + const auto outstanding = pipeline.Outstanding(); + std::atomic finished{false}; + std::atomic waited{false}; + std::thread watchdog([&] { + const auto start = std::chrono::steady_clock::now(); + while (!finished.load()) { + if (std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) { + waited = true; + releaseAll(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + const bool vouched = AgcDriver::ShaderMemory::Vouched(&Driver::QueryPendingSpan, low, bytes, stamp); + finished = true; + watchdog.join(); + const bool queued = !waited.load() && pipeline.Outstanding() == outstanding; + if (!queued || vouched != expected) releaseAll(); + Require(queued, "image table vouch under queued draws: " + what + ": the vouch waited for the queued draws"); + Require(vouched == expected, "image table vouch under queued draws: " + what + (expected ? ": the table was not vouched for" : ": the table was still vouched for")); + }; + const auto settle = [&](std::size_t outstanding) { + const auto start = std::chrono::steady_clock::now(); + while (pipeline.Outstanding() != outstanding) { + if (std::chrono::steady_clock::now() - start > std::chrono::seconds(20)) { + releaseAll(); + Require(false, "image table vouch under queued draws: the released draws were never committed"); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }; + vouch("an idle pipeline", true); + elsewhere.Enqueue(block.At(OutputOffset), block.At(OutputOffset) + 4096u); + vouch("a queued draw writing outside the table", true); + between.Enqueue(low + 32u, low + WideStride); + vouch("a queued draw writing between two records", false); + into.Enqueue(WideRecord(block, 7u), WideRecord(block, 7u) + 32u); + elsewhere.release = true; + between.release = true; + settle(1u); + vouch("a queued draw writing into a record", false); + into.release = true; + pipeline.Drain(DrawPipeline::DrainReason::Capture); + vouch("committed draws that wrote nothing", true); + DrawPipeline::Active() = false; + std::puts("image table vouch under queued draws passed"); +} + void SetBudgets() { #ifdef _WIN32 _putenv_s("APS5_RESOURCE_CACHE_TABLE_ELEMENTS", "8192"); @@ -162,7 +344,7 @@ void SetBudgets() { } -int main() { +int main(int argc, char** argv) { SetBudgets(); try { { @@ -171,6 +353,16 @@ int main() { } GuestBlock block; Prepare(block); + if (argc == 2 && std::string(argv[1]) == "--vouched-lookups") { + VouchedLookups(block); + AgcDriverShutdown_nid_postfix(); + return 0; + } + if (argc == 2 && std::string(argv[1]) == "--queued-draws") { + VouchUnderQueuedDraws(block); + AgcDriverShutdown_nid_postfix(); + return 0; + } TablesChangingPerDispatch(block); SrtMovingPerDispatch(block); AgcDriverShutdown_nid_postfix(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableReuse.cpp index cf52853a18..d5a6b46847 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableReuse.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableReuse.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -44,11 +45,7 @@ constexpr std::uint32_t Type2D = 9; constexpr std::size_t TexturesOffset = 65536; constexpr std::size_t BlockBytes = TexturesOffset + 2 * 65536; constexpr std::size_t UnimportableBytes = BlockBytes - 256; -#ifdef _WIN32 -constexpr std::uint64_t CpuWriteResolutions = 2; -#else constexpr std::uint64_t CpuWriteResolutions = 1; -#endif alignas(256) std::array Keys{}; alignas(256) std::array Output{}; @@ -207,6 +204,18 @@ Step Since(const AgcDriver::Graphics::ImageTableReuseCounts& before) { return {now.reused - before.reused, now.resolved - before.resolved}; } +void ExpectWrites(const AgcDriver::Graphics::ImageTableReuseCounts& before, std::uint64_t kept, std::uint64_t made, const std::string& what) { + static const bool keptSets = std::getenv("APS5_KEPT_DESCRIPTOR_SETS") == nullptr || std::string(std::getenv("APS5_KEPT_DESCRIPTOR_SETS")) != "0"; + const auto now = AgcDriver::Graphics::ImageTableReuses(); + const auto keptNow = now.writesKept - before.writesKept; + const auto madeNow = now.writesMade - before.writesMade; + if (!keptSets) { + Require(keptNow == 0u && madeNow == kept + made, "image table reuse: " + what + ": table descriptors were kept with kept sets turned off"); + return; + } + Require(keptNow == kept && madeNow == made, "image table reuse: " + what + ": " + std::to_string(keptNow) + " table descriptor writes kept and " + std::to_string(madeNow) + " made, expected " + std::to_string(kept) + " and " + std::to_string(made)); +} + void Expect(const AgcDriver::Graphics::ImageTableReuseCounts& before, std::uint64_t reused, std::uint64_t resolved, const std::string& what) { const auto step = Since(before); Require(step.reused == reused && step.resolved == resolved, "image table reuse: " + what + ": " + std::to_string(step.reused) + " reused and " + std::to_string(step.resolved) + " resolved tables, expected " + std::to_string(reused) + " and " + std::to_string(resolved)); @@ -271,12 +280,15 @@ void RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& block, const Gu return; } Expect(before, 2, 1, "three dispatches of one table in one epoch"); + if (std::getenv("APS5_NO_TABLE_REPLAY") == nullptr) Require(AgcDriver::Graphics::ImageTableReuses().replayed - before.replayed == 2u, "image table reuse: three dispatches of one table in one epoch did not replay the bound table twice"); + ExpectWrites(before, 3, 0, "three dispatches of one table in one epoch"); before = AgcDriver::Graphics::ImageTableReuses(); AgcDriver::Graphics::ClearCachedTextures(device.Device()); Require(Dispatch(device, result, 7) == Row(block.Texels(7)), "image table reuse: the table after the texture cache was cleared sampled the wrong entry"); Require(Dispatch(device, result, 9) == Row(block.Texels(9)), "image table reuse: the table resolved after the texture cache was cleared sampled the wrong entry"); Expect(before, 1, 1, "a kept table whose textures the texture cache dropped"); + ExpectWrites(before, 1, 1, "a kept table whose textures the texture cache dropped"); before = AgcDriver::Graphics::ImageTableReuses(); std::vector stored(Width * Height); @@ -303,6 +315,7 @@ void RunTests(AgcDriver::VulkanDevice& device, const GuestBlock& block, const Gu Require(Dispatch(device, moved, 19) == Row(block.Texels(Moved)), "image table reuse: the moved table sampled the wrong entry when reused"); Require(Dispatch(device, result, 21) == Row(block.Texels(21)), "image table reuse: the original table sampled the wrong entry when reused"); Expect(before, 2, 2, "two tables that differ in one record"); + ExpectWrites(before, 0, 4, "two tables that differ in one record, alternating"); SetOutput(block.Texels(13)); const auto writer = Compile(device, block); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 544b856619..aaa4b364e1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -5,8 +5,11 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" -#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/AsyncLog.hpp" #include "../NarrowConstantStoreFixture.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" @@ -36,6 +39,9 @@ #define NOMINMAX #endif #include +#include +#else +#include #endif namespace { @@ -720,7 +726,12 @@ void LazyRegistration() { header.registers[0].value = static_cast(invalidAddress >> 8u); header.registers[1].value = static_cast(invalidAddress >> 40u); AgcDriverRegisterShader_nid_postfix(&header.shader); - ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); }, "s_code_end"); + AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); + { + AgcDriver::DriverDetail::ShaderPreparationTransaction transaction; + const auto& state = transaction.Read(*registered()); + Require(state.deferred && state.entries.empty(), "the pipeline ABI of a lazily registered shader it cannot prepare did not leave it to its dispatch"); + } commands[2] = header.registers[0].value; commands[5] = header.registers[1].value; sceAgcDriverSubmitAcb(0x20, &packet); @@ -784,11 +795,429 @@ void DeferredUndecodableRegistration() { ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "unsupported MIMG opcode"); } +class StderrCapture { +public: + StderrCapture() { + AgcDriver::FlushAsyncLog_nid_no_patch(); + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + AgcDriver::FlushAsyncLog_nid_no_patch(); + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +template +std::string Logged(TAction action) { + StderrCapture capture; + try { + action(); + } catch (...) { + static_cast(capture.Finish()); + throw; + } + return capture.Finish(); +} + +std::size_t Deferrals(const std::string& log) { + constexpr std::string_view line = "could not be prepared when the title created its pipeline"; + std::size_t count = 0; + for (auto found = log.find(line); found != std::string::npos; found = log.find(line, found + line.size())) ++count; + return count; +} + +struct AbiHeader { + Shader shader{}; + std::array registers{}; + std::array context{}; + ShaderSpecialRegs specials{}; + ShaderUserData users{}; +}; + +std::uint32_t ProgramLow(const void* code) { + return static_cast(reinterpret_cast(code) >> 8u); +} + +std::uint32_t ProgramHigh(const void* code) { + return static_cast(reinterpret_cast(code) >> 40u); +} + +template +void DescribeAbi(AbiHeader& header, const std::array& code, std::uint8_t type, std::initializer_list registers, std::initializer_list context) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(AbiHeader); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.type = type; + header.shader.user_data = &header.users; + std::copy(registers.begin(), registers.end(), header.registers.begin()); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = static_cast(registers.size()); + std::copy(context.begin(), context.end(), header.context.begin()); + header.shader.cx_registers = header.context.data(); + header.shader.num_cx_registers = static_cast(context.size()); +} + +alignas(256) constexpr std::array UserDataCaller{ + 0x7e020280u, 0xbe940383u, 0xbe8e0300u, 0xbe8f0301u, 0xbe8e210eu, 0xbe8e0302u, 0xbe8f0303u, 0xbe8e210eu, + 0x80948114u, 0xbf078014u, 0xbf85fff7u, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, 0xbf9f0000u, + 0x30306c73u, 0x00000040u, 0x00000008u}; +alignas(256) constexpr std::array AddFiveCallee{0x4a020285u, 0xbe80200eu}; +alignas(256) constexpr std::array AddSixtyFourCallee{0x4a0202c0u, 0xbe80200eu}; +alignas(256) std::array CallerOutput{}; +alignas(256) constexpr std::array WaterfallStoreCode{ + 0x7e020280u, 0x7e040280u, 0x7e080280u, 0x7e0a0280u, 0x7e0c0280u, 0x7e0e0280u, 0xbe86047eu, 0xf40c0200u, + 0xfa000000u, 0xd4c10000u, 0x00020083u, 0xbf8cc07fu, 0xf0200f08u, 0x00020401u, 0x8afe007eu, 0xbf89fff7u, + 0xbefe0406u, 0xbf810000u}; +alignas(256) constexpr std::array WaterfallPixelCode{ + 0xc80a0002u, 0x7e000280u, 0x7e020280u, 0xbe86047eu, 0xf40c0200u, 0xfa000000u, 0xd4c10000u, 0x00020483u, + 0xbf8cc07fu, 0xf0000f08u, 0x00020400u, 0x8afe007eu, 0xbf89fff7u, 0xbefe0406u, 0xbf8c3f70u, 0xf800180fu, + 0x07060504u, 0xbf810000u}; +alignas(256) constexpr std::array WaterfallVertexCode{ + 0x7e020280u, 0x7e040280u, 0xbe86037eu, 0xf40c0404u, 0xfa000000u, 0xd4c10008u, 0x00020083u, 0xbf8cc07fu, + 0xf0000f08u, 0x00040401u, 0x8a7e087eu, 0xbf89fff7u, 0xbefe0306u, 0xbf8a0000u, 0xbf8c3f70u, 0xf80008cfu, + 0x07060504u, 0xbf810000u}; +alignas(256) constexpr std::array OtherWaterfallVertexCode = WaterfallVertexCode; +alignas(256) constexpr std::array BarrierFreeWaterfallVertexCode{ + 0x7e020280u, 0x7e040280u, 0xbe86037eu, 0xf40c0404u, 0xfa000000u, 0xd4c10008u, 0x00020083u, 0xbf8cc07fu, + 0xf0000f08u, 0x00040401u, 0x8a7e087eu, 0xbf89fff7u, 0xbefe0306u, 0xbf800000u, 0xbf8c3f70u, 0xf80008cfu, + 0x07060504u, 0xbf810000u}; +alignas(256) constexpr std::array FrontCode{0x7e0002ffu, 0, 0x7e0202ffu, 0, 0x7e0402ffu, 0, 0x7e0602ffu, 0x3f800000u, 0xf80008cfu, 0x03020100u, 0xbf810000u}; +alignas(256) constexpr std::array UndecodablePixelCode{0xf1949f05u, 0x00060018u, 0xbf810000u}; +alignas(256) constexpr std::array PassthroughPixelCode{0xc8020002u, 0xf800180fu, 0x00000000u, 0xbf810000u}; + +void DescribeWaterfallPixel(AbiHeader& pixel) { + DescribeAbi(pixel, WaterfallPixelCode, 1, {{0x8, ProgramLow(WaterfallPixelCode.data())}, {0x9, ProgramHigh(WaterfallPixelCode.data())}, {0xa, 0}, {0xb, 2u << 1u}}, {{0x1b6, 1}, {0x1b3, 2}, {0x1b4, 2}, {0x191, 0}, {0x1c5, 4}}); +} + +void DescribeFront(AbiHeader& vertex) { + DescribeAbi(vertex, FrontCode, 2, {{0xc8, ProgramLow(FrontCode.data())}, {0xc9, ProgramHigh(FrontCode.data())}, {0x8b, 0}, {0x8a, 0}, {0xca, 0}, {0xcb, 0}, {0xcc, 0}}, {{0x2d5, 0x2000}}); + vertex.shader.specials = &vertex.specials; + vertex.specials.dispatch_modifier = 0x8000; +} + +void DescribeWaterfallVertex(AbiHeader& vertex, const std::array& code) { + DescribeAbi(vertex, code, 2, {{0xc8, ProgramLow(code.data())}, {0xc9, ProgramHigh(code.data())}, {0x8a, 0}, {0x8b, 2u << 1u}}, {{0x1ff, 64}, {0x2ab, 4}, {0x2ce, 0}}); + vertex.shader.specials = &vertex.specials; + vertex.specials = {{0x25b, 0x8040}, {0x2d5, 0x00402000}, 0, {}, {}, {0x29b, 2}, {0x260, 0}}; +} + +struct FetchingHeader { + AbiHeader abi; + std::array directOffsets{}; + ShaderSemantic semantic{}; +}; + +void DescribeStaticFetch(FetchingHeader& vertex) { + vertex.abi.shader.header_size = sizeof(FetchingHeader); + for (auto& reg : vertex.abi.registers) { + if (reg.offset == 0x8b) reg.value = 4u << 1u; + } + vertex.directOffsets.fill(0xffffu); + vertex.directOffsets[8] = 0; + vertex.directOffsets[10] = 2; + vertex.abi.users.direct_resource_offset = vertex.directOffsets.data(); + vertex.abi.users.direct_resource_count = static_cast(vertex.directOffsets.size()); + vertex.semantic.size_in_elements = 4; + vertex.semantic.static_vb_index = 1; + vertex.abi.shader.input_semantics = &vertex.semantic; + vertex.abi.shader.num_input_semantics = 1; +} + +void AbiDeferredCompute() { + static AbiHeader caller; + DescribeAbi(caller, UserDataCaller, 0, {{0x20c, ProgramLow(UserDataCaller.data())}, {0x20d, ProgramHigh(UserDataCaller.data())}, {0x207, 64}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 8u << 1u}}, {}); + caller.shader.specials = &caller.specials; + caller.specials.dispatch_modifier = 0x8000u; + AgcDriverRegisterShader_nid_postfix(&caller.shader); + const auto before = ShaderRecompiler::FrontEndBuilds(); + AgcDriverResolveShaderAbi_nid_postfix(&caller.shader, {}, {}); + const auto attempted = ShaderRecompiler::FrontEndBuilds(); + AgcDriverResolveShaderAbi_nid_postfix(&caller.shader, {}, {}); + Require(attempted > before && ShaderRecompiler::FrontEndBuilds() == attempted, "a compute static ABI deferred to its dispatch was prepared again for the same state"); + const auto first = reinterpret_cast(AddFiveCallee.data()); + const auto second = reinterpret_cast(AddSixtyFourCallee.data()); + const auto output = reinterpret_cast(CallerOutput.data()); + const std::array users{static_cast(first), static_cast(first >> 32u), static_cast(second), static_cast(second >> 32u), + static_cast(output), static_cast((output >> 32u) & 0xffffu), static_cast(sizeof(CallerOutput)), 0x31016facu}; + std::vector commands; + for (std::size_t index = 0; index < caller.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, caller.registers[index].offset, caller.registers[index].value}); + for (std::uint32_t index = 0; index < users.size(); ++index) commands.insert(commands.end(), {0xc0017600u, 0x240u + index, users[index]}); + commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); + CallerOutput.fill(0xcafebabeu); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(output, sizeof(CallerOutput)); + Require(std::ranges::all_of(CallerOutput, [](std::uint32_t lane) { return lane == 3u * (5u + 64u); }), "a compute shader its static ABI deferred did not run its user-data callees at its dispatch"); + AgcDriverShutdown_nid_postfix(); +} + +void AbiDeferredRegistrationFailure() { + static AbiHeader store; + DescribeAbi(store, WaterfallStoreCode, 0, {{0x20c, ProgramLow(WaterfallStoreCode.data())}, {0x20d, ProgramHigh(WaterfallStoreCode.data())}, {0x207, 64}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 2u << 1u}}, {}); + store.shader.specials = &store.specials; + static AbiHeader front; + DescribeFront(front); + static AbiHeader pixel; + DescribeWaterfallPixel(pixel); + pixel.context[0] = {0x1b6, 0}; + static AbiHeader vertex; + DescribeWaterfallVertex(vertex, BarrierFreeWaterfallVertexCode); + static AbiHeader interpolated; + DescribeWaterfallPixel(interpolated); + for (auto* header : {&store, &front, &pixel, &vertex, &interpolated}) AgcDriverRegisterShader_nid_postfix(&header->shader); + const std::array primitive{{{0x242, 4}}}; + const std::array frontStages{&front.shader, &pixel.shader}; + const std::array vertexStages{&vertex.shader, &interpolated.shader}; + const auto resolve = [&] { + AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader, &vertex.shader}); + AgcDriverResolveShaderAbi_nid_postfix(&store.shader, {}, {}); + AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17); + AgcDriverResolveGraphicsStagesAbi_nid_postfix(frontStages, {}, primitive); + AgcDriverResolveGraphicsStagesAbi_nid_postfix(vertexStages, {}, primitive); + AgcDriverResolveGraphicsAbi_nid_postfix(&vertex.shader, &interpolated.shader, 17); + scope.Commit(); + }; + auto log = Logged(resolve); + Require(Deferrals(log) == 3 && log.find("is not a valid runtime value") != std::string::npos, "stages whose registration preparation failed were not deferred once each, with the stored reason, when the title created their pipelines"); + log = Logged(resolve); + Require(Deferrals(log) == 0, "stages whose registration preparation failed were logged again for the same pipelines"); + std::vector commands; + for (std::size_t index = 0; index < store.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, store.registers[index].offset, store.registers[index].value}); + commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "is not a valid runtime value"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "is not a valid runtime value"); +} + +void AbiDeferredPixel() { + static AbiHeader pixel; + DescribeWaterfallPixel(pixel); + AgcDriverRegisterShader_nid_postfix(&pixel.shader); + const auto before = ShaderRecompiler::FrontEndBuilds(); + auto log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); }); + const auto attempted = ShaderRecompiler::FrontEndBuilds(); + log += Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); }); + Require(attempted > before && ShaderRecompiler::FrontEndBuilds() == attempted, "a pixel static ABI deferred to its draw was prepared again for the same state"); + Require(Deferrals(log) == 1 && log.find("is not a valid runtime value") != std::string::npos, "a pixel shader its static ABI cannot prepare was not logged once with its reason"); + { + AgcDriver::ShaderPreparationScope scope({&pixel.shader}); + const std::array interpolant{{{0x191, 0x20}}}; + log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, interpolant, {}); }); + scope.Commit(); + } + Require(ShaderRecompiler::FrontEndBuilds() > attempted && Deferrals(log) == 0, "a deferred pixel shader was not prepared once more for another state, or was logged again"); + static AbiHeader front; + DescribeFront(front); + AgcDriverRegisterShader_nid_postfix(&front.shader); + std::vector commands; + for (const auto* header : {&front, &pixel}) { + for (std::size_t index = 0; index < header->shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, header->registers[index].offset, header->registers[index].value}); + for (std::size_t index = 0; index < header->shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, header->context[index].offset, header->context[index].value}); + } + commands.insert(commands.end(), {0xc0016900u, 0x8e, 0xfu, 0xc0016900u, 0x8f, 0xfu, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a deferred pixel shader was not submitted"); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "is not a valid runtime value"); + ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); }, "is not a valid runtime value"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "is not a valid runtime value"); +} + +void AbiDeferredStages() { + static AbiHeader nested; + static AbiHeader direct; + DescribeWaterfallVertex(nested, WaterfallVertexCode); + DescribeWaterfallVertex(direct, OtherWaterfallVertexCode); + AgcDriverRegisterShader_nid_postfix(&nested.shader); + AgcDriverRegisterShader_nid_postfix(&direct.shader); + const std::array primitive{{{0x242, 4}}}; + const auto before = ShaderRecompiler::FrontEndBuilds(); + std::string log; + { + AgcDriver::ShaderPreparationScope scope({&nested.shader}); + log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&nested.shader, {}, primitive); }); + scope.Commit(); + } + const auto nestedAttempt = ShaderRecompiler::FrontEndBuilds(); + Require(nestedAttempt > before && Deferrals(log) == 1, "a vertex shader the static ABI resolves through its graphics stages was not deferred to its draw"); + log = Logged([&] { AgcDriverResolveShaderAbi_nid_postfix(&nested.shader, {}, primitive); }); + Require(ShaderRecompiler::FrontEndBuilds() == nestedAttempt && Deferrals(log) == 0, "a deferred vertex static ABI was prepared or logged again for the same state"); + const std::array stages{&direct.shader}; + log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive); }); + const auto directAttempt = ShaderRecompiler::FrontEndBuilds(); + Require(directAttempt > nestedAttempt && Deferrals(log) == 1, "a graphics stage its ABI cannot prepare was not deferred to its draw"); + log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive); }); + Require(ShaderRecompiler::FrontEndBuilds() == directAttempt && Deferrals(log) == 0, "a deferred graphics stage ABI was prepared or logged again for the same state"); + AgcDriverShutdown_nid_postfix(); +} + +void AbiDeferredRectangle() { + static AbiHeader front; + static AbiHeader undecodable; + static AbiHeader pixel; + DescribeFront(front); + DescribeAbi(undecodable, UndecodablePixelCode, 1, {{0x8, ProgramLow(UndecodablePixelCode.data())}, {0x9, ProgramHigh(UndecodablePixelCode.data())}, {0xa, 0}, {0xb, 0}}, {}); + DescribeWaterfallPixel(pixel); + for (auto* header : {&front, &undecodable, &pixel}) AgcDriverRegisterShader_nid_postfix(&header->shader); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &undecodable.shader, 17); + AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, {}, {}); + { + AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader}); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17); + scope.Commit(); + } + AgcDriverShutdown_nid_postfix(); +} + +void AbiDeferredUndecodableState() { + static AbiHeader front; + DescribeFront(front); + front.context[0] = {0x2ce, 3}; + front.specials = {{0x25b, 0x8040}, {0x2d5, 0x2030}, 0x8000, {}, {}, {0x29b, 1}, {0x262, 0}}; + AgcDriverRegisterShader_nid_postfix(&front.shader); + const std::array stages{&front.shader}; + const std::array context{front.specials.vgt_shader_stages_en, front.specials.vgt_gs_out_prim_type}; + const std::array primitive{front.specials.ge_cntl, front.specials.ge_user_vgpr_en, ShaderRegister{0x242, 4}}; + const auto before = ShaderRecompiler::FrontEndBuilds(); + auto log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, context, primitive); }); + Require(Deferrals(log) == 1 && log.find("VGT_SHADER_STAGES_EN=0x2030: ") != std::string::npos, "a graphics stage whose pipeline state does not decode was not deferred with the decode's reason"); + log = Logged([&] { AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, context, primitive); }); + Require(ShaderRecompiler::FrontEndBuilds() == before && Deferrals(log) == 0, "a graphics stage deferred for a pipeline state that does not decode was prepared or logged again"); + std::vector commands; + for (std::size_t index = 0; index < front.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, front.registers[index].offset, front.registers[index].value}); + commands.insert(commands.end(), {0xc0016900u, 0x2d5, 0x2000u, 0xc0017600u, 0x8, 0, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + log = Logged([&] { + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a graphics stage deferred at its pipeline ABI was not submitted"); + AgcDriverWaitIdle_nid_postfix(); + }); + Require(ShaderRecompiler::FrontEndBuilds() > before && log.find("has no artifact prepared at registration") == std::string::npos, "a graphics stage deferred at its pipeline ABI was not prepared at its first draw, or was reported there as missing a registration artifact"); + AgcDriverShutdown_nid_postfix(); +} + +void AbiDeferredVertexInputs() { + static FetchingHeader fetching; + DescribeWaterfallVertex(fetching.abi, WaterfallVertexCode); + DescribeStaticFetch(fetching); + AgcDriverRegisterShader_nid_postfix(&fetching.abi.shader); + const std::array stages{&fetching.abi.shader}; + const std::array primitive{{{0x242, 4}}}; + const auto resolve = [&] { + AgcDriver::ShaderPreparationScope scope({&fetching.abi.shader}); + AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, {}, primitive); + scope.Commit(); + }; + const auto before = ShaderRecompiler::FrontEndBuilds(); + auto log = Logged(resolve); + Require(Deferrals(log) == 1 && log.find("statically bound vertex attributes are not implemented") != std::string::npos, "a graphics stage whose vertex inputs its pipeline ABI cannot decode was not deferred with the decode's reason"); + log = Logged(resolve); + Require(ShaderRecompiler::FrontEndBuilds() == before && Deferrals(log) == 0, "a graphics stage deferred for vertex inputs that do not decode was prepared or logged again"); + alignas(16) static std::array tables{}; + const auto address = reinterpret_cast(tables.data()); + std::vector commands; + for (std::size_t index = 0; index < fetching.abi.shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, fetching.abi.registers[index].offset, fetching.abi.registers[index].value}); + for (std::uint32_t index = 0; index < 4; ++index) commands.insert(commands.end(), {0xc0017600u, 0x8cu + index, static_cast(index % 2u == 0u ? address : address >> 32u)}); + for (std::size_t index = 0; index < fetching.abi.shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, fetching.abi.context[index].offset, fetching.abi.context[index].value}); + for (const auto reg : {fetching.abi.specials.vgt_shader_stages_en, fetching.abi.specials.vgt_gs_out_prim_type}) commands.insert(commands.end(), {0xc0016900u, reg.offset, reg.value}); + for (const auto reg : {fetching.abi.specials.ge_cntl, fetching.abi.specials.ge_user_vgpr_en}) commands.insert(commands.end(), {0xc0017900u, reg.offset, reg.value}); + commands.insert(commands.end(), {0xc0017600u, 0x8, 0, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 4, 0xc0012d00u, 3, 2}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw of a graphics stage deferred for its vertex inputs was not submitted"); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "statically bound vertex attributes are not implemented"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "statically bound vertex attributes are not implemented"); +} + +void AbiDeferredRectanglePairs() { + static AbiHeader front; + DescribeFront(front); + static AbiHeader pixel; + DescribeAbi(pixel, PassthroughPixelCode, 1, {{0x8, ProgramLow(PassthroughPixelCode.data())}, {0x9, ProgramHigh(PassthroughPixelCode.data())}, {0xa, 0}, {0xb, 0}}, {{0x1b6, 1}, {0x1b3, 2}, {0x1b4, 2}, {0x191, 0x420}, {0x1c5, 4}}); + static AbiHeader unresolved; + DescribeWaterfallPixel(unresolved); + for (auto* header : {&front, &pixel, &unresolved}) AgcDriverRegisterShader_nid_postfix(&header->shader); + const std::array passthrough{pixel.context[3]}; + const auto resolve = [&] { + AgcDriver::ShaderPreparationScope scope({&front.shader, &pixel.shader}); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, nullptr, 17); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &unresolved.shader, 17); + AgcDriverResolveShaderAbi_nid_postfix(&pixel.shader, passthrough, {}); + AgcDriverResolveGraphicsAbi_nid_postfix(&front.shader, &pixel.shader, 17); + scope.Commit(); + }; + auto log = Logged(resolve); + Require(Deferrals(log) == 1 && log.find("custom per-vertex interpolation is unsupported") != std::string::npos, "a rectangle pair its pipeline ABI cannot build was not deferred once with the builder's reason"); + log = Logged(resolve); + Require(Deferrals(log) == 0, "a rectangle pair deferred at its pipeline ABI was logged again"); + const auto draw = [&](bool withPixel) { + std::vector commands; + for (const auto* header : {&front, &pixel}) { + if (header == &pixel && !withPixel) continue; + for (std::size_t index = 0; index < header->shader.num_sh_registers; ++index) commands.insert(commands.end(), {0xc0017600u, header->registers[index].offset, header->registers[index].value}); + for (std::size_t index = 0; index < header->shader.num_cx_registers; ++index) commands.insert(commands.end(), {0xc0016900u, header->context[index].offset, header->context[index].value}); + } + if (!withPixel) commands.insert(commands.end(), {0xc0017600u, 0x8, 0}); + commands.insert(commands.end(), {0xc0016900u, 0x8e, 0xfu, 0xc0016900u, 0x8f, 0xfu, 0xc0016900u, 0xd, 0x00100010u, 0xc0016900u, 0x204, 0x80000u, 0xc0016900u, 0x1c3, 4, 0xc0017900u, 0x242, 17, 0xc0012d00u, 3, 2}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "a rect-list draw was not submitted"); + AgcDriverWaitIdle_nid_postfix(); + }; + draw(false); + ExpectFailure([&] { draw(true); }, "custom per-vertex interpolation is unsupported"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "custom per-vertex interpolation is unsupported"); +} + } int main(int argc, char** argv) { try { - Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup" || std::string_view(argv[1]) == "--lazy")), "invalid test arguments"); + const std::string_view abiMode = argc == 2 ? argv[1] : ""; + const bool abiDeferred = abiMode == "--abi-deferred-compute" || abiMode == "--abi-deferred-pixel" || abiMode == "--abi-deferred-stages" || abiMode == "--abi-deferred-rectangle" || abiMode == "--abi-deferred-undecodable-state" || abiMode == "--abi-deferred-registration-failure" || abiMode == "--abi-deferred-vertex-inputs" || abiMode == "--abi-deferred-rectangle-pairs"; + Require(argc == 1 || abiDeferred || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable" || std::string_view(argv[1]) == "--registered-workgroup" || std::string_view(argv[1]) == "--lazy")), "invalid test arguments"); + if (abiDeferred) { + if (!OpenVulkanTestDevice()) return VulkanTestSkipped; + if (abiMode == "--abi-deferred-compute") AbiDeferredCompute(); + else if (abiMode == "--abi-deferred-pixel") AbiDeferredPixel(); + else if (abiMode == "--abi-deferred-stages") AbiDeferredStages(); + else if (abiMode == "--abi-deferred-undecodable-state") AbiDeferredUndecodableState(); + else if (abiMode == "--abi-deferred-registration-failure") AbiDeferredRegistrationFailure(); + else if (abiMode == "--abi-deferred-vertex-inputs") AbiDeferredVertexInputs(); + else if (abiMode == "--abi-deferred-rectangle-pairs") AbiDeferredRectanglePairs(); + else AbiDeferredRectangle(); + std::cout << "shader ABI deferral tests passed: " << abiMode << '\n'; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--lazy") { if (!OpenVulkanTestDevice()) return VulkanTestSkipped; LazyRegistration(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp new file mode 100644 index 0000000000..ec90020045 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp @@ -0,0 +1,173 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::size_t BlockBytes = 1u << 20u; +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 128; +constexpr std::size_t ImageBytes = Width * Height * 4u; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Initial = 0x3f800000u; +constexpr std::uint32_t Stored = 0x11111111u; +constexpr std::uint32_t Overwritten = 0x22222222u; +constexpr std::size_t OverwrittenDword = 0x100u / 4u; +constexpr std::size_t FarDword = 100u * Width; + +alignas(256) constexpr std::array StoreImageCode{0x7e000280u, 0x7e020280u, 0x7e0402ffu, Stored, 0xf0200108u, 0x00010200u, 0x7e0202ffu, 0x00000064u, 0xf0200108u, 0x00010200u, 0xbf810000u}; +alignas(256) constexpr std::array StoreGlobalCode{0x7e000208u, 0x7e020209u, 0x7e0402ffu, Overwritten, 0xdc708100u, 0x007d0200u, 0xbf810000u}; + +class GuestBlock { +public: + explicit GuestBlock(bool watched) : watched(watched) { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + if (watched) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "storage flush ordering: cannot allocate the guest block"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), BlockBytes), "storage flush ordering: the guest block is not write-watched"); + std::fill(Words(), Words() + BlockBytes / 4u, Initial); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + AgcDriver::GuestMemory::CollectWritesUncached(Address(), BlockBytes); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + if (watched) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint32_t* Words() const { return reinterpret_cast(block); } + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + +private: + bool watched; + std::uint8_t* block = nullptr; +}; + +ShaderRecompiler::RecompileResult Compile(std::span code, std::span userData, const ShaderRecompiler::SpirvTarget& target) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +std::vector ImageUserData(std::uint64_t base) { + std::vector userData(12, 0u); + userData[4] = static_cast(base >> 8u); + userData[5] = static_cast((base >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Width - 1u) & 3u) << 30u); + userData[6] = ((Width - 1u) >> 2u) | ((Height - 1u) << 14u); + userData[7] = 0xfacu | (Type2D << 28u); + return userData; +} + +std::vector GlobalUserData(std::uint64_t base) { + std::vector userData(10, 0u); + userData[8] = static_cast(base); + userData[9] = static_cast(base >> 32u); + return userData; +} + +bool AddressBasedStorePending(std::uint64_t image, bool open) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + const auto* recorder = AgcDriver::Graphics::Recorder::Active(); + const auto read = recorder != nullptr ? recorder->DescribePendingRead(image + ImageBytes, 4u) : std::nullopt; + return read.has_value() && read->kind == AgcDriver::Graphics::Recorder::ReadKind::AddressBased && read->open == open; +} + +void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, std::size_t offset, bool submit, const char* name) { + const auto* words = guest.Words() + offset / 4u; + const auto image = guest.Address() + offset; + const auto target = device.Target(); + const auto storeImage = Compile(StoreImageCode, ImageUserData(image), target); + const auto storeGlobal = Compile(StoreGlobalCode, GlobalUserData(image), target); + device.Dispatch(storeImage, 1u, 1u, 1u, {}, reinterpret_cast(StoreImageCode.data())); + device.Dispatch(storeGlobal, 1u, 1u, 1u, {}, reinterpret_cast(StoreGlobalCode.data())); + if (submit) { + std::uint64_t serial = 0; + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + serial = device.SubmitAndEpoch(); + } + device.WaitRecorded(serial); + } + const std::string what = std::string("storage flush ordering, ") + name + ": "; + Require(AddressBasedStorePending(image, !submit), what + "the address-based store was not pending in the expected batch at the flush"); + Require(AgcDriver::Graphics::StorageTexture::FlushPending(image, ImageBytes, nullptr, "test"), what + "the image store left no results pending"); + device.WaitIdle(); + Require(words[FarDword] == Stored, what + "the pending image's texel away from the newer write was not stored"); + Require(words[OverwrittenDword] == Overwritten, what + "the pending image's store put its older texels over a newer address-based store"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const bool watched = AgcDriver::GuestMemory::WriteWatched(); + GuestBlock guest(watched); + if (!watched) std::puts("guest memory has no write watch: the writer-in-flight case is skipped"); + else if (!device->CanWaitUnlocked()) std::puts("the recorder cannot wait for the GPU without running completions: the writer-in-flight case is skipped"); + else Run(*device, guest, 0, true, "writer in flight"); + Run(*device, guest, BlockBytes / 2u, false, "writer in the open batch"); + std::puts("storage flush ordering tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceIme/Export.cpp b/core/libs/prx/libSceIme/Export.cpp index 8d2f2748ee..575463cf73 100644 --- a/core/libs/prx/libSceIme/Export.cpp +++ b/core/libs/prx/libSceIme/Export.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -67,6 +68,7 @@ std::mutex g_keyboardMutex; std::set g_openKeyboards; std::mutex g_imeMutex; std::optional g_session; +std::atomic g_textSink{nullptr}; void PanelSize(const Param& param, uint32_t* width, uint32_t* height) { const uint32_t scale = (param.option & OptionUseOver2K) != 0 ? 2 : 1; @@ -284,13 +286,31 @@ int ValidateOpen(const Param* param, const ExtendedParam* extended) { extern "C" { void ImePublishInput_nid_postfix(const ImeHostInput& input) { - std::lock_guard lock(g_imeMutex); - if (g_session) Apply(*g_session, input); + { + std::lock_guard lock(g_imeMutex); + if (g_session) { + Apply(*g_session, input); + return; + } + } + const ImeHostTextSink sink = g_textSink.load(); + if (sink == nullptr) return; + std::u16string units; + if (input.kind == ImeHostInputKind::Text) { + std::size_t size = 0; + while (size < sizeof(input.text) && input.text[size] != '\0') ++size; + units = DecodeUtf8(input.text, size); + } + sink(input.kind, units.data(), units.size()); } bool ImeCapturesKeyboard_nid_postfix() { std::lock_guard lock(g_imeMutex); - return g_session.has_value(); + return g_session.has_value() || g_textSink.load() != nullptr; +} + +void ImeSetHostTextSink_nid_no_patch(ImeHostTextSink sink) { + g_textSink = sink; } int APS5_VABI sceImeOpen_nid_postfix(const Param* param, const ExtendedParam* extended) { diff --git a/core/libs/prx/libSceIme/include/ImeInput.hpp b/core/libs/prx/libSceIme/include/ImeInput.hpp index 4212dd61ea..3d27316f20 100644 --- a/core/libs/prx/libSceIme/include/ImeInput.hpp +++ b/core/libs/prx/libSceIme/include/ImeInput.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEIME_INCLUDE_IMEINPUT_HPP #define CORE_LIBS_PRX_LIBSCEIME_INCLUDE_IMEINPUT_HPP +#include #include enum class ImeHostInputKind : std::uint32_t { @@ -23,4 +24,8 @@ struct ImeHostInput { extern "C" void ImePublishInput_nid_postfix(const ImeHostInput& input); extern "C" bool ImeCapturesKeyboard_nid_postfix(); +using ImeHostTextSink = void (*)(ImeHostInputKind kind, const char16_t* text, std::size_t length); + +extern "C" void ImeSetHostTextSink_nid_no_patch(ImeHostTextSink sink); + #endif diff --git a/core/libs/prx/libSceImeDialog/CMakeLists.txt b/core/libs/prx/libSceImeDialog/CMakeLists.txt new file mode 100644 index 0000000000..69efa4be3f --- /dev/null +++ b/core/libs/prx/libSceImeDialog/CMakeLists.txt @@ -0,0 +1,17 @@ +add_library(libSceImeDialog SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp) +target_include_directories(libSceImeDialog PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libSceImeDialog PRIVATE libSceIme) +configure_windows_unwind(libSceImeDialog) + +if(BUILD_TESTING) + add_test_executable(guest_ime_dialog_tests ${LIBS_INCLUDE_DIR}/tests/GuestImeDialog.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../libSceVideoOut/src/ImeTextInput.cpp) + target_include_directories(guest_ime_dialog_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_SDL2_INCLUDE_DIR}) + target_link_libraries(guest_ime_dialog_tests PRIVATE libSceImeDialog libSceIme libc) + configure_windows_unwind(guest_ime_dialog_tests) + add_test(NAME guest_ime_dialog COMMAND guest_ime_dialog_tests) + if(WIN32) + set_tests_properties(guest_ime_dialog PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceImeDialog/Export.cpp b/core/libs/prx/libSceImeDialog/Export.cpp index 84fe1c7e23..71df834abb 100644 --- a/core/libs/prx/libSceImeDialog/Export.cpp +++ b/core/libs/prx/libSceImeDialog/Export.cpp @@ -1,9 +1,11 @@ -#include +#include #include #include -#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libSceIme/include/ImeInput.hpp" namespace { @@ -11,21 +13,105 @@ constexpr int StatusNone = 0; constexpr int StatusRunning = 1; constexpr int StatusFinished = 2; constexpr std::uint32_t EndStatusOk = 0; +constexpr std::uint32_t EndStatusUserCanceled = 1; constexpr std::uint32_t EndStatusAborted = 2; +constexpr std::uint32_t MaxTextLength = 2048; +constexpr int ErrorBusy = static_cast(0x80bc0001u); constexpr int ErrorNotOpened = static_cast(0x80bc0002u); +constexpr int ErrorInvalidMaxTextLength = static_cast(0x80bc0016u); +constexpr int ErrorInvalidInputTextBuffer = static_cast(0x80bc0017u); +constexpr int ErrorInvalidAddress = static_cast(0x80bc0031u); -std::atomic status{StatusNone}; -std::atomic polls{0}; -std::atomic endStatus{EndStatusOk}; +struct Dialog { + int status = StatusNone; + std::uint32_t endStatus = EndStatusOk; + char16_t* buffer = nullptr; + std::uint32_t maxLength = 0; + std::u16string text; + std::size_t caret = 0; +}; + +std::mutex g_mutex; +Dialog g_dialog; + +bool IsHighSurrogate(char16_t unit) { return unit >= 0xd800 && unit <= 0xdbff; } +bool IsLowSurrogate(char16_t unit) { return unit >= 0xdc00 && unit <= 0xdfff; } + +std::size_t UnitsBefore(const Dialog& dialog) { + if (dialog.caret == 0) return 0; + return dialog.caret >= 2 && IsLowSurrogate(dialog.text[dialog.caret - 1]) && IsHighSurrogate(dialog.text[dialog.caret - 2]) ? 2 : 1; +} + +std::size_t UnitsAfter(const Dialog& dialog) { + if (dialog.caret >= dialog.text.size()) return 0; + return dialog.caret + 1 < dialog.text.size() && IsHighSurrogate(dialog.text[dialog.caret]) && IsLowSurrogate(dialog.text[dialog.caret + 1]) ? 2 : 1; +} + +void Finish(std::uint32_t endStatus) { + g_dialog.endStatus = endStatus; + g_dialog.status = StatusFinished; + ImeSetHostTextSink_nid_no_patch(nullptr); +} + +void Insert(Dialog& dialog, const char16_t* units, std::size_t length) { + std::u16string accepted; + for (std::size_t i = 0; i < length; ++i) { + if (units[i] != u'\n' && units[i] != u'\r') accepted.push_back(units[i]); + } + std::size_t count = std::min(accepted.size(), dialog.maxLength - dialog.text.size()); + if (count > 0 && count < accepted.size() && IsHighSurrogate(accepted[count - 1])) --count; + dialog.text.insert(dialog.caret, accepted, 0, count); + dialog.caret += count; +} + +void Receive(ImeHostInputKind kind, const char16_t* units, std::size_t length) { + std::lock_guard lock(g_mutex); + Dialog& dialog = g_dialog; + if (dialog.status != StatusRunning) return; + switch (kind) { + case ImeHostInputKind::Text: + Insert(dialog, units, length); + return; + case ImeHostInputKind::Backspace: { + const std::size_t count = UnitsBefore(dialog); + dialog.caret -= count; + dialog.text.erase(dialog.caret, count); + return; + } + case ImeHostInputKind::Delete: + dialog.text.erase(dialog.caret, UnitsAfter(dialog)); + return; + case ImeHostInputKind::Left: + dialog.caret -= UnitsBefore(dialog); + return; + case ImeHostInputKind::Right: + dialog.caret += UnitsAfter(dialog); + return; + case ImeHostInputKind::Home: + dialog.caret = 0; + return; + case ImeHostInputKind::End: + dialog.caret = dialog.text.size(); + return; + case ImeHostInputKind::Enter: + std::copy(dialog.text.begin(), dialog.text.end(), dialog.buffer); + dialog.buffer[dialog.text.size()] = u'\0'; + Finish(EndStatusOk); + return; + case ImeHostInputKind::Escape: + Finish(EndStatusUserCanceled); + return; + } +} } extern "C" { int APS5_VABI sceImeDialogAbort(void) { - if (status.load() != StatusRunning) return ErrorNotOpened; - endStatus = EndStatusAborted; - status = StatusFinished; + std::lock_guard lock(g_mutex); + if (g_dialog.status != StatusRunning) return ErrorNotOpened; + Finish(EndStatusAborted); return 0; } @@ -50,47 +136,42 @@ int APS5_VABI sceImeDialogGetPanelSizeExtended(const Param* param, const Extende } int APS5_VABI sceImeDialogGetResult(Result* result) { - std::fprintf(stderr, "[ime] GetResult (status %d)\n", status.load()); - if (result == nullptr || status.load() != StatusFinished) return ErrorNotOpened; + std::lock_guard lock(g_mutex); + if (result == nullptr || g_dialog.status != StatusFinished) return ErrorNotOpened; *result = Result{}; - result->endstatus = endStatus.load(); + result->endstatus = g_dialog.endStatus; return 0; } int APS5_VABI sceImeDialogGetStatus(void) { - if (status.load() == StatusRunning && ++polls > 1) status = StatusFinished; - static std::atomic traced{0}; - const int count = traced.fetch_add(1); - if (count < 8 || count % 2000 == 0) std::fprintf(stderr, "[ime] GetStatus #%d -> %d\n", count, status.load()); - return status.load(); + std::lock_guard lock(g_mutex); + return g_dialog.status; } int APS5_VABI sceImeDialogInit(const Param* param, const ExtendedParam* extended) { (void)extended; - static std::atomic reported{0}; - bool filled = false; - if (param != nullptr && param->input_text_buffer != nullptr && param->max_text_length != 0 && param->input_text_buffer[0] == u'\0') { - static constexpr char16_t defaultText[] = u"Slayer"; - std::uint32_t length = 0; - while (defaultText[length] != u'\0' && length < param->max_text_length) { - param->input_text_buffer[length] = defaultText[length]; - ++length; - } - param->input_text_buffer[length] = u'\0'; - filled = true; - } - std::fprintf(stderr, "[ime] Init #%d\n", reported.load()); - if (reported.fetch_add(1) == 0) std::fprintf(stderr, "[ime] dialog requested (max text length %u): finishing at once with %s\n", param != nullptr ? param->max_text_length : 0u, filled ? "a default text" : "the text unchanged"); - polls = 0; - endStatus = EndStatusOk; - status = StatusRunning; + std::lock_guard lock(g_mutex); + if (g_dialog.status != StatusNone) return ErrorBusy; + if (param == nullptr) return ErrorInvalidAddress; + if (param->max_text_length == 0 || param->max_text_length > MaxTextLength) return ErrorInvalidMaxTextLength; + if (param->input_text_buffer == nullptr) return ErrorInvalidInputTextBuffer; + g_dialog = Dialog{}; + g_dialog.buffer = param->input_text_buffer; + g_dialog.maxLength = param->max_text_length; + std::size_t length = 0; + while (length < param->max_text_length && param->input_text_buffer[length] != u'\0') ++length; + g_dialog.text.assign(param->input_text_buffer, length); + g_dialog.caret = length; + g_dialog.status = StatusRunning; + ImeSetHostTextSink_nid_no_patch(&Receive); return 0; } int APS5_VABI sceImeDialogTerm(void) { - std::fprintf(stderr, "[ime] Term (status %d)\n", status.load()); - if (status.load() == StatusNone) return ErrorNotOpened; - status = StatusNone; + std::lock_guard lock(g_mutex); + if (g_dialog.status == StatusNone) return ErrorNotOpened; + ImeSetHostTextSink_nid_no_patch(nullptr); + g_dialog = Dialog{}; return 0; } diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 2ad1bc257b..67711a6d58 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -887,6 +887,14 @@ int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) { std::lock_guard lk(g_mutex); g_socks.emplace(descriptor, connection); } + if (listener.nonblock) { + const int enabled = 1; + if (sceNetSetsockopt(descriptor, NET_SOL_SOCKET, NET_SO_NBIO, &enabled, sizeof(enabled)) != 0) { + const int error = *errno_slot(); + sceNetSocketClose(descriptor); + return fail(error); + } + } if (addr && !native_to_guest_address(peer, addr, addrlen)) { sceNetSocketClose(descriptor); return fail(NET_EAFNOSUPPORT); diff --git a/core/libs/prx/libSceRegMgr/CMakeLists.txt b/core/libs/prx/libSceRegMgr/CMakeLists.txt new file mode 100644 index 0000000000..e10c382e33 --- /dev/null +++ b/core/libs/prx/libSceRegMgr/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceRegMgr SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceRegMgr PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceRegMgr) + +if(BUILD_TESTING) + add_test_executable(guest_regmgr_tests ${LIBS_INCLUDE_DIR}/tests/GuestRegMgr.cpp) + target_include_directories(guest_regmgr_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_regmgr_tests PRIVATE libSceRegMgr libc) + configure_windows_unwind(guest_regmgr_tests) + add_test(NAME guest_regmgr COMMAND guest_regmgr_tests) + if(WIN32) + set_tests_properties(guest_regmgr PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceRegMgr/Export.cpp b/core/libs/prx/libSceRegMgr/Export.cpp new file mode 100644 index 0000000000..d71daed4b0 --- /dev/null +++ b/core/libs/prx/libSceRegMgr/Export.cpp @@ -0,0 +1,14 @@ +#include +#include "prx/libc/include/General.hpp" + +extern "C" { + +std::int32_t APS5_VABI sceRegMgrNonSysGetInt(std::int64_t key, std::int32_t unk, std::int32_t* data) { + (void)key; + (void)unk; + (void)data; + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceSsl/CMakeLists.txt b/core/libs/prx/libSceSsl/CMakeLists.txt index 99095ec97a..cb39ff0e4c 100644 --- a/core/libs/prx/libSceSsl/CMakeLists.txt +++ b/core/libs/prx/libSceSsl/CMakeLists.txt @@ -85,9 +85,14 @@ if(BUILD_TESTING) add_test(NAME guest_ssl_transport_${table} COMMAND guest_ssl_transport_tests ${table}) set_tests_properties(guest_ssl_transport_${table} PROPERTIES TIMEOUT 30) endforeach() + add_test_executable(guest_ssl_ps5_readers_tests ${LIBS_INCLUDE_DIR}/tests/GuestSslPs5Readers.cpp) + target_include_directories(guest_ssl_ps5_readers_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_ssl_ps5_readers_tests PRIVATE libSceSsl libc) + configure_windows_unwind(guest_ssl_ps5_readers_tests) + add_test(NAME guest_ssl_ps5_readers COMMAND guest_ssl_ps5_readers_tests) if(WIN32) target_link_libraries(guest_ssl_transport_tests PRIVATE bcrypt ws2_32) - set_tests_properties(guest_ssl guest_ssl_load guest_ssl_transport_kernel guest_ssl_transport_net PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(guest_ssl guest_ssl_load guest_ssl_ps5_readers guest_ssl_transport_kernel guest_ssl_transport_net PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceSsl/Export.cpp b/core/libs/prx/libSceSsl/Export.cpp index 5ed1cc3d57..5d7394fdb3 100644 --- a/core/libs/prx/libSceSsl/Export.cpp +++ b/core/libs/prx/libSceSsl/Export.cpp @@ -11,12 +11,14 @@ #include "prx/libkernel/Socket/include/SocketPoll.hpp" #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -1053,8 +1055,26 @@ int APS5_VABI sceSslGetFingerprint(int sslCtxId, void* sslCert, int hashType, un return 0; } -int APS5_VABI sceSslGetPem(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceSslGetPem(int sslCtxId, void* sslCert, char* buf, std::size_t* bufLen) { + if (!bufLen) return ErrorInvalidArg; + std::lock_guard lock(State().mutex); + auto& context = GetContext(sslCtxId); + const auto* crt = RequireCertificate(__func__, sslCtxId, sslCert); + PoolScope scope(context.pool); + std::size_t required = 0; + const auto encode = [&](unsigned char* output, std::size_t capacity, std::size_t* written) { + return mbedtls_pem_write_buffer("-----BEGIN CERTIFICATE-----\n", "-----END CERTIFICATE-----\n", crt->raw.p, crt->raw.len, output, capacity, written); + }; + const int sizing = encode(nullptr, 0, &required); + if (sizing != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) CheckBackend(sizing, "SSL PEM sizing failed"); + if (!buf) { + *bufLen = required; + return 0; + } + if (*bufLen < required) throw std::length_error("SSL PEM buffer is too small"); + std::size_t written = 0; + CheckBackend(encode(reinterpret_cast(buf), required, &written), "SSL PEM encoding failed"); + *bufLen = written; return 0; } diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index b8e4ba2743..e630686244 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -106,7 +106,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endforeach() - foreach(flipMode IN ITEMS eop eop_main eop_aftergpu) + foreach(flipMode IN ITEMS eop eop_main eop_aftergpu cpuflip) add_test(NAME video_out_flip_${flipMode} COMMAND video_out_flip_tests ${flipMode}) set_tests_properties(video_out_flip_${flipMode} PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "SDL_VIDEODRIVER=${ANYPS5_VIDEO_OUT_TEST_DRIVER}") if(WIN32) @@ -114,6 +114,12 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endforeach() + add_test(NAME video_out_flip_cpuflip_in_order COMMAND video_out_flip_tests cpuflip) + set_tests_properties(video_out_flip_cpuflip_in_order PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "SDL_VIDEODRIVER=${ANYPS5_VIDEO_OUT_TEST_DRIVER};APS5_CPU_FLIP_IN_ORDER=1") + if(WIN32) + set_tests_properties(video_out_flip_cpuflip_in_order PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() if(BUILD_TESTING AND NOT WIN32) diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index 96b972a435..88c960113f 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -28,7 +28,6 @@ #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" -#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/PreciseWait.hpp" #include "prx/libc/include/Shutdown.hpp" @@ -440,10 +439,7 @@ int VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flip // A title that does not pace on flipPendingNum can run ahead of the presenter now that the queue // worker no longer waits per flip; a full queue is the documented error, not a failure. if (flipQueue->reservations.load() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) return VIDEO_OUT_ERROR_FLIP_QUEUE_FULL; - std::array words{AgcDriver::FlipPacketHeader, static_cast(handle), static_cast(index), static_cast(flipMode), static_cast(static_cast(flipArg)), static_cast(static_cast(flipArg) >> 32u)}; - Packet packet{words.data(), static_cast(words.size()), 0, {}}; - const auto result = sceAgcDriverSubmitDcb(&packet); - require(result == 0, "driver rejected flip submission"); + AgcDriverSubmitFlip_nid_postfix(AgcDriver::FlipInfo{static_cast(handle), index, static_cast(flipMode), flipArg}); return 0; } diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index a7072e5e51..8f3e865c3b 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -225,6 +225,57 @@ void testDeferredBuffer() { LibcRunShutdown_nid_postfix(); } +void testCpuFlipOrder() { + const bool inOrder = std::getenv("APS5_CPU_FLIP_IN_ORDER") != nullptr; + const int handle = sceVideoOutOpen(255, 0, 0, nullptr); + auto cfg = VideoOutDriver::Get().GetConfig(handle); + std::vector allocation(65536 + 65535); + const auto storage = alignedBuffer(allocation); + VideoOutBuffers buffer{storage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 64, 64, 0, 0, 0); + check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffer registration failed"); + const auto flipsWithin = [&](int count, std::chrono::milliseconds limit) { + std::unique_lock lock(cfg->mutex); + const bool done = cfg->vblankCond.wait_for(lock, limit, [&] { return cfg->failure || cfg->flipStatus.count == count; }); + if (cfg->failure) std::rethrow_exception(cfg->failure); + return done; + }; + const auto shownArgument = [&] { + std::lock_guard lock(cfg->mutex); + return cfg->flipStatus.flipArg; + }; + auto first = std::make_shared(); + AgcDriverRegisterVideoOutput_nid_postfix(7, first); + std::array firstHeld{0xc004105c, 7, 0xfffffffeu, 1, 0, 0}; + Packet firstPacket{firstHeld.data(), 6, 0, {}}; + check(sceAgcDriverSubmitDcb(&firstPacket) == 0, "the first gated submission was not accepted"); + std::thread flipThread([&] { check(sceVideoOutSubmitFlip(handle, 0, 1, 5) == 0, "the flip thread's flip was not accepted"); }); + flipThread.join(); + const bool ahead = flipsWithin(1, std::chrono::milliseconds(inOrder ? 1000 : 8000)); + if (inOrder) check(!ahead, "with APS5_CPU_FLIP_IN_ORDER a flip completed while a queue-0 submission ahead of it was held"); + else check(ahead, "a flip from a thread that never submitted queue-0 work waited for a held queue-0 submission"); + first->Release(); + check(flipsWithin(1, std::chrono::seconds(30)), "the flip thread's flip did not complete"); + check(shownArgument() == 5, "the flip thread's flip presented the wrong request"); + auto second = std::make_shared(); + AgcDriverRegisterVideoOutput_nid_postfix(8, second); + std::array secondHeld{0xc004105c, 8, 0xfffffffeu, 1, 0, 0}; + Packet secondPacket{secondHeld.data(), 6, 0, {}}; + check(sceAgcDriverSubmitDcb(&secondPacket) == 0, "the second gated submission was not accepted"); + check(sceVideoOutSubmitFlip(handle, 0, 1, 6) == 0, "the submitting thread's flip was not accepted"); + check(!flipsWithin(2, std::chrono::milliseconds(1000)), "a flip from the thread that submitted queue-0 work completed ahead of that work"); + second->Release(); + check(flipsWithin(2, std::chrono::seconds(30)), "the submitting thread's flip did not complete"); + check(shownArgument() == 6, "the submitting thread's flip presented the wrong request"); + AgcDriverWaitIdle_nid_postfix(); + sceVideoOutUnregisterBuffers(handle, 0); + sceVideoOutClose(handle); + AgcDriverUnregisterVideoOutput_nid_postfix(7, first); + AgcDriverUnregisterVideoOutput_nid_postfix(8, second); + LibcRunShutdown_nid_postfix(); +} + void testUnregisteredBuffer() { const int handle = sceVideoOutOpen(255, 0, 0, nullptr); auto cfg = VideoOutDriver::Get().GetConfig(handle); @@ -975,6 +1026,7 @@ int main(int argc, char** argv) { else if (argc == 2 && std::string(argv[1]) == "deferred") testDeferredBuffer(); else if (argc == 2 && std::string(argv[1]) == "unregistered") testUnregisteredBuffer(); else if (argc == 2 && std::string(argv[1]) == "ps4") testPs4Buffers(); + else if (argc == 2 && std::string(argv[1]) == "cpuflip") testCpuFlipOrder(); else testLifetime(argc == 2 && std::string(argv[1]) == "reopen"); std::puts("VideoOut flip tests passed"); return 0; diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 0638bf7ef8..e51a53ce1e 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -45,6 +45,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/FloatBuiltins.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/HostEntropy.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Backtrace.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/DinkumwareAtomics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Time.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/AtomicBuiltins.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FileAndHeap.cpp @@ -402,6 +403,15 @@ if(BUILD_TESTING) set_tests_properties(guest_environment PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_dinkumware_atomics_tests ${LIBS_INCLUDE_DIR}/tests/GuestDinkumwareAtomics.cpp) + target_include_directories(guest_dinkumware_atomics_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_dinkumware_atomics_tests PRIVATE libc) + configure_windows_unwind(guest_dinkumware_atomics_tests) + add_test(NAME guest_dinkumware_atomics COMMAND guest_dinkumware_atomics_tests) + if(WIN32) + set_tests_properties(guest_dinkumware_atomics PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_hostname_tests ${LIBS_INCLUDE_DIR}/tests/GuestHostname.cpp) target_include_directories(guest_hostname_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_hostname_tests PRIVATE libc) diff --git a/core/libs/prx/libc/include/FilesystemError.hpp b/core/libs/prx/libc/include/FilesystemError.hpp index 038b5f04f7..37a4f7ea0e 100644 --- a/core/libs/prx/libc/include/FilesystemError.hpp +++ b/core/libs/prx/libc/include/FilesystemError.hpp @@ -23,7 +23,7 @@ inline int FilesystemError(const std::error_code& error) { return 5; } -std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesFile); +std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, bool replacesExisting); inline int ResolvePathError_nid_no_patch(const char* path, std::filesystem::path& resolved, bool module = false, bool archived = false) { try { diff --git a/core/libs/prx/libc/src/DinkumwareAtomics.cpp b/core/libs/prx/libc/src/DinkumwareAtomics.cpp new file mode 100644 index 0000000000..f8aecb774e --- /dev/null +++ b/core/libs/prx/libc/src/DinkumwareAtomics.cpp @@ -0,0 +1,175 @@ +#include +#include +#include + +#include "prx/libc/include/General.hpp" + +namespace { + +constexpr int MemoryOrderSeqCst = 5; + +template +void Store(volatile TValue* target, TValue value) { + __atomic_store_n(target, value, __ATOMIC_SEQ_CST); +} + +template +TValue Load(volatile TValue* target) { + return __atomic_load_n(target, __ATOMIC_SEQ_CST); +} + +bool IsValidOrder(int order) { + return order >= 0 && order <= MemoryOrderSeqCst; +} + +template +TValue Exchange(volatile TValue* target, TValue value) { + return __atomic_exchange_n(target, value, __ATOMIC_SEQ_CST); +} + +template +int CompareExchange(volatile TValue* target, TValue* expected, TValue desired) { + return __atomic_compare_exchange_n(target, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ? 1 : 0; +} + +void LockFlag(volatile unsigned char* flag) { + while (__atomic_exchange_n(flag, static_cast(1), __ATOMIC_ACQUIRE) != 0) { + } +} + +void UnlockFlag(volatile unsigned char* flag) { + __atomic_store_n(flag, static_cast(0), __ATOMIC_SEQ_CST); +} + +void CopyBytes(volatile void* destination, const volatile void* source, std::size_t size) { + std::memcpy(const_cast(destination), const_cast(source), size); +} + +int LockedCompareExchange(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired) { + LockFlag(flag); + void* object = const_cast(target); + const bool equal = std::memcmp(object, expected, size) == 0; + if (equal) { + std::memcpy(object, desired, size); + } else { + std::memcpy(expected, object, size); + } + UnlockFlag(flag); + return equal ? 1 : 0; +} + +} + +extern "C" { + +void APS5_VABI _Atomic_store_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { Store(target, value); } +void APS5_VABI _Atomic_store_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { Store(target, value); } +void APS5_VABI _Atomic_store_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { Store(target, value); } +void APS5_VABI _Atomic_store_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int) { Store(target, value); } + +std::uint8_t APS5_VABI _Atomic_load_1_nid_postfix(volatile std::uint8_t* target, int) { return Load(target); } +std::uint16_t APS5_VABI _Atomic_load_2_nid_postfix(volatile std::uint16_t* target, int) { return Load(target); } +std::uint64_t APS5_VABI _Atomic_load_8_nid_postfix(volatile std::uint64_t* target, int order) { + if (order == 3 || order == 4 || !IsValidOrder(order)) return 0; + return Load(target); +} + +std::uint8_t APS5_VABI _Atomic_exchange_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return Exchange(target, value); } +std::uint16_t APS5_VABI _Atomic_exchange_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return Exchange(target, value); } +std::uint32_t APS5_VABI _Atomic_exchange_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return Exchange(target, value); } +std::uint64_t APS5_VABI _Atomic_exchange_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? Exchange(target, value) : 0; } + +int APS5_VABI _Atomic_compare_exchange_strong_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t* expected, std::uint8_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_strong_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t* expected, std::uint16_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_strong_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t* expected, std::uint32_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_strong_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t* expected, std::uint64_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_weak_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t* expected, std::uint8_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_weak_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t* expected, std::uint16_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_weak_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t* expected, std::uint64_t desired, int, int) { + return CompareExchange(target, expected, desired); +} + +std::uint8_t APS5_VABI _Atomic_fetch_add_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST); } +std::uint16_t APS5_VABI _Atomic_fetch_add_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST); } +std::uint64_t APS5_VABI _Atomic_fetch_add_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_add(target, value, __ATOMIC_SEQ_CST) : 0; } +std::uint8_t APS5_VABI _Atomic_fetch_sub_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST); } +std::uint16_t APS5_VABI _Atomic_fetch_sub_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST); } +std::uint64_t APS5_VABI _Atomic_fetch_sub_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_sub(target, value, __ATOMIC_SEQ_CST) : 0; } +std::uint8_t APS5_VABI _Atomic_fetch_and_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); } +std::uint16_t APS5_VABI _Atomic_fetch_and_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); } +std::uint32_t APS5_VABI _Atomic_fetch_and_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST); } +std::uint64_t APS5_VABI _Atomic_fetch_and_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_and(target, value, __ATOMIC_SEQ_CST) : 0; } +std::uint8_t APS5_VABI _Atomic_fetch_or_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); } +std::uint16_t APS5_VABI _Atomic_fetch_or_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); } +std::uint32_t APS5_VABI _Atomic_fetch_or_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST); } +std::uint64_t APS5_VABI _Atomic_fetch_or_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_or(target, value, __ATOMIC_SEQ_CST) : 0; } +std::uint8_t APS5_VABI _Atomic_fetch_xor_1_nid_postfix(volatile std::uint8_t* target, std::uint8_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); } +std::uint16_t APS5_VABI _Atomic_fetch_xor_2_nid_postfix(volatile std::uint16_t* target, std::uint16_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); } +std::uint32_t APS5_VABI _Atomic_fetch_xor_4_nid_postfix(volatile std::uint32_t* target, std::uint32_t value, int) { return __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST); } +std::uint64_t APS5_VABI _Atomic_fetch_xor_8_nid_postfix(volatile std::uint64_t* target, std::uint64_t value, int order) { return IsValidOrder(order) ? __atomic_fetch_xor(target, value, __ATOMIC_SEQ_CST) : 0; } + +int APS5_VABI _Atomic_is_lock_free_1_nid_postfix() { return 1; } +int APS5_VABI _Atomic_is_lock_free_2_nid_postfix() { return 1; } +int APS5_VABI _Atomic_is_lock_free_4_nid_postfix() { return 1; } +int APS5_VABI _Atomic_is_lock_free_8_nid_postfix() { return 1; } + +int APS5_VABI _Atomic_flag_test_and_set_nid_postfix(volatile unsigned char* flag, int) { + return __atomic_exchange_n(flag, static_cast(1), __ATOMIC_SEQ_CST); +} + +void APS5_VABI _Atomic_flag_clear_nid_postfix(volatile unsigned char* flag, int) { + __atomic_store_n(flag, static_cast(0), __ATOMIC_SEQ_CST); +} + +void APS5_VABI _Atomic_thread_fence_nid_postfix(int order) { + if (order == MemoryOrderSeqCst) __atomic_thread_fence(__ATOMIC_SEQ_CST); +} + +void APS5_VABI _Atomic_signal_fence_nid_postfix(int) { +} + +void APS5_VABI _Atomic_copy_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, const volatile void* source, int) { + LockFlag(flag); + CopyBytes(target, source, size); + UnlockFlag(flag); +} + +void APS5_VABI _Atomic_exchange_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, volatile void* value, int) { + LockFlag(flag); + auto* object = static_cast(const_cast(target)); + auto* other = static_cast(const_cast(value)); + for (std::size_t i = 0; i < size; ++i) { + const unsigned char byte = object[i]; + object[i] = other[i]; + other[i] = byte; + } + UnlockFlag(flag); +} + +int APS5_VABI _Atomic_compare_exchange_strong_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired, int, int) { + return LockedCompareExchange(flag, size, target, expected, desired); +} + +int APS5_VABI _Atomic_compare_exchange_weak_nid_postfix(volatile unsigned char* flag, std::size_t size, volatile void* target, void* expected, const void* desired, int, int) { + return LockedCompareExchange(flag, size, target, expected, desired); +} + +} diff --git a/core/libs/prx/libc/src/Rename.cpp b/core/libs/prx/libc/src/Rename.cpp index d7f2126c76..f5d9545b5c 100644 --- a/core/libs/prx/libc/src/Rename.cpp +++ b/core/libs/prx/libc/src/Rename.cpp @@ -10,11 +10,11 @@ #include #endif -std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, [[maybe_unused]] bool replacesFile) { +std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination, [[maybe_unused]] bool replacesExisting) { namespace fs = std::filesystem; std::error_code error; #ifdef _WIN32 - if (replacesFile) { + if (replacesExisting) { const auto target = fs::absolute(destination, error); if (error) return error; struct RenameInfo { DWORD flags; HANDLE root; DWORD nameBytes; WCHAR name[1]; }; @@ -28,7 +28,7 @@ std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, c info->flags = replaceExisting | posixSemantics | (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) ? ignoreReadOnly : 0); info->nameBytes = static_cast(nameBytes); std::memcpy(info->name, target.c_str(), nameBytes); - const auto handle = CreateFileW(source.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, nullptr); + const auto handle = CreateFileW(source.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, nullptr); if (handle == INVALID_HANDLE_VALUE) return std::error_code(static_cast(GetLastError()), std::system_category()); constexpr auto renameInfoEx = static_cast(22); if (!SetFileInformationByHandle(handle, renameInfoEx, info, static_cast(buffer.size()))) error = std::error_code(static_cast(GetLastError()), std::system_category()); @@ -42,7 +42,6 @@ std::error_code RenameNative_nid_no_patch(const std::filesystem::path& source, c static int RenamePath_nid_no_patch(const std::filesystem::path& source, const std::filesystem::path& destination) { constexpr int GuestEnoent = 2; - constexpr int GuestEio = 5; constexpr int GuestEnotdir = 20; constexpr int GuestEisdir = 21; constexpr int GuestEinval = 22; @@ -90,14 +89,7 @@ static int RenamePath_nid_no_patch(const std::filesystem::path& source, const st return error ? FilesystemError(error) : GuestEnotempty; } } -#ifdef _WIN32 - if (!sameFile && sourceIsDirectory && destinationIsDirectory) { - error.clear(); - if (!fs::remove(destination, error)) return error ? FilesystemError(error) : GuestEio; - } -#endif - - error = RenameNative_nid_no_patch(source, destination, destinationExists && !sourceIsDirectory && !destinationIsDirectory); + error = RenameNative_nid_no_patch(source, destination, destinationExists && (!sourceIsDirectory || !sameFile)); return error ? FilesystemError(error) : 0; } diff --git a/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp index c23d288aa9..3d685d640a 100644 --- a/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp +++ b/core/libs/prx/libc/src/specifics/windows/CrashReport.cpp @@ -381,6 +381,9 @@ LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) { if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && record->NumberParameters >= 2) { const char* kind = record->ExceptionInformation[0] == 0 ? "read" : record->ExceptionInformation[0] == 1 ? "write" : "execute"; Report(" %s of 0x%016llx\n", kind, static_cast(record->ExceptionInformation[1])); + std::uint8_t trap[2]; + if (record->ExceptionInformation[0] == 0 && record->ExceptionInformation[1] == ~0ull && ReadCode(context->Rip, trap, sizeof(trap)) == sizeof(trap) && trap[0] == 0xcd && trap[1] == 0x45) + Report(" guest abort() (int 0x45)\n"); } Report(" rax %016llx rbx %016llx rcx %016llx rdx %016llx\n", context->Rax, context->Rbx, context->Rcx, context->Rdx); Report(" rsi %016llx rdi %016llx rbp %016llx rsp %016llx\n", context->Rsi, context->Rdi, context->Rbp, context->Rsp); diff --git a/core/libs/prx/libkernel/Aio/src/Aio.cpp b/core/libs/prx/libkernel/Aio/src/Aio.cpp index 7471d272aa..9926ac1dd6 100644 --- a/core/libs/prx/libkernel/Aio/src/Aio.cpp +++ b/core/libs/prx/libkernel/Aio/src/Aio.cpp @@ -134,9 +134,9 @@ int SubmitCommands(KernelAioRwRequest* req, std::int32_t size, bool write, std:: } int SubmitCommandsMultiple(KernelAioRwRequest* req, std::int32_t size, bool write, std::int32_t* id, const char* name) { + if (req != nullptr && id != nullptr && size > AioRequestNumMax) throw std::runtime_error(std::string(name) + ": more than 128 requests per batch is not modelled"); const int error = ValidateRequests(req, size, id); if (error != 0) return error; - if (size > AioRequestNumMax) throw std::runtime_error(std::string(name) + ": more than 128 requests per batch is not modelled"); for (std::int32_t i = 0; i < size; ++i) { const std::int32_t queue = AllocateId(); SetState(queue, RunRequest(req[i], write) ? AioCompleted : AioAborted); diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 2ffe176dcd..808ed7eb37 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -238,7 +238,7 @@ if(BUILD_TESTING) endif() add_test(NAME guest_dynamic_loader COMMAND guest_dynamic_loader_tests $) set_tests_properties(guest_dynamic_loader PROPERTIES - WORKING_DIRECTORY "$") + WORKING_DIRECTORY "$" RESOURCE_LOCK guest_dynamic_loader_files) if(WIN32) set_tests_properties(guest_dynamic_loader PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") @@ -271,7 +271,7 @@ if(BUILD_TESTING) target_link_options(guest_dynamic_loader_fixed_base_tests PRIVATE -no-pie -fuse-ld=bfd) add_test(NAME guest_dynamic_loader_fixed_base COMMAND guest_dynamic_loader_fixed_base_tests $) set_tests_properties(guest_dynamic_loader_fixed_base PROPERTIES - WORKING_DIRECTORY "$") + WORKING_DIRECTORY "$" RESOURCE_LOCK guest_dynamic_loader_files) endif() add_test_executable(guest_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestModuleInfo.cpp) target_include_directories(guest_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -587,6 +587,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_start PROPERTIES TIMEOUT 60 RUN_SERIAL TRUE ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_join_detach_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadJoinDetach.cpp) + target_include_directories(guest_pthread_join_detach_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_join_detach_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_join_detach_tests) + add_test(NAME guest_pthread_join_detach COMMAND guest_pthread_join_detach_tests) + set_tests_properties(guest_pthread_join_detach PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_join_detach PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp) target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc) diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 8370224ea9..c45f5cedd5 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -123,7 +123,20 @@ int _mapFlexible(void** addr, size_t len, int prot, int flags) { const size_t configured = _configuredFlexibleSize(); const size_t images = GuestImageSegmentBytes_nid_no_patch(); std::lock_guard lock(g_flexibleLock); - const size_t used = images + _flexibleUsedLocked(); + size_t replaced = 0; + constexpr int MapFixed = 0x10; + constexpr int MapNoOverwrite = 0x80; + if (addr != nullptr && *addr != nullptr && (flags & (MapFixed | MapNoOverwrite)) == MapFixed) { + const auto start = reinterpret_cast(*addr); + if (len <= std::numeric_limits::max() - start) { + const auto end = start + len; + for (const auto& [rangeStart, rangeLength] : g_flexibleRanges) { + const auto rangeEnd = rangeStart + rangeLength; + if (rangeStart < end && rangeEnd > start) replaced += std::min(end, rangeEnd) - std::max(start, rangeStart); + } + } + } + const size_t used = images + _flexibleUsedLocked() - replaced; if (len > configured - std::min(configured, used)) { std::fprintf(stderr, "[memory] flexible memory exhausted: request 0x%zx with 0x%zx of 0x%zx in use\n", len, used, configured); return SCE_KERNEL_ERROR_ENOMEM; diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 41a60f43b6..292d52406c 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -1323,24 +1323,8 @@ int64_t APS5_VABI pwritev_nid_postfix(int d, const KernelIovec* iov, int iovcnt, int APS5_VABI sceKernelRename(const char* from, const char* to) { if (!from || !to) return SceErrorFromErrno(GUEST_EFAULT); - if (*from == '\0' || *to == '\0') return SceErrorFromErrno(GUEST_ENOENT); - std::filesystem::path source; - if (const int error = ResolvePathError_nid_no_patch(from, source)) return SceKernelError(error); - std::error_code error; - const auto sourceStatus = std::filesystem::status(source, error); - if (!std::filesystem::exists(sourceStatus)) return SceErrorFromErrno(GUEST_ENOENT); - std::filesystem::path destination; - if (const int result = ResolvePathError_nid_no_patch(to, destination)) return SceKernelError(result); - bool replacesFile = false; -#ifdef _WIN32 - const auto destinationStatus = std::filesystem::symlink_status(destination, error); - replacesFile = !std::filesystem::is_directory(sourceStatus) && std::filesystem::exists(destinationStatus) && !std::filesystem::is_directory(destinationStatus); -#endif - error = RenameNative_nid_no_patch(source, destination, replacesFile); - if (error) return SceErrorFromErrno(GUEST_EIO); - RecordWrittenPath_nid_no_patch(source); - RecordWrittenPath_nid_no_patch(destination); - return 0; + const int result = RenameGuestPath_nid_no_patch(from, to); + return result == 0 ? 0 : SceKernelError(errno); } int APS5_VABI sceKernelRmdir(const char* path) { diff --git a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp index aa9a90cb4b..fdead458de 100644 --- a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp +++ b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp @@ -2,6 +2,7 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libkernel/Module/ModuleArgs.hpp" #include +#include #include #include #include @@ -39,6 +40,9 @@ struct Module { void* native = nullptr; bool owned = true; bool global = false; +#ifdef _WIN32 + std::uintptr_t globalOrder = 0; +#endif ~Module() { if (owned && native) { #ifdef _WIN32 @@ -193,7 +197,21 @@ void* APS5_VABI dlopen_nid_postfix(const char* path, int flags) { #endif std::lock_guard lock(modulesMutex); const auto handle = nextHandle++; - modules.emplace(handle, std::move(module)); +#ifdef _WIN32 + module->globalOrder = module->global ? handle : 0; + for (const auto& [key, entry] : modules) { + if (entry->native == module->native && entry->globalOrder != 0 && + (module->globalOrder == 0 || entry->globalOrder < module->globalOrder)) module->globalOrder = entry->globalOrder; + } +#endif + modules.emplace(handle, module); +#ifdef _WIN32 + for (const auto& [key, entry] : modules) { + if (entry->native != module->native) continue; + entry->global = module->globalOrder != 0; + entry->globalOrder = module->globalOrder; + } +#endif return reinterpret_cast(handle); } catch (const std::exception& error) { Error(error.what()); return nullptr; } } @@ -205,6 +223,10 @@ void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name) { std::lock_guard lock(modulesMutex); if (handle == nullptr || handle == reinterpret_cast(static_cast(-2))) { for (const auto& [key, module] : modules) if (module->global) search.push_back(module); +#ifdef _WIN32 + std::sort(search.begin(), search.end(), [](const auto& left, const auto& right) { return left->globalOrder < right->globalOrder; }); + search.erase(std::unique(search.begin(), search.end(), [](const auto& left, const auto& right) { return left->native == right->native; }), search.end()); +#endif } else { auto found = modules.find(reinterpret_cast(handle)); if (found == modules.end()) { Error("dlsym: invalid or unsupported module handle"); return nullptr; } diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index 44c987b41e..676048d2fb 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -144,6 +144,7 @@ struct PthreadPrivate { bool _adopted; Pthread* _handleStorage; std::atomic _joined; + bool _joining = false; std::mutex _join_mtx; TimedWait::Condition _join_cv; std::atomic cancelPending{false}; diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index 20a62ac8cb..2758f37088 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -886,9 +886,22 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { ThreadCancel::Check(); { std::unique_lock lock(thread->_join_mtx); - ThreadCancel::WaitUntil(thread->_join_cv, lock, std::nullopt, [&] { return thread->_finished.load(); }); + if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; + if (thread->_joining) return SCE_KERNEL_ERROR_EOPNOTSUPP; + thread->_joining = true; + while (!thread->_finished.load()) { + ThreadCancel::Wait(thread->_join_cv, lock, std::nullopt); + if (ThreadCancel::Requested()) { + thread->_joining = false; + lock.unlock(); + ThreadCancel::Check(); + lock.lock(); + thread->_joining = true; + } + } + thread->_joining = false; + if (thread->_joined.exchange(true, std::memory_order_acq_rel)) return SCE_KERNEL_ERROR_ESRCH; } - if (thread->_joined.exchange(true, std::memory_order_acq_rel)) return SCE_KERNEL_ERROR_ESRCH; #ifdef _WIN32 if (WaitForSingleObject(thread->nativeHandle, INFINITE) != WAIT_OBJECT_0) throw std::system_error(GetLastError(), std::system_category(), "Joining guest thread"); @@ -905,9 +918,11 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { int APS5_VABI scePthreadDetach(Pthread thread) { if (!thread) return SCE_KERNEL_ERROR_EINVAL; - if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; - if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_EINVAL; - thread->_detached = true; + { + std::lock_guard lock(thread->_join_mtx); + if (thread->_detached || thread->_joining || thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_EINVAL; + thread->_detached = true; + } #ifdef _WIN32 ReleaseThread(thread); #else diff --git a/core/libs/prx/libkernel_sys/Export.cpp b/core/libs/prx/libkernel_sys/Export.cpp index a8746955a5..f616bb5e5f 100644 --- a/core/libs/prx/libkernel_sys/Export.cpp +++ b/core/libs/prx/libkernel_sys/Export.cpp @@ -385,10 +385,17 @@ int APS5_VABI chroot_nid_postfix(const char* path) { int APS5_VABI symlink_nid_postfix(const char* target, const char* created) { if (target == nullptr || created == nullptr) return static_cast(Fail(GuestEfault)); if (*target == '\0' || *created == '\0') return static_cast(Fail(GuestEnoent)); - if (*target == '/') { + const std::filesystem::path contents(target); + if (contents.has_root_name() || contents.has_root_directory()) { NotImplemented_nid_no_patch("symlink: absolute targets"); return -1; } + for (const auto& part : contents) { + if (part == "..") { + NotImplemented_nid_no_patch("symlink: parent-directory targets"); + return -1; + } + } std::filesystem::path native; if (const int error = ResolvePathError_nid_no_patch(created, native)) return static_cast(Fail(error)); std::error_code error; @@ -398,7 +405,6 @@ int APS5_VABI symlink_nid_postfix(const char* target, const char* created) { if (std::filesystem::exists(std::filesystem::symlink_status(native, error))) return static_cast(Fail(GuestEexist)); #endif if (!std::filesystem::is_directory(native.parent_path(), error)) return static_cast(Fail(GuestEnoent)); - const std::filesystem::path contents(target); const bool directory = std::filesystem::is_directory(native.parent_path() / contents, error); #ifdef _WIN32 DWORD flags = SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; diff --git a/core/libs/tests/GuestAio.cpp b/core/libs/tests/GuestAio.cpp index 96d8030630..cad5b3c0b7 100644 --- a/core/libs/tests/GuestAio.cpp +++ b/core/libs/tests/GuestAio.cpp @@ -328,6 +328,16 @@ int main(int argc, char** argv) { Check(sceKernelAioSubmitWriteCommandsMultiple(writeBatch, 0, 0, writeIds) == SCE_KERNEL_ERROR_EINVAL); KernelAioRwRequest noResult{0, 1, readBack.data(), nullptr, batchFd}; Check(sceKernelAioSubmitWriteCommandsMultiple(&noResult, 1, 0, writeIds) == SCE_KERNEL_ERROR_EFAULT); + std::array unread{}; + std::array unreadIds{}; + for (const auto submit : {sceKernelAioSubmitReadCommandsMultiple, sceKernelAioSubmitWriteCommandsMultiple}) { + bool limited = false; + try { submit(unread.data(), 129, 0, unreadIds.data()); } catch (const std::runtime_error&) { limited = true; } + Check(limited); + } + std::int32_t unreadId = 0; + Check(sceKernelAioSubmitReadCommands(unread.data(), 129, 0, &unreadId) == SCE_KERNEL_ERROR_EFAULT); + Check(sceKernelAioSubmitWriteCommands(unread.data(), 129, 0, &unreadId) == SCE_KERNEL_ERROR_EFAULT); Check(sceKernelAioWaitRequests(readIds, 2, nullptr, 1, nullptr) == SCE_KERNEL_ERROR_EFAULT); Check(sceKernelAioWaitRequests(nullptr, 2, readStates, 1, nullptr) == SCE_KERNEL_ERROR_EFAULT); Check(sceKernelAioWaitRequests(readIds, -1, readStates, 1, nullptr) == SCE_KERNEL_ERROR_EINVAL); diff --git a/core/libs/tests/GuestAudioIn.cpp b/core/libs/tests/GuestAudioIn.cpp index 9b39a837c3..7dc3e6ff92 100644 --- a/core/libs/tests/GuestAudioIn.cpp +++ b/core/libs/tests/GuestAudioIn.cpp @@ -81,6 +81,14 @@ void TestValidation() { Require(ReportsNotImplemented("sceAudioInIsPadEmbeddedMic", [] { sceAudioInIsPadEmbeddedMic(1); })); } +void InputSound(int handle, std::vector& block) { + for (int attempt = 0; attempt < 200; ++attempt) { + Require(sceAudioInInput(handle, block.data()) == 256); + if (std::any_of(block.begin(), block.end() - 1, [](std::int16_t sample) { return sample != 0; })) return; + } + Require(false); +} + void TestCapture() { const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_in_capture-" + std::to_string(std::random_device{}()) + ".raw"); std::vector recorded(256 * 2 * 4); @@ -95,7 +103,7 @@ void TestCapture() { std::vector block(256 * 2 + 1, 0x5555); std::vector captured; for (int i = 0; i < 4; ++i) { - Require(sceAudioInInput(handle, block.data()) == 256); + InputSound(handle, block); captured.insert(captured.end(), block.begin(), block.end() - 1); Require(block.back() == 0x5555); } @@ -105,7 +113,7 @@ void TestCapture() { Require(sceAudioInClose(handle) == 0); const int async = sceAudioInAsyncOpen(user, 0, 0, 256, 48000, 2); Require(async == handle && sceAudioInGetSilentState(async) == 0); - Require(sceAudioInInput(async, block.data()) == 256); + InputSound(async, block); Require(std::equal(block.begin(), block.end() - 1, recorded.begin()) && block.back() == 0x5555); Require(sceAudioInClose(async) == 0); std::filesystem::remove(path); diff --git a/core/libs/tests/GuestDinkumwareAtomics.cpp b/core/libs/tests/GuestDinkumwareAtomics.cpp new file mode 100644 index 0000000000..8e45d76902 --- /dev/null +++ b/core/libs/tests/GuestDinkumwareAtomics.cpp @@ -0,0 +1,120 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +void APS5_VABI _Atomic_store_1_nid_postfix(volatile std::uint8_t*, std::uint8_t, int); +void APS5_VABI _Atomic_store_8_nid_postfix(volatile std::uint64_t*, std::uint64_t, int); +std::uint16_t APS5_VABI _Atomic_load_2_nid_postfix(volatile std::uint16_t*, int); +std::uint64_t APS5_VABI _Atomic_load_8_nid_postfix(volatile std::uint64_t*, int); +std::uint32_t APS5_VABI _Atomic_exchange_4_nid_postfix(volatile std::uint32_t*, std::uint32_t, int); +std::uint64_t APS5_VABI _Atomic_exchange_8_nid_postfix(volatile std::uint64_t*, std::uint64_t, int); +int APS5_VABI _Atomic_compare_exchange_strong_2_nid_postfix(volatile std::uint16_t*, std::uint16_t*, std::uint16_t, int, int); +int APS5_VABI _Atomic_compare_exchange_weak_8_nid_postfix(volatile std::uint64_t*, std::uint64_t*, std::uint64_t, int, int); +std::uint8_t APS5_VABI _Atomic_fetch_add_1_nid_postfix(volatile std::uint8_t*, std::uint8_t, int); +std::uint64_t APS5_VABI _Atomic_fetch_add_8_nid_postfix(volatile std::uint64_t*, std::uint64_t, int); +std::uint16_t APS5_VABI _Atomic_fetch_sub_2_nid_postfix(volatile std::uint16_t*, std::uint16_t, int); +std::uint32_t APS5_VABI _Atomic_fetch_and_4_nid_postfix(volatile std::uint32_t*, std::uint32_t, int); +std::uint8_t APS5_VABI _Atomic_fetch_or_1_nid_postfix(volatile std::uint8_t*, std::uint8_t, int); +std::uint64_t APS5_VABI _Atomic_fetch_xor_8_nid_postfix(volatile std::uint64_t*, std::uint64_t, int); +int APS5_VABI _Atomic_is_lock_free_4_nid_postfix(); +int APS5_VABI _Atomic_flag_test_and_set_nid_postfix(volatile unsigned char*, int); +void APS5_VABI _Atomic_flag_clear_nid_postfix(volatile unsigned char*, int); +void APS5_VABI _Atomic_thread_fence_nid_postfix(int); +void APS5_VABI _Atomic_signal_fence_nid_postfix(int); +void APS5_VABI _Atomic_copy_nid_postfix(volatile unsigned char*, std::size_t, volatile void*, const volatile void*, int); +void APS5_VABI _Atomic_exchange_nid_postfix(volatile unsigned char*, std::size_t, volatile void*, volatile void*, int); +int APS5_VABI _Atomic_compare_exchange_strong_nid_postfix(volatile unsigned char*, std::size_t, volatile void*, void*, const void*, int, int); +int APS5_VABI _Atomic_compare_exchange_weak_nid_postfix(volatile unsigned char*, std::size_t, volatile void*, void*, const void*, int, int); +} + +static void Require(bool value) { if (!value) std::abort(); } + +constexpr int Relaxed = 0, Acquire = 2, Release = 3, AcqRel = 4, SeqCst = 5; + +static void CheckTyped() { + std::uint8_t byte = 0; + _Atomic_store_1_nid_postfix(&byte, 7, SeqCst); + Require(byte == 7 && _Atomic_fetch_add_1_nid_postfix(&byte, 250, Relaxed) == 7 && byte == 1); + Require(_Atomic_fetch_or_1_nid_postfix(&byte, 0x80, AcqRel) == 1 && byte == 0x81); + std::uint16_t half = 5; + Require(_Atomic_load_2_nid_postfix(&half, Release) == 5 && _Atomic_fetch_sub_2_nid_postfix(&half, 6, SeqCst) == 5 && half == 0xffff); + std::uint16_t expectedHalf = 1; + Require(_Atomic_compare_exchange_strong_2_nid_postfix(&half, &expectedHalf, 2, SeqCst, Relaxed) == 0 && expectedHalf == 0xffff); + Require(_Atomic_compare_exchange_strong_2_nid_postfix(&half, &expectedHalf, 2, SeqCst, Relaxed) == 1 && half == 2); + std::uint32_t word = 0xff0f; + Require(_Atomic_fetch_and_4_nid_postfix(&word, 0xf0ff, Acquire) == 0xff0f && word == 0xf00f); + Require(_Atomic_exchange_4_nid_postfix(&word, 9, 6) == 0xf00f && word == 9); + std::uint64_t wide = 0; + _Atomic_store_8_nid_postfix(&wide, 0x123456789ull, Release); + Require(_Atomic_load_8_nid_postfix(&wide, Acquire) == 0x123456789ull); + Require(_Atomic_load_8_nid_postfix(&wide, Release) == 0 && _Atomic_load_8_nid_postfix(&wide, AcqRel) == 0); + Require(_Atomic_fetch_add_8_nid_postfix(&wide, 1, SeqCst) == 0x123456789ull && wide == 0x12345678aull); + Require(_Atomic_fetch_add_8_nid_postfix(&wide, 1, 6) == 0 && wide == 0x12345678aull); + Require(_Atomic_exchange_8_nid_postfix(&wide, 1, 7) == 0 && wide == 0x12345678aull); + Require(_Atomic_fetch_xor_8_nid_postfix(&wide, 0xf, Relaxed) == 0x12345678aull && wide == 0x123456785ull); + std::uint64_t expectedWide = 0x123456785ull; + Require(_Atomic_compare_exchange_weak_8_nid_postfix(&wide, &expectedWide, 3, SeqCst, SeqCst) == 1 && wide == 3); + Require(_Atomic_is_lock_free_4_nid_postfix() == 1); +} + +static void CheckFlagsAndFences() { + unsigned char flag = 0; + Require(_Atomic_flag_test_and_set_nid_postfix(&flag, SeqCst) == 0 && flag == 1); + Require(_Atomic_flag_test_and_set_nid_postfix(&flag, Relaxed) == 1); + _Atomic_flag_clear_nid_postfix(&flag, Release); + Require(flag == 0); + _Atomic_thread_fence_nid_postfix(SeqCst); + _Atomic_signal_fence_nid_postfix(SeqCst); +} + +struct Wide { + std::uint64_t parts[3]; +}; + +static void CheckGeneric() { + unsigned char lock = 0; + Wide target{{1, 2, 3}}, source{{4, 5, 6}}, expected{{1, 2, 3}}; + _Atomic_copy_nid_postfix(&lock, sizeof(Wide), &target, &source, SeqCst); + Require(std::memcmp(&target, &source, sizeof(Wide)) == 0 && lock == 0); + Wide other{{7, 8, 9}}; + _Atomic_exchange_nid_postfix(&lock, sizeof(Wide), &target, &other, SeqCst); + Require(target.parts[0] == 7 && other.parts[0] == 4 && lock == 0); + const Wide desired{{10, 11, 12}}; + Require(_Atomic_compare_exchange_strong_nid_postfix(&lock, sizeof(Wide), &target, &expected, &desired, SeqCst, SeqCst) == 0); + Require(std::memcmp(&expected, &target, sizeof(Wide)) == 0 && lock == 0); + Require(_Atomic_compare_exchange_weak_nid_postfix(&lock, sizeof(Wide), &target, &expected, &desired, SeqCst, SeqCst) == 1); + Require(std::memcmp(&target, &desired, sizeof(Wide)) == 0 && lock == 0); +} + +static void CheckConcurrentGeneric() { + unsigned char lock = 0; + Wide counter{{0, 0, 0}}; + constexpr int Iterations = 20000; + auto worker = [&] { + for (int i = 0; i < Iterations; ++i) { + Wide expected; + _Atomic_copy_nid_postfix(&lock, sizeof(Wide), &expected, &counter, SeqCst); + Wide next; + do { + next = Wide{{expected.parts[0] + 1, expected.parts[1] + 2, expected.parts[2] + 3}}; + } while (!_Atomic_compare_exchange_strong_nid_postfix(&lock, sizeof(Wide), &counter, &expected, &next, SeqCst, SeqCst)); + } + }; + std::thread first(worker); + std::thread second(worker); + first.join(); + second.join(); + Require(counter.parts[0] == 2 * Iterations && counter.parts[1] == 4 * Iterations && counter.parts[2] == 6 * Iterations); +} + +int main() { + CheckTyped(); + CheckFlagsAndFences(); + CheckGeneric(); + CheckConcurrentGeneric(); + return 0; +} diff --git a/core/libs/tests/GuestDynamicLoader.cpp b/core/libs/tests/GuestDynamicLoader.cpp index 6ad61bd721..3a83d086a9 100644 --- a/core/libs/tests/GuestDynamicLoader.cpp +++ b/core/libs/tests/GuestDynamicLoader.cpp @@ -35,6 +35,70 @@ static bool ThrowsRuntimeError(TFunction function) { } return false; } +#ifdef _WIN32 +static void CheckGlobalLookupOrder(const char* path) { + const std::filesystem::path other = "anyps5-global-order-for-test.guest.prx"; + std::filesystem::copy_file(path, other, std::filesystem::copy_options::overwrite_existing); + void* local = dlopen_nid_postfix(path, 2); + void* earlierGlobal = dlopen_nid_postfix(other.string().c_str(), 2 | 0x100); + void* promoted = dlopen_nid_postfix(path, 2 | 0x100); + Require(local && earlierGlobal && promoted); + void* firstSymbol = dlsym_nid_postfix(local, "GuestModuleAdd"); + void* earlierSymbol = dlsym_nid_postfix(earlierGlobal, "GuestModuleAdd"); + Require(firstSymbol && earlierSymbol && firstSymbol != earlierSymbol); + const std::array scopes{nullptr, reinterpret_cast(-2)}; + for (void* scope : scopes) Require(dlsym_nid_postfix(scope, "GuestModuleAdd") == earlierSymbol); + Require(dlclose_nid_postfix(promoted) == 0); + for (void* scope : scopes) Require(dlsym_nid_postfix(scope, "GuestModuleAdd") == earlierSymbol); + Require(dlclose_nid_postfix(local) == 0); + Require(dlclose_nid_postfix(earlierGlobal) == 0); + std::filesystem::remove(other); +} +static bool CheckGlobalVisibility(const char* path, bool globalFirst) { + using Add = int (APS5_VABI *)(int, int); + const std::array scopes{nullptr, reinterpret_cast(-2)}; + const auto requireHidden = [&] { + for (void* scope : scopes) { + Require(dlsym_nid_postfix(scope, "GuestModuleAdd") == nullptr); + Require(dlerror_nid_postfix() != nullptr); + } + }; + requireHidden(); + void* first = dlopen_nid_postfix(path, 2 | (globalFirst ? 0x100 : 0)); + Require(first != nullptr); + if (!globalFirst) requireHidden(); + void* second = dlopen_nid_postfix(path, 2 | (globalFirst ? 0 : 0x100)); + Require(second != nullptr && second != first); + auto add = reinterpret_cast(dlsym_nid_postfix(first, "GuestModuleAdd")); + Require(add && add(19, 23) == 42); + Require(dlsym_nid_postfix(second, "GuestModuleAdd") == reinterpret_cast(add)); + for (void* scope : scopes) Require(dlsym_nid_postfix(scope, "GuestModuleAdd") == reinterpret_cast(add)); + void* global = globalFirst ? first : second; + void* local = globalFirst ? second : first; + Require(dlclose_nid_postfix(global) == 0); + Require(dlsym_nid_postfix(local, "GuestModuleAdd") == reinterpret_cast(add)); + Require(add(20, 22) == 42); + bool visible = true; + for (void* scope : scopes) { + const bool found = dlsym_nid_postfix(scope, "GuestModuleAdd") == reinterpret_cast(add); + if (!found) { + std::fprintf(stderr, "RTLD_GLOBAL lost after closing its handle: global_first=%d scope=%s; surviving handle returns %d\n", + globalFirst, scope == nullptr ? "NULL" : "-2", add(20, 22)); + dlerror_nid_postfix(); + visible = false; + } + } + Require(dlclose_nid_postfix(local) == 0); + requireHidden(); + void* reloaded = dlopen_nid_postfix(path, 2); + Require(reloaded != nullptr); + auto reloadedAdd = reinterpret_cast(dlsym_nid_postfix(reloaded, "GuestModuleAdd")); + Require(reloadedAdd && reloadedAdd(21, 21) == 42); + requireHidden(); + Require(dlclose_nid_postfix(reloaded) == 0); + return visible; +} +#endif int main(int argc, char** argv) { Require(argc == 2); Require(dlerror_nid_postfix() == nullptr); @@ -53,6 +117,12 @@ int main(int argc, char** argv) { dlerror_nid_postfix(); void* executable = dlopen_nid_postfix(nullptr, 2); Require(executable != nullptr && dlclose_nid_postfix(executable) == 0); +#ifdef _WIN32 + CheckGlobalLookupOrder(argv[1]); + const bool localFirstVisible = CheckGlobalVisibility(argv[1], false); + const bool globalFirstVisible = CheckGlobalVisibility(argv[1], true); + Require(localFirstVisible && globalFirstVisible); +#endif void* module = dlopen_nid_postfix(argv[1], 2 | 0x100); Require(module != nullptr); using Add = int (APS5_VABI *)(int, int); diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index a28a0ef2fc..91559bb71c 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -615,6 +615,63 @@ int main() { Require(rename_nid_postfix(name(targetFile).c_str(), name(targetFile).c_str()) == 0); Require(rename_nid_postfix(name(sourceDirectory).c_str(), name(emptyDirectory).c_str()) == 0); Require(!std::filesystem::exists(sourceDirectory) && std::filesystem::is_directory(emptyDirectory / "child")); + const auto kernelSource = area / "kernel_source"; + const auto kernelTarget = area / "kernel_target"; + Require(std::filesystem::create_directories(kernelSource / "child")); + Require(std::filesystem::create_directory(kernelTarget)); + { std::ofstream stream(kernelSource / "marker"); stream << "kernel directory contents"; } + const int directoryResult = sceKernelRename(name(kernelSource).c_str(), name(kernelTarget).c_str()); + if (directoryResult != 0) + std::fprintf(stderr, "kernel empty-directory replacement result=0x%08x, expected=0\n", static_cast(directoryResult)); + Require(directoryResult == 0); + Require(!std::filesystem::exists(kernelSource) && std::filesystem::is_directory(kernelTarget / "child")); + { std::ifstream stream(kernelTarget / "marker"); std::string contents; std::getline(stream, contents); + Require(contents == "kernel directory contents"); } + const int parentResult = sceKernelRename(name(targetFile).c_str(), absentParent.c_str()); + Require(parentResult == static_cast(0x80020002u)); + Require(std::filesystem::is_regular_file(targetFile) && !std::filesystem::exists(area / "absent")); + const int prefixResult = sceKernelRename(name(targetFile).c_str(), name(targetFile / "target").c_str()); + Require(prefixResult == static_cast(0x80020014u)); + Require(std::filesystem::is_regular_file(targetFile)); + const int fileDirectoryResult = sceKernelRename(name(targetFile).c_str(), name(fullDirectory).c_str()); + Require(fileDirectoryResult == static_cast(0x80020015u)); + Require(std::filesystem::is_regular_file(targetFile) && std::filesystem::is_regular_file(fullDirectory / "entry")); + const int directoryFileResult = sceKernelRename(name(kernelTarget).c_str(), name(targetFile).c_str()); + Require(directoryFileResult == static_cast(0x80020014u)); + Require(std::filesystem::is_regular_file(kernelTarget / "marker") && std::filesystem::is_regular_file(targetFile)); + const int nonemptyResult = sceKernelRename(name(kernelTarget).c_str(), name(fullDirectory).c_str()); + Require(nonemptyResult == static_cast(0x80020042u)); + Require(std::filesystem::is_regular_file(kernelTarget / "marker") && std::filesystem::is_regular_file(fullDirectory / "entry")); + const auto descendant = kernelTarget / "child" / "moved"; + const int descendantResult = sceKernelRename(name(kernelTarget).c_str(), name(descendant).c_str()); + Require(descendantResult == static_cast(0x80020016u)); + Require(std::filesystem::is_directory(kernelTarget / "child") && !std::filesystem::exists(descendant)); + { std::ifstream stream(targetFile); std::string contents; std::getline(stream, contents); Require(contents == "source"); } + { std::ifstream stream(fullDirectory / "entry"); std::string contents; std::getline(stream, contents); Require(contents == "entry"); } + { std::ifstream stream(kernelTarget / "marker"); std::string contents; std::getline(stream, contents); + Require(contents == "kernel directory contents"); } +#ifdef _WIN32 + for (const bool kernel : {true, false}) { + const auto lockedSource = area / (kernel ? "locked_kernel_source" : "locked_libc_source"); + const auto preservedTarget = area / (kernel ? "preserved_kernel_target" : "preserved_libc_target"); + Require(std::filesystem::create_directory(lockedSource)); + Require(std::filesystem::create_directory(preservedTarget)); + const auto marker = lockedSource / "marker"; + { std::ofstream stream(marker); stream << "preserved source contents"; } + const HANDLE held = CreateFileW(marker.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + Require(held != INVALID_HANDLE_VALUE); + const int result = kernel ? sceKernelRename(name(lockedSource).c_str(), name(preservedTarget).c_str()) : + rename_nid_postfix(name(lockedSource).c_str(), name(preservedTarget).c_str()); + Require(CloseHandle(held) != 0); + Require(result != 0 && std::filesystem::is_regular_file(marker)); + if (!std::filesystem::is_directory(preservedTarget)) + std::fprintf(stderr, "failed %s directory rename removed its destination\n", kernel ? "kernel" : "libc"); + Require(std::filesystem::is_directory(preservedTarget) && std::filesystem::is_empty(preservedTarget)); + { std::ifstream stream(marker); std::string contents; std::getline(stream, contents); + Require(contents == "preserved source contents"); } + } +#endif for (const bool kernel : {false, true}) { const auto incoming = name(area / "incoming.txt"); const auto occupied = name(area / "occupied.txt"); diff --git a/core/libs/tests/GuestFlexibleMemoryBudget.cpp b/core/libs/tests/GuestFlexibleMemoryBudget.cpp index ced1a258d5..cba381e734 100644 --- a/core/libs/tests/GuestFlexibleMemoryBudget.cpp +++ b/core/libs/tests/GuestFlexibleMemoryBudget.cpp @@ -22,6 +22,14 @@ std::uint64_t GuestImageSegmentBytes_nid_no_patch(); void* APS5_VABI dlopen_nid_postfix(const char*, int); char* APS5_VABI dlerror_nid_postfix(); int APS5_VABI dlclose_nid_postfix(void*); +std::size_t APS5_VABI sceKernelGetDirectMemorySize(); +int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); +int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); +int APS5_VABI sceKernelMapDirectMemory2(void**, std::size_t, int, int, int, std::int64_t, std::size_t); +int APS5_VABI sceKernelMapNamedDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t, const char*); +int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry*, int, int*, int); +int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); +int APS5_VABI sceKernelVirtualQuery(const void*, int, VirtualQueryInfo*, std::uint64_t); } static void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -115,13 +123,124 @@ int main(int argc, char** argv) { Require(sceKernelMapFlexibleMemory(&oversized, before + PS5_PAGE_SIZE, 3, 0) == SCE_KERNEL_ERROR_ENOMEM && oversized == nullptr); Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before); if (mode == "limited") { + const auto checkBudget = [&](std::size_t expected) { + std::size_t current = 0; + Require(sceKernelAvailableFlexibleMemorySize(¤t) == 0); + std::printf("Fixed flexible mapping: available=%zu expected=%zu\n", current, expected); + std::fflush(stdout); + Require(current == expected); + }; + constexpr auto page = PS5_PAGE_SIZE; + void* original = nullptr; + Require(sceKernelMapFlexibleMemory(&original, page * 4, 3, 0) == 0); + auto* bytes = static_cast(original); + for (std::size_t i = 0; i < 4; ++i) bytes[i * page] = static_cast(17 + i); + auto* middle = static_cast(original) + page; + void* replacement = middle; + Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == middle); + Require(bytes[0] == 17 && bytes[page] == 0 && bytes[page * 2] == 19 && bytes[page * 3] == 20); + checkBudget(before - page * 4); + bytes[page] = 23; + Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x90) == SCE_KERNEL_ERROR_ENOMEM); + Require(bytes[page] == 23); + checkBudget(before - page * 4); + replacement = original; + Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == original); + Require(bytes[0] == 0 && bytes[page] == 23 && bytes[page * 3] == 20); + checkBudget(before - page * 4); + Require(sceKernelMapFlexibleMemory(&replacement, page * 3, 3, 0x10) == 0 && replacement == original); + Require(bytes[0] == 0 && bytes[page] == 0 && bytes[page * 2] == 0 && bytes[page * 3] == 20); + checkBudget(before - page * 4); + Require(sceKernelReleaseFlexibleMemory(middle, page * 2) == 0); + checkBudget(before - page * 2); + replacement = middle; + Require(sceKernelMapFlexibleMemory(&replacement, page * 3, 3, 0x10) == 0 && replacement == middle); + Require(bytes[page] == 0 && bytes[page * 3] == 0); + checkBudget(before - page * 4); + Require(sceKernelReleaseFlexibleMemory(original, page * 4) == 0); + checkBudget(before); const auto length = before & ~(PS5_PAGE_SIZE - 1); void* full = nullptr; Require(sceKernelMapFlexibleMemory(&full, length, 3, 0) == 0); void* extra = nullptr; Require(sceKernelMapFlexibleMemory(&extra, PS5_PAGE_SIZE, 3, 0) == SCE_KERNEL_ERROR_ENOMEM && extra == nullptr); + static_cast(full)[0] = 31; + replacement = full; + Require(sceKernelMapFlexibleMemory(&replacement, length, 3, 0x90) == SCE_KERNEL_ERROR_ENOMEM); + Require(static_cast(full)[0] == 31); + checkBudget(before - length); + auto* fullBytes = static_cast(full); + fullBytes[page] = 37; + fullBytes[length - page] = 41; + auto* fullMiddle = static_cast(full) + page; + replacement = fullMiddle; + Require(sceKernelMapFlexibleMemory(&replacement, page, 3, 0x10) == 0 && replacement == fullMiddle); + Require(fullBytes[0] == 31 && fullBytes[page] == 0 && fullBytes[length - page] == 41); + checkBudget(before - length); + replacement = static_cast(full) + length - page; + Require(sceKernelMapFlexibleMemory(&replacement, page * 2, 3, 0x10) == SCE_KERNEL_ERROR_ENOMEM); + Require(fullBytes[0] == 31 && fullBytes[length - page] == 41); + checkBudget(before - length); + replacement = full; + Require(sceKernelMapFlexibleMemory(&replacement, length + page, 3, 0x10) == SCE_KERNEL_ERROR_ENOMEM); + Require(static_cast(full)[0] == 31); + checkBudget(before - length); + Require(sceKernelMapFlexibleMemory(&replacement, length, 3, 0x10) == 0 && replacement == full); + Require(static_cast(full)[0] == 0); + checkBudget(before - length); Require(sceKernelReleaseFlexibleMemory(full, length) == 0); Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before); + for (int api = 0; api < 4; ++api) { + constexpr auto directPage = PS5_PAGE_SIZE; + const auto checkDirectBudget = [&](std::size_t expected) { + std::size_t current = 0; + Require(sceKernelAvailableFlexibleMemorySize(¤t) == 0); + std::printf("Direct replacement API=%d available=%zu expected=%zu\n", api, current, expected); + std::fflush(stdout); + Require(current == expected); + }; + void* original = nullptr; + Require(sceKernelMapFlexibleMemory(&original, directPage * 4, 3, 0) == 0); + auto* bytes = static_cast(original); + bytes[0] = 17; + bytes[directPage * 4 - 1] = 29; + bytes[directPage] = 37; + bytes[directPage * 3 - 1] = 41; + std::int64_t physical = -1; + Require(sceKernelAllocateDirectMemory(0, sceKernelGetDirectMemorySize(), directPage * 2, directPage, 0, &physical) == 0); + const auto mapDirect = [&](void*& address, int flags) { + if (api == 0) return sceKernelMapDirectMemory(&address, directPage * 2, 3, flags, physical, directPage); + if (api == 1) return sceKernelMapDirectMemory2(&address, directPage * 2, 0, 3, flags, physical, directPage); + if (api == 2) return sceKernelMapNamedDirectMemory(&address, directPage * 2, 3, flags, physical, directPage, "budget direct"); + KernelBatchMapEntry entry{address, static_cast(physical), directPage * 2, 3, 0, 0, 0}; + int processed = -1; + const int result = sceKernelBatchMap2(&entry, 1, &processed, flags); + Require(processed == (result == 0 ? 1 : 0)); + address = entry.start; + return result; + }; + void* middle = static_cast(original) + directPage; + Require(mapDirect(middle, 0x90) == SCE_KERNEL_ERROR_ENOMEM); + Require(middle == static_cast(original) + directPage); + checkDirectBudget(before - directPage * 4); + Require(bytes[0] == 17 && bytes[directPage * 4 - 1] == 29); + Require(bytes[directPage] == 37 && bytes[directPage * 3 - 1] == 41); + Require(mapDirect(middle, 0x10) == 0 && middle == static_cast(original) + directPage); + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(middle, 0, &info, sizeof(info)) == 0 && info.is_direct && !info.is_flexible); + Require(bytes[0] == 17 && bytes[directPage * 4 - 1] == 29); + checkDirectBudget(before - directPage * 2); + const auto recovered = before - directPage * 2; + void* reused = nullptr; + Require(sceKernelMapFlexibleMemory(&reused, recovered, 3, 0) == 0); + static_cast(reused)[recovered - 1] = 31; + Require(sceKernelReleaseFlexibleMemory(reused, recovered) == 0); + checkDirectBudget(before - directPage * 2); + Require(sceKernelReleaseDirectMemory(physical, directPage * 2) == 0); + checkDirectBudget(before - directPage * 2); + Require(sceKernelReleaseFlexibleMemory(original, directPage * 4) == 0); + checkDirectBudget(before); + } } if (module) { Require(dlclose_nid_postfix(module) == 0); diff --git a/core/libs/tests/GuestImeDialog.cpp b/core/libs/tests/GuestImeDialog.cpp new file mode 100644 index 0000000000..3bfec70576 --- /dev/null +++ b/core/libs/tests/GuestImeDialog.cpp @@ -0,0 +1,158 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libSceIme/include/ImeInput.hpp" +#include "prx/libSceVideoOut/include/ImeTextInput.hpp" +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceImeDialogInit(const Param*, const ExtendedParam*); +int APS5_VABI sceImeDialogGetStatus(void); +int APS5_VABI sceImeDialogGetResult(Result*); +int APS5_VABI sceImeDialogAbort(void); +int APS5_VABI sceImeDialogTerm(void); +} + +namespace { + +constexpr int StatusNone = 0; +constexpr int StatusRunning = 1; +constexpr int StatusFinished = 2; +constexpr std::uint32_t EndStatusOk = 0; +constexpr std::uint32_t EndStatusUserCanceled = 1; +constexpr std::uint32_t EndStatusAborted = 2; +constexpr int ErrorBusy = static_cast(0x80bc0001u); +constexpr int ErrorNotOpened = static_cast(0x80bc0002u); +constexpr int ErrorInvalidMaxTextLength = static_cast(0x80bc0016u); +constexpr int ErrorInvalidInputTextBuffer = static_cast(0x80bc0017u); +constexpr int ErrorInvalidAddress = static_cast(0x80bc0031u); + +void Require(bool value) { if (!value) std::abort(); } + +constexpr unsigned WindowId = 7; +ImeTextInput g_window; + +void Type(const char* utf8) { + SDL_Event event{}; + event.type = SDL_TEXTINPUT; + event.text.windowID = WindowId; + std::strncpy(event.text.text, utf8, sizeof(event.text.text) - 1); + g_window.HandleEvent(event, WindowId); +} + +void Press(SDL_Scancode scancode) { + SDL_Event event{}; + event.type = SDL_KEYDOWN; + event.key.windowID = WindowId; + event.key.keysym.scancode = scancode; + g_window.HandleEvent(event, WindowId); +} + +Param DialogParam(char16_t* buffer, std::uint32_t maxLength) { + Param param{}; + param.max_text_length = maxLength; + param.input_text_buffer = buffer; + return param; +} + +std::uint32_t EndStatus() { + Result result{}; + Require(sceImeDialogGetResult(&result) == 0); + return result.endstatus; +} + +void Open(const Param& param) { + Require(sceImeDialogInit(¶m, nullptr) == 0); + Require(sceImeDialogGetStatus() == StatusRunning); + Require(ImeCapturesKeyboard_nid_postfix()); +} + +void Close() { + Require(sceImeDialogTerm() == 0); + Require(sceImeDialogGetStatus() == StatusNone); +} + +} + +int main() { + char16_t buffer[16] = u"Hi"; + const Param param = DialogParam(buffer, 8); + Require(!ImeCapturesKeyboard_nid_postfix()); + Require(sceImeDialogGetStatus() == StatusNone); + Require(sceImeDialogTerm() == ErrorNotOpened); + + Open(param); + for (int poll = 0; poll < 8; ++poll) Require(sceImeDialogGetStatus() == StatusRunning); + Require(sceImeDialogInit(¶m, nullptr) == ErrorBusy); + Result early{}; + Require(sceImeDialogGetResult(&early) == ErrorNotOpened); + Type("\xc3\xa9"); + Type("\xf0\x9f\x98\x80"); + Require(std::u16string(buffer) == u"Hi"); + Press(SDL_SCANCODE_RETURN); + Require(sceImeDialogGetStatus() == StatusFinished); + Require(!ImeCapturesKeyboard_nid_postfix()); + Require(EndStatus() == EndStatusOk); + Require(std::u16string(buffer) == u"Hi\u00e9\U0001F600"); + Require(sceImeDialogAbort() == ErrorNotOpened); + Close(); + Require(sceImeDialogTerm() == ErrorNotOpened); + + buffer[0] = u'\0'; + Open(param); + Type("abc"); + Press(SDL_SCANCODE_HOME); + Press(SDL_SCANCODE_DELETE); + Press(SDL_SCANCODE_END); + Type("\xf0\x9f\x98\x80"); + Press(SDL_SCANCODE_BACKSPACE); + Press(SDL_SCANCODE_LEFT); + Type("X"); + Type("1234"); + Press(SDL_SCANCODE_END); + Type("\xf0\x9f\x98\x80"); + Type("98"); + Type("Z"); + Press(SDL_SCANCODE_RETURN); + Require(EndStatus() == EndStatusOk); + Require(std::u16string(buffer) == u"bX1234c9"); + Close(); + + buffer[0] = u'\0'; + Open(param); + Press(SDL_SCANCODE_RETURN); + Require(EndStatus() == EndStatusOk); + Require(buffer[0] == u'\0'); + Close(); + + std::memcpy(buffer, u"keep", sizeof(u"keep")); + Open(param); + Type("zz"); + Press(SDL_SCANCODE_ESCAPE); + Require(sceImeDialogGetStatus() == StatusFinished); + Require(EndStatus() == EndStatusUserCanceled); + Require(std::u16string(buffer) == u"keep"); + Close(); + + Open(param); + Require(sceImeDialogAbort() == 0); + Require(sceImeDialogGetStatus() == StatusFinished); + Require(!ImeCapturesKeyboard_nid_postfix()); + Require(EndStatus() == EndStatusAborted); + Type("ignored"); + Press(SDL_SCANCODE_RETURN); + Require(std::u16string(buffer) == u"keep"); + Close(); + + Require(sceImeDialogInit(nullptr, nullptr) == ErrorInvalidAddress); + const Param empty = DialogParam(buffer, 0); + Require(sceImeDialogInit(&empty, nullptr) == ErrorInvalidMaxTextLength); + const Param tooLong = DialogParam(buffer, 2049); + Require(sceImeDialogInit(&tooLong, nullptr) == ErrorInvalidMaxTextLength); + const Param noBuffer = DialogParam(nullptr, 8); + Require(sceImeDialogInit(&noBuffer, nullptr) == ErrorInvalidInputTextBuffer); + Require(sceImeDialogGetStatus() == StatusNone); + Require(!ImeCapturesKeyboard_nid_postfix()); +} diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp index 14e3137646..c33f09d129 100644 --- a/core/libs/tests/GuestKernelSys.cpp +++ b/core/libs/tests/GuestKernelSys.cpp @@ -88,6 +88,44 @@ bool ChrootThrows(const std::string& path) { return false; } +bool SymlinkTargetThrows(const char* target, const std::string& created) { + try { + symlink_nid_postfix(target, created.c_str()); + } catch (const std::runtime_error&) { + return true; + } + return false; +} + +void CheckSymlinkTargetRestrictions(const std::filesystem::path& root) { + const auto created = (root / "rejected").string(); + for (const auto& target : {std::string(".."), std::string("../x"), std::string("a/../b"), std::string("a/../../x"), std::string("/abs")}) + Require(SymlinkTargetThrows(target.c_str(), created)); +#ifdef _WIN32 + for (const auto& target : {std::string("C:/"), std::string("\\\\server\\share")}) + Require(SymlinkTargetThrows(target.c_str(), created)); +#endif + const auto links = [](const std::string& created, const char* target) { + char buffer[16]; + const auto length = std::strlen(target); + return readlink_nid_postfix(created.c_str(), buffer, sizeof(buffer)) == static_cast(length) && + std::memcmp(buffer, target, length) == 0; + }; + const auto nested = (root / "accepted-nested").string(); + const int link = symlink_nid_postfix("sub/dir", nested.c_str()); + if (link == 0) { + Require(links(nested, "sub/dir")); + const auto dots = (root / "accepted-dots").string(); + Require(symlink_nid_postfix("a..b", dots.c_str()) == 0 && links(dots, "a..b")); + const auto ellipsis = (root / "accepted-ellipsis").string(); + Require(symlink_nid_postfix("...", ellipsis.c_str()) == 0 && links(ellipsis, "...")); + Require(std::filesystem::remove(nested) && std::filesystem::remove(dots) && std::filesystem::remove(ellipsis)); + } else { + Require(link == -1 && Error() == 1); + std::fprintf(stderr, "symbolic links unavailable: the symlink target success cases not tested\n"); + } +} + void CheckPathconf(const std::filesystem::path& root, const std::string& file) { const auto directory = root.generic_string(); for (const auto& path : {file, directory}) { @@ -415,6 +453,7 @@ int main() { Require(symlink_nid_postfix("original.bin", (root / "missing" / "symbolic").string().c_str()) == -1 && Error() == 2); Require(symlink_nid_postfix(nullptr, symbolic.c_str()) == -1 && Error() == 14); Require(symlink_nid_postfix("original.bin", "") == -1 && Error() == 2); + CheckSymlinkTargetRestrictions(root); const int base = sceKernelOpen(root.string().c_str(), SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0); Require(base >= 0); const int relative = openat_nid_postfix(base, "file.txt", SCE_KERNEL_O_RDONLY); diff --git a/core/libs/tests/GuestPthreadJoinDetach.cpp b/core/libs/tests/GuestPthreadJoinDetach.cpp new file mode 100644 index 0000000000..48a982989e --- /dev/null +++ b/core/libs/tests/GuestPthreadJoinDetach.cpp @@ -0,0 +1,75 @@ +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +extern "C" int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +extern "C" int APS5_VABI scePthreadDetach(Pthread thread); + +static constexpr int SCE_OK = 0; + +static void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message); + std::exit(1); + } +} + +struct Joinee { + Pthread thread = nullptr; + std::atomic release{false}; +}; + +struct Joiner { + Pthread target = nullptr; + void* result = nullptr; + int status = -1; +}; + +static void* APS5_VABI Wait(void* opaque) { + auto* joinee = static_cast(opaque); + while (!joinee->release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return opaque; +} + +static void* APS5_VABI Join(void* opaque) { + auto* joiner = static_cast(opaque); + joiner->status = scePthreadJoin(joiner->target, &joiner->result); + return nullptr; +} + +static bool Joining(Pthread thread) { + std::lock_guard lock(thread->_join_mtx); + return thread->_joining; +} + +int main() { + Joinee joinee; + Require(scePthreadCreate(&joinee.thread, nullptr, Wait, &joinee, "joinee") == SCE_OK, "create the joinee"); + Joiner joiner{joinee.thread}; + Pthread joinerThread = nullptr; + Require(scePthreadCreate(&joinerThread, nullptr, Join, &joiner, "joiner") == SCE_OK, "create the joiner"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!Joining(joinee.thread)) { + Require(std::chrono::steady_clock::now() < deadline, "the joiner waits in scePthreadJoin"); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + Require(scePthreadDetach(joinee.thread) == SCE_KERNEL_ERROR_EINVAL, "detaching a thread with a joiner returns EINVAL"); + Require(scePthreadJoin(joinee.thread, nullptr) == SCE_KERNEL_ERROR_EOPNOTSUPP, "a second joiner gets EOPNOTSUPP"); + joinee.release.store(true); + Require(scePthreadJoin(joinerThread, nullptr) == SCE_OK, "join the joiner"); + Require(joiner.status == SCE_OK && joiner.result == &joinee, "the first join completes with the return value"); + + Joinee detached; + Require(scePthreadCreate(&detached.thread, nullptr, Wait, &detached, "detached") == SCE_OK, "create a thread to detach"); + Require(scePthreadDetach(detached.thread) == SCE_OK, "detach a thread without a joiner"); + Require(scePthreadJoin(detached.thread, nullptr) == SCE_KERNEL_ERROR_EINVAL, "joining a detached thread returns EINVAL"); + detached.release.store(true); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + return 0; +} diff --git a/core/libs/tests/GuestRegMgr.cpp b/core/libs/tests/GuestRegMgr.cpp new file mode 100644 index 0000000000..dfdf074535 --- /dev/null +++ b/core/libs/tests/GuestRegMgr.cpp @@ -0,0 +1,27 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +extern "C" { +std::int32_t APS5_VABI sceRegMgrNonSysGetInt(std::int64_t key, std::int32_t unk, std::int32_t* data); +} + +static void Require(bool value) { if (!value) std::abort(); } + +template +static bool ReportsNotImplemented(const char* name, TCall call) { + try { + call(); + } catch (const std::runtime_error& error) { + return std::string(error.what()) == std::string(name) + " not implemented"; + } + return false; +} + +int main() { + std::int32_t value = 0; + Require(ReportsNotImplemented("sceRegMgrNonSysGetInt", [&] { sceRegMgrNonSysGetInt(0, 0, &value); })); + Require(ReportsNotImplemented("sceRegMgrNonSysGetInt", [&] { sceRegMgrNonSysGetInt(-1, 1, nullptr); })); +} diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 464bcd0e32..52599d3c22 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -956,6 +956,71 @@ static void CheckUnspecifiedIpv6() { Require(sceNetSocketClose(receiver) == 0); } +static void CheckAcceptedNonblocking(int family, bool nonblocking) { + const auto addressSize = family == 2 ? 16u : 28u; + std::array address{}; + address[0] = static_cast(addressSize); + address[1] = static_cast(family); + if (family == 2) { address[4] = 127; address[7] = 1; } + else address[23] = 1; + const int listener = sceNetSocket("accept-mode", family, 1, 6); + Require(listener >= 0 && sceNetBind_nid_postfix(listener, address.data(), addressSize) == 0); + Require(sceNetListen(listener, 1) == 0); + const int mode = nonblocking ? 1 : 0; + Require(sceNetSetsockopt(listener, 0xffff, 0x1200, &mode, sizeof(mode)) == 0); + std::uint32_t size = address.size(); + Require(sceNetGetsockname(listener, address.data(), &size) == 0 && size == addressSize); + const int client = sceNetSocket("accept-client", family, 1, 6); + Require(client >= 0 && sceNetConnect(client, address.data(), addressSize) == 0); + int accepted = -1; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + do { + accepted = sceNetAccept(listener, nullptr, nullptr); + if (accepted >= 0) break; + Require(nonblocking && Failed(accepted, 35)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < deadline); + Require(accepted >= 0); + int inherited = -1; + size = sizeof(inherited); + Require(sceNetGetsockopt(accepted, 0xffff, 0x1200, &inherited, &size) == 0 && size == sizeof(inherited)); + const int status = fcntl_nid_postfix(accepted, 3); + std::printf("Accepted IPv%d mode=%d SO_NBIO=%d F_GETFL=%d\n", family == 2 ? 4 : 6, mode, inherited, status); + std::fflush(stdout); + Require(inherited == mode && status == (2 | (nonblocking ? 4 : 0))); + const int changed = 1 - mode; + Require(sceNetSetsockopt(listener, 0xffff, 0x1200, &changed, sizeof(changed)) == 0); + size = sizeof(inherited); + Require(sceNetGetsockopt(accepted, 0xffff, 0x1200, &inherited, &size) == 0 && inherited == mode); + struct ReceiveResult { std::int64_t bytes; int error; char value; }; + auto receive = std::async(std::launch::async, [accepted] { + char byte = '?'; + const auto received = sceNetRecv(accepted, &byte, sizeof(byte), 0); + return ReceiveResult{received, *sceNetErrnoLoc(), byte}; + }); + const bool ready = receive.wait_for(nonblocking ? std::chrono::milliseconds(2000) : std::chrono::milliseconds(50)) == std::future_status::ready; + const char payload = 'n'; + Require(sceNetSend(client, &payload, sizeof(payload), 0) == sizeof(payload)); + const auto received = receive.get(); + Require(ready == nonblocking); + if (nonblocking) { + Require(received.bytes == static_cast(0x80410123) && received.error == 35 && received.value == '?'); + char byte = '?'; + std::int64_t count = -1; + const auto arrival = std::chrono::steady_clock::now() + std::chrono::seconds(2); + do { + count = sceNetRecv(accepted, &byte, sizeof(byte), 0); + if (count >= 0) break; + Require(Failed(count, 35)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < arrival); + Require(count == sizeof(byte) && byte == payload); + } else { + Require(received.bytes == sizeof(payload) && received.value == payload); + } + for (const int descriptor : {accepted, client, listener}) Require(sceNetSocketClose(descriptor) == 0); +} + static void CheckIpv6SocketsAreIpv6Only() { const int inet = sceNetSocket("ipv4-any", 2, 2, 17); Require(inet >= 0); @@ -1081,6 +1146,11 @@ int main() { CheckUnboundDatagramTimeoutAfterAddress(); CheckUnspecifiedIpv6(); if (!ipv6Unavailable) CheckIpv6SocketsAreIpv6Only(); + for (const int family : {2, 28}) { + if (family == 28 && ipv6Unavailable) continue; + CheckAcceptedNonblocking(family, false); + CheckAcceptedNonblocking(family, true); + } CheckDatagramDisconnect(); CheckCloseKernelSocket(); diff --git a/core/libs/tests/GuestSslLoad.cpp b/core/libs/tests/GuestSslLoad.cpp index 4cb3a27c3e..95e12521f7 100644 --- a/core/libs/tests/GuestSslLoad.cpp +++ b/core/libs/tests/GuestSslLoad.cpp @@ -33,6 +33,7 @@ int APS5_VABI sceSslClose(int); int APS5_VABI sceSslGetNotBefore(int, void*, SceSsl::RtcTick*); int APS5_VABI sceSslGetNotAfter(int, void*, SceSsl::RtcTick*); int APS5_VABI sceSslGetFingerprint(int, void*, int, unsigned char*); +int APS5_VABI sceSslGetPem(int, void*, char*, std::size_t*); } namespace { @@ -278,6 +279,23 @@ void CertificateReaders() { Throws([&] { sceSslGetFingerprint(id, fake, 3, sha256); }); Throws([&] { sceSslGetFingerprint(9999, cert, 3, sha256); }); + std::size_t pemLen = 0; + Check(sceSslGetPem(id, cert, nullptr, &pemLen) == 0, "query pem size failed"); + Check(pemLen > 0, "pem length is 0"); + std::vector pemBuf(pemLen); + std::size_t writtenLen = pemBuf.size(); + Check(sceSslGetPem(id, cert, pemBuf.data(), &writtenLen) == 0, "write pem failed"); + Check(writtenLen == pemLen, "written pem length mismatch"); + Check(std::strstr(pemBuf.data(), "-----BEGIN CERTIFICATE-----\n") != nullptr, "pem header missing"); + Check(std::strstr(pemBuf.data(), "-----END CERTIFICATE-----\n") != nullptr, "pem footer missing"); + + char smallPem[8]{}; + std::size_t smallPemLen = sizeof(smallPem); + Throws([&] { sceSslGetPem(id, cert, smallPem, &smallPemLen); }); + Check(sceSslGetPem(id, cert, pemBuf.data(), nullptr) == static_cast(0x8095177a), "null pem len accepted"); + Throws([&] { sceSslGetPem(id, nullptr, pemBuf.data(), &writtenLen); }); + Throws([&] { sceSslGetPem(id, fake, pemBuf.data(), &writtenLen); }); + Throws([&] { sceSslClose(1); }); Check(sceSslFreeCaList(id, &handles) == 0, "free ca list failed"); @@ -288,6 +306,7 @@ void CertificateReaders() { Throws([&] { sceSslGetNotBefore(id, cert, &begin); }); Throws([&] { sceSslGetNotAfter(id, cert, &limit); }); Throws([&] { sceSslGetFingerprint(id, cert, 3, sha256); }); + Throws([&] { sceSslGetPem(id, cert, pemBuf.data(), &writtenLen); }); const int id2 = sceSslInit_nid_postfix(1024 * 1024); Check(sceSslLoadCert(id2, 1, &pointer, nullptr, nullptr) == 0, "CA load failed"); @@ -304,6 +323,7 @@ void CertificateReaders() { Throws([&] { sceSslGetNotBefore(id2, cert2, &begin); }); Throws([&] { sceSslGetNotAfter(id2, cert2, &limit); }); Throws([&] { sceSslGetFingerprint(id2, cert2, 3, sha256); }); + Throws([&] { sceSslGetPem(id2, cert2, pemBuf.data(), &writtenLen); }); Check(sceSslTerm_nid_postfix(id2) == 0, "term failed"); Throws([&] { sceSslGetNameEntryCount(id2, subject2); }); Throws([&] { sceSslFreeSslCertName(id2, subject2); }); diff --git a/core/libs/tests/GuestSslPs5Readers.cpp b/core/libs/tests/GuestSslPs5Readers.cpp new file mode 100644 index 0000000000..7f7bf6e909 --- /dev/null +++ b/core/libs/tests/GuestSslPs5Readers.cpp @@ -0,0 +1,103 @@ +#include "prx/libSceSsl/SslTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "SslCertificateFixture.hpp" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceSslInit_nid_postfix(std::uint64_t); +int APS5_VABI sceSslTerm_nid_postfix(int); +int APS5_VABI sceSslLoadCert(int, int, SceSsl::Data**, SceSsl::Data*, SceSsl::Data*); +int APS5_VABI sceSslUnloadCert(int); +int APS5_VABI sceSslGetCaList(int, SceSsl::CaList*); +int APS5_VABI sceSslFreeCaList(int, SceSsl::CaList*); +int APS5_VABI sceSslGetSerialNumber(int, void*, unsigned char*, std::size_t*); +void* APS5_VABI sceSslGetSubjectName(int, void*); +void* APS5_VABI sceSslGetIssuerName(int, void*); +int APS5_VABI sceSslGetNameEntryCount(int, void*); +int APS5_VABI sceSslGetNameEntryInfo(int, void*, int, char*, std::size_t, char*, std::size_t, std::size_t*); +int APS5_VABI sceSslFreeSslCertName(int, void*); +int APS5_VABI sceSslGetPem(int, void*, char*, std::size_t*); +} + +namespace { + +void Check(bool value, const char* message) { + if (!value) throw std::runtime_error(message); +} + +template +void Throws(Function function) { + try { function(); } catch (const std::exception&) { return; } + throw std::runtime_error("unsupported reader arguments were accepted"); +} + +void Readers() { + const int context = sceSslInit_nid_postfix(1024 * 1024); + const int other = sceSslInit_nid_postfix(1024 * 1024); + SceSsl::Data data{reinterpret_cast(const_cast(RootPem)), std::strlen(RootPem)}; + auto* pointer = &data; + Check(sceSslLoadCert(context, 1, &pointer, nullptr, nullptr) == 0, "certificate load failed"); + SceSsl::CaList list{}; + Check(sceSslGetCaList(context, &list) == 0 && list.num == 1, "certificate handle absent"); + void* certificate = list.certs[0]; + std::size_t length = SIZE_MAX; + Check(sceSslGetSerialNumber(context, certificate, nullptr, &length) == 0 && length == 3, "PS5 serial size query failed"); + std::array serial; + serial.fill(0xa5); + Check(sceSslGetSerialNumber(context, certificate, serial.data(), &length) == 0 && length == 3, "PS5 serial copy failed"); + Check(serial == std::array{0x00, 0x80, 0xa1, 0xa5, 0xa5}, "serial copy changed its bytes or guards"); + serial.fill(0xa5); + length = 2; + Check(sceSslGetSerialNumber(context, certificate, serial.data(), &length) == static_cast(0x8095177a) && length == 3, "short serial buffer did not report its required size"); + Check(std::all_of(serial.begin(), serial.end(), [](auto byte) { return byte == 0xa5; }), "short serial buffer was modified"); + Throws([&] { sceSslGetSerialNumber(other, certificate, nullptr, &length); }); + + auto* subject = sceSslGetSubjectName(context, certificate); + auto* issuer = sceSslGetIssuerName(context, certificate); + Check(subject && issuer && subject != issuer, "certificate name absent"); + Check(sceSslGetNameEntryCount(context, subject) == 3 && sceSslGetNameEntryCount(context, issuer) == 3, "PS5 name entry count failed"); + Throws([&] { sceSslGetSubjectName(other, certificate); }); + Throws([&] { sceSslGetNameEntryCount(other, subject); }); + Throws([&] { sceSslFreeSslCertName(other, subject); }); + std::array oid{}, value{}; + length = SIZE_MAX; + Check(sceSslGetNameEntryInfo(context, subject, 1, oid.data(), oid.size(), value.data(), value.size(), &length) == 0, "PS5 eight-argument name query failed"); + Check(std::strcmp(oid.data(), "O") == 0 && std::strcmp(value.data(), "AnyPS5 Tests") == 0 && length == 12, "name output or full-width length changed"); + + length = SIZE_MAX; + Check(sceSslGetPem(context, certificate, nullptr, &length) == 0 && length == std::strlen(RootPem) + 1, "PS5 PEM size query failed"); + std::vector pem(length + 2, static_cast(0x5a)); + Check(sceSslGetPem(context, certificate, pem.data(), &length) == 0 && length == std::strlen(RootPem) + 1, "PS5 PEM copy failed"); + Check(std::strcmp(pem.data(), RootPem) == 0 && pem[length] == 0x5a && pem[length + 1] == 0x5a, "PEM copy changed bytes or guards"); + std::array small{'x', 'x', 'x', 'x'}; + length = 2; + Throws([&] { sceSslGetPem(context, certificate, small.data(), &length); }); + Check(std::all_of(small.begin(), small.end(), [](char byte) { return byte == 'x'; }), "short PEM buffer was modified"); + Throws([&] { sceSslGetPem(other, certificate, nullptr, &length); }); + Check(sceSslUnloadCert(context) == 0, "unload failed"); + Check(sceSslGetSerialNumber(context, certificate, nullptr, &length) == 0 && length == 3, "CA snapshot lost its certificate on unload"); + Check(sceSslFreeCaList(context, &list) == 0, "CA list free failed"); + Throws([&] { sceSslGetSerialNumber(context, certificate, nullptr, &length); }); + Check(sceSslGetNameEntryCount(context, subject) == 3, "owned name depended on freed certificate storage"); + Check(sceSslFreeSslCertName(context, subject) == 0, "name free failed"); + Throws([&] { sceSslGetNameEntryCount(context, subject); }); + Check(sceSslTerm_nid_postfix(context) == 0, "context term failed"); + Throws([&] { sceSslGetNameEntryCount(context, issuer); }); + Check(sceSslTerm_nid_postfix(other) == 0, "other context term failed"); +} + +} + +int main() { + try { Readers(); } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/WindowsExceptions.cpp b/core/libs/tests/WindowsExceptions.cpp index 19a4bb2e86..d7adc8c92e 100644 --- a/core/libs/tests/WindowsExceptions.cpp +++ b/core/libs/tests/WindowsExceptions.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -74,7 +75,7 @@ static int RunThrowChild() { return 100; } -static void testUnhandledThrowSite() { +static std::string RunChild(const wchar_t* argument, DWORD& exitCode) { SECURITY_ATTRIBUTES attributes{}; attributes.nLength = sizeof(attributes); attributes.bInheritHandle = TRUE; @@ -84,7 +85,6 @@ static void testUnhandledThrowSite() { RequireHandle(SetHandleInformation(readPipe, HANDLE_FLAG_INHERIT, 0) != 0); std::array image{}; RequireHandle(GetModuleFileNameW(nullptr, image.data(), static_cast(image.size())) != 0); - const wchar_t* argument = L"\" throw-uncaught"; std::array command{}; const std::size_t imageLength = std::wcslen(image.data()); const std::size_t argumentLength = std::wcslen(argument); @@ -110,22 +110,47 @@ static void testUnhandledThrowSite() { } CloseHandle(readPipe); RequireHandle(WaitForSingleObject(child.hProcess, 20000) == WAIT_OBJECT_0); - DWORD exitCode = 0; RequireHandle(GetExitCodeProcess(child.hProcess, &exitCode) != 0); CloseHandle(child.hProcess); output[outputLength < output.size() ? outputLength : output.size() - 1] = 0; + return std::string(output.data()); +} + +static void testUnhandledThrowSite() { + DWORD exitCode = 0; + const std::string output = RunChild(L"\" throw-uncaught", exitCode); RequireHandle(exitCode == 0xC0000409); - const char* probeMarker = std::strstr(output.data(), "probe "); + const char* probeMarker = std::strstr(output.c_str(), "probe "); RequireHandle(probeMarker != nullptr); const auto probe = static_cast(std::strtoull(probeMarker + 6, nullptr, 16)); RequireHandle(probe != 0); - const char* marker = std::strstr(output.data(), "thrown from "); + const char* marker = std::strstr(output.c_str(), "thrown from "); RequireHandle(marker != nullptr); const auto logged = static_cast(std::strtoull(marker + 13, nullptr, 16)); RequireHandle(logged != 0); if (logged <= probe || logged - probe >= 0x1000) throw std::runtime_error("unhandled exception named the wrapper instead of the throw site"); } +[[gnu::noinline, noreturn]] static void GuestAbortProbe() { + __asm__ volatile("int $0x45"); + __builtin_unreachable(); +} + +static int RunAbortChild() { + std::fprintf(stderr, "abort probe\n"); + std::fflush(stderr); + GuestAbortProbe(); +} + +static void testGuestAbortReport() { + DWORD exitCode = 0; + const std::string output = RunChild(L"\" guest-abort", exitCode); + char code[16]; + std::snprintf(code, sizeof(code), "0x%08lx", static_cast(exitCode)); + if (exitCode != 0xC0000005 || output.find("guest abort() (int 0x45)") == std::string::npos) + throw std::runtime_error(std::string("guest abort child exited with ") + code + " and printed:\n" + output); +} + static int destroyed; struct Guard { ~Guard() { ++destroyed; } }; static void ThrowNested() { @@ -273,10 +298,12 @@ static void testUnwindThroughRuntimeModules() { int main(int argc, char** argv) { if (argc > 1) { if (std::strcmp(argv[1], "throw-uncaught") == 0) return RunThrowChild(); + if (std::strcmp(argv[1], "guest-abort") == 0) return RunAbortChild(); return 100; } TestTypeInfoVtables(); testUnhandledThrowSite(); + testGuestAbortReport(); testExceptionPointer(); testFutureException(); testMadeExceptionPointer(); diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index a5ccd31294..1d271f3dbb 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -169,15 +169,21 @@ function(add_shader_recompiler target runtimeTarget) target_compile_features(${target} PUBLIC cxx_std_20) + if(ANYPS5_ENABLE_SPIRV_TOOLS) + target_sources(${target} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvOptimizer.cpp) + endif() + set(shaderCacheGeneratedDir "${CMAKE_CURRENT_BINARY_DIR}/${target}_generated") set(shaderCacheVersionSource "${shaderCacheGeneratedDir}/ShaderCacheVersion.cpp") file(GLOB_RECURSE shaderCacheRecompilerSources CONFIGURE_DEPENDS - "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.hpp" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.h" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.cmake" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/CMakeLists.txt" ) + get_target_property(shaderCacheCompiledSources ${target} SOURCES) + list(APPEND shaderCacheRecompilerSources ${shaderCacheCompiledSources}) + list(REMOVE_DUPLICATES shaderCacheRecompilerSources) set(shaderCacheKeySources "${PROJECT_SOURCE_DIR}/core/libs/prx/libc/include/HostThreadLocal.hpp") foreach(source IN LISTS shaderCacheKeySources) if(NOT EXISTS "${source}") @@ -279,6 +285,7 @@ function(add_shader_recompiler target runtimeTarget) LinkUserDataCalls FindTableCalls LinkCalls + LoadStamp WriteFileAtomically ) set(shaderCacheManifest) @@ -328,8 +335,15 @@ function(add_shader_recompiler target runtimeTarget) ) if(ANYPS5_ENABLE_SPIRV_TOOLS) - target_sources(${target} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/SpirvOptimizer.cpp) target_link_libraries(${target} PRIVATE SPIRV-Tools-opt SPIRV-Tools-static) endif() target_compile_definitions(${target} PUBLIC ANYPS5_ENABLE_SPIRV_TOOLS=$) + if(BUILD_TESTING AND ANYPS5_ENABLE_SPIRV_TOOLS AND NOT ANYPS5_USE_SYSTEM_DEPS AND NOT TARGET spirv_validator_exit_tests) + add_test_executable(spirv_validator_exit_tests "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/tests/SpirvValidatorExit.cpp") + target_link_libraries(spirv_validator_exit_tests PRIVATE SPIRV-Tools-static) + set_target_properties(spirv_validator_exit_tests PROPERTIES CXX_EXTENSIONS OFF) + configure_windows_unwind(spirv_validator_exit_tests) + add_test(NAME spirv_validator_exit COMMAND spirv_validator_exit_tests) + set_tests_properties(spirv_validator_exit PROPERTIES TIMEOUT 30) + endif() endfunction() diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index df27ddc83b..da36be7eae 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -214,6 +214,7 @@ private: append(key, value.fetchBufferReg); append(key, value.fetchEmbedded); append(key, value.paClVsOutCntl); + append(key, value.depthRemapMode); if (value.resourcesNum > value.resources.size()) throw std::runtime_error("Shader cache: invalid vertex resource count"); append(key, value.resourcesNum); for (std::uint32_t i = 0; i < value.resourcesNum; ++i) { @@ -263,6 +264,7 @@ private: append(key, value.srgbDecodeFormats); append(key, value.narrowSubgroupClock); append(key, value.float64DenormPreserve); + append(key, value.depthRangeRemap); append(key, value.registrationModules); append(key, value.registrationWordLimit); append(key, value.vertexSubgroups); diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index bd635a59d1..91f598d70d 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -422,7 +422,7 @@ ShaderPixelStageInfo readPixelInfo(Reader& reader, std::uint32_t version) { info.positionLocation = readPixelPositionLocation(reader); for (std::uint8_t& location : info.sampleLocations) location = reader.ReadU8(); } - if (version >= 35u) { + if (version >= 36u) { for (std::uint32_t& order : info.sampleOrder) order = reader.ReadU32(); } if (version < 5u) { @@ -798,7 +798,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(35u); + writer.WriteU32(36u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -824,6 +824,10 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteBool(request.target.float16DenormPreserve); writer.WriteU32Span(request.context.linkedCallFallbacks); writer.WriteBool(request.target.spannedWaves); + writer.WriteBool(request.target.depthRangeRemap); + if (request.context.vertex.has_value()) { + writer.WriteU32(request.context.vertex->depthRemapMode); + } return base64Encode(buffer); } @@ -832,7 +836,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 35u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 36u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -869,6 +873,13 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const result.request.context.linkedCallFallbacks = result.linkedCallFallbacks; } if (version >= 32u) result.request.target.spannedWaves = reader.ReadBool(); + if (version >= 35u) { + result.request.target.depthRangeRemap = reader.ReadBool(); + if (result.vertex.has_value()) { + result.vertex->depthRemapMode = reader.ReadU32(); + result.request.context.vertex = result.vertex; + } + } if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data"); return result; } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index bdd3727ea1..08b2529bc0 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -14,7 +14,11 @@ #include #include #include +#include +#include +#include #include +#include #include #include @@ -28,6 +32,64 @@ struct DescriptorValue { bool operator==(const DescriptorValue& other) const = default; }; +template +class SharedValues { +public: + SharedValues() = default; + explicit SharedValues(std::vector values) : block(values.empty() ? nullptr : std::make_shared(std::move(values))) {} + + [[nodiscard]] std::size_t size() const { return block != nullptr ? block->values.size() : 0u; } + [[nodiscard]] bool empty() const { return block == nullptr; } + [[nodiscard]] const T* data() const { return block != nullptr ? block->values.data() : nullptr; } + [[nodiscard]] const T* begin() const { return data(); } + [[nodiscard]] const T* end() const { return data() + size(); } + [[nodiscard]] const T& operator[](std::size_t index) const { return block->values[index]; } + [[nodiscard]] const T& at(std::size_t index) const { + if (index >= size()) throw std::out_of_range("SharedValues::at"); + return block->values[index]; + } + [[nodiscard]] const T& front() const { return at(0); } + [[nodiscard]] const T& back() const { return at(size() - 1u); } + [[nodiscard]] std::span Span() const { return {data(), size()}; } + [[nodiscard]] std::vector Copy() const { return {begin(), end()}; } + [[nodiscard]] bool Shares(const SharedValues& other) const { return block == other.block; } + + [[nodiscard]] std::uint64_t Hash() const { + constexpr std::uint64_t Basis = 0xcbf29ce484222325ull; + if (block == nullptr) return Basis; + if (const auto known = block->hash.load(std::memory_order_relaxed); known != 0u) return known; + std::uint64_t hash = Basis; + const auto mix = [&](std::uint64_t value) { hash = (hash ^ value) * 0x100000001b3ull; }; + mix(block->values.size()); + for (const auto& value : block->values) { + if constexpr (std::is_same_v) { + mix(value.dwordCount); + mix(value.availableDwordMask); + for (const auto dword : value.dwords) mix(dword); + } else if constexpr (std::is_integral_v) { + mix(value); + } else { + mix(value.first); + mix(value.second); + } + } + hash = hash != 0u ? hash : 1u; + block->hash.store(hash, std::memory_order_relaxed); + return hash; + } + + friend bool operator==(const SharedValues& left, const SharedValues& right) { return left.block == right.block || std::ranges::equal(left.Span(), right.Span()); } + friend bool operator==(const SharedValues& left, const std::vector& right) { return std::ranges::equal(left.Span(), right); } + +private: + struct Block { + explicit Block(std::vector values) : values(std::move(values)) {} + std::vector values; + mutable std::atomic hash{0}; + }; + std::shared_ptr block; +}; + enum class UniformFillKind { None, Buffer, Image }; struct UniformFill { @@ -47,15 +109,15 @@ struct ImageTableColumnSnapshot { std::uint32_t keys = 0; std::uint32_t fault = 0; bool outside = false; - std::vector codes; - std::vector> readRanges; + SharedValues codes; + SharedValues words; + SharedValues> readRanges; bool operator==(const ImageTableColumnSnapshot& other) const = default; }; struct ImageTableSnapshot { std::vector tables; - std::vector words; std::vector> ranges; std::uint64_t shader = 0; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index 7de44b5980..1425a5ec58 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -6,6 +6,7 @@ #include "ImageTableAbi.hpp" #include #include +#include #include #include #include @@ -42,6 +43,16 @@ struct ResolvedImageTables { std::uint32_t faults = 0; }; +struct ImageTableReuseCounts { + std::uint64_t unwritten = 0; + std::uint64_t compared = 0; + std::uint64_t wordsRead = 0; + std::uint64_t shared = 0; + std::uint64_t columns = 0; + std::uint64_t resolutions = 0; + std::uint64_t resolutionsShared = 0; +}; + struct RuntimeImageModeMatch { std::optional mode; ImageTableAbi::PoisonReason reason = ImageTableAbi::PoisonReason::InvalidFormat; @@ -62,12 +73,14 @@ public: static std::uint32_t RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span modes); static void PrepareImageModes(ShaderInfo& info); static const char* UnnormalizedSampleMismatch(const ImageResource& image); + static std::shared_ptr ResolveImageTablesShared(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey); static bool NullImageDescriptor(const ImageResource& image, const DescriptorValue& descriptor); static ResolvedImageTables ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey = 0); [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; static void CompactPlan(IrResourcePlan& plan); void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const; static std::uint64_t SpecializationNanoseconds(); + static ImageTableReuseCounts TableReuse(); static std::uint64_t EvaluateNanoseconds(); static std::uint64_t MaterializeNanoseconds(); }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index 8279b48ef4..8ab74ff208 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -69,6 +69,10 @@ struct ShaderStageRuntime { } }; +struct ShaderDepthRemap { + std::uint32_t mode = 0; +}; + struct ShaderClipSpaceTransform { float scale[2] = {}; float offset[2] = {}; @@ -137,6 +141,7 @@ struct ShaderVertexInputInfo { std::uint32_t hostSubgroupSize = 64; std::uint32_t paClVsOutCntl = 0; ShaderClipSpaceTransform clipSpace; + ShaderDepthRemap depthRemap; ShaderMeshInputInfo mesh; ShaderTessellationInputInfo tess; bool fetchExternal = false; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index 8d345285de..79269617fe 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -4,7 +4,9 @@ #include "IntermediateRepresentation/IrProgram.hpp" #include "UserDataUse.hpp" +#include #include +#include #include #include #include @@ -15,6 +17,51 @@ using SrtMemoryReader = bool (*)(void* userData, std::uint64_t address, std::uin using SrtMemoryProbe = bool (*)(void* userData, std::uint64_t address, std::uint64_t bytes); using SrtMemoryBulkReader = bool (*)(void* userData, std::uint64_t address, std::uint32_t* values, std::uint64_t count); +struct SrtWriteStamp { + std::uint64_t generation = 0; + std::uint64_t layout = 0; +}; + +struct SrtStampCell { + std::atomic sequence{0}; + std::atomic generation{0}; + std::atomic layout{0}; +}; + +struct SrtGuestWords { + std::vector> runs; + std::vector words; + std::shared_ptr stamp; + std::uint64_t low = 0; + std::uint64_t high = 0; + bool ordered = false; +}; + +inline SrtWriteStamp LoadStamp(const SrtGuestWords& words) { + if (words.stamp == nullptr) return {}; + const auto& cell = *words.stamp; + const auto sequence = cell.sequence.load(std::memory_order_acquire); + if ((sequence & 1u) != 0u) return {}; + const SrtWriteStamp stamp{cell.generation.load(std::memory_order_relaxed), cell.layout.load(std::memory_order_relaxed)}; + std::atomic_thread_fence(std::memory_order_acquire); + return cell.sequence.load(std::memory_order_relaxed) == sequence ? stamp : SrtWriteStamp{}; +} + +inline void StoreStamp(const SrtGuestWords& words, const SrtWriteStamp& stamp) { + if (words.stamp == nullptr || stamp.generation == 0u) return; + auto& cell = *words.stamp; + auto sequence = cell.sequence.load(std::memory_order_relaxed); + if ((sequence & 1u) != 0u || !cell.sequence.compare_exchange_strong(sequence, sequence + 1u, std::memory_order_relaxed)) return; + std::atomic_thread_fence(std::memory_order_release); + cell.generation.store(stamp.generation, std::memory_order_relaxed); + cell.layout.store(stamp.layout, std::memory_order_relaxed); + cell.sequence.store(sequence + 2u, std::memory_order_release); +} + +using SrtMemoryStamper = SrtWriteStamp (*)(void* userData, std::uint64_t address, std::uint64_t bytes); +using SrtMemoryUnchanged = bool (*)(void* userData, std::uint64_t address, std::uint64_t bytes, const SrtWriteStamp& stamp); +using SrtMemorySupplier = bool (*)(void* userData, const std::shared_ptr& words); + // The guest addresses a walk dereferenced (Detail::Evaluator::EvaluateRawRead): the leaf read of // each pure flat slot (IrResourcePlan::pureFlatSlots) as (flat offset, address), set by the // flattened loop through `leaf`/`leafSlot` around that slot's evaluation, and every other raw read @@ -36,6 +83,9 @@ struct SrtRuntime { SrtMemoryProbe accessible = nullptr; SrtMemoryProbe pendingWrite = nullptr; SrtMemoryBulkReader readWords = nullptr; + SrtMemoryStamper stampWrites = nullptr; + SrtMemoryUnchanged unchangedSince = nullptr; + SrtMemorySupplier supplyWords = nullptr; SrtUserDataUse* userDataUse = nullptr; }; diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 82904cf201..f55d87d756 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -435,6 +435,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l physical.bufferAtomic.push_back(buffer.atomic); physical.bufferWritten.push_back(buffer.written || buffer.atomic); physical.bufferRead.push_back(buffer.read); + physical.bufferExtent.push_back(buffer.maxByteExtent); if (buffer.written || buffer.atomic) ++entry.writtenBuffers; else ++entry.readOnlyBuffers; } diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 4c3056a18e..3ef4eb3f09 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -5,6 +5,7 @@ #include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "RdnaDecoder/RdnaImageOpDecoder.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" +#include "ThreadOwned.hpp" #include "ImageTableAbi.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -332,6 +334,8 @@ struct TableCounters { std::array, static_cast(NarrowSkip::Count)> narrowSkips{}; std::atomic narrowPositions{0}; std::atomic selected{0}; + std::atomic reusedUnwritten{0}; + std::atomic reusedCompared{0}; std::atomic nanoseconds{0}; std::atomic lastReport{0}; }; @@ -341,6 +345,21 @@ TableCounters& tableCounters() { return counters; } +struct TableReuseTotals { + std::atomic unwritten{0}; + std::atomic compared{0}; + std::atomic wordsRead{0}; + std::atomic shared{0}; + std::atomic columns{0}; + std::atomic resolutions{0}; + std::atomic resolutionsShared{0}; +}; + +TableReuseTotals& tableReuseTotals() { + static TableReuseTotals totals; + return totals; +} + void reportTables() { if (!MaterializeProfiled()) return; auto& counters = tableCounters(); @@ -366,13 +385,22 @@ void reportTables() { skips += " " + std::string(NarrowSkipNames[reason]) + "=" + std::to_string(count); if (reason == static_cast(NarrowSkip::KeyScan)) skips += " (up to " + std::to_string(counters.narrowPositions.exchange(0, std::memory_order_relaxed)) + " positions, APS5_IMAGE_TABLE_KEY_SCAN " + std::to_string(TableKeyScanLimit()) + ")"; } - std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped%s%s, %.1f ms\n", + std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped%s%s, reused %llu unwritten / %llu compared, %.1f ms\n", static_cast(snapshots), static_cast(counters.tables.exchange(0, std::memory_order_relaxed)), static_cast(counters.keys.exchange(0, std::memory_order_relaxed)), static_cast(counters.selected.exchange(0, std::memory_order_relaxed)), poison.empty() ? " none" : poison.c_str(), - static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), skips.empty() ? "" : ":", skips.c_str(), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); + static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), skips.empty() ? "" : ":", skips.c_str(), static_cast(counters.reusedUnwritten.exchange(0, std::memory_order_relaxed)), static_cast(counters.reusedCompared.exchange(0, std::memory_order_relaxed)), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); } enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped, Carried }; +struct ColumnContents { + SharedValues codes; + SharedValues words; + SharedValues> readRanges; + std::uint32_t poisoned = 0; + std::uint32_t unmapped = 0; + std::uint32_t carried = 0; +}; + struct ColumnRecords { std::uint64_t base = 0; std::uint64_t size = 0; @@ -381,9 +409,12 @@ struct ColumnRecords { std::uint32_t fault = 0; bool narrowed = false; bool outside = false; + bool sealed = false; std::vector states; std::vector words; + std::vector distinct; std::vector> readRanges; + ColumnContents contents; }; struct TableTrace { @@ -407,6 +438,49 @@ void traceTable(const IrResourcePlan& plan, std::uint32_t table, const TableColu std::fprintf(stderr, "[image-table] shader 0x%llx table %u (%s): base 0x%llx size 0x%llx stride 0x%x addend 0x%x offset 0x%x dwords %u, %u records, %u keys%s, %u structural poison\n", static_cast(plan.shaderHash), table, column.sampler ? "sampler" : "image", static_cast(trace.base), static_cast(trace.size), column.stride, column.addend, column.offset, column.dwordCount, trace.records, trace.keys, trace.narrowed ? " (narrowed)" : "", trace.poison); } +bool TableSnapshotReuse() { + static const bool enabled = std::getenv("APS5_NO_TABLE_SNAPSHOT_REUSE") == nullptr; + return enabled; +} + +struct ColumnKey { + std::uint64_t base = 0; + std::uint64_t size = 0; + std::uint32_t records = 0; + std::uint32_t keys = 0; + std::uint32_t stride = 0; + std::uint32_t addend = 0; + std::uint32_t offset = 0; + std::uint32_t dwordCount = 0; + std::uint32_t maxKey = 0; + std::uint32_t foldedAddend = 0; + std::uint32_t foldedCarry = 0; + bool address = false; + bool sampler = false; + + bool operator==(const ColumnKey& other) const = default; +}; + +struct ColumnMemo { + ColumnKey key; + std::shared_ptr read; + ColumnContents contents; + std::uint64_t low = 0; + std::uint64_t high = 0; + + std::size_t Footprint() const { + return read->words.size() + read->runs.size() * 3u + contents.codes.size() + contents.words.size() * 9u + contents.readRanges.size() * 4u; + } +}; + +constexpr std::size_t ColumnMemoEntries = 16; +constexpr std::size_t ColumnMemoWords = std::size_t{1} << 22u; + +std::vector& ColumnMemos() { + thread_local std::vector* slot = nullptr; + return ThreadOwned(slot); +} + template struct SlotScratch { std::vector slots; @@ -521,6 +595,11 @@ public: } } + void Clear() { + lease.Clear(); + entries = 0; + } + void Insert(std::uint64_t hash, std::uint32_t value) { if ((entries + 1u) * 2u > lease.Slots().size()) grow(); place(lease.Slots(), hash, value); @@ -761,29 +840,67 @@ private: low = std::min(low, address); high = std::max(high, address + sizeof(std::uint32_t)); } - records.words.front() = WordsIndex(value); + records.words.front() = WordsIndex(records, value); if (low < high) tables.ranges.emplace_back(low, high - low); } - std::uint32_t WordsIndex(const DescriptorValue& value) { + std::uint32_t WordsIndex(ColumnRecords& records, const DescriptorValue& value) { const auto hash = hashWords(value.dwords, (static_cast(value.dwordCount) << 8u) | value.availableDwordMask); - if (const auto found = wordsIndex.Find(hash, [&](std::uint32_t index) { return tables.words[index] == value; })) return *found; - const auto index = static_cast(tables.words.size()); - tables.words.push_back(value); + if (const auto found = wordsIndex.Find(hash, [&](std::uint32_t index) { return records.distinct[index] == value; })) return *found; + const auto index = static_cast(records.distinct.size()); + records.distinct.push_back(value); wordsIndex.Insert(hash, index); return index; } + void Seal(ColumnRecords& records) { + auto& contents = records.contents; + std::vector codes(records.keys, 0u); + for (std::uint32_t key = 0; key < records.keys; key++) { + switch (records.states[key]) { + case RecordState::Read: + codes[key] = records.words[key]; + continue; + case RecordState::OutsideDomain: + codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::OutsideDomain)); + break; + case RecordState::Unmapped: + codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::Unmapped)); + contents.unmapped++; + break; + case RecordState::Carried: + codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::OffsetCarry)); + contents.carried++; + break; + } + contents.poisoned++; + } + contents.codes = SharedValues(std::move(codes)); + contents.words = SharedValues(std::move(records.distinct)); + contents.readRanges = SharedValues>(std::move(records.readRanges)); + records.sealed = true; + tableReuseTotals().columns.fetch_add(1, std::memory_order_relaxed); + } + ColumnRecords& Column(std::uint32_t source) { if (const auto found = columns.find(source); found != columns.end()) { return found->second; } auto& records = columns[source]; - const auto& column = *plan.descriptorSources.at(source).tableColumn; + ReadColumn(*plan.descriptorSources.at(source).tableColumn, source, records); + if (!records.sealed) Seal(records); + return records; + } + + void ReadColumn(const TableColumn& column, std::uint32_t source, ColumnRecords& records) { + stamp = {}; + comparedRuns.clear(); + comparedWords.clear(); + wordsIndex.Clear(); const bool active = source >= activeSources.size() || activeSources[source] != 0u; if (!active) { records.fault = ImageTableAbi::PoisonCode(static_cast(PoisonReason::Inactive)); - return records; + return; } if (column.address) { DescriptorValue pointer; @@ -794,13 +911,13 @@ private: const auto heap = EvaluateBuffer(column.heapSource); if (heap.Type() != 0u) { records.fault = ImageTableAbi::PoisonCode(static_cast(PoisonReason::NotBuffer)); - return records; + return; } records.base = heap.Base48(); records.size = heap.GetSize(); if (column.selected) { Select(column, records); - return records; + return; } records.records = ImageTableAbi::ScalarBufferRecords(column.addend, column.stride, column.offset + (column.indirect ? 4u : 0u), records.size); if (records.size != 0u) tables.ranges.emplace_back(records.base, records.size); @@ -808,13 +925,16 @@ private: records.records = std::min(records.records, ImageTableAbi::KeyRecords(column.maxKey, column.stride)); records.keys = std::min(records.records, ImageTableAbi::MaxKeys); records.outside = records.keys < records.records; - records.states.assign(records.keys, RecordState::Read); if (!column.keyDomains.empty() && records.keys != 0u) { Narrow(column, records); } - records.words.assign(records.keys, 0u); + const auto unread = [&] { + if (!records.narrowed) records.states.assign(records.keys, RecordState::Read); + records.words.assign(records.keys, 0u); + }; const auto end = records.base + records.size; if (column.indirect) { + unread(); for (std::uint32_t record = 0; record < records.keys; record++) { if (records.states[record] != RecordState::Read) continue; const auto offset = column.addend + record * column.stride; @@ -853,11 +973,20 @@ private: past = std::max(past, *address + sizeof(std::uint32_t)); } if (records.states[record] != RecordState::Read) continue; - records.words[record] = WordsIndex(value); + records.words[record] = WordsIndex(records, value); tables.ranges.emplace_back(first, past - first); } - return records; + return; } + const bool memoized = TableSnapshotReuse() && !records.narrowed && records.keys != 0u; + const ColumnKey key{records.base, records.size, records.records, records.keys, column.stride, column.addend, column.offset, column.dwordCount, column.maxKey, column.foldedAddend, column.foldedCarry, column.address, column.sampler}; + if (memoized && Reuse(key, column, records, false)) return; + if (memoized && !column.address && records.size != 0u && runtime.stampWrites != nullptr) { + const auto span = std::min(records.size, static_cast(column.addend) + column.offset + static_cast(records.keys) * column.stride + column.dwordCount * 4u); + stamp = runtime.stampWrites(runtime.userContext, records.base, span); + } + if (memoized && Reuse(key, column, records, true)) return; + unread(); constexpr auto Unread = std::numeric_limits::max(); std::vector whole(records.keys, Unread); std::vector>> split; @@ -926,15 +1055,12 @@ private: } if (records.states[record] == RecordState::Read) split.emplace_back(record, slots); } - std::vector read(total); - std::uint32_t at = 0; - for (const auto& [address, count] : runs) { - if (runtime.readWords != nullptr) { - if (!runtime.readWords(runtime.userContext, address, read.data() + at, count)) throw std::runtime_error("image table: failed to read guest memory at 0x" + hexText(address)); - } else { - for (std::uint32_t word = 0; word < count; word++) read[at + word] = Read(address + word * 4u); - } - at += count; + std::vector read; + if (!comparedWords.empty() && runs == comparedRuns) { + read = std::move(comparedWords); + } else { + read.resize(total); + ReadRuns(runs, read); } auto next = split.begin(); for (std::uint32_t record = 0; record < records.keys; record++) { @@ -953,7 +1079,7 @@ private: } ++next; } - records.words[record] = WordsIndex(value); + records.words[record] = WordsIndex(records, value); } if (column.address && low < high) tables.ranges.emplace_back(low, high - low); std::sort(records.readRanges.begin(), records.readRanges.end()); @@ -967,7 +1093,116 @@ private: } } records.readRanges.resize(ranges); - return records; + Seal(records); + if (memoized) Remember(key, records, std::move(runs), std::move(read), column.address && low < high ? low : 0u, column.address && low < high ? high : 0u); + } + + void ReadRuns(const std::vector>& runs, std::vector& read) const { + std::size_t at = 0; + for (const auto& [address, count] : runs) { + if (runtime.readWords != nullptr) { + if (!runtime.readWords(runtime.userContext, address, read.data() + at, count)) throw std::runtime_error("image table: failed to read guest memory at 0x" + hexText(address)); + } else { + for (std::uint32_t word = 0; word < count; word++) read[at + word] = Read(address + word * 4u); + } + at += count; + } + tableReuseTotals().wordsRead.fetch_add(read.size(), std::memory_order_relaxed); + } + + bool WholePageAccessible(std::uint64_t address) { + const auto [found, inserted] = wholePages.try_emplace(address & ~(PageBytes - 1u), false); + if (inserted) found->second = Accessible(address & ~(PageBytes - 1u), PageBytes); + return found->second; + } + + bool Unwritten(const ColumnMemo& memo) const { + if (runtime.unchangedSince == nullptr || runtime.supplyWords == nullptr || memo.read->low >= memo.read->high) return false; + const auto vouched = LoadStamp(*memo.read); + if (vouched.generation == 0u || !runtime.unchangedSince(runtime.userContext, memo.read->low, memo.read->high - memo.read->low, vouched)) return false; + return runtime.supplyWords(runtime.userContext, memo.read); + } + + bool Compared(const TableColumn& column, const ColumnRecords& records, ColumnMemo& memo) { + const auto end = records.base + records.size; + for (const auto& [address, count] : memo.read->runs) { + const auto past = address + static_cast(count) * sizeof(std::uint32_t); + for (auto page = address & ~(PageBytes - 1u); page < past; page += PageBytes) { + const auto first = std::max(page, address); + if (column.address ? !WholePageAccessible(first) : !PageAccessible(first, records.base, end)) return false; + } + } + std::vector read(memo.read->words.size()); + ReadRuns(memo.read->runs, read); + if (read != memo.read->words) { + comparedRuns = memo.read->runs; + comparedWords = std::move(read); + return false; + } + StoreStamp(*memo.read, stamp); + return true; + } + + void Apply(const TableColumn& column, ColumnRecords& records, const ColumnMemo& memo) { + records.contents = memo.contents; + records.sealed = true; + if (column.address && memo.low < memo.high) tables.ranges.emplace_back(memo.low, memo.high - memo.low); + } + + bool Reuse(const ColumnKey& key, const TableColumn& column, ColumnRecords& records, bool compare) { + auto& memos = ColumnMemos(); + const auto found = std::find_if(memos.begin(), memos.end(), [&](const ColumnMemo& entry) { return entry.key == key; }); + if (found == memos.end()) return false; + auto& counters = tableCounters(); + auto& totals = tableReuseTotals(); + if (!compare && Unwritten(*found)) { + counters.reusedUnwritten.fetch_add(1, std::memory_order_relaxed); + totals.unwritten.fetch_add(1, std::memory_order_relaxed); + } else if (compare && Compared(column, records, *found)) { + counters.reusedCompared.fetch_add(1, std::memory_order_relaxed); + totals.compared.fetch_add(1, std::memory_order_relaxed); + } else { + return false; + } + Apply(column, records, *found); + totals.shared.fetch_add(1, std::memory_order_relaxed); + std::rotate(memos.begin(), found, found + 1); + return true; + } + + void Remember(const ColumnKey& key, const ColumnRecords& records, std::vector>&& runs, std::vector&& words, std::uint64_t low, std::uint64_t high) { + if (records.contents.poisoned != records.contents.carried) return; + auto& memos = ColumnMemos(); + auto found = std::find_if(memos.begin(), memos.end(), [&](const ColumnMemo& entry) { return entry.key == key; }); + if (found == memos.end()) found = memos.emplace(memos.end()); + auto& memo = *found; + memo.key = key; + std::uint64_t first = std::numeric_limits::max(); + std::uint64_t past = 0; + std::uint64_t total = 0; + bool ordered = !runs.empty(); + for (const auto& [address, count] : runs) { + const auto bytes = static_cast(count) * sizeof(std::uint32_t); + ordered = ordered && count != 0u && address % sizeof(std::uint32_t) == 0u && address >= past && address <= std::numeric_limits::max() - bytes; + first = std::min(first, address); + past = std::max(past, address + bytes); + total += count; + } + ordered = ordered && total == words.size(); + memo.read = std::make_shared(SrtGuestWords{std::move(runs), std::move(words), std::make_shared(), first < past ? first : 0u, first < past ? past : 0u, ordered}); + StoreStamp(*memo.read, stamp); + memo.contents = records.contents; + memo.low = low; + memo.high = high; + if (memo.Footprint() > ColumnMemoWords) { + memos.erase(found); + return; + } + std::rotate(memos.begin(), found, found + 1); + std::size_t kept = 0; + std::size_t footprint = 0; + while (kept < memos.size() && kept < ColumnMemoEntries && footprint + memos[kept].Footprint() <= ColumnMemoWords) footprint += memos[kept++].Footprint(); + memos.resize(kept); } void Snapshot(std::uint32_t source, std::uint32_t table) { @@ -980,32 +1215,15 @@ private: out.keys = records.keys; out.fault = records.fault; out.outside = records.outside; - out.codes.assign(records.keys, 0u); - out.readRanges = records.readRanges; - TableTrace trace{records.base, records.size, records.records, records.keys, 0u, records.narrowed}; + out.codes = records.contents.codes; + out.words = records.contents.words; + out.readRanges = records.contents.readRanges; auto& counters = tableCounters(); if (records.fault != 0u) counters.poison[records.fault & ~ImageTableAbi::PoisonFlag].fetch_add(1, std::memory_order_relaxed); - for (std::uint32_t key = 0; key < records.keys; key++) { - switch (records.states[key]) { - case RecordState::Read: - out.codes[key] = records.words[key]; - continue; - case RecordState::OutsideDomain: - out.codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::OutsideDomain)); - break; - case RecordState::Unmapped: - out.codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::Unmapped)); - counters.poison[static_cast(PoisonReason::Unmapped)].fetch_add(1, std::memory_order_relaxed); - break; - case RecordState::Carried: - out.codes[key] = ImageTableAbi::PoisonCode(static_cast(PoisonReason::OffsetCarry)); - counters.poison[static_cast(PoisonReason::OffsetCarry)].fetch_add(1, std::memory_order_relaxed); - break; - } - trace.poison++; - } + if (records.contents.unmapped != 0u) counters.poison[static_cast(PoisonReason::Unmapped)].fetch_add(records.contents.unmapped, std::memory_order_relaxed); + if (records.contents.carried != 0u) counters.poison[static_cast(PoisonReason::OffsetCarry)].fetch_add(records.contents.carried, std::memory_order_relaxed); counters.keys.fetch_add(records.keys, std::memory_order_relaxed); - if (TableTraced()) traceTable(plan, table, column, trace); + if (TableTraced()) traceTable(plan, table, column, {records.base, records.size, records.records, records.keys, records.contents.poisoned, records.narrowed}); } const IrResourcePlan& plan; @@ -1017,6 +1235,9 @@ private: HashScratchIndex wordsIndex{threadScratch()}; std::unordered_map pages; std::unordered_map wholePages; + SrtWriteStamp stamp; + std::vector> comparedRuns; + std::vector comparedWords; std::vector keyWords; }; @@ -1042,7 +1263,9 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, } snapshot.userData.assign(runtime.userData.begin(), runtime.userData.begin() + plan.userDataCount); - std::vector values; + struct ValuesStorage {}; + auto& values = HostThreadLocal, ValuesStorage>(); + values.clear(); const auto evaluateStarted = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources, &snapshot.srtPoison, &snapshot.nullRootReads); if (MaterializeProfiled()) evaluateNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - evaluateStarted).count()), std::memory_order_relaxed); @@ -1332,6 +1555,59 @@ std::uint32_t decodeTableEntry(const ImageResource& image, std::span reducingSamplers(const ShaderInfo& info, const ResourceSnapshot& snapshot) { + std::vector reducing(info.images.size(), false); + for (const auto& pair : info.sampledPairs) { + if (info.samplers.at(pair.sampler).table != NoTable) continue; + const auto& words = snapshot.samplers.at(pair.sampler).dwords; + if (reducesBetweenTexels(words[0], words[2])) reducing.at(pair.image) = true; + } + return reducing; +} + +bool TableResolutionReuse() { + static const bool enabled = std::getenv("APS5_NO_TABLE_RESOLUTION_REUSE") == nullptr; + return enabled; +} + +struct ResolutionMemoEntry { + std::uint64_t memoKey = 0; + std::vector tables; + std::vector reducing; + std::shared_ptr resolved; + std::size_t footprint = 0; +}; + +struct ResolutionMemo { + std::mutex mutex; + std::vector entries; +}; + +ResolutionMemo& resolutionMemo() { + static ResolutionMemo memo; + return memo; +} + +constexpr std::size_t ResolutionMemoEntries = 256; +constexpr std::size_t ResolutionMemoVariantEntries = 4; +constexpr std::size_t ResolutionMemoWords = std::size_t{1} << 23u; + +bool sameColumns(const std::vector& left, const std::vector& right) { + if (left.size() != right.size()) return false; + for (std::size_t index = 0; index < left.size(); index++) { + const auto& a = left[index]; + const auto& b = right[index]; + if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || !a.codes.Shares(b.codes) || !a.words.Shares(b.words)) return false; + } + return true; +} + +std::size_t resolutionFootprint(const ResolvedImageTables& resolved, const std::vector& tables) { + std::size_t footprint = resolved.map.size() + (resolved.elements.size() * sizeof(ImageTableElement) + resolved.samplers.size() * sizeof(ImageTableSamplerElement) + resolved.poison.size() * sizeof(ImageTablePoisonRecord)) / sizeof(std::uint32_t); + for (const auto& table : tables) footprint += 16u + table.codes.size() + table.words.size() * 9u; + return footprint; +} + } ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey) { @@ -1376,14 +1652,10 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i }; HashScratchIndex elementIndex{threadScratch()}; std::vector directSamplers(info.images.size(), 0u); - std::vector reducingSampler(info.images.size(), false); for (const auto& pair : info.sampledPairs) { - const auto& sampler = info.samplers.at(pair.sampler); - if (sampler.table != NoTable) continue; - directSamplers.at(pair.image) |= 1u << pair.sampler; - const auto& words = snapshot.samplers.at(pair.sampler).dwords; - if (reducesBetweenTexels(words[0], words[2])) reducingSampler.at(pair.image) = true; + if (info.samplers.at(pair.sampler).table == NoTable) directSamplers.at(pair.image) |= 1u << pair.sampler; } + const auto reducingSampler = reducingSamplers(info, snapshot); const auto imageRecord = [&](std::uint32_t resource, const DescriptorValue& words) { const auto& image = info.images[resource]; const auto& modes = info.runtimeImageModes[resource]; @@ -1456,7 +1728,9 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i return offset; }; std::vector outside(count, false); - DenseScratchMap records(threadScratch(), tables.words.size()); + std::size_t recordCodes = 0; + for (const auto& column : tables.tables) recordCodes = std::max(recordCodes, column.words.size()); + DenseScratchMap records(threadScratch(), recordCodes); const auto fill = [&](std::uint32_t resource, std::uint32_t table, bool sampler) { const auto& column = tables.tables.at(table); const auto header = Abi::TableHeader(table); @@ -1486,7 +1760,7 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i if (const auto known = records.Find(code)) { mapped = *known; } else { - const auto& words = tables.words[code]; + const auto& words = column.words.at(code); if (sampler || (words.availableDwordMask & (1u << 3u)) == 0u || !nullImageDescriptor(info.images[resource], words)) mapped = sampler ? samplerRecord(resource, words) : imageRecord(resource, words); records.Set(code, mapped); } @@ -1517,6 +1791,42 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i return out; } +std::shared_ptr ResourceMaterializer::ResolveImageTablesShared(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey) { + if (memoKey == 0u || snapshot.tables.tables.empty() || !TableResolutionReuse()) return std::make_shared(ResolveImageTables(info, snapshot, memoKey)); + auto reducing = reducingSamplers(info, snapshot); + auto& memo = resolutionMemo(); + auto& totals = tableReuseTotals(); + { + std::lock_guard lock(memo.mutex); + const auto found = std::find_if(memo.entries.begin(), memo.entries.end(), [&](const ResolutionMemoEntry& entry) { return entry.memoKey == memoKey && entry.reducing == reducing && sameColumns(entry.tables, snapshot.tables.tables); }); + if (found != memo.entries.end()) { + std::rotate(memo.entries.begin(), found, found + 1); + totals.resolutionsShared.fetch_add(1, std::memory_order_relaxed); + return memo.entries.front().resolved; + } + } + auto resolved = std::make_shared(ResolveImageTables(info, snapshot, memoKey)); + totals.resolutions.fetch_add(1, std::memory_order_relaxed); + const auto footprint = resolutionFootprint(*resolved, snapshot.tables.tables); + if (footprint > ResolutionMemoWords) return resolved; + std::lock_guard lock(memo.mutex); + memo.entries.insert(memo.entries.begin(), ResolutionMemoEntry{memoKey, snapshot.tables.tables, std::move(reducing), resolved, footprint}); + std::size_t kept = 0; + std::size_t variantEntries = 0; + std::size_t total = 0; + for (std::size_t index = 0; index < memo.entries.size(); index++) { + auto& entry = memo.entries[index]; + const bool sameVariant = entry.memoKey == memoKey; + if ((sameVariant && variantEntries == ResolutionMemoVariantEntries) || kept == ResolutionMemoEntries || total + entry.footprint > ResolutionMemoWords) continue; + if (sameVariant) variantEntries++; + total += entry.footprint; + if (kept != index) memo.entries[kept] = std::move(entry); + kept++; + } + memo.entries.resize(kept); + return resolved; +} + std::vector ResourceMaterializer::RuntimeImageModes(const ImageResource& image) { const bool table = image.table != NoTable; if (table && image.tableOperation == TableOperation::Unsupported) return {}; @@ -2018,7 +2328,9 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR const auto materializeStarted = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; SrtWalker walker; ResourceSnapshot nextSnapshot; - std::vector activeSources; + struct ActiveSourcesStorage {}; + auto& activeSources = HostThreadLocal, ActiveSourcesStorage>(); + activeSources.clear(); try { materializeSnapshot(plan, runtime, walker, nextSnapshot, activeSources); TableSnapshotter(plan, runtime, walker, activeSources, nextSnapshot.tables).Run(); @@ -2040,6 +2352,11 @@ std::uint64_t ResourceMaterializer::SpecializationNanoseconds() { return specializationNanoseconds.load(std::memory_order_relaxed); } +ImageTableReuseCounts ResourceMaterializer::TableReuse() { + auto& totals = tableReuseTotals(); + return {totals.unwritten.load(std::memory_order_relaxed), totals.compared.load(std::memory_order_relaxed), totals.wordsRead.load(std::memory_order_relaxed), totals.shared.load(std::memory_order_relaxed), totals.columns.load(std::memory_order_relaxed), totals.resolutions.load(std::memory_order_relaxed), totals.resolutionsShared.load(std::memory_order_relaxed)}; +} + std::uint64_t ResourceMaterializer::EvaluateNanoseconds() { return evaluateNanoseconds.load(std::memory_order_relaxed); } diff --git a/core/shader/recompiler/PatchSpirvTools.cmake b/core/shader/recompiler/PatchSpirvTools.cmake index 69cb4bcfc7..4acc117b3b 100644 --- a/core/shader/recompiler/PatchSpirvTools.cmake +++ b/core/shader/recompiler/PatchSpirvTools.cmake @@ -120,3 +120,49 @@ if(NOT spirvToolsReplacements EQUAL 1) message(FATAL_ERROR "SPIRV-Tools optimizer sources have changed; review the local memory fix") endif() set_property(TARGET SPIRV-Tools-opt PROPERTY SOURCES "${spirvToolsPatchedSources}") + +get_target_property(spirvToolsValidatorSourceDir SPIRV-Tools-static SOURCE_DIR) +set(spirvToolsValidatorSource "${spirvToolsValidatorSourceDir}/val/validate_id.cpp") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${spirvToolsValidatorSource}") +file(READ "${spirvToolsValidatorSource}" spirvToolsValidator) +string(FIND "${spirvToolsValidator}" "#include " spirvToolsValidatorInclude) +if(spirvToolsValidatorInclude EQUAL -1) + message(FATAL_ERROR "SPIRV-Tools validator includes have changed; review the local lifetime fix") +endif() +string(REPLACE "#include " "#include \n#include " spirvToolsValidator "${spirvToolsValidator}") +foreach(table IN ITEMS allow deny) + set(oldDeclaration "static std::unordered_set instruction_${table}_set{") + set(newDeclaration "static constexpr spv::Op instruction_${table}_set[]{") + set(oldLookup "instruction_${table}_set.count(opcode)") + set(newLookup "(std::find(std::begin(instruction_${table}_set),\n std::end(instruction_${table}_set), opcode) !=\n std::end(instruction_${table}_set))") + string(FIND "${spirvToolsValidator}" "${oldDeclaration}" declarationPosition) + string(FIND "${spirvToolsValidator}" "${oldLookup}" lookupPosition) + if(declarationPosition EQUAL -1 OR lookupPosition EQUAL -1) + message(FATAL_ERROR "SPIRV-Tools validator type-operand tables have changed; review the local lifetime fix") + endif() + string(REPLACE "${oldDeclaration}" "${newDeclaration}" spirvToolsValidator "${spirvToolsValidator}") + string(REPLACE "${oldLookup}" "${newLookup}" spirvToolsValidator "${spirvToolsValidator}") +endforeach() +set(spirvToolsPatchedValidator "${spirvToolsPatchedDir}/validate_id.cpp") +file(CONFIGURE OUTPUT "${spirvToolsPatchedValidator}" CONTENT "${spirvToolsValidator}" @ONLY) +foreach(validatorTarget IN ITEMS SPIRV-Tools-static SPIRV-Tools-shared) + if(NOT TARGET ${validatorTarget}) + continue() + endif() + get_target_property(validatorSources ${validatorTarget} SOURCES) + set(patchedValidatorSources) + set(validatorReplacements 0) + foreach(source IN LISTS validatorSources) + get_filename_component(sourcePath "${source}" ABSOLUTE BASE_DIR "${spirvToolsValidatorSourceDir}") + if(sourcePath STREQUAL spirvToolsValidatorSource) + list(APPEND patchedValidatorSources "${spirvToolsPatchedValidator}") + math(EXPR validatorReplacements "${validatorReplacements} + 1") + else() + list(APPEND patchedValidatorSources "${source}") + endif() + endforeach() + if(NOT validatorReplacements EQUAL 1) + message(FATAL_ERROR "SPIRV-Tools validator sources have changed; review the local lifetime fix") + endif() + set_property(TARGET ${validatorTarget} PROPERTY SOURCES "${patchedValidatorSources}") +endforeach() diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index e47d9879be..35059904f2 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -30,6 +30,8 @@ inline constexpr std::uint32_t RasterPrimitive = 8233u; inline constexpr std::uint32_t LayerExported = 8234u; inline constexpr std::uint32_t StencilReferenceOffset = 8235u; inline constexpr std::uint32_t ReversedBlend = 8236u; +inline constexpr std::uint32_t DepthRemapBase = 8240u; +inline constexpr std::uint32_t DepthRemapWords = 1u; inline constexpr std::uint32_t HeapCountBase = 8256u; inline constexpr std::uint32_t MipCountBase = 8320u; inline constexpr std::uint32_t DescriptorIndexBase = 16384u; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index ebac71147c..bfdeb070e2 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -557,6 +557,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) targetOptions.nonConstantImageOffsets = request.target.nonConstantImageOffsets; targetOptions.narrowSubgroupClock = request.target.narrowSubgroupClock; targetOptions.float64DenormPreserve = request.target.float64DenormPreserve; + targetOptions.depthRangeRemap = request.target.depthRangeRemap; targetOptions.registrationModules = registration; targetOptions.fragmentDescriptorSet = request.target.registrationModules; targetOptions.float16DenormPreserve = request.target.float16DenormPreserve; @@ -935,6 +936,13 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil if (variant.info.info.rasterPrimitive) moduleKey.push_back(static_cast(request.context.pixel->rasterPrimitive)); if (variant.info.info.layerExported) moduleKey.push_back(request.context.pixel->layerExported ? 1u : 0u); } + const bool depthRemap = request.target.depthRangeRemap && request.shader.stage == ShaderStage::Vertex && request.context.vertex.has_value(); + if (depthRemap) { + if (request.context.vertex->depthRemapMode != 0u && request.layout.pushConstantOffsetBytes % (RuntimeAbi::PushConstantDwords * 4u) + request.layout.pushConstantSizeBytes > DepthRemapPushOffsetBytes) { + throw std::runtime_error("a remapped vertex shader's push data must end before the depth remap words"); + } + moduleKey.push_back(request.context.vertex->depthRemapMode); + } result.vertexAttributes.reserve(result.vertexInputs.size()); std::array vertexClasses{}; std::array runtimeSelectors{}; @@ -963,7 +971,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil if (artifact.registrationModule) runtimeSelectors.at(input.location / RuntimeAbi::VertexSelectorsPerDword) |= (selectors & 0xfffu) << (input.location % RuntimeAbi::VertexSelectorsPerDword * 16u); } } - if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty() || request.context.pixel) { + if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty() || request.context.pixel || depthRemap) { { std::shared_lock lock(plan->mutex); if (const auto found = plan->modules.find(moduleKey); found != plan->modules.end()) entry = found->second; @@ -985,6 +993,9 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil if (variant.info.info.rasterPrimitive) constants.push_back({PipelineSpecialization::RasterPrimitive, moduleKey[index++]}); if (variant.info.info.layerExported) constants.push_back({PipelineSpecialization::LayerExported, moduleKey[index++]}); } + if (depthRemap) { + for (std::uint32_t word = 0; word < PipelineSpecialization::DepthRemapWords; ++word) constants.push_back({PipelineSpecialization::DepthRemapBase + word, moduleKey[index++]}); + } if (!artifact.vertexInputPatches.empty()) { for (const auto& input : result.vertexInputs) { const auto selectors = moduleKey[index++]; @@ -1016,12 +1027,14 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil } else { result.spirv = artifact.spirv; } - const auto tables = ResourceMaterializer::ResolveImageTables(variant.info.info, snapshot, artifact.variantId); + const auto resolved = ResourceMaterializer::ResolveImageTablesShared(variant.info.info, snapshot, artifact.variantId); + const auto& tables = *resolved; BindingAllocationResult bindings; DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request), &tables); result.workgroupMemoryDwords = WorkgroupMemoryStrideDwords(variant.info.info); result.bindings = std::move(bindings.bindings); result.pushConstants = std::move(bindings.pushConstants); + result.readsDepthRemapPush = depthRemap; if (!artifact.stencilReferences.empty()) { if (artifact.stencilReferenceDword * sizeof(std::uint32_t) != result.pushConstants.size() || result.pushConstants.size() + sizeof(std::uint32_t) > request.layout.pushConstantSizeBytes) throw std::runtime_error("stencil replay push constant lies outside the stage range"); result.pushConstants.resize(result.pushConstants.size() + sizeof(std::uint32_t)); @@ -1046,7 +1059,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil for (const auto& poison : tables.poison) result.imageTablePoison.push_back({poison.words, poison.dwordCount, poison.resource, static_cast(poison.reason)}); result.imageTableRanges = snapshot.tables.ranges; result.imageTableReadRanges.reserve(snapshot.tables.tables.size()); - for (const auto& table : snapshot.tables.tables) result.imageTableReadRanges.push_back(table.readRanges); + for (const auto& table : snapshot.tables.tables) result.imageTableReadRanges.push_back(table.readRanges.Copy()); result.imageTableShader = snapshot.tables.shader; result.imageTableFaults = tables.faults; } @@ -1246,14 +1259,10 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(table.keys); mix(table.fault); mix(table.outside ? 1u : 0u); - mixWords(table.codes); - mix(table.readRanges.size()); - for (const auto& [base, size] : table.readRanges) { - mix(base); - mix(size); - } + mix(table.codes.Hash()); + mix(table.words.Hash()); + mix(table.readRanges.Hash()); } - mixDescriptors(tables.words); mix(tables.ranges.size()); for (const auto& [base, size] : tables.ranges) { mix(base); diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 6a41b81710..e3348d8d71 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -212,6 +212,7 @@ struct ShaderVertexStageInfo { std::uint32_t fetchBufferReg; bool fetchEmbedded; std::uint32_t paClVsOutCntl = 0; + std::uint32_t depthRemapMode = 0; }; struct ShaderFloatMode { @@ -287,6 +288,7 @@ struct SpirvTarget { bool float16DenormPreserve = false; bool spannedWaves = false; bool robustBufferAccess2 = false; + bool depthRangeRemap = false; }; struct BindingLayout { @@ -343,6 +345,8 @@ inline constexpr std::uint32_t MeshDrawPushOffsetBytes = 104; inline constexpr std::uint32_t MeshDrawPushBytes = 24; inline constexpr std::uint32_t TessellationLevelPushOffsetBytes = 120; inline constexpr std::uint32_t TessellationLevelPushBytes = 8; +inline constexpr std::uint32_t DepthRemapPushOffsetBytes = 104; +inline constexpr std::uint32_t DepthRemapPushBytes = 8; inline constexpr std::uint32_t MeshArgumentAddressDword = 4; inline constexpr std::uint32_t MeshArgumentIndexCountDword = 3; inline constexpr std::uint32_t MeshArgumentFirstIndexDword = 4; @@ -451,6 +455,7 @@ struct DescriptorBinding { std::vector imageUnnormalizedBaseLevel; std::vector imageSamplers; std::vector bufferRead; + std::vector bufferExtent; }; struct ImageTableEntryPoison { @@ -641,6 +646,7 @@ struct ShaderInvocation { std::uint64_t specializationId = 0; std::vector bindings; std::vector pushConstants; + bool readsDepthRemapPush = false; std::vector vertexAttributes; std::uint32_t poisonedSrtReads = 0; std::vector imageTablePoison; diff --git a/core/shader/recompiler/ShaderCacheVersion.cmake b/core/shader/recompiler/ShaderCacheVersion.cmake index 81be546106..847a513496 100644 --- a/core/shader/recompiler/ShaderCacheVersion.cmake +++ b/core/shader/recompiler/ShaderCacheVersion.cmake @@ -26,7 +26,6 @@ endforeach() list(SORT sources) list(SORT guard) get_filename_component(rootReal "${ROOT}" REALPATH) -get_filename_component(recompilerReal "${RECOMPILER_DIR}" REALPATH) set(sourcesReal) foreach(source IN LISTS sources) get_filename_component(real "${source}" REALPATH) @@ -70,9 +69,8 @@ foreach(source IN LISTS sources) if(NOT resolved) continue() endif() - string(FIND "${resolved}" "${recompilerReal}/" resolvedInRecompiler) string(FIND "${resolved}" "${rootReal}/3rdparty/" resolvedInThirdParty) - if(resolvedInRecompiler EQUAL 0 OR resolvedInThirdParty EQUAL 0 OR resolved IN_LIST sourcesReal) + if(resolvedInThirdParty EQUAL 0 OR resolved IN_LIST sourcesReal) continue() endif() file(RELATIVE_PATH relativeResolved "${rootReal}" "${resolved}") diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index c51d4f63ec..f830045b52 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -85,9 +85,9 @@ namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) static_assert(sizeof(CompiledShaderArtifact) == 240, "CompiledShaderArtifact changed: update the artifact encoder"); -static_assert(sizeof(ShaderInvocation) == 200, "ShaderInvocation changed: update the invocation encoder"); -static_assert(sizeof(RecompileResult) == 472, "RecompileResult changed: update EncodeResult and DecodeResult"); -static_assert(sizeof(DescriptorBinding) == 632, "DescriptorBinding changed: update the binding encoder"); +static_assert(sizeof(ShaderInvocation) == 208, "ShaderInvocation changed: update the invocation encoder"); +static_assert(sizeof(RecompileResult) == 480, "RecompileResult changed: update EncodeResult and DecodeResult"); +static_assert(sizeof(DescriptorBinding) == 656, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 16, "FragmentParameter changed: update the parameter encoder"); @@ -430,6 +430,7 @@ void encodeBinding(Writer& writer, const DescriptorBinding& binding) { writer.Flags(binding.samplerCoordinatesIgnored); writer.Values(std::span(binding.imageSamplers)); writer.Flags(binding.bufferRead); + writer.Values(std::span(binding.bufferExtent)); } void decodeBinding(Reader& reader, DescriptorBinding& binding) { @@ -458,6 +459,7 @@ void decodeBinding(Reader& reader, DescriptorBinding& binding) { reader.Flags(binding.samplerCoordinatesIgnored); reader.Values(binding.imageSamplers); reader.Flags(binding.bufferRead); + reader.Values(binding.bufferExtent); } void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { @@ -563,6 +565,7 @@ void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { writer.List(invocation.bindings, encodeBinding); writer.Value(invocation.pushConstants.size()); for (const auto byte : invocation.pushConstants) writer.Value(static_cast(byte)); + writer.Value(invocation.readsDepthRemapPush); writer.List(invocation.vertexAttributes, [](Writer& out, const VertexAttribute& attribute) { out.Value(attribute.location); out.Value(attribute.components); @@ -602,6 +605,7 @@ void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { reader.Values(pushConstants); invocation.pushConstants.resize(pushConstants.size()); if (!pushConstants.empty()) std::memcpy(invocation.pushConstants.data(), pushConstants.data(), pushConstants.size()); + reader.Value(invocation.readsDepthRemapPush); reader.List(invocation.vertexAttributes, 32, [](Reader& in, VertexAttribute& attribute) { in.Value(attribute.location); in.Value(attribute.components); @@ -1276,6 +1280,8 @@ constexpr std::string_view NeutralSwitches[] = { "APS5_NO_PERF_ITERATION_SELECTION", "APS5_NO_PERF_LAYOUT_PAIR", "APS5_NO_RESULT_MEMO", + "APS5_NO_TABLE_RESOLUTION_REUSE", + "APS5_NO_TABLE_SNAPSHOT_REUSE", }; const std::vector& switchKey() { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index 7f7ccbdec5..385638dd21 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -22,6 +22,7 @@ struct SpirvTargetOptions { bool nonConstantImageOffsets = false; bool narrowSubgroupClock = false; bool float64DenormPreserve = false; + bool depthRangeRemap = false; bool registrationModules = false; bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index ad302bb257..0eb2d84225 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -105,6 +105,8 @@ struct SpirvEmitterState { std::uint32_t spannedExchangeVariable = 0; bool narrowSubgroupClock = false; bool float64DenormPreserve = false; + bool depthRangeRemap = false; + std::uint32_t depthRemapClipIndex = 0; bool registrationModules = false; bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 050316a6f7..b946eccbcf 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -257,6 +257,7 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.nonConstantImageOffsets = target.nonConstantImageOffsets; state.narrowSubgroupClock = target.narrowSubgroupClock; state.float64DenormPreserve = target.float64DenormPreserve; + state.depthRangeRemap = target.depthRangeRemap; state.registrationModules = target.registrationModules; state.fragmentDescriptorSet = target.fragmentDescriptorSet; state.float16DenormPreserve = target.float16DenormPreserve; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index ad0ada9b66..89d3b5dfd3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -327,6 +327,11 @@ void DefineOutputs(SpirvEmitterState& state) { DefineMeshOutputs(state); return; } + const bool depthRemap = state.depthRangeRemap && state.program.Resources().stage == IrShaderStage::Vertex; + if (depthRemap) { + state.depthRemapClipIndex = clipDistanceCount; + clipDistanceCount += 2u; + } const auto stage = state.program.Resources().stage; const bool exportsPosition = std::ranges::any_of(state.outputs, [](const SpirvOutputBinding& output) { return output.kind == StageOutputKind::Position; }); if (exportsPosition && (stage == IrShaderStage::Vertex || stage == IrShaderStage::TessellationEvaluation) && clipDistanceCount + cullDistanceCount < MaxClipDistances) { @@ -343,6 +348,7 @@ void DefineOutputs(SpirvEmitterState& state) { } return variable; }; + if (depthRemap) BuiltIn(state.clipDistanceVariable, F32ArrayType(state, clipDistanceCount), "gl_ClipDistance", spv::BuiltInClipDistance); for (auto& binding : state.outputs) { switch (binding.kind) { case StageOutputKind::Position: @@ -413,7 +419,7 @@ void DefineOutputs(SpirvEmitterState& state) { void DefineDescriptors(SpirvEmitterState& state) { const IrBindingLayout& layout = state.program.Metadata().bindings; const IrShaderStage stage = state.program.Resources().stage; - if (layout.UsesPushData() || state.stencilReplay || stage == IrShaderStage::Mesh || stage == IrShaderStage::TessellationControl) { + if (layout.UsesPushData() || state.stencilReplay || stage == IrShaderStage::Mesh || stage == IrShaderStage::TessellationControl || (state.depthRangeRemap && stage == IrShaderStage::Vertex)) { const auto type = PushConstantBlockType(state); state.pushConstantVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassPushConstant, type), spv::StorageClassPushConstant); state.module.AddName(type, "BufferResource"); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 6bec6505ba..e9351a46f4 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -597,6 +597,59 @@ std::uint32_t ConvertClipCoordinate(SpirvEmitterState& state, std::uint32_t coor return ndc; } +std::uint32_t RemapPositionDepth(SpirvEmitterState& state, std::uint32_t position) { + const auto& remap = state.inputInfo.vertex->depthRemap; + const auto f32 = TypeF32(state); + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + std::array components{}; + for (std::uint32_t i = 0; i < 4u; i++) { + components.at(i) = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, f32, components.at(i), position, i); + } + const auto mode = state.module.SpecializationConstant(u32, PipelineSpecialization::DepthRemapBase, remap.mode); + if (state.pushConstantVariable == 0u) throw std::runtime_error("depth remap push constants were not declared"); + const auto pushFloat = [&](std::uint32_t dword) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0u), ConstantU32(state, DepthRemapPushOffsetBytes / 4u + dword)); + const auto bits = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, bits, pointer); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpBitcast, f32, value, bits); + return value; + }; + const auto scale = pushFloat(0u); + const auto offset = pushFloat(1u); + const auto scaled = state.module.AllocateId(); + state.module.AddFunction(spv::OpFMul, f32, scaled, components[2], scale); + const auto shifted = state.module.AllocateId(); + state.module.AddFunction(spv::OpFMul, f32, shifted, components[3], offset); + const auto remapped = state.module.AllocateId(); + state.module.AddFunction(spv::OpFAdd, f32, remapped, scaled, shifted); + const auto active = state.module.AllocateId(); + state.module.AddFunction(spv::OpINotEqual, boolean, active, mode, ConstantU32(state, 0u)); + const auto z = Select(state, f32, active, remapped, components[2]); + const auto symmetric = state.module.AllocateId(); + state.module.AddFunction(spv::OpIEqual, boolean, symmetric, mode, ConstantU32(state, 2u)); + const auto nearSymmetric = state.module.AllocateId(); + state.module.AddFunction(spv::OpFAdd, f32, nearSymmetric, components[3], components[2]); + const auto near = Select(state, f32, symmetric, nearSymmetric, components[2]); + const auto far = state.module.AllocateId(); + state.module.AddFunction(spv::OpFSub, f32, far, components[3], components[2]); + const auto inside = ConstantF32Value(state, 1.0f); + const auto storeDistance = [&](std::uint32_t index, std::uint32_t distance) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassOutput, f32), pointer, state.clipDistanceVariable, ConstantU32(state, index)); + state.module.AddFunction(spv::OpStore, pointer, Select(state, f32, active, distance, inside)); + }; + if (state.clipDistanceVariable == 0u) throw std::runtime_error("depth remap clip distances were not declared"); + storeDistance(state.depthRemapClipIndex, near); + storeDistance(state.depthRemapClipIndex + 1u, far); + const auto converted = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4u), converted, components[0], components[1], z, components[3]); + return converted; +} + std::uint32_t ConvertPositionToClipSpace(SpirvEmitterState& state, std::uint32_t position) { if (state.inputInfo.vertex == nullptr) { throw std::runtime_error("vertex input info is missing for clip-space conversion"); @@ -1628,6 +1681,9 @@ void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { if (state.inputInfo.vertex->clipSpace.enabled) { value = ConvertPositionToClipSpace(state, value); } + if (state.depthRangeRemap && state.program.Resources().stage == IrShaderStage::Vertex) { + value = RemapPositionDepth(state, value); + } } if (exp.kind == ExportTargetKind::Position && state.zeroPositionClipIndex) { StoreZeroPositionClipDistance(state, value); diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index f9109f7a2a..c4748189a2 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -175,6 +175,7 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues vertexStorage.hostSubgroupSize = hostSubgroupSize; vertexStorage.fetchEmbedded = vertex.fetchEmbedded; vertexStorage.paClVsOutCntl = vertex.paClVsOutCntl; + vertexStorage.depthRemap = {vertex.depthRemapMode}; vertexStorage.fetchExternal = false; vertexStorage.fetchAttribReg = static_cast(vertex.fetchAttribReg); vertexStorage.fetchBufferReg = static_cast(vertex.fetchBufferReg); diff --git a/core/shader/recompiler/tests/RequestSerializerTests.cpp b/core/shader/recompiler/tests/RequestSerializerTests.cpp index b3319787f8..bd57cbedf3 100644 --- a/core/shader/recompiler/tests/RequestSerializerTests.cpp +++ b/core/shader/recompiler/tests/RequestSerializerTests.cpp @@ -207,11 +207,11 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 35u; + upgraded[4] = 36u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 258u, {0u, 0u, 0u}); if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(upgraded); if (fixture.tessellation) upgraded.insert(upgraded.end() - 35u, 5u, 0u); - upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u, 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u + 5u, 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); auto truncated = bytes; truncated.pop_back(); @@ -246,11 +246,11 @@ int main() { layerRejected = std::string_view(error.what()) == "RequestSerializer: truncated data"; } Require(layerRejected); - layered[4] = 35u; + layered[4] = 36u; layered.insert(layered.begin() + 259u, {0u, 0u}); InsertPixelDefaultsAfterFrontFace(layered); if (fixture.tessellation) layered.insert(layered.end() - 35u, 5u, 0u); - layered.insert(layered.end(), sizeof(std::uint64_t) + 1u, 0u); + layered.insert(layered.end(), sizeof(std::uint64_t) + 1u + 5u, 0u); Require(serializer.Serialize(layerReplay.request) == EncodeBase64(layered)); } } @@ -271,10 +271,10 @@ int main() { Require(replay.request.context.pixel->rasterPrimitive == ShaderRecompiler::RasterPrimitive::Rectangle); Require(replay.request.context.vertex->paClVsOutCntl == 0x00050a05u); Require(replay.request.target.float16DenormPreserve == preserve); - bytes[4] = 35u; + bytes[4] = 36u; bytes.insert(bytes.begin() + 259u, {0u, 0u}); InsertPixelDefaultsAfterFrontFace(bytes); - bytes.insert(bytes.end(), sizeof(std::uint64_t) + 1u, 0u); + bytes.insert(bytes.end(), sizeof(std::uint64_t) + 1u + 5u, 0u); Require(serializer.Serialize(replay.request) == EncodeBase64(bytes)); auto truncated23 = DecodeBase64(encoded23); @@ -395,10 +395,10 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 35u; + upgraded[4] = 36u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 259u, {0u, 0u}); if (fixture.pixel) InsertPixelDefaultsAfterFrontFace(upgraded); - upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u, 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t) + 1u + 5u, 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); for (const bool dither : {false, true}) { if (!fixture.pixel && dither) continue; @@ -408,12 +408,12 @@ int main() { const auto replay25 = serializer.Deserialize(EncodeBase64(main25)); Require(!fixture.pixel || (replay25.request.context.pixel->layerExported == layer && replay25.request.context.pixel->ditheredAlphaToCoverage == dither && replay25.request.context.pixel->reversedBlend == 0u)); Require(replay25.request.target.float16DenormPreserve == preserve && (!fixture.tessellation || replay25.request.graphics->tessellation->primitiveGroupSize == 37u)); - main25[4] = 35u; + main25[4] = 36u; if (fixture.pixel) { main25.insert(main25.begin() + 260u, 0u); InsertPixelDefaultsAfterFrontFace(main25); } - main25.insert(main25.end(), sizeof(std::uint64_t) + 1u, 0u); + main25.insert(main25.end(), sizeof(std::uint64_t) + 1u + 5u, 0u); Require(serializer.Serialize(replay25.request) == EncodeBase64(main25)); } } @@ -427,9 +427,9 @@ int main() { const auto replay = serializer.Deserialize(encoded); Require(replay.request.target.float16DenormPreserve == preserve); auto bytes = DecodeBase64(encoded); - Require(bytes.size() >= 8u && bytes[4] == 35u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); + Require(bytes.size() >= 8u && bytes[4] == 36u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); auto future = bytes; - future[4] = 36u; + future[4] = 37u; bool futureRejected = false; try { static_cast(serializer.Deserialize(EncodeBase64(future))); diff --git a/core/shader/recompiler/tests/SpirvValidatorExit.cpp b/core/shader/recompiler/tests/SpirvValidatorExit.cpp new file mode 100644 index 0000000000..2bab5af5c9 --- /dev/null +++ b/core/shader/recompiler/tests/SpirvValidatorExit.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::array, 3> modules; + +bool Validate(const std::vector& words) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + return tools.Validate(words); +} + +bool Check(const char* phase) { + const bool function = Validate(modules[0]); + const bool phi = Validate(modules[1]); + const bool invalidRejected = !Validate(modules[2]); + std::fprintf(stderr, "%s: function=%d phi=%d invalidRejected=%d\n", phase, function, phi, invalidRejected); + return function && phi && invalidRejected; +} + +void ValidateAtExit() { + if (!Check("exit validation")) std::_Exit(EXIT_FAILURE); +} + +} + +int main() { + if (std::atexit(ValidateAtExit) != 0) return EXIT_FAILURE; + const std::string prefix = R"(OpCapability Shader +OpMemoryModel Logical GLSL450 +OpEntryPoint GLCompute %main "main" +OpExecutionMode %main LocalSize 1 1 1 +%void = OpTypeVoid +%fn = OpTypeFunction %void +%bool = OpTypeBool +%true = OpConstantTrue %bool +%uint = OpTypeInt 32 0 +%one = OpConstant %uint 1 +%two = OpConstant %uint 2 +)"; + const std::string function = "%main = OpFunction %void None %fn\n%entry = OpLabel\n"; + const std::string tail = "OpReturn\nOpFunctionEnd\n"; + const std::string phi = R"(OpSelectionMerge %merge None +OpBranchConditional %true %left %right +%left = OpLabel +OpBranch %merge +%right = OpLabel +OpBranch %merge +%merge = OpLabel +%value = OpPhi %uint %one %left %two %right +)"; + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + if (!tools.Assemble(prefix + function + tail, &modules[0]) || + !tools.Assemble(prefix + function + phi + tail, &modules[1]) || + !tools.Assemble(prefix + "%main = OpFunction %void None %void\n%entry = OpLabel\n" + tail, &modules[2])) return EXIT_FAILURE; + return Check("warm validation") ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 8c88a9ed06..733cb672ed 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -123,6 +123,7 @@ flowchart LR - `APS5_LAZY_SHADER_PREPARE=1` retains registration validation and decoding but discards the temporary plan without queueing preparation. The snapshot is deferred, not pending; draws and dispatches prepare and cache it on first use. Explicit ABI resolution still prepares its requested stages. This opt-in mode takes precedence over synchronous registration preparation and trades background CPU/memory use for possible first-use stalls; disk caches and shader output are unchanged. - On devices with graphics pipeline libraries (`SpirvTarget::registrationModules`), a vertex or pixel artifact is emitted in registration form: the buffer V# words, image modes, swizzles and mip ranges, depth-compare state, export mappings and vertex component selectors it used to specialize on per draw are read at run time (image `ResourceMetadata` and a few state dwords in the shader data), descriptor heaps keep every element in place (an unused one gets a null or placeholder descriptor), and the pixel stage's bindings live in descriptor set 1 so each stage's library depends only on its own set layout (`INDEPENDENT_SETS`). An artifact above `registrationWordLimit` SPIR-V words is re-emitted specialized, and `CompiledShaderArtifact::registrationModule` records which form it has. For artifacts in registration form, `RegistrationResult` materializes the module once, and `PrebuildStageLibrary` builds its pre-rasterization or fragment-shader library on a separate one-thread pool (`APS5_PREBUILD_THREADS`) with a fixed default state, pairing it with a stub stage. A later draw with that state finds both libraries in the store and only links. - A draw or dispatch whose state matches no artifact prepared at registration (a title that builds its static ABI state itself, without `sceAgcDriverResolve*Abi`) gets one prepared at use by `SourceHandleFor` or `InvocationFor`, logged once per artifact and kept in the snapshot for the next use. +- A stage that `sceAgcDriverResolve*Abi` cannot prepare for the state the title creates its pipeline with (a compute shader that calls user-data pointers, a descriptor the resource plan rejects) is left to its first use like a stage deferred at registration: `ResolveShaderAbi` and `PrepareGraphicsStages` catch the failure inside their own preparation transaction, so the `ShaderPreparationScope` the title's call runs in still commits, log it once per shader, mark the shader deferred and record the ABI key, so a repeated call prepares nothing; `ResolvePreparedGraphics` skips a deferred stage's rectangles until a draw has prepared it. A draw or dispatch that cannot prepare the stage either fails as before. Driver failures, `ProcessShutdown` and a failure stored by the shader's registration preparation still propagate from the ABI call. - A compute shader whose `s_swappc_b64` calls a user-data pointer cannot be linked at registration, because the callee's address is only known at dispatch. `InvocationFor` in [`ShaderRegistry.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) reads each callee from guest memory, appends one copy per call after the caller (`LinkUserDataCalls`) and prepares that linked code, so a different callee gets its own prepared entry. - A draw's `Graphics::Pipeline` comes from `CachedPipeline`. When the device has `VK_EXT_graphics_pipeline_library` with fast linking, `VK_EXT_extended_dynamic_state` and `VK_KHR_dynamic_rendering`, draws render with `vkCmdBeginRenderingKHR` instead of render passes and the pipeline is linked from four libraries (vertex input, pre-rasterization shaders, fragment shader, fragment output) kept per device in [`PipelineLibrary.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp). A library built for one pipeline is reused by the next: another blend, vertex layout or target format builds only that part (the shader libraries name no attachment format), and cull mode, front face, depth and stencil state are dynamic, so changing them only links. The fast link is what the draw waits for; a background thread then links the same libraries with link-time optimization and the pipeline binds that one from the next draw on. Push constants then use a 256-byte block with a 128-byte slot per side: the pre-rasterization stages stack from byte 0, the pixel shader always starts at byte 128 (`StagePushOffset`), so its library does not depend on how much push data its vertex shader uses. Pipelines with per-face depth bias, a separate alpha blend into an 8_8 target with the alternate component order (both draw twice), rect lists, geometry shaders or a stage without a variant id are created whole, still with dynamic rendering; devices without the extensions (or with `APS5_NO_GPL=1`) keep render passes and whole pipelines. - A draw whose `PA_SC_AA_CONFIG` selects 2, 4 or 8 samples renders into host-only multisampled images. [`MultisampleTarget.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp) keeps one image per device, surface address, format, extent and sample count (`MultisampleTargetView`); the pipeline, render pass and library keys carry `State::samples`, and a multisampled depth target is a multisampled depth surface. The guest sample planes and FMASK are never read or written, so the registry also guards the target's guest range: a CPU read (`GuestMemory::SetReadGuard`), a DMA copy, and a sampled or storage image read of it throw. A `CB_COLOR_CONTROL` resolve mode over a multisampled CB0 runs as a render pass whose resolve attachment is CB1's resident image (`ResolveMultisampleTarget`). A zero fill of the target's CMASK (`NoteColorMetadataFill`, from fill kernels and CP DMA fills) clears the image at its next use. @@ -155,9 +156,11 @@ A T# or S# that a shader loads from a buffer at a runtime offset (`key * stride A T# whose eight dwords and record offset the SRT walker can evaluate is planned as a selected column (`TableColumn::selected`): the offset is kept as a descriptor source (`TableColumn::selection`) and the shader indexes entry 0 of a one-record table. The column stays a whole table when the shader may write the memory its offset is read from at run time: the buffer it is read from, or any memory when the read or a write goes through a pointer or a GPU-only descriptor. Samplers are not selected: an S# the walker can evaluate is a direct sampler. When the selected columns would take a shader past `ShaderInfo::MaxImages` or `ShaderInfo::MaxSampledPairs`, `PlanTableColumns` plans the columns of one heap unselected again, the heap with the most selected columns first, until both fit. -At every invocation, `TableSnapshotter` in [`ResourceMaterializer.cpp`](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) reads each column's records from guest memory into `ResourceSnapshot::tables`: heap base and size, the record count (at most `ImageTableAbi::MaxKeys`), one code per record (an index into the distinct descriptor words, or a poison code) and the ranges it read. A selected column evaluates its offset and reads only that record, as the scalar loads would: dwords past the heap read zeros, an unmapped record or an offset that carries past 32 bits is poison, and an offset the walker cannot read faults the table instead of throwing. Its heap base moves to the record, so the shader's fault address and the written-heap check cover that record alone. Any other column narrows to the records its keys select, plus record 0, when every access has a key domain: a key buffer of at most 256 positions (`APS5_IMAGE_TABLE_KEY_SCAN`), or a value the SRT walker evaluates from user data, SRT reads and constants (which, for an image, selects its record before it could narrow). A loop-carried, lane-reduced or masked key keeps the whole column, so the cost of an invocation grows with the column's records and its distinct T#s. The snapshot is part of the invocation, not of the compiled variant: a changed table or key changes neither the SPIR-V nor the variant's disk key. +At every invocation, `TableSnapshotter` in [`ResourceMaterializer.cpp`](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) reads each column's records from guest memory into `ResourceSnapshot::tables`: heap base and size, the record count (at most `ImageTableAbi::MaxKeys`), one code per record (an index into the column's distinct descriptor words, or a poison code) and the ranges it read. A selected column evaluates its offset and reads only that record, as the scalar loads would: dwords past the heap read zeros, an unmapped record or an offset that carries past 32 bits is poison, and an offset the walker cannot read faults the table instead of throwing. Its heap base moves to the record, so the shader's fault address and the written-heap check cover that record alone. Any other column narrows to the records its keys select, plus record 0, when every access has a key domain: a key buffer of at most 256 positions (`APS5_IMAGE_TABLE_KEY_SCAN`), or a value the SRT walker evaluates from user data, SRT reads and constants (which, for an image, selects its record before it could narrow). A loop-carried, lane-reduced or masked key keeps the whole column, so the cost of an invocation grows with the column's records and its distinct T#s. The snapshot is part of the invocation, not of the compiled variant: a changed table or key changes neither the SPIR-V nor the variant's disk key. -`ResolveImageTables` turns the snapshot into the map buffer (`DescriptorRole::ImageTableMap`, layout in [`ImageTableAbi.hpp`](../../core/shader/recompiler/ImageTableAbi.hpp)): a header, eight words per table (map start, key count, heap base and size, the code for keys past the snapshot, a table fault) and one record per key (flags with the decode mode, the swizzle dword and the element for images; flags and element for samplers). The shader reads `map[key]` and switches over the image's prepared runtime modes. Each distinct T# becomes one element, up to `ImageElementBudget` and the device's capacity. An entry that does not decode, is multisampled or over budget, pairs a reducing sampler with a point-only mode, is an unnormalized or reducing S#, or whose surface has no mapped byte is poisoned: it reads zeros and reports a fault through the BDA fault buffer only when a lane selects it (`APS5_IMAGE_TABLE_STRICT=1` throws at the snapshot instead). The driver's `resolveImageTables` in [`ShaderResources.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) looks up or creates a texture per element and keeps the last four resolved tables per thread for reuse within one collect epoch while their memory is unchanged. Cached objects grow with their tables, so the resource cache keeps at most `ResourceCache::TableElementBudget()` table elements (65,536, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) and evicts its least recently used table objects past that, and the dispatch cache evicts its least recently used entries while their captured words exceed `Driver::DispatchCacheBytes()` (256 MiB, `APS5_DISPATCH_CACHE_BYTES`). +`TableSnapshotter` keeps the last 16 column snapshots per thread (at most 4 Mi words in all, least recently used first out; `APS5_NO_TABLE_SNAPSHOT_REUSE=1` turns them off), keyed by heap base and size, record shape and key count: the words read and the column a snapshot holds, its codes, distinct T#s and read ranges. A narrowed, selected or pointer-indirect column, or one with an unmapped record, is not kept. The column is built once and never changed (`SharedValues`): a later capture of the same column shares it with no per-record work and no copy, each snapshot holding a reference, and the result memo hashes its contents once (`SharedValues::Hash`). A column read again after a write is a new one; the snapshots that share the old one keep it. When the runtime vouches that nothing wrote the column's runs since a stamp taken before the column's first read, the column is not read again: its words are handed to the capture as one shared block (`SrtRuntime::supplyWords`, `SrtGuestWords`): `ShaderMemory` keeps a reference to it and reports its runs in the captured regions the draw cache compares, split at page boundaries as a read reports them, without copying them into its pages, and moves them into pages only when a later read in the same `ShaderMemory` touches them. When the draw or dispatch cache sets a fresh capture against a kept one, words both copied from the same place of one such block are equal without a compare (`SameCapturedWords`); only the words around them are compared. A capture holds its blocks weakly for this, so the identity of a released block never matches a new one. `ShaderMemory` vouches through the write tracker: the range was stamped by `GuestMemory::CollectWrites` before any page of it was fetched, no block of it was stamped since (a collected CPU store, the driver's own stores, `MarkWritten`), no recorded GPU work writes it, and no guest mapping changed. The stamp and the check both call `GuestMemory::FlushGpuWrites` over the range first, as a read through `GuestMemory` does, so storage image results kept on the GPU, unit shadows and queued labels over it are in guest memory before the tracker answers. A CPU store between two collects of one collect epoch is seen at the next epoch, as for every other stamp. Otherwise, outside the write watch for example, the runs are read again and the snapshot is reused only when they are equal word for word; when they differ, the column is walked afresh from the words just read. + +`ResolveImageTables` turns the snapshot into the map buffer (`DescriptorRole::ImageTableMap`, layout in [`ImageTableAbi.hpp`](../../core/shader/recompiler/ImageTableAbi.hpp)): a header, eight words per table (map start, key count, heap base and size, the code for keys past the snapshot, a table fault) and one record per key (flags with the decode mode, the swizzle dword and the element for images; flags and element for samplers). The shader reads `map[key]` and switches over the image's prepared runtime modes. A result memo miss takes the resolution of an earlier one by reference when the variant, the reducing state of the direct samplers paired with table images and every column match, a column's codes and distinct T#s by identity (the same shared values, so equal copies resolve again): `ResolveImageTablesShared` keeps at most 4 resolutions per variant and 256 in all within 8 Mi words, counting the columns they keep alive, least recently used first out, and `APS5_NO_TABLE_RESOLUTION_REUSE=1` turns it off. Each distinct T# becomes one element, up to `ImageElementBudget` and the device's capacity. An entry that does not decode, is multisampled or over budget, pairs a reducing sampler with a point-only mode, is an unnormalized or reducing S#, or whose surface has no mapped byte is poisoned: it reads zeros and reports a fault through the BDA fault buffer only when a lane selects it (`APS5_IMAGE_TABLE_STRICT=1` throws at the snapshot instead). The driver's `resolveImageTables` in [`ShaderResources.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) looks up or creates a texture per element and keeps the last four resolved tables per thread for reuse within one collect epoch while their memory is unchanged. Cached objects grow with their tables, so the resource cache keeps at most `ResourceCache::TableElementBudget()` table elements (65,536, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) and evicts its least recently used table objects past that, and the dispatch cache evicts its least recently used entries while their captured words exceed `Driver::DispatchCacheBytes()` (256 MiB, `APS5_DISPATCH_CACHE_BYTES`). The disk cache stores the table and table operation of each image and sampler, the image shapes of the table binding, the invocation's table poison list, read ranges and fault count, and the table column of each descriptor source in the front-end plan, with its key domains and its selection. diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 9f3445e9cd..21835908fc 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -41,6 +41,8 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them | | `APS5_NO_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes a fresh result for every recompile instead of reusing the result memo | | `APS5_NO_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | runs the runtime-source walk of a plan in the IR evaluator instead of the compiled tape | +| `APS5_NO_TABLE_RESOLUTION_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | stops a result memo miss from reusing an unchanged image table's resolution, so the table is resolved again | +| `APS5_NO_TABLE_SNAPSHOT_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | turns off the image table snapshot reuse, so every capture reads and compares each table column | | `APS5_NO_WINDOW_NARROWING` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp` | | indexes every slot of the shader's vector window instead of only the slots the program uses | | `APS5_PROBE` | string | `core/shader/recompiler/Translation/src/DispatchInstructions.cpp` | | pc:vgpr[:shift] probe that copies that VGPR into v255 after the instruction so image stores write it as a per-lane trace | | `APS5_RAYTRACING` | string | `core/shader/recompiler/Translation/src/DispatchInstructions.cpp` | | selects the image_bvh_intersect_ray behaviour: strict throws and miss writes 0xffffffff | @@ -153,6 +155,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_IGNORE_DEPTH_TEST` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp` | | renders depth- and stencil-tested draws without a depth target as if the tests always passed | | `APS5_INDIRECT_VERTEX_MIB` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` | 64 | cap in MiB on a vertex attribute extent, above which a GPU-side indirect draw resolves on the CPU | | `APS5_INSERT_COMPARE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp` | | compares a fresh capture's regions against the recorder before inserting it into the cache | +| `APS5_KEPT_DESCRIPTOR_SETS` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | 8 | number of freed dedicated image table descriptor sets kept for a later build with the same layout and sizes; 0 keeps none | | `APS5_KEYFLIP_DROP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | drops a pending unit's results on a clear-code to uncompressed key flip, as before | | `APS5_KEYS_FILL_CLEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | clears an image from DCC keys whose fill pattern it matches (only when set to 1) | | `APS5_LABEL_FLUSH_US` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp` | 1000 | age in microseconds at which a pending label batch is recorded or submitted | @@ -287,6 +290,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_STORAGE_DEDUPE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | looks each identical consecutive storage descriptor up, and refreshes its image, separately | | `APS5_NO_SYNC_SITES` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | leaves only the per-source and per-thread sync counts, with no per-site attribution | | `APS5_NO_SYNC_THROUGH` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | makes every pending-write sync a full Sync instead of the targeted unlocked wait | +| `APS5_NO_TABLE_REPLAY` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | makes a reused bound image table rerun each element's checks instead of replaying its recorded bind outcomes | | `APS5_NO_TABLE_REUSE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | disables reusing a materialized image table | | `APS5_NO_TARGET_WRITEBACK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | skips writing render targets back to guest memory after the draw | | `APS5_NO_TEMPLATE_DATA_REFRESH` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | keys the ShaderData and FlattenedSrt words into compute keys instead of refreshing a template's data buffers | @@ -300,6 +304,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_UNREGISTERED_DROP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | stores a storage image's pending results even when its memory is no longer registered, instead of dropping them | | `APS5_NO_VALIDATE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | validates every draw instead of serving the validation memo | | `APS5_NO_VALIDATE_SKIP` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | syncs every pending write instead of skipping validation on written-element evidence | +| `APS5_NO_VOUCHED_VALIDATION` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp` | | validates a cached draw or dispatch through its per-run checks instead of its table block's vouch | | `APS5_NO_WAIT_OVERLAP_SUBMIT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` | | disables the unlocked wait that submits recorded work while overlapping writes are open | | `APS5_NO_WORDWISE_CAPTURE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp` | | fetches every captured page whole instead of reading pending pages word by word | | `APS5_NO_WORKER_AFFINITY` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/WorkerAffinity.cpp` | | leaves the queue workers and presenter unpinned (affinity mask 0) | diff --git a/docs/dev/debt/libSceAgc.md b/docs/dev/debt/libSceAgc.md index 83a5cbca52..1e442fe0da 100644 --- a/docs/dev/debt/libSceAgc.md +++ b/docs/dev/debt/libSceAgc.md @@ -15,8 +15,9 @@ - [sceAgcCbCondWrite](../../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) (libSceAgc) - argument order and the 9-dword COND_WRITE layout follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs), which agree on them; SharpProspero lists another order. The write space is accepted only as 1 (memory), the one value both encode the same way, and other values throw. - [sceAgcAcbAtomicGds_0900](../../../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - signature, control/mode bit mapping and the 11-dword ATOMIC_GDS (0x1d) layout follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/3c7e8072f0d7684e930454794fd5c02056946d73/src/game_compat.c) only, which derives them from the library's disassembly; KytyPS5 and shadPS4 have no body for it. Not confirmed on hardware or a title (PPSA07286 imports it, nothing was run), and the driver has no ATOMIC_GDS handler (opcode 0x1d is not executed as an atomic GDS operation) - [sceAgcGetIsTrinityMode](../../../core/libs/prx/libSceAgc/Misc/src/Platform.cpp) (libSceAgc) - signature from the only PPSA26344 call (a pointer to a one-byte flag, result ignored); a null pointer throws because its error code is unknown -- [sceAgcDcbAtomicGds_0900](../../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - name from its NID (`zARR5aCmkoY`), the DCB counterpart of `sceAgcAcbAtomicGds_0900`, signature unknown +- [sceAgcDcbAtomicGds_0900](../../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - and `sceAgcDcbAtomicGds` (forwards to it) - signature, control/mode bit mapping and the 11-dword ATOMIC_GDS (0x1d) layout are those of `sceAgcAcbAtomicGds_0900`, which follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/3c7e8072f0d7684e930454794fd5c02056946d73/src/game_compat.c) only; KytyPS5 and shadPS4 have no body for it. Not confirmed on hardware or a title, and the driver has no ATOMIC_GDS handler - [sceAgcDcbContextStateOpGetSize](../../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - sizes from KytyPS5 only (5, 27, 27 and 32 dwords), equal to what `sceAgcDcbContextStateOp_0100` writes; KytyPS5 returns 0 for an operation above 3, here it throws +- [sceAgcDcbContextStateOp](../../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - delegates to `sceAgcDcbContextStateOp_0100` (same packets as KytyPS5's `AgcDcbContextStateOp`); KytyPS5 returns nullptr for an operation above 3, here it throws; the packet words were not compared with a real console - [sceAgcBranchPatchSetThenTarget_0300, sceAgcBranchPatchSetElseTarget_0300](../../../core/libs/prx/libSceAgc/Patch/src/Branch.cpp) (libSceAgc) - names from their NIDs (`7Wa3aeJgeVU`, `rP5xLdOf26k`); the `(packet, cachePolicy, target, sizeInDwords)` signature is inferred from PPSA28416's calls (a branch packet, 0, an aligned target, 4), with the policy stored where `sceAgcCbBranch` puts it - [sceAgcDcbSet{Cx,Sh,Uc}RegistersIndirectGetSize](../../../core/libs/prx/libSceAgc/DcbState/src/Registers.cpp) (libSceAgc) - the count argument is ignored: PPSA28416 calls them with leftover register contents (0xf5f5039c), so the real functions likely take none; the size is 20 bytes either way - [sceAgcUpdateInterpolantMapping_0100](../../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - name from its NID (`k0E7vkgqAuE`); the `(regs, vs, ps)` arguments are the declared ones, behaviour unknown diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index cd63352c21..4703cfeb23 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -30,6 +30,7 @@ - [Strided buffer overhang](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a strided buffer descriptor that ends less than one stride past the end of the readable registered allocation holding its base is bound only up to that allocation's end, even when another readable allocation starts right there. This assumes the bytes past the end are the rounded-up `num_records` of the last record (RADV rounds it up), which no fetch inside the buffer reads; a guest that really keeps a vertex split across two adjacent allocations would read zeros for the cut tail. Taken from the RetroArch Dolphin core case in #2912, not measured on hardware. - [Buffer descriptor CACHE_SWIZZLE hint](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - bit 30 of word 1 in a buffer resource descriptor (`words[1] & 0x40000000u`) is normalized to zero when binding guest buffers in `ShaderResources::addGuestBuffer`. On GFX10 (gfx1013), bit 30 of V# word 1 is `CACHE_SWIZZLE` (Mesa's `gfx10-rsrc.json`: STRIDE [29:16], CACHE_SWIZZLE [30], SWIZZLE_ENABLE [31]), a cache-bank hint with no effect on the address or data, which is dropped here. - [Buffer descriptors past the range limit](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a buffer V# whose size is over `maxStorageBufferRange` is bound only up to the end of the readable registered allocation holding its base when that allocation ends before the limit, and as the empty placeholder buffer, like a null V#, when no registered allocation holds its base; a V# over an allocation that reaches the limit still throws. Titles leave such words in descriptor slots their shaders do not reach on the path taken (PPSA10528 binds `c8010002 2c6bec20 c0053c00 04000013`, about 33 TB from an address above the user address space). What the console returns for an access past the bound range is not known; here the device's robust buffer access decides. Follows the review of #3202, not measured on hardware. +- [Vertex fetch overhang](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - a zero-padded vertex fetch whose valid range (`num_records * stride`) ends less than one stride past the end of the readable memory at its base copies only the readable bytes and reads the rest as zeros, instead of failing the draw; a longer overhang still fails. This extends the strided buffer overhang rule above (the last record is the rounded-up `num_records`) to the staging copy and is not measured on hardware. The observed case is a fetch of 14858 records of 44 bytes that ends 8 bytes past the end of the 0xa0000-byte mapping holding it. - [Zero-stride vertex fetch range](../../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - for a stride-0 V# with OOB_SELECT 2 the whole fetch reads memory unless NUM_RECORDS is 0; with OOB_SELECT 3 it reads zero when NUM_RECORDS is 0 and throws when the fetch only partly fits. Rules taken from the GFX10 OOB_SELECT table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c), not measured on hardware. Whether OOB_SELECT 3 checks each dword of a multi-dword fetch separately is unknown. OOB_SELECT 0 and 1 keep the structured record checks. - [Resource registration](../../../core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp) (libSceAgcDriver) - registration always fails with `RESOURCE_REGISTRATION_UNAVAILABLE`, so the owner and resource queries, `sceAgcDriverSetResourceUserData`, `sceAgcDriverFindResourcesPublic` and `sceAgcDriverUnregisterAllResourcesForOwner` return the same error without reading their arguments. Their parameter lists follow the libSceGnmDriver counterparts and are unverified. `sceAgcDriverGetResourceRegistrationMaxNameLength` writes 0xfc and returns 0, as [prosper](https://github.com/mattias800/prosper) does from a title's registration configuration; the `uint32_t*` parameter matches the SDK wrapper `sce::Agc::ResourceRegistration::getMaxNameLength(unsigned int*)`. sharpemu writes 256 and KytyPS5 returns `RESOURCE_REGISTRATION_UNAVAILABLE`, so the value is unverified; a null output throws - [sceAgcDriverRegisterMultipleResources](../../../core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp) uses the existing unavailable-registration state and leaves caller memory untouched. Its four arguments (uint32 handle output, uint32 owner, descriptor pointer, uint32 count) follow the original PPSA34502 / 01.010.000 codzm call. The caller builds 40-byte descriptors and explicitly accepts 0x8A6C9018 with an invalid output handle. Enabled registration and console argument-validation precedence are unverified. @@ -178,7 +179,7 @@ - [16-bit depth with stencil](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) is stored as `D24_UNORM_S8_UINT`, or `D32_SFLOAT_S8_UINT`, on a device without `D16_UNORM_S8_UINT` as an attachment. Depth comparisons, clears and the depth image see the wider format's precision, so results at the 16-bit quantization boundary can differ from the console; the constant depth bias is scaled by 256 for D24, and a non-zero constant on the D32 float fallback throws. Not measured on console hardware. - [Depth surfaces created over storage writes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver): a depth surface created without HTILE copies its depth plane from the storage image at the plane's address, a pending one first, when that image has the plane's extent, one level, a 2D view or a 2D array view (its layer 0 is copied) and a 16-bit format for a 16-bit plane or a 32-bit format for a 32-bit plane. A multisampled surface, a surface the device stores in another format, a surface with HTILE and a surface without such an image start from `DB_DEPTH_CLEAR`, and the stencil plane from `DB_STENCIL_CLEAR`; CPU writes to the plane's memory are not read. Not measured on the console, where the plane is that memory. - [Flip packets raise the graphics end-of-pipe event](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) (libSceAgcDriver) - a flip delivers the queue's end-of-pipe event (`sceAgcDriverAddEqEvent`) once the work recorded before it completes, as a `RELEASE_MEM` with `INT_SEL` does, but only on a queue that has never delivered an interrupt from such a release: the queue is latched the first time a release interrupt reaches the event queue, and flips on that queue deliver nothing while its own release interrupts keep arriving. The latch is cleared when the queue has flipped five times without another interrupt from its own releases: the fifth flip delivers the event and unlatched the queue, so a title that waits on the flip event after latching recovers only if it submits further flips. The bound cannot be by time: the next release interrupt can arrive seconds later than a suppressed flip across loading stalls, and delivering an unexpected flip event on a queue that pairs flips with releases can crash; no fixed window short enough to unblock a waiting title keeps such a title alive. Whether the console raises the event from the flip itself or from an interrupt its flip sequence contains is not measured; nor is how the console behaves for a title that waits on the flip event and also raises release interrupts on the same queue. -- [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less 2 GiB (imports and type-3 mirrors share it; #167's rule). A new import beyond it retires the least recently used imports not used in the last 256 lookups; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. A refused import is tried again at its next use. The 64 MiB window, the 256-lookup protection and the 2 GiB reserve are choices, not measured. +- [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less the larger of 2 GiB and half that heap (imports, type-3 mirrors and buffers and textures placed in host memory share it; on a 16 GiB machine whose host heap is half the memory, a 2 GiB reserve left too little for the rest). A new import beyond it retires the least recently used imports not used in the last 256 lookups; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. A refused import is tried again at its next use. The 64 MiB window, the 256-lookup protection and the 2 GiB reserve are choices, not measured. - [Registration preparation at submit](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) - `Driver::Submit` waits for every registration-time shader preparation queued before the call, not only those of the shaders the commands use, so a submission behind a long queue of unrelated preparations waits for all of them on the submitting thread. Waiting there instead of in the queue worker keeps the GPU side of a submission short: a title that sees a submission take seconds treats the GPU as hung (Goat Simulator 3, PPSA04159, then stops submitting and its garbage collector fails a 10 s timeout). On the console shaders are compiled ahead of time, so neither wait exists; how long a title tolerates a slow submit call is not known. - [Resource cache entries holding released textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a sampled texture the texture cache drops while something else holds it, or makes without caching (`admitTexture` refusing a first use), is marked released, and `ResourceCache::Insert` refuses descriptor sets that hold one and removes the entries that do. The removal waits for the next insert: until then `Find` still returns those entries and their textures stay alive. One process-wide flag starts the sweep, so while the sampled cache is over its budget every insert scans all entries (1,024 at start, up to 16,384 after growth, or `APS5_RESOURCE_CACHE_ENTRIES`, each with its textures); that cost is not measured. When `ReleaseTextureCaches` runs on device memory exhaustion, the sampled textures these entries hold are only marked released and are freed at the next insert, so the immediate allocation retry gets no memory back from them. A draw or dispatch whose textures the cache refused builds its descriptor sets again on each use for as long as the cache stays full; the count bounds (`APS5_RESOURCE_CACHE_ENTRIES`, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) still bound nothing by bytes, and sampled textures made outside the texture cache (the disabled cache, depth-surface copies) are never marked. Measured on one title on one RTX 4090: with the default bounds, sampled textures held outside the cache peaked at 2,836 MiB (7,830 MiB before) and device-local heap 0 stayed under its 22.9 GB budget for the whole run (26,357 MiB before, which aborted with `VK_ERROR_OUT_OF_DEVICE_MEMORY` at the wolf scene). Whether the extra descriptor-set builds cost frame time elsewhere is not measured. @@ -186,3 +187,4 @@ - [Unclamped normalized color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - CB_COLOR_INFO targets with BLEND_CLAMP clear (bit 15) on UNORM and SNORM formats are accepted when blending is disabled (ENABLE clear or BLEND_BYPASS set). Vulkan converts unblended shader values and clamps to the normalized range automatically; blending into an unclamped normalized target remains unmodeled because Vulkan always clamps inputs prior to blending. - [Thick SW_4KB_S_X volumes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) use AMD addrlib's GFX10 non-RB+ 16-pipe `GFX10_SW_4K_S3_X_PATINFO` equations for 1, 2, 4, 8 and 16-byte elements, including packed mip tails. The layout follows [AMD PAL c5e80007](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp); it has not been measured on console hardware. Other thick swizzle modes outside the supported families still throw. Mipmapped 3D color targets remain refused by the color-register decoder. +- [Viewport depth range remap](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - on a device without `VK_EXT_depth_range_unrestricted` a viewport depth range outside [0, 1] (Minecraft PPSA17221 draws its world with `PA_CL_VPORT_ZSCALE` 1, `ZOFFSET` 0 in negative-one-to-one clip space) is folded into the vertex shader: `RemapPositionDepth` scales clip z so z/w is the console's depth, the pipeline's depth clamp applies the console's `PA_SC_VPORT_ZMIN/ZMAX` clipped to [0, 1] (exact for a normalized depth target or none; Minecraft clamps its float depth target to [-1, 1], so a depth below 0 is stored as 0 there, which only a later test or read at exactly 0 can tell, and the first such draw is reported once), and two extra `gl_ClipDistance` elements restore the near and far planes depth clamping removes. The mode is a pipeline specialization constant (`PipelineSpecialization::DepthRemapBase`); scale and offset are read from the draw's push block (two words at byte 104, `DepthRemapPushOffsetBytes`, so a remapped draw's stage push data ends before it), so draws that differ only in them share one vertex module and one pipeline. The remap serves the vertex path only; mesh and tessellation draws with such a range still throw. Whether the console clamps before or after its depth bias and how it treats a clamp range narrower than the transform's are not measured. diff --git a/docs/dev/debt/libSceImeDialog.md b/docs/dev/debt/libSceImeDialog.md new file mode 100644 index 0000000000..01003701d8 --- /dev/null +++ b/docs/dev/debt/libSceImeDialog.md @@ -0,0 +1,9 @@ +# libSceImeDialog technical debt + +### Silent stubs + +- [sceImeDialogInit](../../../core/libs/prx/libSceImeDialog/Export.cpp) (libSceImeDialog) shows no dialog: the text is typed on the host keyboard, which [ImeTextInput](../../../core/libs/prx/libSceVideoOut/src/ImeTextInput.cpp) forwards from the game window through libSceIme while no IME panel is open. Text, Backspace, Delete, Left, Right, Home and End edit the text, Enter accepts it (end status 0) and Escape cancels (end status 1). The controller cannot type, the title, placeholder, type and options are not shown or applied, and line breaks cannot be typed. While the dialog runs, key presses do not reach the keyboard-to-pad bindings of [PadInput](../../../core/libs/prx/libSceVideoOut/src/PadInput.cpp) + +### Unknown function info + +- [sceImeDialogInit, sceImeDialogGetStatus, sceImeDialogGetResult, sceImeDialogAbort, sceImeDialogTerm](../../../core/libs/prx/libSceImeDialog/Export.cpp) (libSceImeDialog) - the statuses (none 0, running 1, finished 2), the end statuses (ok 0, user canceled 1, aborted 2) and the errors of `sceImeDialogInit` for an open dialog (`0x80bc0001`), a null parameter (`0x80bc0031`), a maximum text length outside 1 to 2048 (`0x80bc0016`) and a null text buffer (`0x80bc0017`) follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/b2d7ec7d31f9ad9426db93afe079db8679b3e1e0/src/core/libraries/ime/ime_dialog.cpp); none is measured on a PS5. A result, abort or term on a dialog that is not finished, running or open returns `0x80bc0002`, where shadPS4 returns `0x80bc0105` to `0x80bc0107`, and `sceImeDialogTerm` also closes a running dialog, which shadPS4 refuses. The text reaches the title's buffer only when Enter accepts it diff --git a/docs/dev/debt/libSceNet.md b/docs/dev/debt/libSceNet.md index e101cbca34..1af8c1ec19 100644 --- a/docs/dev/debt/libSceNet.md +++ b/docs/dev/debt/libSceNet.md @@ -10,6 +10,8 @@ ### Functional +- [sceNetAccept](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) inherits the listener's non-blocking mode through the existing `SO_NBIO` setter, as the libkernel socket path does and FreeBSD's [`sonewconn`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/uipc_socket.c#L571) copies the socket state. The mode is sampled when accept starts; changes between connection creation and accept are not modelled. Not measured on a console. + - [sceNetConnect to AF_UNSPEC](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) dissolves a datagram association as FreeBSD 11.4's `udp_disconnect()` does, which resets the local address to the wildcard address and keeps the port. On Linux a socket bound to an explicit port keeps its local address as well (`127.0.0.1:` stays `127.0.0.1:`), because a bound socket cannot be bound again; one bound to port 0 is bound again to the wildcard address with the port it had - [sceNetSetsockopt](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) sets `SO_RCVTIMEO` and `SO_SNDTIMEO` as the host socket's timeouts. On Windows a receive or send on a stream socket that times out fails with `ETIMEDOUT`, where FreeBSD returns `EAGAIN` or the partial count: Winsock leaves the connection in an indeterminate state after a timeout and reports no partial count ([SOL_SOCKET options](https://learn.microsoft.com/en-us/windows/win32/winsock/sol-socket-socket-options)), so FreeBSD's behaviour needs the timeout waited for without the host option. A receive on a datagram socket that times out fails with `EAGAIN` - [sceNetSetsockopt, sceNetGetsockopt](../../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) take `SO_SNDTIMEO` and `SO_RCVTIMEO` (0x1005, 0x1006) as a 16-byte `struct timeval`, as FreeBSD 11.4's [`sosetopt` and `sogetopt`](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/sys/kern/uipc_socket.c#L2657-L2690) do, and Sony's 0x1105 and 0x1106 as an `int` of microseconds. A [console log of ProsperoLight](https://github.com/blackbearreloaded/ProsperoLight/issues/49) (firmware 12.20) shows receives timing out every 100 ms after it gave 0x1006 a `{0, 100000}` timeval. The errors for a value shorter than a timeval (`EINVAL`) or out of range (`EDOM`), and `sceNetGetsockopt` cutting the timeval to the caller's length, follow FreeBSD's `sooptcopyin` and `sooptcopyout` and are not measured on a console diff --git a/docs/dev/debt/libSceRegMgr.md b/docs/dev/debt/libSceRegMgr.md new file mode 100644 index 0000000000..467ad2850e --- /dev/null +++ b/docs/dev/debt/libSceRegMgr.md @@ -0,0 +1,5 @@ +# libSceRegMgr technical debt + +### Unknown function info + +- [sceRegMgrNonSysGetInt](../../../core/libs/prx/libSceRegMgr/Export.cpp) (libSceRegMgr, NID `dKeshzt29G4`) - imported by Astro Bot (PPSA21564), which fails to load without libSceRegMgr.prx (issue 4720); the `(int64_t key, int32_t unk, int32_t* data)` signature follows [ChonkyStation4](https://github.com/liuk7071/ChonkyStation4/blob/c101a23ecbc15e56c6e30d5dd24d3ed8880a957a/ChonkyStation4/OS/Libraries/SceRegMgr/SceRegMgr.cpp#L38-L47), not measured on a console, so every call throws diff --git a/docs/dev/debt/libSceSsl.md b/docs/dev/debt/libSceSsl.md index 8138ae133c..8b1772a8a4 100644 --- a/docs/dev/debt/libSceSsl.md +++ b/docs/dev/debt/libSceSsl.md @@ -7,3 +7,4 @@ - [sceSslGetSerialNumber, sceSslGetSubjectName, sceSslGetIssuerName, sceSslGetNameEntryCount, sceSslGetNameEntryInfo, sceSslFreeSslCertName, sceSslGetNotBefore, sceSslGetNotAfter, sceSslGetFingerprint](../../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - signatures and context ownership follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/60977b29103452b9681ae10c7a9011d44965bf16/src/SharpProspero/Interop/Net/Http.cs#L293-L350), with hash types matching its `SceSslFingerprintHashAlgorithm` enum. Operations take a context handle as their first argument and operate on certificate and name handles scoped to that context. `sceSslGetSerialNumber` copies into a caller-allocated output buffer or queries the required length when the buffer is null or capacity is zero; a buffer smaller than the serial number returns `SSL_ERROR_INVALID_ARG` (0x8095177a). Mbed TLS parses the certificate's serial number, validity period (converted to microseconds since Gregorian year 0001 epoch), fingerprint digests and X.509 subject and issuer distinguished names, extracting OID attribute short names (or numeric strings) and value strings. An unknown context, unknown or expired certificate handle or invalid certificate name handle throws; null output pointers, unsupported hash types or out-of-range entry indices return `SSL_ERROR_INVALID_ARG` (0x8095177a). The error codes, the zero-capacity length query, the NUL-terminated truncation of name entry strings with the full value length in `valueLen`, and the lifetime of name handles (released only by `sceSslFreeSslCertName` or when the context is terminated) have not been measured on a PS5 - [sceSslLoadCert, sceSslGetCaList, sceSslFreeCaList, sceSslGetCaCerts, sceSslFreeCaCerts, sceSslUnloadCert](../../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - signatures and layouts follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Net/Http.cs). PPSA15508/01.002.000 calls `sceSslLoadCert` with one 1,408-byte PEM CA and null client certificate/key arguments; this confirms that call shape, not console error behavior. Mbed TLS 3.6.7 parses DER/PEM chains and checks an optional client certificate/key pair. Imports are retained per context and committed together; a failed import preserves existing credentials. CA queries return independently owned snapshots; unload clears active credentials while outstanding snapshots remain valid. Appending repeated CA imports, snapshot lifetime across unload, memory overhead and error precedence have not been measured on a PS5. Invalid loads, malformed material, exhausted pools and unsupported keys throw rather than inventing console error codes. No root bundle is shipped; an empty context still returns `NOT_FOUND`. The existing null-list error, unknown-context behavior and legacy clearing of foreign raw CA lists remain unchanged. Certificate reader queries reflect active certificates or outstanding snapshots. Loading credentials does not start a TLS connection; `sceSslConnect` verifies the server against the loaded CAs and offers the loaded client certificate - [sceSslEnableVerifyOption, sceSslDisableVerifyOption, sceSslSetMinSslVersion, sceSslSetAlpn, sceSslGetAlpnSelected](../../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - signatures follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/60977b29103452b9681ae10c7a9011d44965bf16/src/SharpProspero/Interop/Net/Http.cs), the source of the other connection signatures. The verify options select which verification failures `sceSslConnect` reports (see the connection entry above), with the bit values SharpProspero's `SceSslVerifyOption` gives: `SERVER_VERIFY` 0x1, `CN_CHECK` 0x4, `NOT_AFTER_CHECK` 0x8, `NOT_BEFORE_CHECK` 0x10, `KNOWN_CA_CHECK` 0x20. The initial options are all five checks (0x3D), so a connection whose title never calls these fails closed; what the console starts with is unknown, and the one source, the HTTPS layer of [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/4865dd92ca6b6bb907748f125883093af038f53b/src/libs/network.cpp#L191), starts its libSceHttp connections with `ssl_flags` 0xA7 (`SERVER_VERIFY`, `CLIENT_VERIFY`, `CN_CHECK`, `KNOWN_CA_CHECK`, SNI), evidence that verification is on by default rather than for these bits. A check that is disabled is not reported, whatever the chain. `sceSslSetMinSslVersion` accepts the `SceSslVersion` values SharpProspero lists, 0 (none) to 5 (TLS 1.2), and throws for others; whether the console accepts a TLS 1.3 value is unknown. Every accepted value gives the same handshake, as Mbed TLS 3 speaks TLS 1.2 and nothing older and TLS 1.3 is compiled out. `sceSslSetAlpn` stores the string up to its terminator and throws for a null list; it is split on commas into the protocols offered in the handshake, as HTTP libraries write ALPN lists, and whether the console expects that or the TLS wire format is unknown. `sceSslGetAlpnSelected` returns the protocol the server selected, or null before the handshake and when none was selected; the pointer is owned by the connection. An unknown connection and a connection whose context was terminated throw, as for `sceSslSetVerifyCallback` +- [sceSslGetPem](../../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - static inspection of PPSA28002 confirms the context-first four-argument call with a caller-owned buffer and an in/out 64-bit length. Mbed TLS serializes PEM including its trailing NUL. Buffers smaller than the required encoding throw; an unknown context or foreign or expired certificate handle throws. The exact console PEM length policy, error precedence and truncation behavior have not been measured on a PS5. Existing serial/name, validity and fingerprint readers and their documented uncertainties remain unchanged diff --git a/docs/dev/debt/libSceVideoOut.md b/docs/dev/debt/libSceVideoOut.md index ce95581f16..ae73a5efc5 100644 --- a/docs/dev/debt/libSceVideoOut.md +++ b/docs/dev/debt/libSceVideoOut.md @@ -21,6 +21,7 @@ - [sceVideoOutOpen](../../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. PPSA26344's only call at `0xCCF5EA` passes `(0xFF, 0, 0, nullptr)`. Whether a non-null param continues past offset 24 is also unknown - [sceVideoOutGetBufferLabelAddress](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) exposes a port-owned contiguous array of 16 aligned 64-bit labels and returns its count. Astro Bot PPSA21564's bundled AGC driver passes a pointer output and addresses each label at `base + index * 8`. The count, zero initialization on registration, and release of the previous displayed buffer when its replacement completes follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/23c41333822fe8eca4c609b43119180af63ae4ac/src/core/libraries/videoout/video_out.cpp) and its driver; PS5 console equivalence has not been measured. Labels live until the port closes, and reopening creates a new array. Error precedence follows the reference for null output and the existing open-port policy for handles. Exact scanout release timing, repeated same-buffer flips and concurrent guest label stores remain unverified. EOP-flip registration and interrupt tokens remain unimplemented; exposing these labels does not implement that separate API. - [GPU flips](../../../core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp) (libSceVideoOut) - a command-buffer flip of a buffer index with no registered buffer when the GPU reaches it presents black and completes like any flip (flip event, count, `flipPendingNum`). The console's behaviour is unknown; PPSA05684 requires that it completes: its renderer submits a flip of buffer 0 while another thread waits for `sceVideoOutIsFlipPending` to reach 0 before registering the buffers, and `flipPendingNum` counts that flip from its submission as in shadPS4 +- [sceVideoOutSubmitFlip](../../../core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp) (libSceVideoOut, libSceAgcDriver) - in the [AGC driver](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp), a CPU flip waits for the graphics-queue submissions ahead of it when the calling thread has submitted graphics-queue work itself: a title that submits a frame and flips it without waiting relies on that order, since emulated GPU work finishes later than the console's. A flip from a thread that has never submitted graphics-queue work is presented without waiting for queued submissions, unless a command-buffer or EOP flip ahead of it has not been reached. Such a thread can only know its frame is finished by waiting for it, as the PS5 RADV WSI's flip thread waits on the frame's fence. Whether the console ever orders a CPU flip behind the graphics queue is not measured; `APS5_CPU_FLIP_IN_ORDER=1` orders every CPU flip - [sceVideoOutAddPreVblankStartEvent](../../../core/libs/prx/libSceVideoOut/src/Event.cpp) (libSceVideoOut) - how long before the vblank the console signals pre-vblank start is unknown. The event is triggered on each emulated vblank tick immediately before the vblank event, with its own count as the event data - [sceVideoOutRegisterBuffers, sceVideoOutGetFlipStatus, sceVideoOutGetVblankStatus](../../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - PS4 callers. A handle is treated as PS4 once buffers are registered through the PS4 `sceVideoOutRegisterBuffers`; from then on its flip and vblank status are written in the PS4 64- and 32-byte layouts, with `tsc` and `submitTsc` 0. A PS4 title that reads a status before it registers buffers gets the larger PS5 layout. The PS4 function accepts only linear A8R8G8B8_SRGB (0x80000000) and A8B8G8R8_SRGB (0x80002200) buffers without options; tiled PS4 buffers throw, as their layout differs from the PS5 tiling. Linear PS4 buffers may be 4 KiB aligned: the OpenOrbis graphics sample registers its second 1920x1080 buffer 8,294,400 bytes after the first. The PS4 alignment rule is not known diff --git a/docs/dev/debt/libc.md b/docs/dev/debt/libc.md index d9c760ef18..116bda7a6f 100644 --- a/docs/dev/debt/libc.md +++ b/docs/dev/debt/libc.md @@ -24,6 +24,7 @@ - [sceLibcMspaceReallocalign](../../../core/libs/prx/libc/src/Mspace.cpp) (libSceLibcInternal, implemented in libc) takes `(void* heap, void* block, size_t alignment, size_t size)`, the order of the console's `libSceLibcInternal.sprx` (SDK 4.5x build of `J03950001`, 0x13880, whose internal routine at 0x12390 treats the third argument as the alignment and, for a null block, passes it to the routine `sceLibcMspaceMemalign` uses). Measured on a PS5 (firmware 13.60) from a homebrew payload: a null block with alignment 4096 and size 100 returns a 4096-aligned block; an alignment of 0 or 3 succeeds and 100 gives a 128-aligned block, as the routine raises it to at least 32 and to the next power of two; an alignment of 2^63 or more returns null, keeps the block and leaves `errno` alone; and a size of 0 returns a block (the same one when it is already suitable) instead of freeing it. A block that is aligned and large enough is kept where it is, without giving back the bytes past the new size as the console's in-place resize does - [acosl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, like the other libc math functions here, not by the console's (FreeBSD msun ld80 code in Sony's `libc.prx`): results can differ from the console's in the last bit, and on Linux glibc sets `errno` to `EDOM` for `acosl` outside [-1, 1] where msun only raises the floating-point exception. `long double` is the x87 80-bit format, as Sony's `frexpl` uses it (`fldt 0x8(%rsp)`, result in `st(0)`) - [atanl, sinl, cosl, tanl, sinhl, coshl, tanhl, expl, exp2l, logl, log10l and powl](../../../core/libs/prx/libc/src/Math.cpp) (libc) - computed by the host's math library, as `acosl` is, not by the console's FreeBSD msun ld80 code: results can differ from the console's in the last bit (glibc's `log10l(1000)` is not exactly 3), and on Linux glibc sets `errno` to `EDOM` or `ERANGE` where msun only raises the floating-point exception. `ldexpl` and `truncl` are exact operations and give the console's result; `logb` and `remainder` give the console's value too, but on Linux glibc sets `errno` (`ERANGE` for `logb(0)`, `EDOM` for `remainder` by zero or of an infinity) where msun only raises the floating-point exception +- [Dinkumware atomics](../../../core/libs/prx/libc/src/DinkumwareAtomics.cpp) (libc) - every operation is sequentially consistent whatever memory order it is given, which is never weaker than the console. Matching the `libc.prx` bundled with PPSA08427 (that binary run side by side on every order from 0 to 7), `_Atomic_load_8` returns 0 for a release, acq_rel or invalid order and the 8-byte exchange and fetch operations return 0 without touching memory for an order above `memory_order_seq_cst`; the generic `_Atomic_copy`, `_Atomic_exchange` and `_Atomic_compare_exchange_*` lock the byte flag passed first, as the console does - [std_execute_once](../../../core/libs/prx/libc/Export.cpp) (libc) - assumed to be the plain-named twin of `std::_Execute_once` (`_ZSt13_Execute_onceRSt9once_flagPFiPvS1_PS1_ES1_`, the callee of Dinkumware `call_once`): it delegates there and gets the [thread.once.callonce] semantics already verified for that symbol (one invocation per flag, concurrent callers block until it completes, and a failing or throwing invocation leaves the flag unready so the next caller runs it). The name equivalence is not confirmed on a title, and the callback convention (called with (null, arg, null), non-zero return meaning success) is the twin's, unverified for this symbol - [__atomic_load, __atomic_store, __atomic_exchange and __atomic_compare_exchange](../../../core/libs/prx/libc/src/AtomicBuiltins.cpp) (libc) - the generic, size-parameterised libatomic entry points are a single `ud2` in the `libc.prx` bundled with PPSA08427 (disassembly), so the console traps on any call; these throw instead - [Dinkumware mutexes and conditions](../../../core/libs/prx/libc/src/Threads.cpp) (libc) use pointer-to-handle arguments, matching the console aerolib signatures reported in [#3079](https://github.com/boykopovar/AnyPS5/pull/3079). The title's `_Mtx_lock` maps `0` to `_Thrd_success` (0), `0x8002000b` to `_Thrd_busy` (3), and other failures to `_Thrd_error` (4); its internal mutex init recognizes recursive flag `0x100`, while every other type uses default attributes. The observed relock failure for a plain mutex suggests those defaults are error-checking, but that has not been confirmed on hardware. The title's `_Mtx_unlock` and `_Cnd_wait` return success regardless of their internal result. The host implementation treats invalid ownership preconditions as no-ops, and throws for invalid handles, host wait errors and destroying a locked mutex as debug guards; those cases are not confirmed against the console. diff --git a/docs/dev/debt/libkernel.md b/docs/dev/debt/libkernel.md index b9348ec44e..4fd7f8576b 100644 --- a/docs/dev/debt/libkernel.md +++ b/docs/dev/debt/libkernel.md @@ -148,7 +148,7 @@ - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it. A thread that holds the address-wait lock of `sceKernelSyncOnAddressWait`, `sceKernelSyncOnAddressWake` and `_umtx_op`, or the park queue lock of the guest mutexes, gets the raise as an APC instead, which runs once it has released the lock or while it waits; the other locks of the emulator are not deferred this way - [sigaltstack](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records each thread's alternate signal stack and reports it back, following FreeBSD's `kern_sigaltstack` (`EINVAL` for flags other than `SS_DISABLE`, `ENOMEM` below `MINSIGSTKSZ` 2048), but handlers never run on it, so `SS_ONSTACK` and `EPERM` are never reported - [sigpending](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) reports the intersection of the process-wide pending set of `sigtimedwait` and the blocked mask of `_sigprocmask`, as FreeBSD's `sys_sigpending` intersects the pending list with `td_sigmask`, and returns -1 with `EFAULT` for a null set. Because `_sigprocmask` keeps only the first 32 signals, signals 33 to 128 are never blocked and never appear in the set, and setting `SIG_IGN` (or `SIG_DFL` on a signal whose default action is to ignore) does not clear a signal that was already marked pending. PPSA99420 imports it -- [scePthreadJoin, scePthreadCreate](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - on a PS5 (firmware 13.60) joining a thread a second time returns `ESRCH` (`0x80020003`), as it does here, but a joined thread is kept until `scePthreadCreate` publishes over the same handle storage, so one host object and its handle are retained per distinct guest handle storage location for the lifetime of the process; a guest that joins into a fresh storage location every time keeps every one of them. A null entry crashed the payload and still throws. Joining the calling thread returns `EDEADLK` and a zeroed attribute creates a default thread, as on the console. `scePthreadDetach` on a joined thread answers `EINVAL`, as FreeBSD does on a thread that is not joinable; the console answer for a detach after a join is unverified. A detached thread is still released when it ends, so joining it after it ended reads freed memory. A handle value copied to a second storage and joined there retains the object under the original storage key, so a later `scePthreadCreate` over that original storage releases an object the copy still points at, and the copy reads freed memory +- [scePthreadJoin, scePthreadCreate](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - on a PS5 (firmware 13.60) joining a thread a second time returns `ESRCH` (`0x80020003`), as it does here, but a joined thread is kept until `scePthreadCreate` publishes over the same handle storage, so one host object and its handle are retained per distinct guest handle storage location for the lifetime of the process; a guest that joins into a fresh storage location every time keeps every one of them. A null entry crashed the payload and still throws. Joining the calling thread returns `EDEADLK` and a zeroed attribute creates a default thread, as on the console. `scePthreadDetach` on a joined thread answers `EINVAL`, as FreeBSD does on a thread that is not joinable; the console answer for a detach after a join is unverified. Detaching a thread while another thread waits to join it returns `EINVAL`, and a second concurrent join returns `EOPNOTSUPP`, as FreeBSD 11.4's [`thr_detach.c`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libthr/thread/thr_detach.c) and [`thr_join.c`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libthr/thread/thr_join.c) check `joiner`; before this PPSA34015 stopped once with `Joining guest thread: Access is denied` when a thread it was joining was released under the join, and which of the two calls released it was not captured. A detached thread is still released when it ends, so joining it after it ended reads freed memory. A handle value copied to a second storage and joined there retains the object under the original storage key, so a later `scePthreadCreate` over that original storage releases an object the copy still points at, and the copy reads freed memory - [Thread cancellation](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) follows FreeBSD libthr (states and types 0/1 and 0/2, `PTHREAD_CANCELED` 1, the `_pthread_cleanup_info` node layout), not measured on the console. A request is acted on only in `scePthreadTestcancel`, the condition variable, `sem` and `scePthreadSem` waits, `scePthreadJoin` and the sleeps, and their POSIX names; file and socket calls are not cancellation points. A sleep is not interrupted: the thread exits when it ends. The asynchronous type acts like the deferred one, except at the two set calls. - [cpuset_setaffinity](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - the calling thread, another live guest thread selected by its guest TID, or the calling process is handled; other levels (ROOT/CPUSET) and other scopes (CPUSET/JAIL) throw as unmodelled, invalid levels or scopes return `EINVAL` and unknown thread or process IDs return `ESRCH`, following [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/sys/kern/kern_cpuset.c#L1281-L1286) thread selection. Bits outside `sceKernelGetAvailableCpumask` are dropped, and a mask with none of those bits fails with `EDEADLK`, as FreeBSD does when the new set has no CPU in common with the parent set. Guest-TID lookup and the update hold the thread registry lock so a join cannot release the target during access. Affinity is recorded for guest CPU reporting and child inheritance, not applied to host scheduling; PS5 behaviour has not been measured - [cpuset_getaffinity](../../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - the calling thread, another live guest thread selected by its guest TID, or the calling process is handled; other levels (ROOT/CPUSET) and other scopes (CPUSET/JAIL) throw as unmodelled, invalid levels or scopes return `EINVAL` and unknown thread or process IDs return `ESRCH`, following [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/sys/kern/kern_cpuset.c#L426-L437) guest-TID selection, not measured on PS5. For the process the mask is the union of the affinities of the live guest threads, as [FreeBSD 11.4](https://github.com/freebsd/freebsd-src/blob/d106002967eff712894ab8b5337bbbff106ce5e0/sys/kern/kern_cpuset.c#L1150-L1156) ORs the mask of every thread in the process; threads the guest did not create through libkernel are not included, and the console result is unverified diff --git a/tools/import_audit.py b/tools/import_audit.py index cb5f9ac8b4..320db9d959 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -410,10 +410,11 @@ def module_files(directory): for file in directory_files(directory, "*"): with file.open("rb") as handle: magic = handle.read(4) - if magic == b"\x7fELF": + if magic == b"\x7fELF" or (magic.startswith(b"MZ") and file.name.endswith(".guest.prx")): yield file elif (file.suffix.lower() in (".prx", ".sprx", ".suprx", ".debug_prx", ".elf") - or magic.startswith(b"MZ") or magic in (b"\x4f\x15\x3d\x1d", b"\x54\x14\xf5\xee")): + or magic.startswith(b"MZ") + or magic in (b"\x4f\x15\x3d\x1d", b"\x54\x14\xf5\xee")): raise AuditError(f"{file}: expected ELF module (first bytes {magic!r})") @@ -431,11 +432,15 @@ def read_modules(directories): raise AuditError(f"ambiguous guest module {file.name}: {previous} and {file} have different contents") continue images[file.name] = (file, digest) - own = elf_imports(file, data) - symbols, needed = own - imports[file.name] = (sorted(symbols), sorted(needed)) - for name in elf_exports(file, data): - exports[name].add(file.name) + if data.startswith(b"MZ"): + imports[file.name] = ([], []) + names = pe_exports(file, data) + else: + symbols, needed = elf_imports(file, data) + imports[file.name] = (sorted(symbols), sorted(needed)) + names = elf_exports(file, data) + for name in names: + exports[name.split(MODULE_SUFFIX, 1)[0]].add(file.name) return imports, {name: sorted(files) for name, files in exports.items()} @@ -739,6 +744,7 @@ def main(argv=None): if __name__ == "__main__": - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8", errors="replace") + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8", errors="replace") raise SystemExit(main()) diff --git a/tools/nid_names.py b/tools/nid_names.py index 82292e7e85..6ca6262fcb 100644 --- a/tools/nid_names.py +++ b/tools/nid_names.py @@ -197,7 +197,8 @@ def main(argv=None): for row in rows: print("%-40s %s" % (row["name"], row["nid"])) return 0 - for nid in args.nid or []: + args.nid = [nid.split("#", 1)[0] for nid in args.nid or []] + for nid in args.nid: if re.fullmatch(r"[A-Za-z0-9+\-]{11}", nid) is None: parser.error(f"invalid NID {nid!r}: expected 11 characters from A-Z, a-z, 0-9, + or -") diff --git a/tools/tests/test_build_eligibility.py b/tools/tests/test_build_eligibility.py index 56e41b71c6..e984e6030c 100644 --- a/tools/tests/test_build_eligibility.py +++ b/tools/tests/test_build_eligibility.py @@ -9,9 +9,11 @@ from pathlib import Path WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/build.yml" HARNESS = r""" +console.error("eligibility: harness-start", process.pid, process.version); const fs = require('fs'); const scenario = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); const script = fs.readFileSync(process.argv[3], 'utf8'); +console.error("eligibility: inputs-read", process.pid); const calls = []; const outputs = {}; const warnings = []; @@ -56,7 +58,10 @@ const core = { }; const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor; new AsyncFunction('github', 'context', 'core', script)(github, context, core).then( - () => process.stdout.write(JSON.stringify({ outputs, warnings, calls, summary })), + () => { + console.error("eligibility: gate-complete", process.pid); + process.stdout.write(JSON.stringify({ outputs, warnings, calls, summary })); + }, (error) => { console.error(error); process.exit(1); } ); """ @@ -93,8 +98,13 @@ class RelinkerEligibilityTests(unittest.TestCase): scenario.write_text(json.dumps({"commits": commits, "search": search, "files": list(files), "manual": manual}), encoding="utf-8") environment = dict(os.environ, PR_NUMBER="7", GITHUB_TRIGGERING_ACTOR="maintainer") - result = subprocess.run([self.node, "harness.js", "scenario.json", "gate.js"], cwd=self.root, - capture_output=True, text=True, env=environment, timeout=90) + try: + result = subprocess.run([self.node, "harness.js", "scenario.json", "gate.js"], cwd=self.root, + capture_output=True, text=True, env=environment, timeout=90) + except subprocess.TimeoutExpired as error: + stdout = error.stdout.decode(errors="replace") if isinstance(error.stdout, bytes) else error.stdout or "" + stderr = error.stderr.decode(errors="replace") if isinstance(error.stderr, bytes) else error.stderr or "" + self.fail(f"Node harness timed out after {error.timeout}s: {self.node}\nstdout:\n{stdout}\nstderr:\n{stderr}") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) return json.loads(result.stdout) diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 85d27593e0..39c0e5e614 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -207,6 +207,18 @@ class Workspace: class ImportAuditTests(unittest.TestCase): + def test_cli_error_paths_are_utf8_under_an_ascii_environment(self): + registry = self.work.root / "\u65e5\u672c\u8a9e.json" + result = subprocess.run( + [sys.executable, str(Path(import_audit.__file__)), str(registry), + "--libs", str(self.work.libs), "--source", str(self.work.source)], + env=dict(os.environ, PYTHONIOENCODING="ascii"), capture_output=True, timeout=20) + self.assertEqual(result.returncode, 2) + self.assertIn(str(registry), result.stderr.decode("utf-8")) + self.assertNotIn(b"Traceback", result.stderr) + self.assertEqual(result.stdout, b"") + + def test_registry_weak_flags_are_booleans(self): registry = self.work.root / "game.json" output = self.work.root / "audit.json" @@ -1334,6 +1346,18 @@ class ImportAuditTests(unittest.TestCase): struct.pack_into(">"%~dp0compiler-invoked" echo %*\nexit /b 99\n', encoding='utf-8') + else: + compiler.write_text('#!/bin/sh\nif [ "$#" -eq 1 ]; then case "$1" in -print-sysroot|--version|-Wl,--help|-Wl,--version) exit 0;; esac; fi\nif [ "$#" -eq 2 ] && [ "$1" = "-v" ] && [ "$2" = "--help" ]; then exit 0; fi\nprintf "%s\\n" "$*" >> "$(dirname "$0")/compiler-invoked"\nexit 99\n', encoding='utf-8') + compiler.chmod(0o755) + mode = 'ON' if spirv_tools else 'OFF' + (root / 'CMakeLists.txt').write_text(f'''cmake_minimum_required(VERSION 3.20) +set(CMAKE_CXX_COMPILER "{compiler.as_posix()}") +set(CMAKE_CXX_COMPILER_ID GNU) +set(CMAKE_CXX_COMPILER_VERSION 15.2.0) +set(CMAKE_CXX_COMPILER_ID_RUN TRUE) +set(CMAKE_CXX_COMPILER_FORCED TRUE) +set(CMAKE_CXX_COMPILER_WORKS TRUE) +set(CMAKE_CXX_ABI_COMPILED TRUE) +set(CMAKE_CXX_STANDARD_COMPUTED_DEFAULT 17) +set(CMAKE_CXX_EXTENSIONS_COMPUTED_DEFAULT ON) +set(CMAKE_CXX_COMPILE_FEATURES cxx_std_20) +project(ShaderCacheSourceFixture LANGUAGES CXX) +set(CMAKE_BUILD_TYPE Release) +set(ANYPS5_ENABLE_SPIRV_TOOLS {mode}) +set(ANYPS5_USE_SYSTEM_DEPS ON) +set(ANYPS5_SYSTEM_DEPS_STAMP fixture) +add_library(fixture_runtime INTERFACE) +add_library(glslang INTERFACE) +add_library(SPIRV INTERFACE) +add_library(SPIRV-Tools-opt INTERFACE) +add_library(SPIRV-Tools-static INTERFACE) +include("${{CMAKE_CURRENT_SOURCE_DIR}}/core/shader/recompiler/CMakeLists.txt") +add_shader_recompiler(shader_recompiler_probe fixture_runtime GUARD_SOURCES "${{CMAKE_CURRENT_SOURCE_DIR}}/driver.cpp") +get_target_property(runtimeSources shader_recompiler_probe SOURCES) +list(JOIN runtimeSources "\\n" runtimeSources) +file(WRITE "${{CMAKE_CURRENT_BINARY_DIR}}/runtime-sources.txt" "${{runtimeSources}}\\n") +''', encoding='utf-8') + self.configure() + + def configure(self, flags=''): + result = subprocess.run([CMAKE, '-S', str(self.root), '-B', str(self.build), '-G', 'Ninja', f'-DCMAKE_CXX_FLAGS={flags}'], + capture_output=True, encoding='utf-8', errors='replace', timeout=15) + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + if (self.root / 'compiler-invoked').exists(): + raise AssertionError('The generator fixture invoked a compiler: ' + (self.root / 'compiler-invoked').read_text(encoding='utf-8')) + self.generated = self.build / 'shader_recompiler_probe_generated' + self.listing = self.generated / 'sources.txt' + ninja = (self.build / 'build.ninja').read_text(encoding='utf-8') + command = next(line for line in ninja.splitlines() if line.startswith(' COMMAND = ') and 'ShaderCacheVersion.cmake' in line) + self.extra = re.search(r'"-DEXTRA=([^"]*)"', command).group(1) + self.compiled = [] + for line in ninja.splitlines(): + match = re.match(r'build .*: CXX_COMPILER__shader_recompiler_probe_[^ ]+ (.*?) \|\|', line) + if match: + self.compiled.append(Path(match.group(1).replace('$:', ':').replace('$ ', ' '))) + + def version(self, success=True): + self.calls += 1 + output = self.build / f'probe-version-{self.calls}.cpp' + result = subprocess.run([CMAKE, f'-DROOT={self.root.as_posix()}', + f'-DRECOMPILER_DIR={self.recompiler.as_posix()}', + f'-DLIST_FILE={self.listing.as_posix()}', f'-DOUTPUT={output.as_posix()}', + f'-DEXTRA={self.extra}', '-P', str(self.recompiler / 'ShaderCacheVersion.cmake')], + capture_output=True, encoding='utf-8', errors='replace', timeout=10) + if not success: + if result.returncode == 0 or output.exists(): + raise AssertionError('The generator accepted an unkeyed dependency') + return result.stdout + result.stderr + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + return re.search(r'SourceVersion = 0x([0-9a-f]+)ull', output.read_text(encoding='utf-8')).group(1) + + def sources(self): + return {Path(line.split(' ', 1)[1]) for line in self.listing.read_text(encoding='utf-8').splitlines() if line.startswith('sources ')} + + +class ShaderCacheRuntimeSourceTests(unittest.TestCase): + def fixture(self, spirv_tools=False): + temporary = tempfile.TemporaryDirectory(prefix='shader_cache_sources_') + self.addCleanup(temporary.cleanup) + return Fixture(Path(temporary.name), spirv_tools) + + def test_compiler_metadata_queries_and_compilation_guard(self): + fixture = self.fixture() + allowed = [('-print-sysroot',), ('--version',), ('-v', '--help'), ('-Wl,--help',), ('-Wl,--version',)] + rejected = [('-v', '--help', 'fixture.cpp'), ('-v', '--help', '-c', 'fixture.cpp'), + ('--help', '-v'), ('-v',), ('--help',), ('-c', 'fixture.cpp'), + ('fixture.cpp', '-o', 'fixture.exe'), ('--version', 'fixture.cpp'), + ('-print-sysroot', 'fixture.cpp'), ('-Wl,--help', 'fixture.cpp'), ('-Wl,--version', '-c', 'fixture.cpp')] + script = fixture.root / 'metadata.cmake' + lines = ['cmake_minimum_required(VERSION 3.20)', + f'set(CMAKE_CXX_COMPILER "{fixture.compiler.as_posix()}")', + 'include(Platform/Apple-GNU)', + 'cmake_gnu_set_sysroot_flag(CXX)', + 'cmake_gnu_set_osx_deployment_target_flag(CXX)'] + for args in allowed: + arguments = ' '.join(f'"{arg}"' for arg in args) + lines += [f'execute_process(COMMAND "${{CMAKE_CXX_COMPILER}}" {arguments} RESULT_VARIABLE result)', + 'if(NOT result EQUAL 0)', 'message(FATAL_ERROR "Metadata query was rejected")', 'endif()'] + marker = fixture.root / 'compiler-invoked' + lines += [f'if(EXISTS "{marker.as_posix()}")', + 'message(FATAL_ERROR "Compiler metadata reached the compilation guard")', 'endif()'] + for args in rejected: + arguments = ' '.join(f'"{arg}"' for arg in args) + lines += [f'execute_process(COMMAND "${{CMAKE_CXX_COMPILER}}" {arguments} RESULT_VARIABLE result)', + 'if(NOT result EQUAL 99)', 'message(FATAL_ERROR "Unexpected compiler arguments were accepted")', 'endif()'] + script.write_text('\n'.join(lines) + '\n', encoding='utf-8') + result = subprocess.run([CMAKE, '-P', str(script)], capture_output=True, + encoding='utf-8', errors='replace', timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(marker.read_text(encoding='utf-8').splitlines()), len(rejected)) + + def test_runtime_source_graph_off_and_on(self): + for enabled in (False, True): + with self.subTest(spirv_tools=enabled): + fixture = self.fixture(enabled) + generated = fixture.generated / 'ShaderCacheVersion.cpp' + selected = {Path(line) for line in (fixture.build / 'runtime-sources.txt').read_text(encoding='utf-8').splitlines()} + self.assertEqual(set(fixture.compiled), selected) + self.assertNotIn(generated, fixture.sources()) + self.assertEqual({path for path in fixture.sources() if path.suffix in ('.cpp', '.cc', '.cxx')}, selected - {generated}) + self.assertIn(fixture.recompiler / 'tests/DummyShaders.cpp', fixture.sources()) + self.assertEqual(fixture.recompiler / 'SpirvBackend/src/SpirvOptimizer.cpp' in fixture.sources(), enabled) + fixture.version() + + def test_uncompiled_test_sources_do_not_change_version(self): + fixture = self.fixture() + previous = fixture.version() + for name in ('CacheFileReadTests.cpp', 'RequestSerializerTests.cpp'): + path = fixture.recompiler / 'tests' / name + self.assertNotIn(path, fixture.sources()) + path.write_bytes(path.read_bytes() + b'\n') + self.assertEqual(fixture.version(), previous, name) + (fixture.recompiler / 'tests/NewFixture.cpp').write_text('int new_fixture;\n', encoding='utf-8') + fixture.configure() + self.assertEqual(fixture.version(), previous) + + def test_runtime_sources_headers_and_cmake_change_version(self): + fixture = self.fixture() + previous = fixture.version() + for name in ('Recompiler.cpp', 'tests/DummyShaders.cpp', 'tests/DummyShaders.hpp', 'BdaAbi.hpp', 'CMakeLists.txt', 'ShaderCacheVersion.cmake'): + with self.subTest(source=name): + path = fixture.recompiler / name + before = path.read_bytes() + path.write_bytes(before + b'\n') + self.assertNotEqual(fixture.version(), previous) + path.write_bytes(before) + self.assertEqual(fixture.version(), previous) + + def test_flags_change_version(self): + fixture = self.fixture() + previous = fixture.version() + fixture.configure('-DSHADER_CACHE_FIXTURE=1') + self.assertNotEqual(fixture.version(), previous) + + def test_optimizer_content_only_changes_enabled_version(self): + for enabled in (False, True): + with self.subTest(spirv_tools=enabled): + fixture = self.fixture(enabled) + previous = fixture.version() + optimizer = fixture.recompiler / 'SpirvBackend/src/SpirvOptimizer.cpp' + optimizer.write_bytes(optimizer.read_bytes() + b'\n') + self.assertEqual(fixture.version() == previous, not enabled) + + def test_uncompiled_internal_include_is_rejected(self): + fixture = self.fixture() + source = fixture.recompiler / 'Recompiler.cpp' + source.write_text('#include "tests/CacheFileReadTests.cpp"\n' + source.read_text(encoding='utf-8'), encoding='utf-8') + self.assertIn('does not hash', fixture.version(False)) + + def test_unhashed_external_include_is_rejected(self): + fixture = self.fixture() + (fixture.root / 'external.hpp').write_text('int external_fixture;\n', encoding='utf-8') + source = fixture.recompiler / 'Recompiler.cpp' + source.write_text('#include "../../../external.hpp"\n' + source.read_text(encoding='utf-8'), encoding='utf-8') + self.assertIn('does not hash', fixture.version(False)) + + def test_unreviewed_driver_call_is_rejected(self): + fixture = self.fixture() + fixture.guard.write_text('ShaderRecompiler::UnreviewedFixtureCall();\n', encoding='utf-8') + self.assertIn('has not been reviewed', fixture.version(False)) + + +unittest.main() diff --git a/tools/tests/test_trace_digest.py b/tools/tests/test_trace_digest.py index 2072a7a4d5..c0b7b67112 100644 --- a/tools/tests/test_trace_digest.py +++ b/tools/tests/test_trace_digest.py @@ -500,6 +500,25 @@ class IntervalDigestTests(unittest.TestCase): self.assertEqual(trace_digest.sparkline([0.0, 0.0]), "..") self.assertEqual(trace_digest.sparkline([]), "") + def test_largest_phase_does_not_depend_on_json_key_order(self): + phases = [("Driver.Completion", 1.0), ("Driver.Packet", 7.0), ("Driver.Present", 2.0)] + for offset in range(len(phases)): + for sort_keys in (False, True): + with self.subTest(offset=offset, sort_keys=sort_keys): + line = interval_line(0.0, 30.0, 20.0, 5.0, + top_phases_ms_per_frame=dict(phases[offset:] + phases[:offset])) + self.path.write_text(json.dumps(line, sort_keys=sort_keys) + "\n", encoding="utf-8") + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(trace_digest.main([str(self.path), "--json"]), 0) + result = json.loads(output.getvalue()) + self.assertEqual(result["intervals"][0]["top_phase"], + {"name": "Driver.Packet", "ms_per_frame": 7.0}) + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.assertEqual(trace_digest.main([str(self.path)]), 0) + self.assertIn("| Driver.Packet 7.0 |", output.getvalue()) + def test_command_line_table_and_json(self): output = io.StringIO() with contextlib.redirect_stdout(output): diff --git a/tools/tests/test_unwrap_self.py b/tools/tests/test_unwrap_self.py index 05436dd626..0d5bf8caf0 100644 --- a/tools/tests/test_unwrap_self.py +++ b/tools/tests/test_unwrap_self.py @@ -14,6 +14,9 @@ PROGRAM_TABLE_OFFSET = 0x40 PT_LOAD = 1 PT_DYNAMIC = 2 PT_TLS = 7 +PT_NOTE = 4 +SCE_COMMENT = 0x6FFFFF00 +SCE_VERSION = 0x6FFFFF01 FIRST_REGION = 0x400 HEADER = b"\x7fELF\x02\x01\x01\x09" + b"\x00" * 56 @@ -147,6 +150,24 @@ class UnwrapTest(unittest.TestCase): self.assertEqual(tls[2], read_only[2] + 0x80) self.assertEqual(tls[5], 0) + def test_segments_after_the_recorded_comment_are_read_from_the_container(self): + segments = default_segments() + [(SCE_COMMENT, 4, 0x15A0, 0x0, 0x20, 0x10), (PT_NOTE, 4, 0x15C0, 0x0, 0x18, 4), + (SCE_VERSION, 4, 0x15D8, 0x0, 0x10, 1)] + records = default_records() + [(0x410004, 0xA80, 0x20), (0x302804, 0xAA0, 0x20)] + payloads = dict(default_payloads()) + payloads[0xAA0] = pattern(11, 0x20) + payloads[0xAC0] = pattern(13, 0x10) + result, output = self.convert(build_container(segments, records, payloads)) + self.assertEqual(result.returncode, 0, result.stderr) + data, headers = read_program_headers(output) + comment = [header for header in headers if header[0] == SCE_COMMENT][0] + version = [header for header in headers if header[0] == SCE_VERSION][0] + note = [header for header in headers if header[0] == PT_NOTE][0] + self.assertEqual(data[comment[2]:comment[2] + comment[5]], pattern(11, 0x20)) + self.assertEqual(version[5], 0x10) + self.assertEqual(data[version[2]:version[2] + version[5]], pattern(13, 0x10)) + self.assertEqual(note[5], 0) + def test_declared_offsets_are_used_when_the_container_has_no_records(self): segments = [(PT_LOAD, 1, 0x800, 0x0, 0x40, 0x1000), (PT_LOAD, 4, 0x900, 0x1000, 0x40, 0x1000)] payloads = {0x800: pattern(3, 0x40), 0x900: pattern(5, 0x40)} diff --git a/tools/trace_digest.py b/tools/trace_digest.py index b12acff8d3..6c5a424c21 100644 --- a/tools/trace_digest.py +++ b/tools/trace_digest.py @@ -280,11 +280,12 @@ def digest_intervals(lines): for line in lines: workers = line.get("worker_busy_pct", {}) busiest = max(workers.items(), key=lambda entry: entry[1]) if workers else None - phases = list(line.get("top_phases_ms_per_frame", {}).items()) + phases = line.get("top_phases_ms_per_frame", {}) + largest = max(phases.items(), key=lambda entry: entry[1]) if phases else None rows.append({"start_s": line["interval_start_s"], "end_s": line["interval_end_s"], "presents": line["presents"], "frame_ms": line["frame_ms"], "gpu_busy_ms_per_present": line["gpu_busy_ms_per_present"], "busiest_worker": {"thread": busiest[0], "pct": busiest[1]} if busiest else None, - "top_phase": {"name": phases[0][0], "ms_per_frame": phases[0][1]} if phases else None, + "top_phase": {"name": largest[0], "ms_per_frame": largest[1]} if largest else None, "trace_dropped": line.get("trace_dropped", False)}) first = lines[0] if lines else {} keys = ("commit", "dirty", "env", "gpu", "driver", "os", "resolution") diff --git a/tools/unwrap_self.py b/tools/unwrap_self.py index 5e16158026..7225820aaa 100644 --- a/tools/unwrap_self.py +++ b/tools/unwrap_self.py @@ -8,9 +8,14 @@ differ inside a container: - The inner ELF program headers carry stale offsets. A SELF inserts its own metadata blocks between the segments, so the segment data sits at the offsets listed in the SELF segment-record table at file offset 0x40, records of 0x20 - bytes holding flags, data offset, size and the size again. Segments the table - does not cover are placed by the vaddr to file mapping of the PT_LOAD that - contains them; segments without a vaddr carry the shift of the last mapped one. + bytes holding flags, data offset, size and the size again. A record with flag + 0x800 holds the data of the program header whose index is in flags bits 20 + and up. Segments the table does not cover are placed by the vaddr to file + mapping of the PT_LOAD that contains them. Segments without a vaddr that + follow the last recorded one in the ELF are stored right after it, back to + back in ELF order, except a PT_NOTE, which the container leaves out and which + is emitted without file bytes. Other segments without a vaddr carry the shift + of the last placed one. - The segments are packed without honouring p_align, while ELF requires p_offset % p_align == p_vaddr % p_align. Every region is copied to an offset that satisfies it, which the relinker enforces on guest modules. @@ -33,8 +38,11 @@ SELF_MAGIC = 0x1D3D154F SELF_MAGIC_KERNEL = 0xEEF51454 RECORD_OFFSET = 0x40 RECORD_SIZE = 0x20 +RECORD_HAS_SEGMENT = 0x800 +RECORD_SEGMENT_SHIFT = 20 FIRST_REGION = 0x400 PT_LOAD = 1 +PT_NOTE = 4 ENTROPY_LIMIT = 7.5 ENTROPY_SAMPLE = 0x10000 @@ -127,8 +135,29 @@ def parse_elf(data): def place_segments(data, headers, records): loads = [header for header in headers if header["type"] == PT_LOAD] used = set() + placed = set() + for index, (flags, segment, size) in enumerate(records): + target = flags >> RECORD_SEGMENT_SHIFT + if flags & RECORD_HAS_SEGMENT and target < len(headers) and target not in placed and headers[target]["size"] == size: + headers[target]["source"] = segment + placed.add(target) + used.add(index) + recorded = [header for header in headers if header["index"] in placed and header["vaddr"] == 0 and header["type"] != PT_LOAD] + if recorded: + last = max(recorded, key=lambda header: header["offset"]) + cursor = last["source"] + last["size"] + trailing = [header for header in headers if header["index"] not in placed and header["vaddr"] == 0 + and header["type"] != PT_LOAD and header["size"] and header["offset"] > last["offset"]] + for header in sorted(trailing, key=lambda header: header["offset"]): + placed.add(header["index"]) + if header["type"] == PT_NOTE: + header["source"] = 0 + header["size"] = 0 + continue + header["source"] = cursor + cursor += header["size"] for header in loads: - if header["size"] == 0: + if header["size"] == 0 or header["index"] in placed: continue for index, (_, segment, size) in enumerate(records): if index in used: @@ -150,6 +179,10 @@ def place_segments(data, headers, records): for header in headers: if header["type"] == PT_LOAD: continue + if header["index"] in placed: + if header["size"] and (tail is None or header["offset"] > tail[0]): + tail = (header["offset"], header["source"] - header["offset"]) + continue if header["vaddr"] != 0: host = load_for(header["vaddr"], header["size"]) if host is None: