Mirror-bot/AnyPS5
1
0 Fork 0
mirror of https://github.com/boykopovar/AnyPS5.git synced 2026-10-28 12:05:06 +00:00
Code Issues Projects Releases Activity

Merge remote-tracking branch 'origin/main' into fix/socketpair-fstat-inode

Browse Source 
# Conflicts:
#	core/libs/prx/libkernel/Socket/src/Udp.cpp
#	docs/dev/debt/libkernel.md
This commit is contained in:
Retr0 committed 2026-10-26 21:09:20 +02:00
2 parents e7de2a5e37 afc780e681
commit d99208fa84
1026 files changed
+55359 -6137
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/build.yml
+1 -1
View File
@@ -421,7 +421,7 @@ jobs:
linux:
needs: eligibility
if: ${{ (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
if: ${{ !github.event.repository.fork && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.linux)) && needs.eligibility.outputs.full == 'true' }}
runs-on: [self-hosted, linux, x64, anyps5]
timeout-minutes: 60
env:
.github/workflows/input-mapper.yml
+29
View File
@@ -0,0 +1,29 @@
name: Input mapper
on:
push:
branches: [main]
pull_request:
branches: [main]
paths:
- '.github/workflows/input-mapper.yml'
- 'tools/input-mapper/**'
- 'core/libs/prx/libScePad/include/**'
- 'core/libs/prx/libSceVideoOut/src/InputMapping.cpp'
permissions:
contents: read
jobs:
input-mapper:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-node@v7
with:
node-version: '22'
package-manager-cache: false
- run: npm --prefix tools/input-mapper test
.github/workflows/pages-progress.yml
+1
View File
@@ -18,6 +18,7 @@ concurrency:
jobs:
progress:
if: ${{ !github.event.repository.fork }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
.github/workflows/pages.yml
+1 -1
View File
@@ -33,7 +33,7 @@ concurrency:
jobs:
pages:
if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
if: ${{ !github.event.repository.fork && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') }}
runs-on: ubuntu-latest
environment:
name: github-pages
.github/workflows/web-tests.yml
+1 -1
View File
@@ -36,4 +36,4 @@ jobs:
node --check demo/web/output-directory.mjs
node --check demo/web/worker.mjs
- name: Test browser converter helpers
run: node --test demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs
run: node --test demo/web/tests/app.test.mjs demo/web/tests/batch.test.mjs demo/web/tests/output-directory.test.mjs
.gitmodules
+6
View File
@@ -1,3 +1,6 @@
[submodule "3rdparty/jxrlib"]
path = 3rdparty/jxrlib
url = https://github.com/4creators/jxrlib.git
[submodule "3rdparty/SDL2"]
path = 3rdparty/SDL2
url = https://github.com/libsdl-org/SDL.git
@@ -61,3 +64,6 @@
[submodule "3rdparty/vorbis"]
path = 3rdparty/vorbis
url = https://github.com/xiph/vorbis.git
[submodule "3rdparty/nghttp2"]
path = 3rdparty/nghttp2
url = https://github.com/nghttp2/nghttp2.git
3rdparty/SPIRV-Headers
Vendored
+1 -1
Submodule 3rdparty/SPIRV-Headers updated: 01e0577914...b824a462d4.
3rdparty/SPIRV-Tools
Vendored
+1 -1
Submodule 3rdparty/SPIRV-Tools updated: 7f2d9ee926...262bdab481.
3rdparty/jxrlib
Vendored Submodule
+1
Submodule 3rdparty/jxrlib added at f752187986.
3rdparty/nghttp2
Vendored Submodule
+1
Submodule 3rdparty/nghttp2 added at 85e300c79f.
CMakeLists.txt
+34 -5
View File
@@ -136,10 +136,17 @@ if(BUILD_TESTING)
${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py
${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND})
set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20)
add_test(NAME shader_cache_runtime_sources COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND})
set_tests_properties(shader_cache_runtime_sources PROPERTIES TIMEOUT 180)
foreach(shard RANGE 0 6)
if(shard EQUAL 0)
set(shader_cache_sources_test_name shader_cache_runtime_sources)
else()
set(shader_cache_sources_test_name shader_cache_runtime_sources_shard_${shard})
endif()
add_test(NAME ${shader_cache_sources_test_name} COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_shader_cache_runtime_sources.py
${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler ${CMAKE_COMMAND} --shard ${shard}/7)
set_tests_properties(${shader_cache_sources_test_name} PROPERTIES TIMEOUT 180)
endforeach()
add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
@@ -164,6 +171,8 @@ if(BUILD_TESTING)
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
add_test(NAME unwrap_self COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_unwrap_self.py)
add_test(NAME audit_dump COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_audit_dump.py)
foreach(shard RANGE 0 11)
if(shard EQUAL 0)
set(conventions_test_name conventions_function_bodies)
@@ -241,8 +250,19 @@ if(BUILD_TESTING)
add_test(NAME pad_timeline COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pad_timeline.py)
set_tests_properties(pad_timeline PROPERTIES TIMEOUT 20)
set(check_compile_modes_source "${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_modes.py")
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${check_compile_modes_source}")
file(STRINGS "${check_compile_modes_source}" check_compile_modes_cases REGEX "^ def test_")
foreach(test_case IN LISTS check_compile_modes_cases)
string(REGEX REPLACE "^ def (test_[A-Za-z0-9_]+).*" "\\1" test_case "${test_case}")
add_test(NAME check_compile_modes_${test_case} COMMAND ${Python3_EXECUTABLE}
"${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_modes.py" CheckCompileModeTests.${test_case})
set_tests_properties(check_compile_modes_${test_case} PROPERTIES TIMEOUT 30)
endforeach()
add_test(NAME trace_digest COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_trace_digest.py)
add_test(NAME symbolize_samples COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_symbolize_samples.py)
add_test(NAME conventions_bom_shebang COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_bom_shebang.py)
set_tests_properties(conventions_bom_shebang PROPERTIES TIMEOUT 20)
@@ -264,10 +284,17 @@ if(BUILD_TESTING)
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_blob_reads.py)
add_test(NAME conventions_agent_files COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_agent_files.py)
add_test(NAME conventions_alignment COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_alignment.py)
add_test(NAME conventions_pr_template COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_pr_template.py)
add_test(NAME check_compile_preprocessor COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile_preprocessor.py)
set_tests_properties(check_compile_preprocessor PROPERTIES TIMEOUT 20)
add_test(NAME check_compile COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py)
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileTests)
add_test(NAME check_compile_outputs COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_compile.py CheckCompileOutputTests)
add_test(NAME performance_compare COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_performance_compare.py)
set_tests_properties(performance_compare PROPERTIES TIMEOUT 20)
@@ -348,6 +375,8 @@ if(NOT ANYPS5_USE_SYSTEM_DEPS)
endif()
endif()
include(cmake/Jxr.cmake)
set(LIBUSB_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(LIBUSB_INSTALL_TARGETS OFF CACHE BOOL "" FORCE)
set(LIBUSB_BUILD_TESTING OFF CACHE BOOL "" FORCE)
CONTRIBUTING.md
+1
View File
@@ -12,6 +12,7 @@
- Implement the general behaviour of a function, not what one title happens to need.
- Don't add replacements for modules that titles ship themselves in `sce_module/`, `sce_modules/` or `prx/`: engine or middleware modules (Cohtml, FMOD, GOG Galaxy) and SDK libraries that only wrap other system libraries. The relinker converts and loads the title's own module, and a host library with the same name is left out of `DT_NEEDED`. Only system libraries, which reach the kernel or the hardware, are reimplemented in [core/libs/prx](core/libs/prx). A host copy may still exist for a wrapped library that some dumps lack entirely, like libSceNpCppWebApi: the title's own module wins whenever one is shipped, so it is only ever consulted by titles that ship nothing.
- Avoid non-standard extensions (`__attribute__`, etc.) where standard C++ is enough. Helper symbols that must not become NIDs use the `_nid_no_patch` or `_nid_no_patch_cut` suffix.
- `alignas` takes at most 8192: MinGW GCC, the Windows compiler, rejects static objects aligned above it. A larger block is aligned at run time inside a bigger buffer.
- The relinker uses only the C++20 standard library.
- Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system. With `ANYPS5_USE_SYSTEM_DEPS=ON` the packages packaged by the distributions (SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools) come from the system instead; that resolution lives in [cmake/SystemDeps.cmake](cmake/SystemDeps.cmake) and the default build never reads the system.
- Don't add tests that only check that a symbol is exported: a missing export already fails at startup.
cmake/Jxr.cmake
+44
View File
@@ -0,0 +1,44 @@
enable_language(C)
set(JXR_SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/jxrlib)
function(configure_jxr_native target)
target_include_directories(${target} SYSTEM PRIVATE
${JXR_SOURCE_DIR}/common/include ${JXR_SOURCE_DIR}/image/sys ${JXR_SOURCE_DIR}/jxrgluelib)
target_compile_definitions(${target} PRIVATE DISABLE_PERF_MEASUREMENT)
if(WIN32)
target_compile_definitions(${target} PRIVATE WIN32 __in=)
target_compile_options(${target} PRIVATE -include ${JXR_SOURCE_DIR}/image/sys/strcodec.h)
else()
target_compile_definitions(${target} PRIVATE __ANSI__)
target_compile_options(${target} PRIVATE -include ${CMAKE_SOURCE_DIR}/cmake/JxrCompat.h)
endif()
set_target_properties(${target} PROPERTIES C_STANDARD 99 C_STANDARD_REQUIRED ON
POSITION_INDEPENDENT_CODE ON C_VISIBILITY_PRESET hidden)
endfunction()
add_library(jxr STATIC EXCLUDE_FROM_ALL
${JXR_SOURCE_DIR}/image/sys/adapthuff.c
${JXR_SOURCE_DIR}/image/sys/image.c
${JXR_SOURCE_DIR}/image/sys/strcodec.c
${JXR_SOURCE_DIR}/image/sys/strPredQuant.c
${JXR_SOURCE_DIR}/image/sys/strTransform.c
${JXR_SOURCE_DIR}/image/decode/decode.c
${JXR_SOURCE_DIR}/image/decode/JXRTranscode.c
${JXR_SOURCE_DIR}/image/decode/postprocess.c
${JXR_SOURCE_DIR}/image/decode/segdec.c
${JXR_SOURCE_DIR}/image/decode/strdec.c
${JXR_SOURCE_DIR}/image/decode/strInvTransform.c
${JXR_SOURCE_DIR}/image/decode/strPredQuantDec.c
${JXR_SOURCE_DIR}/image/encode/encode.c
${JXR_SOURCE_DIR}/image/encode/segenc.c
${JXR_SOURCE_DIR}/image/encode/strenc.c
${JXR_SOURCE_DIR}/image/encode/strFwdTransform.c
${JXR_SOURCE_DIR}/image/encode/strPredQuantEnc.c
${JXR_SOURCE_DIR}/jxrgluelib/JXRGlue.c
${JXR_SOURCE_DIR}/jxrgluelib/JXRGlueJxr.c
${JXR_SOURCE_DIR}/jxrgluelib/JXRGluePFC.c
${JXR_SOURCE_DIR}/jxrgluelib/JXRMeta.c)
configure_jxr_native(jxr)
if(UNIX)
target_link_libraries(jxr PRIVATE m)
endif()
cmake/JxrCompat.h
+8
View File
@@ -0,0 +1,8 @@
#pragma once
#include <wchar.h>
#include "strcodec.h"
#if defined(__ANSI__) && !defined(WIN32) && !defined(_BIG__ENDIAN_)
U32 _byteswap_ulong(U32 bits);
#endif
core/libs/CMakeLists.txt
+131 -1
View File
@@ -269,6 +269,15 @@ if(BUILD_TESTING)
set_tests_properties(guest_kernel_sys_sdk_version PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
add_test_executable(guest_kernel_sys_console_identity_tests tests/GuestKernelSysConsoleIdentity.cpp)
target_include_directories(guest_kernel_sys_console_identity_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_console_identity_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_console_identity_tests)
add_test(NAME guest_kernel_sys_console_identity COMMAND guest_kernel_sys_console_identity_tests)
if(WIN32)
set_tests_properties(guest_kernel_sys_console_identity PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
add_test_executable(nid_buffer_validation_tests nid/tests/BufferValidation.cpp)
target_include_directories(nid_buffer_validation_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
target_compile_features(nid_buffer_validation_tests PRIVATE cxx_std_20)
@@ -278,6 +287,32 @@ if(BUILD_TESTING)
add_test(NAME nid_buffer_validation COMMAND nid_buffer_validation_tests)
set_tests_properties(nid_buffer_validation PROPERTIES TIMEOUT 10)
add_test_executable(guest_dinkumware_fetch_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/GuestDinkumwareFetch.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/NidCompute.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/Sha1.cpp)
target_include_directories(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
target_link_libraries(guest_dinkumware_fetch_tests PRIVATE ${CMAKE_DL_LIBS})
configure_windows_unwind(guest_dinkumware_fetch_tests)
if(MINGW)
target_link_libraries(guest_dinkumware_fetch_tests PRIVATE libc)
endif()
add_custom_target(guest_dinkumware_fetch_libraries DEPENDS ${LIBC_PATCHED})
add_dependencies(guest_dinkumware_fetch_tests guest_dinkumware_fetch_libraries)
foreach(operation IN ITEMS and or xor)
foreach(width IN ITEMS 1 2 4)
add_test(NAME guest_dinkumware_fetch_${operation}_${width} COMMAND guest_dinkumware_fetch_tests
$<TARGET_FILE_DIR:libc> unpatched ${operation} ${width})
add_test(NAME guest_dinkumware_fetch_${operation}_${width}_patched COMMAND guest_dinkumware_fetch_tests
${PATCHED_DIR} patched ${operation} ${width})
set_tests_properties(guest_dinkumware_fetch_${operation}_${width}
guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_dinkumware_fetch_${operation}_${width} PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(guest_dinkumware_fetch_${operation}_${width}_patched PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:${PATCHED_DIR}")
endif()
endforeach()
endforeach()
add_test_executable(pe_import_first_tests allocator/tests/PeImportFirstTests.cpp allocator/PeImportFirst.cpp)
target_include_directories(pe_import_first_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_compile_features(pe_import_first_tests PRIVATE cxx_std_20)
@@ -329,6 +364,11 @@ if(BUILD_TESTING)
target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++)
endif()
add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR})
if(MINGW)
get_filename_component(patchedLibrariesRuntimeDirectory "${CMAKE_CXX_COMPILER}" DIRECTORY)
set_tests_properties(patched_libraries PROPERTIES
ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${patchedLibrariesRuntimeDirectory}")
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_library(host_libstdcxx_exception_module SHARED tests/HostLibstdcxxExceptionModule.cpp)
target_link_libraries(host_libstdcxx_exception_module PRIVATE ${PATCHED_DIR}/libc.prx)
@@ -366,14 +406,52 @@ if(BUILD_TESTING)
target_link_libraries(guest_kernel_sys_pthread_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_pthread_tests)
add_test(NAME guest_kernel_sys_pthread COMMAND guest_kernel_sys_pthread_tests)
add_test_executable(guest_kernel_sys_constants_tests tests/GuestKernelSysConstants.cpp)
target_include_directories(guest_kernel_sys_constants_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_constants_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_constants_tests)
add_test(NAME guest_kernel_sys_constants COMMAND guest_kernel_sys_constants_tests)
set_tests_properties(guest_kernel_sys_constants PROPERTIES TIMEOUT 20)
add_test_executable(guest_firmware_constant_returns_tests tests/GuestFirmwareConstantReturns.cpp)
target_include_directories(guest_firmware_constant_returns_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_firmware_constant_returns_tests PRIVATE libSceComposite libSceFsInternalForVsh libSceHttpCache libScePlayReady libSceSdma libSceSsl libSceVisionManager libmdbg_syscore libkernel libc)
configure_windows_unwind(guest_firmware_constant_returns_tests)
add_test(NAME guest_firmware_constant_returns COMMAND guest_firmware_constant_returns_tests)
set_tests_properties(guest_firmware_constant_returns PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_firmware_constant_returns PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceComposite>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceFsInternalForVsh>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttpCache>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libScePlayReady>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSdma>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSsl>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceVisionManager>;PATH=path_list_prepend:$<TARGET_FILE_DIR:libmdbg_syscore>")
endif()
if(Python3_Interpreter_FOUND)
add_test(NAME nid_verneed COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_verneed.py $<TARGET_FILE:nid_patcher>)
endif()
if(WIN32)
set_tests_properties(guest_kernel_sys guest_kernel_sys_thread_defaults guest_kernel_sys_barrier guest_kernel_sys_pthread PROPERTIES ENVIRONMENT_MODIFICATION
set_tests_properties(guest_kernel_sys guest_kernel_sys_thread_defaults guest_kernel_sys_barrier guest_kernel_sys_pthread guest_kernel_sys_constants PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
add_test_executable(guest_kernel_sys_threads_tests tests/GuestKernelSysThreads.cpp
nid/src/NidCompute.cpp nid/src/Sha1.cpp)
target_include_directories(guest_kernel_sys_threads_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/nid/include)
target_link_libraries(guest_kernel_sys_threads_tests PRIVATE ${CMAKE_DL_LIBS})
add_custom_target(guest_kernel_sys_threads_libraries DEPENDS
${PATCHED_DIR}/libkernel_sys.prx ${PATCHED_DIR}/libkernel.prx ${LIBC_PATCHED})
add_dependencies(guest_kernel_sys_threads_tests guest_kernel_sys_threads_libraries)
configure_windows_unwind(guest_kernel_sys_threads_tests)
if(MINGW)
target_compile_options(guest_kernel_sys_threads_tests PRIVATE -fasynchronous-unwind-tables)
target_link_options(guest_kernel_sys_threads_tests PRIVATE -static -static-libgcc -static-libstdc++)
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_test(NAME guest_kernel_sys_threads_main_late_load COMMAND guest_kernel_sys_threads_tests ${UNPATCHED_DIR} unpatched main_late_load)
add_test(NAME guest_kernel_sys_threads_main_late_load_patched COMMAND guest_kernel_sys_threads_tests ${PATCHED_DIR} patched main_late_load)
set_tests_properties(guest_kernel_sys_threads_main_late_load guest_kernel_sys_threads_main_late_load_patched PROPERTIES TIMEOUT 20)
endif()
foreach(case IN ITEMS main)
add_test(NAME guest_kernel_sys_threads_${case} COMMAND guest_kernel_sys_threads_tests ${UNPATCHED_DIR} unpatched ${case})
add_test(NAME guest_kernel_sys_threads_${case}_patched COMMAND guest_kernel_sys_threads_tests ${PATCHED_DIR} patched ${case})
set_tests_properties(guest_kernel_sys_threads_${case} guest_kernel_sys_threads_${case}_patched PROPERTIES TIMEOUT 20)
endforeach()
add_test_executable(guest_gnm_neo_packets_tests tests/GuestGnmNeoPackets.cpp)
target_include_directories(guest_gnm_neo_packets_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
@@ -388,6 +466,18 @@ if(BUILD_TESTING)
add_subdirectory(tests/write_watch)
add_subdirectory(tests/guest_arena)
endif()
add_test_executable(guest_kernel_sys_pthread_np_tests tests/GuestKernelSysPthreadNp.cpp)
target_include_directories(guest_kernel_sys_pthread_np_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_pthread_np_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_pthread_np_tests)
foreach(pthreadNpMode IN ITEMS rwlockattr kind)
add_test(NAME guest_kernel_sys_pthread_np_${pthreadNpMode} COMMAND guest_kernel_sys_pthread_np_tests ${pthreadNpMode})
set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_kernel_sys_pthread_np_${pthreadNpMode} PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
endforeach()
add_test_executable(guest_kernel_sys_module_internal_tests tests/GuestKernelSysModuleInternal.cpp)
target_include_directories(guest_kernel_sys_module_internal_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_module_internal_tests PRIVATE libkernel_sys libkernel libc)
@@ -407,6 +497,23 @@ if(BUILD_TESTING)
endif()
add_test(NAME windows_write_watch COMMAND windows_write_watch_tests)
set_tests_properties(windows_write_watch PROPERTIES TIMEOUT 20)
add_test_executable(windows_shared_map_cost_tests tests/WindowsSharedMapCost.cpp)
target_include_directories(windows_shared_map_cost_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_compile_definitions(windows_shared_map_cost_tests PRIVATE _WIN32_WINNT=0x0A00)
if(MINGW)
target_compile_options(windows_shared_map_cost_tests PRIVATE -fasynchronous-unwind-tables)
endif()
add_test(NAME windows_shared_map_cost COMMAND windows_shared_map_cost_tests)
set_tests_properties(windows_shared_map_cost PROPERTIES TIMEOUT 120)
endif()
add_test_executable(guest_kernel_sys_login_tests tests/GuestKernelSysLogin.cpp)
target_include_directories(guest_kernel_sys_login_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_login_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_login_tests)
add_test(NAME guest_kernel_sys_login COMMAND guest_kernel_sys_login_tests)
if(WIN32)
set_tests_properties(guest_kernel_sys_login PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
if(WIN32)
@@ -489,6 +596,17 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceMove>")
endif()
add_test_executable(guest_pthread_create_suspended_tests tests/GuestPthreadCreateSuspended.cpp)
target_include_directories(guest_pthread_create_suspended_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_pthread_create_suspended_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_pthread_create_suspended_tests)
add_test(NAME guest_pthread_create_suspended COMMAND guest_pthread_create_suspended_tests)
set_tests_properties(guest_pthread_create_suspended PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_pthread_create_suspended PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
add_test_executable(guest_np_party_tests tests/GuestNpParty.cpp)
target_include_directories(guest_np_party_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_np_party_tests PRIVATE libSceNpParty libc)
@@ -859,6 +977,18 @@ if(BUILD_TESTING)
endif()
add_test(NAME MemoryPoolTest COMMAND MemoryPoolTest)
add_test_executable(guest_kernel_sys_swapcontext_tests tests/GuestKernelSysSwapcontext.cpp)
target_include_directories(guest_kernel_sys_swapcontext_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_compile_options(guest_kernel_sys_swapcontext_tests PRIVATE -fno-omit-frame-pointer)
target_link_libraries(guest_kernel_sys_swapcontext_tests PRIVATE libkernel_sys libSceLibcInternal libkernel libc)
configure_windows_unwind(guest_kernel_sys_swapcontext_tests)
add_test(NAME guest_kernel_sys_swapcontext COMMAND guest_kernel_sys_swapcontext_tests)
set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_kernel_sys_swapcontext PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
add_test_executable(guest_acm_tests tests/GuestAcm.cpp)
target_include_directories(guest_acm_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_acm_tests PRIVATE libSceAcm libc)
core/libs/SceTypes.hpp
+9
View File
@@ -1502,6 +1502,15 @@ struct SharePlayConnectionInfoA {
std::int32_t visitorUserId;
};
static_assert(sizeof(SharePlayConnectionInfoA) == 72 && offsetof(SharePlayConnectionInfoA, hostAccountId) == 48 && offsetof(SharePlayConnectionInfoA, hostUserId) == 64);
struct SharePlayConnectionInfo {
std::int32_t status;
std::int32_t mode;
NpOnlineId hostOnlineId;
NpOnlineId visitorOnlineId;
std::int32_t hostUserId;
std::int32_t visitorUserId;
};
static_assert(sizeof(SharePlayConnectionInfo) == 56 && offsetof(SharePlayConnectionInfo, hostUserId) == 48);
struct NpId { NpOnlineId online_id; std::uint8_t opaque[4]; };
struct NpCreateAsyncRequestParameter { std::uint8_t opaque[64]; };
struct NpCheckPremiumParameter { std::uint8_t opaque[64]; };
core/libs/nid/include/nid/ElfPatcher.hpp
+12 -1
View File
@@ -22,6 +22,17 @@ struct Elf64_Ehdr {
std::uint16_t e_shstrndx;
};
struct Elf64_Phdr {
std::uint32_t p_type;
std::uint32_t p_flags;
std::uint64_t p_offset;
std::uint64_t p_vaddr;
std::uint64_t p_paddr;
std::uint64_t p_filesz;
std::uint64_t p_memsz;
std::uint64_t p_align;
};
struct Elf64_Shdr {
std::uint32_t sh_name;
std::uint32_t sh_type;
@@ -103,7 +114,7 @@ constexpr std::uint16_t kShnUndef = 0u;
class ElfNidPatcher final : public IBinaryPatcher {
public:
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
};
}
core/libs/nid/include/nid/IBinaryPatcher.hpp
+6 -1
View File
@@ -8,11 +8,16 @@
namespace Nid {
struct PatchOptions {
bool executable = false;
bool onlySce = false;
};
class IBinaryPatcher {
public:
virtual ~IBinaryPatcher() = default;
virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const = 0;
virtual void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const = 0;
};
}
core/libs/nid/include/nid/MachONidPatcher.hpp
+1 -1
View File
@@ -22,7 +22,7 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
class MachONidPatcher final : public IBinaryPatcher {
public:
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
};
}
core/libs/nid/include/nid/NidResolver.hpp
+5
View File
@@ -1,6 +1,7 @@
#ifndef NID_NIDRESOLVER_HPP
#define NID_NIDRESOLVER_HPP
#include <nid/IBinaryPatcher.hpp>
#include <string>
#include <unordered_map>
#include <unordered_set>
@@ -8,6 +9,10 @@
namespace Nid {
bool IsSceImport(const std::string& name);
std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options);
std::string ResolveOneName(const std::string& funcName);
std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports);
core/libs/nid/include/nid/PeNidPatcher.hpp
+1 -1
View File
@@ -39,7 +39,7 @@ struct PeSectionHeader {
class PeNidPatcher final : public IBinaryPatcher {
public:
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options = {}) const override;
};
}
core/libs/nid/src/ElfNidPatcher.cpp
+92 -15
View File
@@ -354,7 +354,7 @@ std::vector<std::uint32_t> CollectVerneedNameOffsets(const std::vector<std::uint
}
void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
using namespace Internal;
if (elf.size() < sizeof(Elf64_Ehdr)) throw std::runtime_error("file too small");
@@ -370,11 +370,13 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
std::size_t sysvHashOffset = 0u, sysvHashSize = 0u;
std::uint32_t sysvHashLink = 0u;
std::size_t versymOffset = 0u;
bool hasDynamic = false;
std::uint32_t dynSymSectionIndex = 0u;
std::uint32_t gnuHashSymOffset = 0u;
for (std::uint16_t i = 0u; i < ehdr.e_shnum; ++i) {
const std::size_t shOffset = static_cast<std::size_t>(ehdr.e_shoff) + i * sizeof(Elf64_Shdr);
const auto shdr = Read<Elf64_Shdr>(elf, shOffset);
if (shdr.sh_type == kShtDynamic) hasDynamic = true;
if (shdr.sh_type == kShtDynsym) {
if (shdr.sh_entsize != sizeof(Elf64_Sym) || shdr.sh_size % sizeof(Elf64_Sym) || shdr.sh_link >= ehdr.e_shnum)
throw std::runtime_error("invalid ELF dynsym layout");
@@ -398,6 +400,19 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
throw std::runtime_error("unsupported .gnu.version_d section: symbol version definitions cannot be preserved");
}
if (options.executable && dynSymOffset == 0u && !hasDynamic) {
if (ehdr.e_phnum) {
if (!ehdr.e_phoff || ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0xffffu)
throw std::runtime_error("unsupported ELF program header layout");
RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
for (std::uint16_t index = 0u; index < ehdr.e_phnum; ++index) {
const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
if (segment.p_type == 2u)
throw std::runtime_error("unsupported ELF PT_DYNAMIC without a .dynsym section");
}
}
return;
}
if (dynSymOffset == 0u) throw std::runtime_error("no .dynsym section");
const std::size_t symCount = dynSymSize / sizeof(Elf64_Sym);
@@ -437,19 +452,26 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
elf.begin() + static_cast<std::ptrdiff_t>(dynStrOffset + dynStrSize)
);
std::vector<std::string> importedNames;
std::vector<std::string> exportedNames;
for (std::size_t i = 1u; i < symCount; ++i) {
const std::size_t symOffset = dynSymOffset + i * sizeof(Elf64_Sym);
const auto sym = Read<Elf64_Sym>(elf, symOffset);
if (sym.st_name == 0u) continue;
const std::uint8_t binding = sym.st_info >> 4u;
if (binding == kStbLocal || sym.st_shndx == kShnUndef) continue;
if (binding == kStbLocal) continue;
if (sym.st_shndx == kShnUndef) {
const auto name = ReadCStr(origDynStr, sym.st_name);
if (!name.empty()) importedNames.push_back(name);
continue;
}
const std::string symName = ReadCStr(origDynStr, sym.st_name);
if (symName.empty()) continue;
exportedNames.push_back(symName);
}
const auto nidMap = ResolveNids(exportedNames, libraryName, excludedExports);
const auto nidMap = options.executable ? std::unordered_map<std::string, std::string>{} : ResolveNids(exportedNames, libraryName, excludedExports);
const auto importMap = ResolveImportNids(importedNames, excludedExports, options);
struct SymbolNameUse {
std::size_t symIndex;
@@ -481,14 +503,11 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
} else if (binding != kStbLocal && sym.st_shndx == kShnUndef) {
const bool isUnversioned = versymOffset == 0u ||
(Read<std::uint16_t>(elf, versymOffset + i * sizeof(std::uint16_t)) <= 1u);
const bool hasNidPostfix = symName.size() >= kNidPostfixLen &&
symName.compare(symName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
const bool hasScePrefix = symName.size() >= 3u &&
std::tolower(static_cast<unsigned char>(symName[0])) == 's' &&
std::tolower(static_cast<unsigned char>(symName[1])) == 'c' &&
std::tolower(static_cast<unsigned char>(symName[2])) == 'e';
if (isUnversioned && (hasNidPostfix || hasScePrefix))
newValue = ResolveOneName(symName);
if (options.executable || isUnversioned) {
newValue = importMap.at(symName);
if (options.executable && newValue != symName && versymOffset != 0u)
Write(elf, versymOffset + i * sizeof(std::uint16_t), std::uint16_t{1});
}
}
uses.push_back(SymbolNameUse{i, sym.st_name, newValue});
@@ -538,11 +557,69 @@ void ElfNidPatcher::PatchNids(std::vector<std::uint8_t>& elf, const std::string&
for (const auto offset : preservedOffsets)
preservedToNewOffset.emplace_back(offset, valueOffsets.at(ReadCStr(origDynStr, offset)));
if (newDynStr.size() > dynStrSize && options.executable) {
if (ehdr.e_phentsize != sizeof(Elf64_Phdr) || ehdr.e_phnum == 0 || ehdr.e_phnum >= 0xfffdu)
throw std::runtime_error("invalid ELF program header table for string growth");
RequireElfRange(elf, ehdr.e_phoff, std::uint64_t{ehdr.e_phnum} * sizeof(Elf64_Phdr), "program headers");
const auto align = [](std::uint64_t value) {
if (value > UINT64_MAX - 4095u) throw std::runtime_error("ELF address overflow");
return (value + 4095u) & ~std::uint64_t{4095u};
};
std::vector<Elf64_Phdr> segments;
std::uint64_t virtualEnd = 0;
bool hasPhdr = false;
for (std::size_t index = 0; index < ehdr.e_phnum; ++index) {
const auto segment = Read<Elf64_Phdr>(elf, ehdr.e_phoff + index * sizeof(Elf64_Phdr));
segments.push_back(segment);
hasPhdr |= segment.p_type == 6;
if (segment.p_type != 1) continue;
if (segment.p_memsz > UINT64_MAX - segment.p_vaddr) throw std::runtime_error("ELF segment address overflow");
virtualEnd = std::max(virtualEnd, segment.p_vaddr + segment.p_memsz);
}
if (!virtualEnd) throw std::runtime_error("ELF executable has no load segment");
if (!hasPhdr) segments.insert(segments.begin(), Elf64_Phdr{6, 4, 0, 0, 0, 0, 0, 8});
const auto fileOffset = align(elf.size());
const auto address = align(virtualEnd);
const auto tableSize = (segments.size() + 1u) * sizeof(Elf64_Phdr);
const auto segmentSize = tableSize + newDynStr.size();
if (segmentSize > UINT64_MAX - address || segmentSize > SIZE_MAX - fileOffset) throw std::runtime_error("ELF string segment overflow");
for (auto& segment : segments) {
if (segment.p_type != 6) continue;
segment = {6, 4, fileOffset, address, address, tableSize, tableSize, 8};
}
segments.push_back({1, 4, fileOffset, address, address, segmentSize, segmentSize, 4096});
elf.resize(static_cast<std::size_t>(fileOffset + segmentSize), 0);
for (std::size_t index = 0; index < segments.size(); ++index)
Write(elf, fileOffset + index * sizeof(Elf64_Phdr), segments[index]);
Write(elf, offsetof(Elf64_Ehdr, e_phoff), fileOffset);
Write(elf, offsetof(Elf64_Ehdr, e_phnum), static_cast<std::uint16_t>(segments.size()));
dynStrOffset = static_cast<std::size_t>(fileOffset + tableSize);
dynStrSize = newDynStr.size();
const auto stringAddress = address + tableSize;
const auto sectionOffset = ehdr.e_shoff + dynStrSectionLink * sizeof(Elf64_Shdr);
auto strings = Read<Elf64_Shdr>(elf, sectionOffset);
strings.sh_offset = dynStrOffset;
strings.sh_addr = stringAddress;
strings.sh_size = dynStrSize;
Write(elf, sectionOffset, strings);
bool foundAddress = false;
bool foundSize = false;
for (std::size_t index = 0; index < ehdr.e_shnum; ++index) {
const auto section = Read<Elf64_Shdr>(elf, ehdr.e_shoff + index * sizeof(Elf64_Shdr));
if (section.sh_type != kShtDynamic) continue;
RequireElfRange(elf, section.sh_offset, section.sh_size, "dynamic");
for (std::uint64_t offset = 0; offset + sizeof(Elf64_Dyn) <= section.sh_size; offset += sizeof(Elf64_Dyn)) {
auto entry = Read<Elf64_Dyn>(elf, section.sh_offset + offset);
if (entry.d_tag == kDtNull) break;
if (entry.d_tag == 5) { entry.d_val = stringAddress; foundAddress = true; }
if (entry.d_tag == 10) { entry.d_val = dynStrSize; foundSize = true; }
Write(elf, section.sh_offset + offset, entry);
}
}
if (!foundAddress || !foundSize) throw std::runtime_error("ELF executable lacks DT_STRTAB or DT_STRSZ");
}
if (newDynStr.size() > dynStrSize) {
throw std::runtime_error(
"rebuilt .dynstr (" + std::to_string(newDynStr.size()) + " bytes) exceeds original section size (" +
std::to_string(dynStrSize) + " bytes) - relinking with a larger section is required"
);
throw std::runtime_error("rebuilt .dynstr exceeds original section size; relinking with a larger section is required");
}
newDynStr.resize(dynStrSize, 0u);
core/libs/nid/src/MachONidPatcher.cpp
+100 -57
View File
@@ -341,15 +341,6 @@ std::string StripUnderscore(const std::string& name) {
return !name.empty() && name[0] == '_' ? name.substr(1) : name;
}
bool IsGuestFacingImport(const std::string& name) {
using namespace Internal;
const bool hasNidPostfix = name.size() >= kNidPostfixLen && name.compare(name.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
const bool hasScePrefix = name.size() >= 3u &&
std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
std::tolower(static_cast<unsigned char>(name[2])) == 'e';
return hasNidPostfix || hasScePrefix;
}
std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, const Image& image) {
std::vector<bool> result;
@@ -364,11 +355,48 @@ std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, c
return result;
}
void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image) {
void WriteLinkeditData(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
if (trie.size() <= oldSize) {
std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
return;
}
std::optional<std::size_t> linkedit;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow linkedit data");
if (cmd != kLcSegment64) continue;
const auto segment = Read<SegmentCommand64>(binary, offset);
if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
}
if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
auto segment = Read<SegmentCommand64>(binary, *linkedit);
if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("linkedit data offset overflow");
binary.resize(newOffset, 0);
binary.insert(binary.end(), trie.begin(), trie.end());
std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
segment.FileSize = binary.size() - segment.FileOffset;
segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
Write(binary, *linkedit, segment);
Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
}
void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
std::size_t commandOffset = 0;
std::optional<LinkeditDataCommand> fixups;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
if (cmd == kLcDyldChainedFixups) fixups = Read<LinkeditDataCommand>(binary, offset);
if (cmd == kLcDyldChainedFixups) {
fixups = Read<LinkeditDataCommand>(binary, offset);
commandOffset = offset;
}
if (options.executable && cmd == kLcCodeSignature) throw std::runtime_error("remove the Mach-O code signature before patching and sign again afterwards");
if ((cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly)) {
const auto info = Read<DyldInfoCommand>(binary, offset);
if (info.BindSize != 0 || info.LazyBindSize != 0 || info.WeakBindSize != 0) throw std::runtime_error("Mach-O bind opcodes are not supported; link with chained fixups");
@@ -400,61 +428,72 @@ void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image)
imports.push_back({raw >> 9, raw & 0xff});
}
}
std::set<std::uint32_t> nameOffsets;
for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
std::set<std::uint32_t> rewritten;
std::vector<std::string> names;
std::vector<std::string> originalNames;
std::vector<bool> selected;
for (const auto& item : imports) {
const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
if (rewritten.contains(item.NameOffset)) continue;
const auto at = symbols + item.NameOffset;
if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
const auto name = Internal::ReadCStr(binary, at);
if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
const auto bare = StripUnderscore(name);
if (bare == name || !IsGuestFacingImport(bare)) continue;
const auto patched = "_" + ResolveOneName(bare);
if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
const auto inside = nameOffsets.upper_bound(item.NameOffset);
if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
std::memcpy(binary.data() + at, patched.data(), patched.size());
std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
rewritten.insert(item.NameOffset);
const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
const bool eligible = options.executable || (bare != name && item.Ordinal != 0 && !special && item.Ordinal <= prx.size() && prx[item.Ordinal - 1]);
originalNames.push_back(name);
selected.push_back(eligible);
if (eligible) names.push_back(bare);
}
}
void WriteTrie(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
if (trie.size() <= oldSize) {
std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
const auto mapping = ResolveImportNids(names, excluded, options);
if (!options.executable) {
std::set<std::uint32_t> nameOffsets;
for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
std::set<std::uint32_t> rewritten;
for (const auto& item : imports) {
const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
if (rewritten.contains(item.NameOffset)) continue;
const auto at = symbols + item.NameOffset;
if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
const auto name = Internal::ReadCStr(binary, at);
if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
const auto bare = StripUnderscore(name);
if (bare == name) continue;
const auto patched = "_" + mapping.at(bare);
if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
const auto inside = nameOffsets.upper_bound(item.NameOffset);
if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
std::memcpy(binary.data() + at, patched.data(), patched.size());
std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
rewritten.insert(item.NameOffset);
}
return;
}
std::optional<std::size_t> linkedit;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow their export trie");
if (cmd != kLcSegment64) continue;
const auto segment = Read<SegmentCommand64>(binary, offset);
if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
if (imports.empty()) return;
std::vector<std::uint8_t> data(binary.begin() + base, binary.begin() + symbolsEnd);
const auto newSymbols = data.size();
if (newSymbols > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
Write(data, offsetof(ChainedFixupsHeader, SymbolsOffset), static_cast<std::uint32_t>(newSymbols));
for (std::size_t index = 0; index < imports.size(); ++index) {
const auto& original = originalNames[index];
const auto name = selected[index] ? (original.starts_with("_") ? "_" : "") + mapping.at(StripUnderscore(original)) : original;
const auto offset = data.size() - newSymbols;
const auto at = header.ImportsOffset + index * entrySize;
if (header.ImportsFormat == kImportFormatAddend64) {
if (offset > 0xffffffffu) throw std::runtime_error("chained symbol offset overflow");
const auto raw = Read<std::uint64_t>(data, at);
Write(data, at, (raw & 0xffffffffu) | (std::uint64_t{offset} << 32));
} else {
if (offset > 0x7fffffu) throw std::runtime_error("chained symbol offset overflow");
const auto raw = Read<std::uint32_t>(data, at);
Write(data, at, (raw & 0x1ffu) | (static_cast<std::uint32_t>(offset) << 9));
}
data.insert(data.end(), name.begin(), name.end());
data.push_back(0);
}
if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
auto segment = Read<SegmentCommand64>(binary, *linkedit);
if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("export trie offset overflow");
binary.resize(newOffset, 0);
binary.insert(binary.end(), trie.begin(), trie.end());
std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
segment.FileSize = binary.size() - segment.FileOffset;
segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
Write(binary, *linkedit, segment);
Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
WriteLinkeditData(binary, image, {commandOffset + 8, commandOffset + 12}, data);
}
}
std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binary) {
@@ -464,8 +503,12 @@ std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binar
return ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
}
void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
const auto image = ParseImage(binary);
if (options.executable) {
PatchChainedImports(binary, image, excludedExports, options);
return;
}
const auto location = FindTrie(binary, image);
if (!location) throw std::runtime_error("no export trie");
auto exports = ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
@@ -486,8 +529,8 @@ void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::st
if (found == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + name);
exports[index].Name = "_" + found->second;
}
WriteTrie(binary, image, *location, TrieBuilder(exports).Build());
PatchChainedImports(binary, ParseImage(binary));
WriteLinkeditData(binary, image, *location, TrieBuilder(exports).Build());
PatchChainedImports(binary, ParseImage(binary), excludedExports, options);
}
}
core/libs/nid/src/NidResolver.cpp
+19 -8
View File
@@ -3,21 +3,32 @@
#include <nid/NidCompute.hpp>
#include <nid/NidPatcherUtils.hpp>
#include <cstdint>
#include <cctype>
#include <stdexcept>
#include <unordered_set>
#include <vector>
namespace Nid {
bool IsSceImport(const std::string& name) {
return name.ends_with(Internal::kNidPostfix) || (name.size() >= 3 &&
std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
std::tolower(static_cast<unsigned char>(name[2])) == 'e');
}
std::unordered_map<std::string, std::string> ResolveImportNids(const std::vector<std::string>& names, const std::unordered_set<std::string>& excluded, const PatchOptions& options) {
const std::unordered_set<std::string> unique(names.begin(), names.end());
auto result = ResolveNids(std::vector<std::string>(unique.begin(), unique.end()), "", excluded);
for (const auto& name : unique) {
if ((!options.executable || options.onlySce) && !IsSceImport(name)) result.at(name) = name;
else if (!options.executable) result.at(name) = ResolveOneName(name);
}
return result;
}
std::string ResolveOneName(const std::string& funcName) {
using namespace Internal;
if (IsNidNoPatchCut(funcName))
return StripNidNoPatchCut(funcName);
if (IsNidNoPatch(funcName))
return funcName;
return ComputeNid(StripNidPostfix(funcName), "");
return ResolveNids({funcName}, "", {}).at(funcName);
}
std::unordered_map<std::string, std::string> ResolveNids(const std::vector<std::string>& exportedNames, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) {
core/libs/nid/src/PeNidPatcher.cpp
+231 -158
View File
@@ -36,7 +36,7 @@ std::size_t RvaToOffset(const std::vector<std::uint8_t>& pe, std::uint32_t rva,
}
void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports, const PatchOptions& options) const {
using namespace Internal;
if (pe.size() < 0x40) throw std::runtime_error("file too small");
@@ -54,7 +54,7 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
const std::size_t sectionTableOffset = coffHeaderOffset + 20u + sizeOfOptionalHeader;
const auto symbolTable = Read<std::uint32_t>(pe, coffHeaderOffset + 8u);
const auto symbolCount = Read<std::uint32_t>(pe, coffHeaderOffset + 12u);
for (std::uint16_t i = 0; i < numberOfSections; ++i) {
for (std::uint16_t i = 0; !options.executable && i < numberOfSections; ++i) {
const auto offset = sectionTableOffset + i * sizeof(PeSectionHeader);
auto section = Read<PeSectionHeader>(pe, offset);
if (pe[offset] != '/' || !symbolTable) continue;
@@ -79,159 +79,162 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
throw std::runtime_error("unsupported optional header magic");
}
const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
if (!options.executable) {
const auto exportDir = Read<PeDataDirectory>(pe, dataDirectoryOffset);
if (exportDir.VirtualAddress == 0u) throw std::runtime_error("no export directory");
const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
const std::size_t exportDirOffset = RvaToOffset(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const auto exportTable = Read<PeExportDirectory>(pe, exportDirOffset);
if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
if (exportTable.NumberOfNames == 0u) throw std::runtime_error("no exported names");
const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const std::size_t namesArrayOffset = RvaToOffset(pe, exportTable.AddressOfNames, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const std::size_t ordinalsArrayOffset = RvaToOffset(pe, exportTable.AddressOfNameOrdinals, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
std::vector<std::string> names(exportTable.NumberOfNames);
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const std::string name = ReadCStr(pe, nameOffset);
if (name.empty()) throw std::runtime_error("empty exported name");
nameRvas[i] = nameRva;
names[i] = name;
}
const auto nidMap = ResolveNids(names, libraryName, excludedExports);
const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
}
if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
throw std::runtime_error("could not determine strings region start");
std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
exportTable.Name - edataSection.VirtualAddress < rawLimit;
if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
struct Forwarder {
std::size_t EntryOffset;
std::string Target;
};
std::vector<Forwarder> forwarders;
std::vector<std::uint32_t> directRvas;
const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
const auto rva = Read<std::uint32_t>(pe, entryOffset);
if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
directRvas.push_back(rva);
continue;
std::vector<std::uint32_t> nameRvas(exportTable.NumberOfNames);
std::vector<std::string> names(exportTable.NumberOfNames);
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * 4u);
const std::size_t nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const std::string name = ReadCStr(pe, nameOffset);
if (name.empty()) throw std::runtime_error("empty exported name");
nameRvas[i] = nameRva;
names[i] = name;
}
const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
throw std::runtime_error("forwarder string outside export directory");
forwarders.push_back({entryOffset, target});
stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
}
for (const auto rva : directRvas) {
if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
}
const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
if (end > rawLimit) throw std::runtime_error("export table outside edata section");
stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
};
reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
std::vector<std::string> finalNames(exportTable.NumberOfNames);
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const auto it = nidMap.find(names[i]);
if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
finalNames[i] = it->second;
}
const auto nidMap = ResolveNids(names, libraryName, excludedExports);
std::vector<std::uint8_t> strings;
std::unordered_map<std::string, std::uint32_t> stringRvas;
const auto storeString = [&](const std::string& name) {
const auto existing = stringRvas.find(name);
if (existing != stringRvas.end()) return existing->second;
const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
stringRvas.emplace(name, rva);
strings.insert(strings.end(), name.begin(), name.end());
strings.push_back(0u);
return rva;
};
std::vector<std::uint32_t> forwarderRvas;
for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
throw std::runtime_error("not enough export directory space to repack forwarders");
const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
std::vector<std::uint32_t> finalNameRvas;
for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
if (requiredSize > rawLimit)
throw std::runtime_error("not enough raw space in edata section to repack export names");
const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
std::uint64_t{edataSection.VirtualAddress} + requiredSize);
for (const auto rva : directRvas) {
if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
throw std::runtime_error("repacked export directory would cover a direct export");
}
const auto directorySize = directoryEnd - exportDir.VirtualAddress;
if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
throw std::runtime_error("repack write out of file bounds");
std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
for (std::size_t index = 0u; index < forwarders.size(); ++index)
Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
const std::size_t edataSectionOffset = FindSectionOffsetByRva(pe, exportDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
const auto edataSection = Read<PeSectionHeader>(pe, edataSectionOffset);
const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
auto patchedSection = edataSection;
patchedSection.VirtualSize = newVirtualSize;
Write(pe, edataSectionOffset, patchedSection);
const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
if (fileAlignment == 0u) throw std::runtime_error("invalid file alignment");
struct ExportName {
std::string Name;
std::uint32_t Rva;
std::uint16_t Ordinal;
};
std::uint32_t stringsRegionStart = std::numeric_limits<std::uint32_t>::max();
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const std::uint32_t rvaFromSection = nameRvas[i] - edataSection.VirtualAddress;
if (rvaFromSection < stringsRegionStart) stringsRegionStart = rvaFromSection;
}
if (stringsRegionStart == std::numeric_limits<std::uint32_t>::max())
throw std::runtime_error("could not determine strings region start");
std::uint32_t rawLimit = ((edataSection.SizeOfRawData + fileAlignment - 1u) / fileAlignment) * fileAlignment;
const auto dllName = ReadCStr(pe, RvaToOffset(pe, exportTable.Name, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
const bool repackDllName = exportTable.Name >= edataSection.VirtualAddress &&
exportTable.Name - edataSection.VirtualAddress < rawLimit;
if (repackDllName) stringsRegionStart = std::min(stringsRegionStart, exportTable.Name - edataSection.VirtualAddress);
struct Forwarder {
std::size_t EntryOffset;
std::string Target;
};
std::vector<Forwarder> forwarders;
std::vector<std::uint32_t> directRvas;
const auto functionsArrayOffset = RvaToOffset(pe, exportTable.AddressOfFunctions, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
for (std::uint32_t index = 0u; index < exportTable.NumberOfFunctions; ++index) {
const auto entryOffset = functionsArrayOffset + index * sizeof(std::uint32_t);
const auto rva = Read<std::uint32_t>(pe, entryOffset);
if (rva < exportDir.VirtualAddress || std::uint64_t{rva} - exportDir.VirtualAddress >= exportDir.Size) {
directRvas.push_back(rva);
continue;
}
const auto target = ReadCStr(pe, RvaToOffset(pe, rva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader));
if (target.size() + 1u > exportDir.Size - (rva - exportDir.VirtualAddress))
throw std::runtime_error("forwarder string outside export directory");
forwarders.push_back({entryOffset, target});
stringsRegionStart = std::min(stringsRegionStart, rva - edataSection.VirtualAddress);
}
for (const auto rva : directRvas) {
if (rva >= edataSection.VirtualAddress && rva - edataSection.VirtualAddress >= stringsRegionStart)
rawLimit = std::min(rawLimit, rva - edataSection.VirtualAddress);
}
const auto reserveTable = [&](std::uint32_t rva, std::uint64_t size) {
if (rva < edataSection.VirtualAddress || rva - edataSection.VirtualAddress >= rawLimit) return;
const auto end = std::uint64_t{rva - edataSection.VirtualAddress} + size;
if (end > rawLimit) throw std::runtime_error("export table outside edata section");
stringsRegionStart = std::max(stringsRegionStart, static_cast<std::uint32_t>(end));
};
reserveTable(exportDir.VirtualAddress, sizeof(PeExportDirectory));
reserveTable(exportTable.AddressOfFunctions, std::uint64_t{exportTable.NumberOfFunctions} * 4u);
reserveTable(exportTable.AddressOfNames, std::uint64_t{exportTable.NumberOfNames} * 4u);
reserveTable(exportTable.AddressOfNameOrdinals, std::uint64_t{exportTable.NumberOfNames} * 2u);
std::vector<std::string> finalNames(exportTable.NumberOfNames);
for (std::uint32_t i = 0u; i < exportTable.NumberOfNames; ++i) {
const auto it = nidMap.find(names[i]);
if (it == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + names[i]);
finalNames[i] = it->second;
}
std::vector<std::uint8_t> strings;
std::unordered_map<std::string, std::uint32_t> stringRvas;
const auto storeString = [&](const std::string& name) {
const auto existing = stringRvas.find(name);
if (existing != stringRvas.end()) return existing->second;
const auto rva = edataSection.VirtualAddress + stringsRegionStart + static_cast<std::uint32_t>(strings.size());
stringRvas.emplace(name, rva);
strings.insert(strings.end(), name.begin(), name.end());
strings.push_back(0u);
return rva;
};
std::vector<std::uint32_t> forwarderRvas;
for (const auto& forwarder : forwarders) forwarderRvas.push_back(storeString(forwarder.Target));
if (!forwarders.empty() && (std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart < exportDir.VirtualAddress ||
std::uint64_t{edataSection.VirtualAddress} + stringsRegionStart + strings.size() > std::uint64_t{exportDir.VirtualAddress} + exportDir.Size))
throw std::runtime_error("not enough export directory space to repack forwarders");
const auto dllNameRva = repackDllName ? storeString(dllName) : exportTable.Name;
std::vector<std::uint32_t> finalNameRvas;
for (const auto& name : finalNames) finalNameRvas.push_back(storeString(name));
const auto requiredSize = std::uint64_t{stringsRegionStart} + strings.size();
if (requiredSize > rawLimit)
throw std::runtime_error("not enough raw space in edata section to repack export names");
const auto directoryEnd = std::max(std::uint64_t{exportDir.VirtualAddress} + exportDir.Size,
std::uint64_t{edataSection.VirtualAddress} + requiredSize);
for (const auto rva : directRvas) {
if (rva != 0u && rva >= exportDir.VirtualAddress && std::uint64_t{rva} < directoryEnd)
throw std::runtime_error("repacked export directory would cover a direct export");
}
const auto directorySize = directoryEnd - exportDir.VirtualAddress;
if (directorySize > std::numeric_limits<std::uint32_t>::max()) throw std::runtime_error("export directory too large");
const auto stringFileOffset = static_cast<std::size_t>(edataSection.PointerToRawData) + stringsRegionStart;
if (stringFileOffset > pe.size() || strings.size() > pe.size() - stringFileOffset)
throw std::runtime_error("repack write out of file bounds");
std::memcpy(pe.data() + stringFileOffset, strings.data(), strings.size());
for (std::size_t index = 0u; index < finalNameRvas.size(); ++index)
Write(pe, namesArrayOffset + index * 4u, finalNameRvas[index]);
for (std::size_t index = 0u; index < forwarders.size(); ++index)
Write(pe, forwarders[index].EntryOffset, forwarderRvas[index]);
Write(pe, exportDirOffset + offsetof(PeExportDirectory, Name), dllNameRva);
Write(pe, dataDirectoryOffset + offsetof(PeDataDirectory, Size), static_cast<std::uint32_t>(directorySize));
const std::uint32_t newVirtualSize = std::max(edataSection.VirtualSize, static_cast<std::uint32_t>(requiredSize));
auto patchedSection = edataSection;
patchedSection.VirtualSize = newVirtualSize;
Write(pe, edataSectionOffset, patchedSection);
struct ExportName {
std::string Name;
std::uint32_t Rva;
std::uint16_t Ordinal;
};
std::vector<ExportName> exportNames;
exportNames.reserve(exportTable.NumberOfNames);
for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
if (ordinal >= exportTable.NumberOfFunctions)
throw std::runtime_error("export name ordinal out of bounds");
const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
}
std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
for (std::size_t i = 0; i < exportNames.size(); ++i) {
if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
}
std::vector<ExportName> exportNames;
exportNames.reserve(exportTable.NumberOfNames);
for (std::size_t i = 0; i < exportTable.NumberOfNames; ++i) {
const auto nameRva = Read<std::uint32_t>(pe, namesArrayOffset + i * sizeof(std::uint32_t));
const auto ordinal = Read<std::uint16_t>(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t));
if (ordinal >= exportTable.NumberOfFunctions)
throw std::runtime_error("export name ordinal out of bounds");
const auto nameOffset = RvaToOffset(pe, nameRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
exportNames.push_back({ReadCStr(pe, nameOffset), nameRva, ordinal});
}
std::sort(exportNames.begin(), exportNames.end(), [](const ExportName& left, const ExportName& right) { return left.Name < right.Name; });
for (std::size_t i = 0; i < exportNames.size(); ++i) {
if (i != 0 && exportNames[i - 1].Name == exportNames[i].Name)
throw std::runtime_error("duplicate patched export name: " + exportNames[i].Name);
Write(pe, namesArrayOffset + i * sizeof(std::uint32_t), exportNames[i].Rva);
Write(pe, ordinalsArrayOffset + i * sizeof(std::uint16_t), exportNames[i].Ordinal);
}
constexpr std::size_t kImportDirIndex = 1u;
@@ -250,6 +253,20 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
std::uint32_t FirstThunk;
};
if (options.executable) {
for (const auto index : {4u, 11u, 13u}) {
const auto directory = Read<PeDataDirectory>(pe, dataDirectoryOffset + index * sizeof(PeDataDirectory));
if (directory.VirtualAddress || directory.Size) throw std::runtime_error("signed, bound or delay-loaded PE imports are not supported in --exec mode");
}
}
struct ImportUse {
std::size_t NameOffset;
std::size_t LookupOffset;
std::size_t AddressOffset;
std::string Name;
};
std::vector<ImportUse> imports;
std::vector<std::string> importNames;
std::size_t importDescOffset = RvaToOffset(pe, importDir.VirtualAddress, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
while (true) {
@@ -286,19 +303,9 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
const std::string funcName = ReadCStr(pe, hintNameOffset + 2u);
if (!funcName.empty()) {
const bool hasNidPostfix = funcName.size() >= kNidPostfixLen &&
funcName.compare(funcName.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
const bool hasScePrefix = funcName.size() >= 3u &&
std::tolower(static_cast<unsigned char>(funcName[0])) == 's' &&
std::tolower(static_cast<unsigned char>(funcName[1])) == 'c' &&
std::tolower(static_cast<unsigned char>(funcName[2])) == 'e';
if (hasNidPostfix || hasScePrefix) {
const std::string nid = ResolveOneName(funcName);
if (nid.size() > funcName.size())
throw std::runtime_error("import NID longer than original name: " + funcName);
std::memcpy(pe.data() + hintNameOffset + 2u, nid.data(), nid.size());
pe[hintNameOffset + 2u + nid.size()] = 0u;
}
const auto addressOffset = RvaToOffset(pe, desc.FirstThunk, peHeaderOffset, numberOfSections, sizeOfOptionalHeader) + thunkOffset - RvaToOffset(pe, thunkRva, peHeaderOffset, numberOfSections, sizeOfOptionalHeader);
imports.push_back({hintNameOffset + 2u, thunkOffset, addressOffset, funcName});
importNames.push_back(funcName);
}
}
@@ -307,6 +314,72 @@ void PeNidPatcher::PatchNids(std::vector<std::uint8_t>& pe, const std::string& l
importDescOffset += sizeof(PeImportDescriptor);
}
const auto importMap = ResolveImportNids(importNames, excludedExports, options);
if (!options.executable) {
for (const auto& item : imports) {
const auto& name = importMap.at(item.Name);
if (name.size() > item.Name.size()) throw std::runtime_error("import NID longer than original name: " + item.Name);
std::memcpy(pe.data() + item.NameOffset, name.c_str(), name.size() + 1);
}
return;
}
std::vector<std::uint8_t> strings;
std::unordered_map<std::string, std::uint32_t> stringOffsets;
for (const auto& item : imports) {
const auto& name = importMap.at(item.Name);
if (name == item.Name || stringOffsets.contains(name)) continue;
if (strings.size() & 1u) strings.push_back(0);
if (strings.size() > 0xffffffffu - name.size() - 3u) throw std::runtime_error("PE import string table overflow");
stringOffsets.emplace(name, static_cast<std::uint32_t>(strings.size()));
strings.insert(strings.end(), {0, 0});
strings.insert(strings.end(), name.begin(), name.end());
strings.push_back(0);
}
if (strings.empty()) return;
const auto fileAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 36u);
const auto sectionAlignment = Read<std::uint32_t>(pe, optionalHeaderOffset + 32u);
const auto align = [](std::uint64_t value, std::uint32_t alignment) {
if (!alignment || (alignment & (alignment - 1u))) throw std::runtime_error("invalid PE alignment");
const auto result = (value + alignment - 1u) & ~std::uint64_t{alignment - 1u};
if (result > 0xffffffffu) throw std::runtime_error("PE section overflow");
return static_cast<std::uint32_t>(result);
};
const auto newHeader = sectionTableOffset + numberOfSections * sizeof(PeSectionHeader);
auto headerLimit = std::size_t{Read<std::uint32_t>(pe, optionalHeaderOffset + 60u)};
std::uint64_t virtualEnd = headerLimit;
for (std::uint16_t index = 0; index < numberOfSections; ++index) {
const auto section = Read<PeSectionHeader>(pe, sectionTableOffset + index * sizeof(PeSectionHeader));
if (section.PointerToRawData) headerLimit = std::min(headerLimit, std::size_t{section.PointerToRawData});
virtualEnd = std::max(virtualEnd, std::uint64_t{section.VirtualAddress} + std::max(section.VirtualSize, section.SizeOfRawData));
}
if (numberOfSections == 0xffffu || newHeader + sizeof(PeSectionHeader) > headerLimit || headerLimit > pe.size()) throw std::runtime_error("no room for PE import section header");
PeSectionHeader section{};
const char sectionName[] = ".nidimp";
std::memcpy(section.Name, sectionName, sizeof(sectionName));
section.VirtualSize = static_cast<std::uint32_t>(strings.size());
section.VirtualAddress = align(virtualEnd, sectionAlignment);
section.PointerToRawData = align(pe.size(), fileAlignment);
section.SizeOfRawData = align(strings.size(), fileAlignment);
section.Characteristics = 0x40000040u;
const auto imageSize = align(std::uint64_t{section.VirtualAddress} + section.VirtualSize, sectionAlignment);
const auto rawEnd = std::uint64_t{section.PointerToRawData} + section.SizeOfRawData;
if (rawEnd > 0xffffffffu) throw std::runtime_error("PE file size overflow");
pe.resize(static_cast<std::size_t>(rawEnd), 0);
std::copy(strings.begin(), strings.end(), pe.begin() + section.PointerToRawData);
Write(pe, newHeader, section);
Write(pe, coffHeaderOffset + 2u, static_cast<std::uint16_t>(numberOfSections + 1u));
Write(pe, optionalHeaderOffset + 56u, imageSize);
Write(pe, optionalHeaderOffset + 64u, std::uint32_t{0});
for (const auto& item : imports) {
const auto& name = importMap.at(item.Name);
if (name == item.Name) continue;
const auto rva = section.VirtualAddress + stringOffsets.at(name);
for (const auto offset : {item.LookupOffset, item.AddressOffset}) {
if (magic == 0x20bu) Write(pe, offset, std::uint64_t{rva});
else Write(pe, offset, rva);
}
}
}
}
core/libs/nid/src/main.cpp
+10 -3
View File
@@ -28,6 +28,7 @@ void _writeFile(const std::string& path, const std::vector<std::uint8_t>& data)
int Run(int argc, char* argv[]) {
if (argc < 3) {
std::cerr << "Usage: nid_patcher <library_name> [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
" nid_patcher --exec [--only-sce] [--preserve-exports <unpatched-library> | --preserve-export-list <export-list>] <file> [file2 ...]\n"
" nid_patcher --write-export-list <unpatched-library> <export-list>\n";
return 1;
}
@@ -46,14 +47,20 @@ int Run(int argc, char* argv[]) {
return 0;
}
const std::string libraryName = argv[1];
Nid::PatchOptions options;
options.executable = std::string(argv[1]) == "--exec";
const std::string libraryName = options.executable ? "" : argv[1];
std::unordered_set<std::string> excludedExports;
std::vector<std::string> paths;
try {
if (!options.executable && libraryName.starts_with("--")) throw std::runtime_error("unknown option: " + libraryName);
bool hasReference = false;
for (int i = 2; i < argc; ++i) {
const std::string argument = argv[i];
if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
if (argument == "--only-sce") {
if (!options.executable || options.onlySce) throw std::runtime_error("--only-sce requires --exec and may appear once");
options.onlySce = true;
} else if (argument == "--preserve-exports" || argument == "--preserve-export-list") {
if (hasReference || i + 1 == argc) throw std::runtime_error(argument + " requires exactly one reference");
hasReference = true;
excludedExports = argument == "--preserve-exports" ? Nid::ReadExportExclusions(argv[++i]) : Nid::ReadExportList(argv[++i]);
@@ -72,7 +79,7 @@ int Run(int argc, char* argv[]) {
try {
auto binary = _readFile(path);
const auto patcher = Nid::MakePatcher(binary);
patcher->PatchNids(binary, libraryName, excludedExports);
patcher->PatchNids(binary, libraryName, excludedExports, options);
_writeFile(path, binary);
std::cout << "OK: " << path << "\n";
} catch (const std::exception& e) {
core/libs/nid/tests/test_elf_duplicates.py
+2 -2
View File
@@ -4,7 +4,7 @@ import sys
import unittest
from pathlib import Path
from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC
from test_symbol_versions import build_image, patch, read_dynsym, IMPORTS, NID_NAME, SHN_UNDEF, STT_FUNC, PATCHER
def nid(name):
@@ -21,6 +21,7 @@ def image(names, addresses=None):
return bytes(build_image(entries, False))
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class ElfDuplicateTests(unittest.TestCase):
def test_exports_patched_to_one_name_are_rejected(self):
cases = (
@@ -65,5 +66,4 @@ class ElfDuplicateTests(unittest.TestCase):
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/nid/tests/test_elf_layout.py
+2 -2
View File
@@ -3,9 +3,10 @@ import sys
import unittest
from pathlib import Path
from test_symbol_versions import build_image, symbols, patch, SHOFF
from test_symbol_versions import build_image, symbols, patch, SHOFF, PATCHER
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class ElfLayoutTests(unittest.TestCase):
def test_declared_layout_must_match_supported_tables(self):
changes = ((5, "B", 2), (0x3a, "H", 32),
@@ -44,5 +45,4 @@ class ElfLayoutTests(unittest.TestCase):
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/nid/tests/test_gnu_hash.py
+3 -2
View File
@@ -3,7 +3,7 @@ import sys
import unittest
from pathlib import Path
from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash, sections
from test_symbol_versions import build_image, symbols, patch, HASH_OFFSET, SHOFF, read_dynsym, gnu_hash, sections, PATCHER
def sysv_hash(name):
@@ -31,6 +31,7 @@ def sysv_lookup(image, name):
return None
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class GnuHashTests(unittest.TestCase):
def image(self, buckets=4, bloom=1, shift=6, symoffset=3):
image = build_image(symbols(False), False)
@@ -95,6 +96,7 @@ class GnuHashTests(unittest.TestCase):
self.assertEqual(struct.unpack_from("<I", after, HASH_OFFSET)[0], 0)
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class SysvHashTests(unittest.TestCase):
def image(self, both, buckets):
entries = symbols(False)
@@ -153,5 +155,4 @@ class SysvHashTests(unittest.TestCase):
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/nid/tests/test_macho_nids.py
+9 -2
View File
@@ -7,6 +7,7 @@ import subprocess
import sys
import tempfile
from pathlib import Path
from test_symbol_versions import find_patcher, PATCHER
SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30])
BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"
@@ -262,7 +263,7 @@ def check_export_list(patcher, directory):
def check_refusals(patcher, directory):
growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)]
result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True))
assert result.returncode != 0 and "signed Mach-O images cannot grow their export trie" in result.stdout + result.stderr, (result.stdout, result.stderr)
assert result.returncode != 0 and "signed Mach-O images cannot grow linkedit data" in result.stdout + result.stderr, (result.stdout, result.stderr)
result, _ = patch(patcher, directory, "opcodes.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [], bind_opcodes=True))
assert result.returncode != 0 and "bind opcodes are not supported" in result.stdout + result.stderr, (result.stdout, result.stderr)
result, _ = patch(patcher, directory, "long.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [("_sceXy", 1)]))
@@ -270,7 +271,13 @@ def check_refusals(patcher, directory):
def main():
patcher = Path(sys.argv[1]).resolve()
if len(sys.argv) > 1:
patcher = Path(sys.argv[1]).resolve()
else:
patcher = PATCHER or find_patcher()
if patcher is None:
print("nid_patcher executable required")
sys.exit(0)
with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-a\u00e7\u00e3o-\u65e5\u672c-") as directory:
check_shrinking(patcher, directory)
check_growing(patcher, directory)
core/libs/nid/tests/test_native_paths.py
+2 -4
View File
@@ -5,16 +5,14 @@ import tempfile
import unittest
from pathlib import Path
from test_macho_nids import build_dylib, exports_of, nid
from test_macho_nids import build_dylib, exports_of, nid, PATCHER
try:
import resource
except ImportError:
resource = None
PATCHER = Path(sys.argv.pop(1)).resolve()
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class NativePathTests(unittest.TestCase):
def invoke(self, *arguments):
return subprocess.run([str(PATCHER), "libtest", *map(str, arguments)],
core/libs/nid/tests/test_pe_references.py
+201 -2
View File
@@ -5,7 +5,7 @@ import tempfile
import unittest
from pathlib import Path
from test_macho_nids import build_dylib, exports_of, nid
from test_macho_nids import build_dylib, exports_of, nid, PATCHER
def reference(*, headers=False, address=0x2000, ordinal=0):
@@ -100,6 +100,7 @@ def forwarding_exports(image, bits):
return string(table[4]), names, targets, addresses, (export_rva, export_size)
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class ReferenceTests(unittest.TestCase):
def patch_with_reference(self, data):
with tempfile.TemporaryDirectory() as tmp:
@@ -135,6 +136,7 @@ class ReferenceTests(unittest.TestCase):
self.assertEqual(image, original)
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class ForwardingExportTests(unittest.TestCase):
def patch(self, image):
with tempfile.TemporaryDirectory() as temporary:
@@ -241,6 +243,203 @@ class ForwardingExportTests(unittest.TestCase):
self.assertEqual(after, image)
class ExecutableImportTests(unittest.TestCase):
names = ["foo", "foo_nid_postfix", "z", "sceExample", "helper_nid_no_patch", "plain_nid_no_patch_cut", "SDL_Init", "scePreserved", "hostFunctionLongName", "alias_nid_disambig12_nid_postfix"]
def pe(self, bits=64):
image = bytearray(2048)
image[:2] = b"MZ"
struct.pack_into("<I", image, 0x3c, 0x40)
optional_size = 240 if bits == 64 else 224
struct.pack_into("<IHHIIIHH", image, 0x40, 0x4550, 0x8664 if bits == 64 else 0x14c, 1, 0, 0, 0, optional_size, 0x22)
struct.pack_into("<H", image, 0x58, 0x20b if bits == 64 else 0x10b)
struct.pack_into("<II", image, 0x58 + 32, 4096, 512)
struct.pack_into("<II", image, 0x58 + 56, 8192, 512)
directory = 0x58 + (112 if bits == 64 else 96)
struct.pack_into("<I", image, directory - 4, 16)
struct.pack_into("<II", image, directory + 8, 0x1000, 40)
struct.pack_into("<8sIIIIIIHHI", image, 0x58 + optional_size, b".idata", 1536, 0x1000, 1536, 512, 0, 0, 0, 0, 0xc0000040)
struct.pack_into("<IIIII", image, 512, 0x1040, 0, 0, 0x1028, 0x10c0)
image[552:561] = b"test.prx\0"
offset = 0x380
for index, name in enumerate(self.names):
entry = name.encode() + b"\0"
struct.pack_into("<H", image, offset, 0)
image[offset + 2:offset + 2 + len(entry)] = entry
for table in (576, 704):
struct.pack_into("<Q" if bits == 64 else "<I", image, table + index * (bits // 8), offset - 512 + 0x1000)
offset += len(entry) + 2
return image
def elf(self):
from test_symbol_versions import build_image, SHOFF
entries = [(None, 1, 0, 0, 0)] + [(name, 1, 0, 2, 0) for name in self.names] + [("application_export", 1, 9, 2, 0x400100)]
import test_symbol_versions
from unittest.mock import patch
with patch.object(test_symbol_versions, "DYNSYM_OFFSET", 0xa00):
image = build_image(entries, False)
struct.pack_into("<H", image, 16, 2)
struct.pack_into("<Q", image, 32, 64)
struct.pack_into("<HH", image, 54, 56, 2)
struct.pack_into("<IIQQQQQQ", image, 64, 1, 5, 0, 0x400000, 0x400000, 4096, 8192, 4096)
struct.pack_into("<IIQQQQQQ", image, 120, 2, 6, 0x700, 0x400700, 0x400700, 48, 48, 8)
struct.pack_into("<H", image, 60, 5)
struct.pack_into("<IIQQQQIIQQ", image, SHOFF + 4 * 64, 0, 6, 3, 0x400700, 0x700, 48, 2, 0, 8, 16)
size = struct.unpack_from("<Q", image, SHOFF + 2 * 64 + 32)[0]
struct.pack_into("<qQqQqQ", image, 0x700, 5, 0x400300, 10, size, 0, 0)
return image
def macho(self):
image = bytearray(build_dylib([("_application_export", 0, 0x100)], [("_" + name, 1 if index % 2 else 2) for index, name in enumerate(self.names)]))
struct.pack_into("<I", image, 12, 2)
return image
def read_names(self, image, kind):
if kind == "elf":
from test_symbol_versions import read_dynsym
return [name for name, section, _ in read_dynsym(image) if name and section == 0]
if kind == "macho":
from test_macho_nids import chained_imports
return [name[1:] for name, _ in chained_imports(image)]
bits = 64 if kind == "pe64" else 32
optional_size = 240 if bits == 64 else 224
count = struct.unpack_from("<H", image, 0x46)[0]
sections = [struct.unpack_from("<8sIIIIIIHHI", image, 0x58 + optional_size + index * 40) for index in range(count)]
result = []
for index in range(len(self.names)):
rva = struct.unpack_from("<Q" if bits == 64 else "<I", image, 576 + index * (bits // 8))[0]
self.assertEqual(rva, struct.unpack_from("<Q" if bits == 64 else "<I", image, 704 + index * (bits // 8))[0])
section = next(s for s in sections if s[2] <= rva < s[2] + s[3])
offset = section[4] + rva - section[2] + 2
result.append(image[offset:image.index(0, offset)].decode())
return result
def test_shared_import_rules_and_filter_in_all_formats(self):
for kind, build in (("pe64", self.pe), ("pe32", lambda: self.pe(32)), ("elf", self.elf), ("macho", self.macho)):
for only_sce in (False, True):
with self.subTest(kind=kind, only_sce=only_sce), tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "arbitrary.extension"
image = build()
path.write_bytes(image)
excluded = Path(directory) / "preserved.txt"
excluded.write_bytes(b"scePreserved\n")
args = [str(PATCHER), "--exec", "--preserve-export-list", str(excluded)]
if only_sce:
args.append("--only-sce")
result = subprocess.run([*args, str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
expected = ["foo", nid("foo"), nid("z"), nid("sceExample"), "helper_nid_no_patch", "plain", "SDL_Init", "scePreserved", nid("hostFunctionLongName"), nid("alias")]
if only_sce:
expected[2] = "z"
expected[5] = "plain_nid_no_patch_cut"
expected[8] = "hostFunctionLongName"
self.assertCountEqual(self.read_names(path.read_bytes(), kind), expected)
if kind == "macho":
self.assertEqual(exports_of(path.read_bytes()), exports_of(image))
if kind == "elf":
from test_symbol_versions import read_dynsym
self.assertIn(("application_export", 9, 0x400100), read_dynsym(path.read_bytes()))
def test_exec_accepts_macho_without_exports(self):
from test_macho_nids import LC_DYLD_EXPORTS_TRIE, command_fields
image = self.macho()
command = command_fields(image)[LC_DYLD_EXPORTS_TRIE][0]
struct.pack_into("<II", image, command + 8, 0, 0)
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stderr)
def test_elf_without_sections_rejects_dynamic_segment_without_writing(self):
image = self.elf()
struct.pack_into("<Q", image, 40, 0)
struct.pack_into("<HH", image, 60, 0, 0)
for flags in ([], ["--only-sce"]):
with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", *flags, str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
self.assertIn("PT_DYNAMIC without a .dynsym section", result.stderr)
self.assertEqual(path.read_bytes(), image)
def test_elf_without_sections_or_dynamic_segment_is_unchanged(self):
image = self.elf()
struct.pack_into("<Q", image, 40, 0)
struct.pack_into("<HH", image, 60, 0, 0)
struct.pack_into("<H", image, 56, 1)
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(path.read_bytes(), image)
def test_elf_short_imports_grow_without_overwriting_bss(self):
from test_symbol_versions import sections
self.names = ["a", "b", "c", "d"]
image = self.elf()
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stderr)
after = path.read_bytes()
self.assertCountEqual(self.read_names(after, "elf"), [nid(name) for name in self.names])
offset = struct.unpack_from("<Q", after, 32)[0]
count = struct.unpack_from("<H", after, 56)[0]
segments = [struct.unpack_from("<IIQQQQQQ", after, offset + index * 56) for index in range(count)]
original = next(segment for segment in segments if segment[0] == 1 and segment[3] == 0x400000)
self.assertEqual(original[5:7], (4096, 8192))
phdr = next(segment for segment in segments if segment[0] == 6)
load = next(segment for segment in segments if segment[0] == 1 and segment[2] == offset)
self.assertEqual(phdr[2:4], load[2:4])
strings = next(section for section in sections(after) if section[1] == 3)
self.assertEqual(struct.unpack_from("<Q", after, 0x708)[0], strings[3])
self.assertEqual(struct.unpack_from("<Q", after, 0x718)[0], strings[5])
self.assertGreater(strings[4], len(image))
def test_pe_own_exports_remain_unchanged(self):
image = self.pe()
struct.pack_into("<II", image, 0x58 + 112, 0x1500, 96)
struct.pack_into("<IIHHIIIIIII", image, 0x700, 0, 0, 0, 0, 0x1540, 1, 1, 1, 0x1528, 0x152c, 0x1530)
struct.pack_into("<IIH", image, 0x728, 0x1100, 0x1550, 0)
image[0x740:0x748] = b"app.exe\0"
image[0x750:0x75b] = b"own_export\0"
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(path.read_bytes()[0x700:0x760], image[0x700:0x760])
def test_unsupported_exec_layouts_fail_without_writing(self):
signed = bytearray(build_dylib([], [("_sceExample", 1)], signed=True))
struct.pack_into("<I", signed, 12, 2)
pe = self.pe()
struct.pack_into("<II", pe, 0x58 + 112 + 13 * 8, 0x1400, 32)
full_headers = self.pe()
struct.pack_into("<I", full_headers, 0x58 + 60, 0x58 + 240 + 40)
for image, message in ((signed, "signature"), (pe, "delay-loaded"), (full_headers, "section header"), (b"invalid", "unrecognized")):
with self.subTest(message=message), tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
path.write_bytes(image)
result = subprocess.run([str(PATCHER), "--exec", str(path)], capture_output=True, text=True, timeout=20)
self.assertEqual(result.returncode, 2)
self.assertIn(message, result.stderr)
self.assertEqual(path.read_bytes(), image)
def test_only_sce_requires_exec_and_failure_does_not_write(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "program"
image = self.pe()
path.write_bytes(image)
for arguments in (["libtest", "--only-sce"], ["--only-sce"], ["--exec", "--only-sce", "--only-sce"]):
result = subprocess.run([str(PATCHER), *arguments, str(path)], capture_output=True, timeout=20)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(path.read_bytes(), image)
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/nid/tests/test_shared_strings.py
+2 -2
View File
@@ -3,9 +3,10 @@ import sys
import unittest
from pathlib import Path
from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSYM_OFFSET, read_dynsym, sections, DEFINITIONS
from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSYM_OFFSET, read_dynsym, sections, DEFINITIONS, PATCHER
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class SharedStringTests(unittest.TestCase):
def test_dynamic_names_keep_original_string_when_export_shares_offset(self):
for tag in (1, 14, 15, 29):
@@ -54,5 +55,4 @@ class SharedStringTests(unittest.TestCase):
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/nid/tests/test_symbol_versions.py
+30 -1
View File
@@ -1,3 +1,4 @@
import os
import re
import struct
import subprocess
@@ -198,8 +199,36 @@ def test_unversioned_library_is_still_patched(patcher):
assert len({entry[0] for entry in after if entry[0]}) == len([entry for entry in after if entry[0]])
def find_patcher():
for index, argument in enumerate(sys.argv[1:], 1):
candidate = Path(argument)
if candidate.stem == "nid_patcher" and candidate.is_file():
sys.argv.pop(index)
return candidate.resolve()
env = os.environ.get("ANYPS5_NID_PATCHER") or os.environ.get("NID_PATCHER")
if env and Path(env).is_file():
return Path(env).resolve()
repo_root = Path(__file__).resolve().parents[4]
for candidate in (
repo_root / "build" / "core" / "libs" / "nid_patcher.exe",
repo_root / "build" / "core" / "libs" / "nid_patcher",
):
if candidate.is_file():
return candidate
return None
PATCHER = find_patcher()
def main():
patcher = Path(sys.argv[1]).resolve()
if len(sys.argv) > 1:
patcher = Path(sys.argv[1]).resolve()
else:
patcher = PATCHER or find_patcher()
if patcher is None:
print("nid_patcher executable required")
sys.exit(0)
test_versioned_library_is_rejected(patcher)
test_unversioned_library_is_still_patched(patcher)
print("NID patcher symbol version tests passed")
core/libs/nid/tests/test_verneed.py
+2 -2
View File
@@ -3,7 +3,7 @@ import sys
import unittest
from pathlib import Path
from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSTR_OFFSET, sections
from test_symbol_versions import build_image, symbols, patch, SHOFF, DYNSTR_OFFSET, sections, PATCHER
VERNEED = 0x580
@@ -22,6 +22,7 @@ def image(*, auxiliary=16, count=1, size=32, version=1, auxnext=0, nextrecord=0)
return data
@unittest.skipUnless(PATCHER is not None and PATCHER.is_file(), "nid_patcher executable required")
class VersionRequirementsTests(unittest.TestCase):
def test_auxiliary_record_cannot_leave_declared_section(self):
self.reject(image(auxiliary=32))
@@ -72,5 +73,4 @@ class VersionRequirementsTests(unittest.TestCase):
if __name__ == "__main__":
PATCHER = Path(sys.argv.pop(1)).resolve()
unittest.main()
core/libs/prx/libSceAgc/Acb/include/Sync.hpp
+1 -1
View File
@@ -9,7 +9,7 @@ extern "C" {
uint32_t* APS5_VABI sceAgcAcbCondExec(CommandBuffer* buf, const volatile uint32_t* address, uint32_t numDwords);
uint32_t APS5_VABI sceAgcAcbCondExecGetSize(void);
std::uint32_t* APS5_VABI sceAgcAcbWaitRegMem(CommandBuffer* buf, std::uint8_t size, std::uint8_t compareFunction, std::uint8_t cachePolicy, const volatile void* address, std::uint64_t reference, std::uint64_t mask, std::uint32_t pollCycles);
std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint8_t size);
std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint32_t size);
std::uint32_t* APS5_VABI sceAgcAcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address);
std::uint64_t APS5_VABI sceAgcAcbEventWriteGetSize(std::uint8_t eventType);
std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize();
core/libs/prx/libSceAgc/Acb/src/Sync.cpp
+2 -2
View File
@@ -25,8 +25,8 @@ std::uint32_t* APS5_VABI sceAgcAcbWaitRegMem(CommandBuffer* buf, std::uint8_t si
return Agc::Command::WriteWait(buf, size, compareFunction, 0, cachePolicy, address, reference, mask, pollCycles, __func__);
}
std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint8_t size) {
return size == 0 ? 56 : 64;
std::uint64_t APS5_VABI sceAgcAcbWaitOnAddressGetSize(std::uint32_t size) {
return size == 0 ? 56 : size == 1 ? 64 : 0;
}
std::uint32_t* APS5_VABI sceAgcAcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address) {
core/libs/prx/libSceAgc/CMakeLists.txt
+13
View File
@@ -293,6 +293,19 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_wait_on_address_size_tests EXCLUDE_FROM_ALL tests/WaitOnAddressSize.cpp)
target_include_directories(agc_wait_on_address_size_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_wait_on_address_size_tests PRIVATE libSceAgc libc)
configure_windows_unwind(agc_wait_on_address_size_tests)
if(BUILD_TESTING)
set_target_properties(agc_wait_on_address_size_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
add_test(NAME agc_wait_on_address_size COMMAND agc_wait_on_address_size_tests)
if(WIN32)
set_tests_properties(agc_wait_on_address_size PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_interpolant_mapping_tests EXCLUDE_FROM_ALL
tests/InterpolantMapping.cpp
Shader/src/InterpolantMapping.cpp
core/libs/prx/libSceAgc/Command/src/Packet.cpp
+10 -4
View File
@@ -115,17 +115,23 @@ std::uint32_t* WriteRegisters(CommandBuffer* buffer, std::uint32_t opcode, const
Require(buffer != nullptr, function, "null command buffer");
return buffer->cursor_up;
}
CheckAddress(reinterpret_cast<std::uintptr_t>(registers), 4, function);
CheckAddress(reinterpret_cast<std::uintptr_t>(registers), 1, function);
const auto* bytes = reinterpret_cast<const std::byte*>(const_cast<const ShaderRegister*>(registers));
const auto entryAt = [&](std::uint32_t index) {
ShaderRegister entry;
std::memcpy(&entry, bytes + static_cast<std::size_t>(index) * sizeof(ShaderRegister), sizeof(entry));
return entry;
};
std::vector<ShaderRegister> snapshot;
if (snapshotAll) {
snapshot.reserve(count);
for (std::uint32_t i = 0; i < count; ++i) {
snapshot.push_back({registers[i].offset, registers[i].value});
snapshot.push_back(entryAt(i));
CheckBits(snapshot.back().offset, 0xffffu, function);
}
}
const auto offsetAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].offset : registers[index].offset; };
const auto valueAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].value : registers[index].value; };
const auto offsetAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].offset : entryAt(index).offset; };
const auto valueAt = [&](std::uint32_t index) { return snapshotAll ? snapshot[index].value : entryAt(index).value; };
std::uint32_t* first = nullptr;
std::uint32_t index = 0;
while (index < count) {
core/libs/prx/libSceAgc/DcbFlow/src/Sync.cpp
+1 -1
View File
@@ -26,7 +26,7 @@ std::uint32_t* APS5_VABI sceAgcDcbWaitRegMem(CommandBuffer* buf, std::uint8_t si
}
uint32_t APS5_VABI sceAgcDcbWaitOnAddressGetSize(uint32_t size) {
return size == 0 ? 56 : 64;
return size == 0 ? 56 : size == 1 ? 64 : 0;
}
std::uint32_t* APS5_VABI sceAgcDcbEventWrite(CommandBuffer* buf, std::uint8_t eventType, const volatile void* address) {
core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp
+3 -1
View File
@@ -13,7 +13,9 @@ uint32_t* APS5_VABI sceAgcDcbContextStateOp(CommandBuffer* buf, uint32_t operati
}
uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation) {
Agc::Command::Require(operation <= 3, __func__, "invalid context state operation");
if (operation > 3) {
return 0;
}
if (operation == 0) {
return 5 * sizeof(std::uint32_t);
}
core/libs/prx/libSceAgc/tests/ContextStateSize.cpp
+8 -4
View File
@@ -47,9 +47,13 @@ void testSizes() {
}
}
void testRejections() {
expectFailure([] { sceAgcDcbContextStateOpGetSize(4); });
expectFailure([] { sceAgcDcbContextStateOpGetSize(0xffffffffu); });
void testInvalidOperations() {
for (const std::uint32_t operation : {4u, 15u, 16u, 0xffu, 0x100u, 0x103u, 0xffffffffu}) {
check(sceAgcDcbContextStateOpGetSize(operation) == 0, "an invalid context state operation did not return size 0");
}
std::array<std::uint32_t, 64> words{};
CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
expectFailure([&] { sceAgcDcbContextStateOp_0100(&buffer, 4); });
}
}
@@ -57,7 +61,7 @@ void testRejections() {
int main() {
try {
testSizes();
testRejections();
testInvalidOperations();
LibcRunShutdown_nid_postfix();
std::puts("AGC context state size tests passed");
return 0;
core/libs/prx/libSceAgc/tests/RegisterCommands.cpp
+22
View File
@@ -6,6 +6,7 @@
#include <algorithm>
#include <array>
#include <cstdio>
#include <cstring>
#include <stdexcept>
namespace {
@@ -99,6 +100,26 @@ void testRangeFromAnyByte() {
}
}
void testListFromAnyByte() {
const std::array writers{sceAgcCbSetShRegistersDirect, sceAgcCbSetUcRegistersDirect};
const std::array opcodes{0x7600u, 0x7900u};
const std::array<ShaderRegister, 3> registers{{{0x10u, 0x11223344u}, {0x11u, 0x55667788u}, {0x30u, 0x99aabbccu}}};
for (std::size_t i = 0; i < writers.size(); ++i) {
for (const std::size_t offset : {1u, 2u, 3u, 5u}) {
alignas(16) std::array<unsigned char, 48> bytes{};
std::memcpy(bytes.data() + offset, registers.data(), sizeof(registers));
Storage storage;
storage.words.fill(0xabcdef01u);
const auto* list = reinterpret_cast<const volatile ShaderRegister*>(bytes.data() + offset);
auto* packet = writers[i](&storage.buffer, list, static_cast<std::uint32_t>(registers.size()));
const std::array<std::uint32_t, 7> expected{0xc0020000u | opcodes[i], 0x10u, 0x11223344u, 0x55667788u, 0xc0010000u | opcodes[i], 0x30u, 0x99aabbccu};
check(packet == storage.words.data() && storage.buffer.cursor_up == packet + expected.size(), "unaligned register list allocation mismatch");
check(std::equal(expected.begin(), expected.end(), packet), "unaligned register list was not read byte for byte");
check(packet[expected.size()] == 0xabcdef01u, "unaligned register list overwrote the following word");
}
}
}
void testIndirect() {
const std::array writers{sceAgcDcbSetCxRegistersIndirect, sceAgcDcbSetShRegistersIndirect, sceAgcDcbSetUcRegistersIndirect};
const std::array sizes{sceAgcDcbSetCxRegistersIndirectGetSize, sceAgcDcbSetShRegistersIndirectGetSize, sceAgcDcbSetUcRegistersIndirectGetSize};
@@ -213,6 +234,7 @@ int main() {
testDirect();
testConfig();
testRangeFromAnyByte();
testListFromAnyByte();
testIndirect();
testIndirectPlaceholder();
testDirectList();
core/libs/prx/libSceAgc/tests/WaitOnAddressSize.cpp
+41
View File
@@ -0,0 +1,41 @@
#include "prx/libSceAgc/Acb/include/Sync.hpp"
#include "prx/libSceAgc/DcbFlow/include/Sync.hpp"
#include "prx/libSceAgc/Command/include/Packet.hpp"
#include <array>
#include <cstdint>
#include <cstdio>
namespace {
struct Storage {
std::array<std::uint32_t, 64> words{};
CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
std::uint64_t written() const { return static_cast<std::uint64_t>(buffer.cursor_up - words.data()) * sizeof(std::uint32_t); }
};
int fail(const char* message) {
std::fprintf(stderr, "%s\n", message);
return 1;
}
}
int main() {
alignas(8) std::uint64_t label = 0;
for (const std::uint32_t size : {0u, 1u}) {
const std::uint64_t expected = size == 0 ? 56 : 64;
if (sceAgcAcbWaitOnAddressGetSize(size) != expected || sceAgcDcbWaitOnAddressGetSize(size) != expected) return fail("wait size mismatch");
Storage acb;
Storage dcb;
sceAgcAcbWaitRegMem(&acb.buffer, static_cast<std::uint8_t>(size), 3, 0, &label, 1, ~0ull, 0x40);
sceAgcDcbWaitRegMem(&dcb.buffer, static_cast<std::uint8_t>(size), 3, 0, 0, &label, 1, ~0ull, 0x40);
if (acb.written() != expected || dcb.written() != expected) return fail("wait size differs from the packets the wait builders write");
}
for (const std::uint32_t size : {2u, 3u, 4u, 8u, 0xffu, 0x100u, 0x101u, 0xffffffffu}) {
if (sceAgcAcbWaitOnAddressGetSize(size) != 0 || sceAgcDcbWaitOnAddressGetSize(size) != 0) return fail("an invalid wait size did not return 0");
}
std::puts("AGC wait size tests passed");
return 0;
}
core/libs/prx/libSceAgcDriver/CMakeLists.txt
+95 -9
View File
@@ -327,6 +327,7 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineJournal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeferredDestruction.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
@@ -805,7 +806,9 @@ if(BUILD_TESTING)
set_tests_properties(agc_import_alias_shared PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
add_test(NAME agc_compared_shared_generations COMMAND agc_write_tracking_tests --compared-shared-generations)
set_tests_properties(agc_compared_shared_generations PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
set_tests_properties(agc_write_tracking agc_write_tracking_metadata_commit agc_resident_direct_remap agc_rewatched_direct_shared agc_import_alias_shared agc_compared_shared_generations PROPERTIES ENVIRONMENT_MODIFICATION
add_test(NAME agc_write_tracking_rewatch_import_pages COMMAND agc_write_tracking_tests --rewatch-import-pages)
set_tests_properties(agc_write_tracking_rewatch_import_pages PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 20)
set_tests_properties(agc_write_tracking agc_write_tracking_metadata_commit agc_resident_direct_remap agc_rewatched_direct_shared agc_import_alias_shared agc_compared_shared_generations agc_write_tracking_rewatch_import_pages PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
@@ -860,6 +863,15 @@ if(BUILD_TESTING)
add_test(NAME agc_fragment_barycentric COMMAND agc_fragment_barycentric_tests)
endif()
add_test_executable(agc_tessellation_pipeline_cache_tests EXCLUDE_FROM_ALL tests/TessellationPipelineCache.cpp)
target_include_directories(agc_tessellation_pipeline_cache_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
set_target_properties(agc_tessellation_pipeline_cache_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_tessellation_pipeline_cache_tests)
if(BUILD_TESTING)
set_target_properties(agc_tessellation_pipeline_cache_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_tessellation_pipeline_cache COMMAND agc_tessellation_pipeline_cache_tests)
endif()
add_test_executable(agc_float32_flush_control_tests EXCLUDE_FROM_ALL tests/Float32FlushControl.cpp)
target_include_directories(agc_float32_flush_control_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
set_target_properties(agc_float32_flush_control_tests PROPERTIES CXX_EXTENSIONS OFF)
@@ -1268,6 +1280,20 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_deferred_destruction_tests EXCLUDE_FROM_ALL tests/DeferredDestruction.cpp)
target_include_directories(agc_deferred_destruction_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_deferred_destruction_tests PRIVATE libSceAgcDriver libc)
set_target_properties(agc_deferred_destruction_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_deferred_destruction_tests)
if(BUILD_TESTING)
set_target_properties(agc_deferred_destruction_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_deferred_destruction COMMAND agc_deferred_destruction_tests)
if(WIN32)
set_tests_properties(agc_deferred_destruction PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_device_memory_fallback_tests EXCLUDE_FROM_ALL tests/DeviceMemoryFallback.cpp)
target_include_directories(agc_device_memory_fallback_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_device_memory_fallback_tests PRIVATE libSceAgcDriver libc)
@@ -1561,8 +1587,12 @@ if(BUILD_TESTING)
set_tests_properties(agc_pipeline_library_off PROPERTIES ENVIRONMENT "APS5_NO_GPL=1")
add_test(NAME agc_pipeline_library_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-only)
set_tests_properties(agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1" SKIP_RETURN_CODE 77)
add_test(NAME agc_pipeline_library_lto_info COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
add_test(NAME agc_pipeline_library_lto_info_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-lto-info-only)
set_tests_properties(agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES SKIP_RETURN_CODE 77)
set_tests_properties(agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1")
if(WIN32)
set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only agc_pipeline_library_lto_info agc_pipeline_library_lto_info_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_pipeline_cache agc_pipeline_cache_file PROPERTIES SKIP_RETURN_CODE 77)
@@ -1572,6 +1602,8 @@ if(BUILD_TESTING)
endif()
add_test(NAME agc_import_watch COMMAND agc_driver_recorder_tests --import-watch-only)
set_tests_properties(agc_import_watch PROPERTIES SKIP_RETURN_CODE 77)
add_test(NAME agc_import_settle_initial_dcc COMMAND agc_driver_recorder_tests --settle-initial-dcc-only)
set_tests_properties(agc_import_settle_initial_dcc PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
if(WIN32)
add_test(NAME agc_import_watch_other_family COMMAND agc_driver_recorder_tests --import-watch-other-family-only)
set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77)
@@ -1589,7 +1621,7 @@ if(BUILD_TESTING)
add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only)
add_test(NAME agc_import_settle_storage_image COMMAND agc_driver_recorder_tests --settle-storage-image-only)
set_tests_properties(agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
set_tests_properties(agc_import_watch agc_import_watch_other_family agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image PROPERTIES ENVIRONMENT_MODIFICATION
set_tests_properties(agc_import_watch agc_import_watch_other_family agc_import_settle_deferred agc_import_settle_deferred_other_family agc_import_settle_first_use agc_import_settle_earlier_import agc_import_settle_retire agc_import_settle_storage_image agc_import_settle_initial_dcc PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
if(WIN32)
@@ -2168,6 +2200,12 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_address_passes_draw_transitions COMMAND agc_driver_address_passes_tests --targets)
set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_READ_PASSES=1;APS5_DRAW_TRANSITIONS=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
if(WIN32)
set_tests_properties(agc_driver_address_passes_draw_transitions PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_target_properties(agc_driver_rcp_hardware_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_driver_rcp_hardware_exact COMMAND agc_driver_rcp_hardware_tests)
set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT "APS5_EXACT_RCP=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
@@ -2212,6 +2250,7 @@ endif()
target_link_libraries(agc_driver_rect_list_corner_tests PRIVATE shader_recompiler_agc_driver)
target_link_libraries(agc_driver_queue_zero_late_interrupt_tests PRIVATE libkernel)
target_link_libraries(agc_driver_linear_target_completion_points_tests PRIVATE libkernel)
target_link_libraries(agc_driver_storage_file_write_tests PRIVATE libkernel)
@@ -2226,6 +2265,16 @@ target_include_directories(agc_driver_image_table_cache_bounds_tests PRIVATE $<T
target_include_directories(agc_driver_draw_cache_relocation_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
if(BUILD_TESTING)
add_test(NAME agc_driver_linear_target_completion_points_sync_flip COMMAND agc_driver_linear_target_completion_points_tests --flip)
set_tests_properties(agc_driver_linear_target_completion_points_sync_flip PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SYNC_FLIP=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_driver_linear_target_completion_points_unpipelined COMMAND agc_driver_linear_target_completion_points_tests --label --interrupt)
set_tests_properties(agc_driver_linear_target_completion_points_unpipelined PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_linear_target_completion_points_sync_flip agc_driver_linear_target_completion_points_unpipelined PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_submit_progress COMMAND agc_driver_submit_progress_tests)
set_tests_properties(agc_submit_progress PROPERTIES
ENVIRONMENT "APS5_SUBMIT_PROGRESS=1;ANYPS5_REQUIRE_VULKAN=1" TIMEOUT 30 SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2287,6 +2336,9 @@ if(BUILD_TESTING)
set(testName "agc_driver_msaa_dcc_clear_${samples}")
add_test(NAME ${testName} COMMAND agc_driver_alpha_to_coverage_tests --dcc-fast-clear ${samples})
list(APPEND multisampledDccClearTests ${testName})
set(testName "agc_driver_msaa_dcc_compared_import_${samples}")
add_test(NAME ${testName} COMMAND agc_driver_alpha_to_coverage_tests --dcc-compared-import ${samples})
list(APPEND multisampledDccClearTests ${testName})
endforeach()
set_tests_properties(agc_driver_msaa_dcc_state ${multisampledDccClearTests} PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2337,6 +2389,7 @@ target_include_directories(agc_driver_reversed_order_blend_tests PRIVATE $<TARGE
target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_float_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_uint_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobManager libSceAgc libkernel)
@@ -2356,6 +2409,8 @@ target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PRO
target_include_directories(agc_driver_depth_range_remap_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_depth_bounds_emulation_draw_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
foreach(stage IN ITEMS vert frag)
set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
@@ -2474,12 +2529,14 @@ if(BUILD_TESTING)
set_tests_properties(agc_shader_srt_read_patching PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse)
set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_table_snapshot_reuse_unmasked COMMAND agc_shader_memory_tests --table-snapshot-reuse)
set_tests_properties(agc_shader_table_snapshot_reuse_unmasked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups)
set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws)
set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60)
if(WIN32)
set_tests_properties(agc_shader_table_snapshot_reuse agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization)
set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2492,6 +2549,12 @@ if(BUILD_TESTING)
add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair)
set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_result_memo_budget COMMAND agc_shader_memory_tests --result-memo-budget)
set_tests_properties(agc_shader_result_memo_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_result_memo_budget_threads COMMAND agc_shader_memory_tests --result-memo-budget-threads)
set_tests_properties(agc_shader_result_memo_budget_threads PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_result_memo_budget_masked COMMAND agc_shader_memory_tests --result-memo-budget-masked)
set_tests_properties(agc_shader_result_memo_budget_masked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_layout_pair COMMAND agc_shader_memory_tests --layout-pair)
add_test(NAME agc_shader_layout_pair_off COMMAND agc_shader_memory_tests --layout-pair)
set_tests_properties(agc_shader_layout_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2499,7 +2562,7 @@ if(BUILD_TESTING)
add_test(NAME agc_shader_entry_lane_proof_budget COMMAND agc_shader_memory_tests --entry-lane-proof-budget)
set_tests_properties(agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_result_memo_budget agc_shader_result_memo_budget_threads agc_shader_result_memo_budget_masked agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_locks_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests)
if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests" OR agcTest STREQUAL "agc_driver_rcp_hardware_tests")
@@ -2721,6 +2784,11 @@ if(BUILD_TESTING)
set_tests_properties(agc_pipeline_journal_long_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_pipeline_library_clear_waits_for_build COMMAND agc_driver_graphics_tests --pipeline-library-clear-waits-for-build)
add_test(NAME agc_pipeline_store_lazy_buffer_pool COMMAND agc_driver_graphics_tests --pipeline-store-lazy-buffer-pool)
add_test(NAME agc_pipeline_store_growth COMMAND agc_driver_graphics_tests --pipeline-store-growth)
add_test(NAME agc_pipeline_store_fixed_bound COMMAND agc_driver_graphics_tests --pipeline-store-growth)
add_test(NAME agc_pipeline_store_replay_growth COMMAND agc_driver_graphics_tests --pipeline-store-replay-growth)
set_tests_properties(agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES TIMEOUT 60 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_pipeline_store_fixed_bound PROPERTIES ENVIRONMENT "APS5_PIPELINE_STORE_ENTRIES=1024")
set_tests_properties(agc_pipeline_journal PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_pipeline_library_clear_waits_for_build PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_pipeline_store_lazy_buffer_pool PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2728,7 +2796,7 @@ if(BUILD_TESTING)
add_test(NAME agc_pipeline_journal_extended_paths COMMAND agc_driver_graphics_tests --pipeline-journal-extended-paths)
set_tests_properties(agc_pipeline_journal_extended_paths PROPERTIES TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_pipeline_journal_long_paths agc_pipeline_journal_extended_paths PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_pipeline_journal agc_pipeline_library_clear_waits_for_build agc_pipeline_store_lazy_buffer_pool agc_pipeline_store_growth agc_pipeline_store_fixed_bound agc_pipeline_store_replay_growth PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_driver_sparse_buffer_holes PROPERTIES TIMEOUT 30)
if(WIN32)
@@ -2756,6 +2824,12 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_address_copy_cap COMMAND agc_driver_graphics_tests --address-copy-cap)
add_test(NAME agc_address_copy_cap_remembered COMMAND agc_driver_graphics_tests --address-copy-cap-remembered)
set_tests_properties(agc_address_copy_cap agc_address_copy_cap_remembered PROPERTIES TIMEOUT 30)
if(WIN32)
set_tests_properties(agc_address_copy_cap agc_address_copy_cap_remembered PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_emulated_color_compare_srgb_decode COMMAND agc_driver_emulated_color_compare_tests)
set_tests_properties(agc_driver_emulated_color_compare_srgb_decode PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_SRGB_SHADER_DECODE=1"
@@ -2854,12 +2928,16 @@ if(BUILD_TESTING)
set_tests_properties(agc_structurizer_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_control_flow_iteration_selection_off COMMAND agc_shader_control_flow_tests)
set_tests_properties(agc_shader_control_flow_iteration_selection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_ITERATION_SELECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_structurizer_dominator_tree_off COMMAND agc_structurizer_tests)
set_tests_properties(agc_structurizer_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_control_flow_dominator_tree_off COMMAND agc_shader_control_flow_tests)
set_tests_properties(agc_shader_control_flow_dominator_tree_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOMINATOR_TREE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_structurizer_forced_fallback COMMAND agc_structurizer_tests)
set_tests_properties(agc_structurizer_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_control_flow_forced_fallback COMMAND agc_shader_control_flow_tests)
set_tests_properties(agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FORCE_STRUCTURIZER_FALLBACK=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_structurizer_intersection_off agc_structurizer_iteration_selection_off agc_shader_control_flow_iteration_selection_off agc_structurizer_dominator_tree_off agc_shader_control_flow_dominator_tree_off agc_structurizer_forced_fallback agc_shader_control_flow_forced_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_gated_dispatch_capture_short_timeout COMMAND agc_driver_gated_dispatch_capture_tests)
set_tests_properties(agc_driver_gated_dispatch_capture_short_timeout PROPERTIES
@@ -3118,7 +3196,7 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_array_layer_target_cpu PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync)
foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable registered_workgroup registered_workgroup_sync dispatch_ahead_prewarm abi_deferred_compute abi_deferred_pixel abi_deferred_stages abi_deferred_rectangle abi_deferred_undecodable_state abi_deferred_registration_failure abi_deferred_vertex_inputs abi_deferred_rectangle_pairs lazy lazy_sync)
set(preparedTest "agc_driver_prepared_shaders_${preparedMode}")
if(preparedMode MATCHES "^abi_deferred_")
string(REPLACE "_" "-" preparedArgument ${preparedMode})
@@ -3133,6 +3211,8 @@ if(BUILD_TESTING)
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --lazy)
elseif(preparedMode STREQUAL "registered_workgroup" OR preparedMode STREQUAL "registered_workgroup_sync")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --registered-workgroup)
elseif(preparedMode STREQUAL "dispatch_ahead_prewarm")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --dispatch-ahead-prewarm)
elseif(preparedMode STREQUAL "debug_failure")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --fail-before-registration)
set_tests_properties(${preparedTest} PROPERTIES WILL_FAIL TRUE TIMEOUT 30)
@@ -3433,6 +3513,12 @@ if(BUILD_TESTING)
TIMEOUT 120)
set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
add_test(NAME agc_driver_frame_replay_concurrent COMMAND agc_driver_frame_replay_tests $<TARGET_FILE:agc_frame_replay> concurrent)
set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-concurrent-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=8"
TIMEOUT 120)
set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
@@ -3956,6 +4042,6 @@ endif()
if(BUILD_TESTING)
get_directory_property(agcRegisteredTests TESTS)
list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal)$")
list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$")
set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
endif()
core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp
+2
View File
@@ -33,6 +33,8 @@ extern "C" void AgcDriverSuspendPoint_nid_postfix();
extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std::uint64_t* completed);
extern "C" std::size_t AgcDriverAwaitedWaits_nid_postfix(AgcDriver::AwaitedWait* waits, std::size_t capacity);
extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix();
extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix();
extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)());
extern "C" void AgcDriverRegisterShader_nid_postfix(const Shader* shader);
extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp
+1 -1
View File
@@ -10,7 +10,7 @@ namespace AgcDriver::DriverDetail {
double TraceMs();
std::uint64_t SkippedShader();
bool SkippedShader(std::uint64_t address);
class RefusedDraw final : public std::runtime_error {
public:
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp
+1
View File
@@ -55,6 +55,7 @@ public:
std::uint64_t DecodeDrains() const { return decodeDrains.load(std::memory_order_relaxed); }
std::optional<std::uint64_t> PendingLabel(std::uint64_t address, std::size_t bytes);
static bool HeldReads(std::uint64_t address, std::size_t bytes);
static std::uint64_t HeldReadQueries();
void ReleaseRunningReads();
static bool MetadataPassesPipelined();
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp
+6 -2
View File
@@ -59,6 +59,8 @@ public:
void Shutdown();
void WaitIdle();
std::uint64_t DispatchesAdoptedAhead() const;
std::uint64_t DispatchesRejectedAhead() const;
void SetAheadAdoptHook(void (*hook)());
void CheckFailure();
void ReportFailure(std::exception_ptr error);
void Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip = false);
@@ -74,7 +76,7 @@ public:
void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output);
void UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output);
void AttachWindow(const PresentationWindow& window);
void Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context);
void Present(const PresentationWindow& window, const DisplayBuffer* buffer, const DisplayBuffer* plane, bool opaque, void (*gpuReady)(void*), void* context);
void ReleaseWindow(void* window);
void RegisterShader(const Shader* shader);
void ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
@@ -317,7 +319,7 @@ private:
std::chrono::steady_clock::time_point resolvedAt{};
};
struct GroupCaptureStats {
std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, memoryMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0, argumentMismatches = 0;
double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0;
std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now();
};
@@ -474,6 +476,8 @@ private:
std::atomic<std::uint64_t> evidenceReads{0};
std::atomic<std::uint64_t> evidenceValidations{0};
std::atomic<std::uint64_t> dispatchesAdoptedAhead{0};
std::atomic<std::uint64_t> dispatchesRejectedAhead{0};
std::atomic<void (*)()> aheadAdoptHook{nullptr};
AgcDriver::Mutex writtenBuffersMutex;
std::deque<WrittenBuffer> writtenBuffers;
core/libs/prx/libSceAgcDriver/Execution/include/FrameRecorder.hpp
+7 -4
View File
@@ -10,6 +10,7 @@
#include <cstdint>
#include <deque>
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
@@ -35,10 +36,10 @@ public:
void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
void RegisterVideoOutput(std::uint32_t handle);
void UnregisterVideoOutput(std::uint32_t handle);
void Submit(const Packet& packet, std::uint32_t queue);
void Flip(const FlipInfo& info);
void Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept);
void Flip(const FlipInfo& info, const std::function<void()>& accept);
void ReserveEopFlip(const FlipInfo& info, std::uint32_t token);
void Suspend();
void Suspend(const std::function<void()>& accept);
void PresentBuffer(const DisplayBuffer& buffer);
void FrameBoundary();
@@ -62,7 +63,8 @@ private:
};
FrameCall make(FrameCallKind kind);
void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {});
void record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders = {}, const std::function<void()>& accept = {});
void recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders);
void emitDelta();
void emitShaderDelta(std::span<const Shader* const> shaders);
void writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added = {});
@@ -72,6 +74,7 @@ private:
void snapshot(const Range& range);
void writeReferenceFrames();
bool abandonIfDiskLow();
void stop(const char* reason);
std::string header() const;
std::vector<Range> committedRanges();
std::vector<Range> newRanges();
core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp
+1 -1
View File
@@ -63,7 +63,7 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
bool WriteWatched();
void Unwatch(std::uint64_t address, std::size_t bytes);
void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);
bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true, bool importMayWriteGuest = true);
void SetUnwatchHook(void (*hook)(std::uint64_t address, std::size_t bytes));
#ifdef _WIN32
using FreshCollectHook = bool (*)(std::uintptr_t address, std::size_t bytes, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context);
core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp
+42 -2
View File
@@ -451,18 +451,30 @@ public:
static void AppendString(std::string& output, std::string_view text) {
output += '"';
for (const char character : text) {
std::size_t index = 0;
while (index < text.size()) {
const char character = text[index];
const auto byte = static_cast<unsigned char>(character);
if (character == '"' || character == '\\') {
output += '\\';
output += character;
} else if (byte < 0x20 || byte >= 0x7f) {
} else if (byte >= 0x80) {
if (const auto length = utf8SequenceLength(text, index); length != 0) {
output.append(text.substr(index, length));
index += length;
continue;
}
char escaped[8];
std::snprintf(escaped, sizeof escaped, "\\u%04x", static_cast<unsigned>(byte));
output += escaped;
} else if (byte < 0x20 || byte == 0x7f) {
char escaped[8];
std::snprintf(escaped, sizeof escaped, "\\u%04x", static_cast<unsigned>(byte));
output += escaped;
} else {
output += character;
}
++index;
}
output += '"';
}
@@ -697,6 +709,34 @@ public:
}
private:
static std::size_t utf8SequenceLength(std::string_view text, std::size_t index) {
const auto lead = static_cast<unsigned char>(text[index]);
std::size_t length = 0;
unsigned char low = 0x80;
unsigned char high = 0xbf;
if (lead >= 0xc2 && lead <= 0xdf) {
length = 2;
} else if (lead >= 0xe0 && lead <= 0xef) {
length = 3;
if (lead == 0xe0) low = 0xa0;
if (lead == 0xed) high = 0x9f;
} else if (lead >= 0xf0 && lead <= 0xf4) {
length = 4;
if (lead == 0xf0) low = 0x90;
if (lead == 0xf4) high = 0x8f;
} else {
return 0;
}
if (text.size() - index < length) return 0;
for (std::size_t offset = 1; offset < length; ++offset) {
const auto next = static_cast<unsigned char>(text[index + offset]);
if (next < low || next > high) return 0;
low = 0x80;
high = 0xbf;
}
return length;
}
static void AppendMachine(std::string& output, const TraceSummary& summary) {
if (summary.host) {
output += ",\"cpu\":";
core/libs/prx/libSceAgcDriver/Execution/include/PresentMode.hpp
+7
View File
@@ -3,6 +3,8 @@
#include <vulkan/vulkan.h>
#include <algorithm>
#include <cstdint>
#include <optional>
#include <span>
#include <stdexcept>
#include <string_view>
@@ -22,6 +24,11 @@ inline VkPresentModeKHR SelectPresentMode(const char* requested, std::span<const
return mode;
}
inline std::optional<std::uint32_t> SelectPresentQueue(std::uint32_t familyQueues, std::uint32_t queuesTaken) {
if (queuesTaken >= familyQueues) return std::nullopt;
return queuesTaken;
}
}
#endif
core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp
+2 -2
View File
@@ -26,8 +26,8 @@ struct PresentationWindow {
}
extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationWindow& window);
extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context);
extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context);
extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context);
extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context);
extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window);
extern "C" void AgcDriverTogglePerformanceOverlay_nid_postfix();
extern "C" void AgcDriverReportFailure_nid_postfix(std::exception_ptr error);
core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp
+19 -3
View File
@@ -4,9 +4,12 @@
#define VK_NO_PROTOTYPES
#include <vulkan/vulkan.h>
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <span>
#include <tuple>
#include <utility>
#include <vector>
@@ -24,8 +27,14 @@ public:
PresentationPass& operator=(const PresentationPass&) = delete;
void Bind(std::size_t slot, VkImage image, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits, std::uint32_t layer = 0);
void BindTexels(std::size_t slot, VkCommandBuffer commands, std::span<const std::byte> texels, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits);
void RecordPresent(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout);
void RecordOverlay(VkCommandBuffer commands, std::size_t slot, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
void RecordDump(VkCommandBuffer commands, std::size_t slot, std::uint32_t stride, VkBuffer destination);
void RecordDumpImage(VkCommandBuffer commands, std::size_t slot, std::uint32_t stride);
void RecordDumpReadback(VkCommandBuffer commands, std::size_t slot, VkBuffer destination);
VkImage DumpImage(std::size_t slot) const;
VkExtent2D DumpImageExtent(std::size_t slot) const;
void Release(std::size_t slot) noexcept;
static bool TenBitFormat(VkFormat format);
static VkExtent2D DumpExtent(std::uint32_t width, std::uint32_t height, std::uint32_t stride);
@@ -48,14 +57,21 @@ private:
VkImageView dumpView = VK_NULL_HANDLE;
VkFramebuffer dumpFramebuffer = VK_NULL_HANDLE;
VkExtent2D dumpExtent{};
VkImage uploadImage = VK_NULL_HANDLE;
VkDeviceMemory uploadMemory = VK_NULL_HANDLE;
VkExtent2D uploadExtent{};
std::unique_ptr<Graphics::Buffer> uploadTexels;
};
const Target& target(VkFormat format, VkImageLayout finalLayout);
const Target& target(VkFormat format, VkImageLayout finalLayout, VkImageLayout loadFrom = VK_IMAGE_LAYOUT_UNDEFINED);
VkImageView createView(VkImage image, VkFormat format, VkImageUsageFlags usage, std::uint32_t layer = 0) const;
VkFramebuffer createFramebuffer(VkRenderPass renderPass, VkImageView view, VkExtent2D extent) const;
VkFramebuffer destinationFramebuffer(Slot& slot, const Target& target, VkImage destination, VkFormat format, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout);
void ensureDumpTarget(Slot& slot, VkExtent2D extent);
void releaseDumpTarget(Slot& slot) noexcept;
void draw(VkCommandBuffer commands, const Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, const std::int32_t (&parameters)[9]) const;
void ensureUploadImage(Slot& slot, VkExtent2D extent);
void releaseUploadImage(Slot& slot) noexcept;
void draw(VkCommandBuffer commands, const Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (&parameters)[9]) const;
void release() noexcept;
Graphics::Context context;
@@ -66,7 +82,7 @@ private:
VkShaderModule vertex = VK_NULL_HANDLE;
VkShaderModule fragment = VK_NULL_HANDLE;
std::unique_ptr<Graphics::Buffer> pqNits;
std::map<std::pair<VkFormat, VkImageLayout>, Target> targets;
std::map<std::tuple<VkFormat, VkImageLayout, VkImageLayout>, Target> targets;
std::vector<Slot> slots;
};
core/libs/prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp
+4
View File
@@ -36,6 +36,8 @@ public:
static std::vector<AspectFitRect> LetterboxBands(std::uint32_t width, std::uint32_t height, std::uint32_t destinationWidth, std::uint32_t destinationHeight);
std::uint32_t SourceWidth() const { return sourceWidth; }
std::uint32_t SourceHeight() const { return sourceHeight; }
VkImage SourceImage() const { return sourceImage; }
VkFormat SourceFormat() const { return sourceFormat; }
private:
void release() noexcept;
@@ -46,6 +48,8 @@ private:
std::uint32_t sourceHeight = 0;
VkImage sourceImage = VK_NULL_HANDLE;
VkDeviceMemory sourceMemory = VK_NULL_HANDLE;
VkDeviceSize sourceBytes = 0;
std::uint32_t sourceMemoryType = 0;
VkImage blackImage = VK_NULL_HANDLE;
VkDeviceMemory blackMemory = VK_NULL_HANDLE;
};
core/libs/prx/libSceAgcDriver/Execution/include/SwapchainImages.hpp
+17
View File
@@ -0,0 +1,17 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINIMAGES_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SWAPCHAINIMAGES_HPP
#include <vulkan/vulkan.h>
#include <algorithm>
#include <cstdint>
namespace AgcDriver {
inline std::uint32_t SwapchainImageCount(const VkSurfaceCapabilitiesKHR& surface) {
const std::uint32_t count = std::max(surface.minImageCount, 3u);
return surface.maxImageCount != 0 ? std::min(count, surface.maxImageCount) : count;
}
}
#endif
core/libs/prx/libSceAgcDriver/Execution/include/TessellationPipelineCache.hpp
+14
View File
@@ -0,0 +1,14 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_TESSELLATIONPIPELINECACHE_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_TESSELLATIONPIPELINECACHE_HPP
#include <vulkan/vulkan.h>
namespace AgcDriver {
inline bool TessellationPipelineCacheUsable(VkDriverId driver) {
return driver != VK_DRIVER_ID_MOLTENVK;
}
}
#endif
core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp
+15 -8
View File
@@ -45,9 +45,11 @@ struct DeviceQueuePlan {
bool sparseBinding = false;
std::uint32_t sparseFamily = VK_QUEUE_FAMILY_IGNORED;
std::uint32_t sparseIndex = 0;
bool present = false;
std::uint32_t presentIndex = 0;
};
DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding);
DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding, bool presents);
class VulkanDevice {
public:
@@ -58,6 +60,8 @@ public:
// Builds the pipeline of a journaled compute result ahead of its first dispatch (the journal
// replay's hook, on its threads): false when the device already holds a pipeline for the module.
bool ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants);
bool HasComputePipeline(const ShaderRecompiler::RecompileResult& shader) const;
void PrebuildComputePipeline(const ShaderRecompiler::RecompileResult& shader, std::uint64_t programAddress = 0);
// Starts the journal replay if the device has not (a headless device starts it at its first
// draw or dispatch) and joins it (tests; the teardown joins it anyway).
void WaitForPipelineJournalReplay();
@@ -226,7 +230,7 @@ public:
bool ProvokingVertexLast() const;
bool GraphicsPipelineLibraries() const;
// A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it
// touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command
// touches the main queue. Presentations are slots (FlipInFlight() + 1, each with its own command
// buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires
// the slots whose fence signaled, oldest first, and waits for the oldest ones until at most
// `keep` are in flight (the only CPU wait of the flip path; it returns the time spent waiting);
@@ -235,7 +239,9 @@ public:
// PresentClear/PresentDisplayBuffer record and submit the frame into the next slot (under the
// mutex; the blit follows the frame's batches in queue order, so the displayed image is always
// complete) and return whether one was submitted; QueuePresent hands the image to the swapchain
// (under the mutex). FinishPresent = RetirePresents(0): the render fence wait of the
// (on the device's presentation queue under that queue's mutex, with the GPU mutex released,
// when the presentation family had a free queue; otherwise on the main queue under the GPU
// mutex). FinishPresent = RetirePresents(0): the render fence wait of the
// synchronous paths (APS5_FLIP_INFLIGHT=0, APS5_SYNC_FLIP, PresentPixels). AcquireImage returns
// false when the swapchain is out of date (the frame is dropped; the next Resize recreates it);
// a present without a prior AcquireImage acquires itself, one whose slot is in flight waits for
@@ -253,11 +259,12 @@ public:
// mutex (the same wait under it is only the fallback when the answer changes in between).
bool PresentWaitsForSlots(const DisplayBuffer* buffer) const;
bool AcquireImage();
bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque);
bool PresentClear(std::uint32_t width, std::uint32_t height, bool opaque, const DisplayBuffer* plane = nullptr);
void PresentPixels(std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels);
bool PresentDisplayBuffer(const DisplayBuffer& buffer);
bool PresentDisplayBuffer(const DisplayBuffer& buffer, const DisplayBuffer* plane = nullptr);
double FinishPresent();
void QueuePresent();
bool PresentsOnOwnQueue() const;
// At the flip packet (under the mutex): the recorder's submissions so far and, under
// APS5_PROFILE_DRAW, how many in flight have not signaled (the frame record's batches_at_flip).
void FlipBatches(std::uint64_t& submissions, std::uint64_t& unsignaled) const;
@@ -268,8 +275,8 @@ public:
// blit-submit times after the flip packet; noted in-place reads checked
// (APS5_FLIP_READ_CHECK=1) and found overwritten before the blit's fence.
struct PresentStatistics {
std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops;
double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs;
std::uint64_t presents, gpuUnread, batchesAheadOfBlit, batchesAfterFlip, readsChecked, readsOverwritten, acquireDrops, planes, keyWriterWaits;
double gpuBusyMs, gpuGapMs, lastSubmitAfterFlipMs, blitSubmitAfterFlipMs, keyWriterWaitMs;
};
static PresentStatistics PresentCounts();
struct PresyncStatistics {
@@ -385,7 +392,7 @@ private:
// The tail of a dispatch's device call from the open batch's command buffer to the completion
// registration: keeps, the template data refresh, barriers, bind, push, dispatch, marks.
void recordDispatch(RecordedDispatch& record);
bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr<Graphics::StorageTexture>& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false, bool residentConvert = false, const VkClearColorValue* uniform = nullptr, std::uint32_t residentLayer = 0, bool passthrough = false);
bool present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display = nullptr, const std::shared_ptr<Graphics::StorageTexture>& resident = nullptr, VkFilter residentFilter = VK_FILTER_LINEAR, bool dumpFrame = false, bool residentConvert = false, const VkClearColorValue* uniform = nullptr, std::uint32_t residentLayer = 0, bool passthrough = false, const DisplayBuffer* plane = nullptr);
bool takeScreenshot(bool dumpFrame);
void recreateSwapchain(std::uint32_t width, std::uint32_t height, bool force);
void refreshSurfaceFormats();
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp
+67 -3
View File
@@ -1,11 +1,19 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
#include <array>
#include <atomic>
#include <charconv>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <mutex>
#include <optional>
#include <set>
#include <stdexcept>
#include <string_view>
#include <system_error>
namespace AgcDriver::DriverDetail {
@@ -14,9 +22,65 @@ double TraceMs() {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - origin).count();
}
std::uint64_t SkippedShader() {
static const std::uint64_t skipped = [] { const char* text = std::getenv("APS5_SKIP_SHADER"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }();
return skipped;
namespace {
constexpr std::size_t MaxSkippedShaders = 256;
std::array<std::atomic<std::uint64_t>, MaxSkippedShaders> skippedShaders{};
std::atomic<std::size_t> skippedShaderCount{0};
std::optional<std::uint64_t> programAddress(std::string_view word) {
if (word.starts_with("0x") || word.starts_with("0X")) word.remove_prefix(2);
std::uint64_t address = 0;
const auto [end, error] = std::from_chars(word.data(), word.data() + word.size(), address, 16);
if (word.empty() || error != std::errc{} || end != word.data() + word.size()) return std::nullopt;
return address;
}
void ReloadSkippedShaders(const char* path) {
static std::mutex reload;
static std::filesystem::file_time_type seen{};
const std::unique_lock lock(reload, std::try_to_lock);
if (!lock.owns_lock()) return;
std::error_code error;
const auto stamp = std::filesystem::last_write_time(path, error);
if (error || stamp == seen) return;
seen = stamp;
std::ifstream file(path);
std::size_t count = 0;
skippedShaderCount.store(0, std::memory_order_release);
for (std::string word; count < MaxSkippedShaders && file >> word;) {
if (word.starts_with('#')) {
std::getline(file, word);
continue;
}
const auto address = programAddress(word);
if (!address) {
std::fprintf(stderr, "[skip-shader] ignoring '%s' in %s: not a hexadecimal program address\n", word.c_str(), path);
continue;
}
if (*address != 0) skippedShaders[count++].store(*address, std::memory_order_relaxed);
}
skippedShaderCount.store(count, std::memory_order_release);
std::fprintf(stderr, "[skip-shader] %zu programs listed in %s\n", count, path);
}
}
bool SkippedShader(std::uint64_t address) {
static const std::uint64_t single = [] { const char* text = std::getenv("APS5_SKIP_SHADER"); return text ? std::strtoull(text, nullptr, 16) : 0ull; }();
static const char* const file = std::getenv("APS5_SKIP_SHADER_FILE");
if (address == 0) return false;
if (address == single) return true;
if (file == nullptr) return false;
static std::atomic<std::chrono::steady_clock::rep> nextCheck{0};
const auto now = std::chrono::steady_clock::now().time_since_epoch();
auto due = nextCheck.load(std::memory_order_relaxed);
if (now.count() >= due && nextCheck.compare_exchange_strong(due, std::chrono::duration_cast<std::chrono::steady_clock::duration>(now + std::chrono::milliseconds(500)).count(), std::memory_order_relaxed)) ReloadSkippedShaders(file);
const auto count = skippedShaderCount.load(std::memory_order_acquire);
for (std::size_t index = 0; index < count; ++index) {
if (skippedShaders[index].load(std::memory_order_relaxed) == address) return true;
}
return false;
}
bool SkipRefusedDraws() {
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp
+28 -1
View File
@@ -16,6 +16,7 @@
#include "Optimization/ResourceProgram.hpp"
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <optional>
#include <stdexcept>
#include <utility>
@@ -57,6 +58,19 @@ void ReportDispatchItems(bool profile) {
std::fprintf(stderr, "[dispatch-item] %llu commit items (%llu recipes rebuilt on the committer); drained instead, cumulative:%s\n", static_cast<unsigned long long>(itemCommits.load(std::memory_order_relaxed)), static_cast<unsigned long long>(itemRebuilds.load(std::memory_order_relaxed)), drained.c_str());
}
bool CapturedCurrent(std::span<const ShaderRecompiler::MemoryRegion> captured, const ShaderSnapshot& code) {
for (const auto& region : captured) {
const auto address = region.guestAddress;
const auto bytes = region.bytes.size();
if (bytes == 0 || (address >= code.codeAddress && address < code.codeAddress + code.code.size() * sizeof(std::uint32_t)) || (!code.header.empty() && address >= code.headerAddress && address < code.headerAddress + code.header.size())) continue;
if (Driver::QueryPendingSpan(address, bytes, {}) != ShaderMemory::PendingWrite::None || !GuestMemory::Accessible(reinterpret_cast<const void*>(address), bytes)) return false;
GuestMemory::GuardRead(address, bytes);
GuestMemory::FlushGpuWrites(address, bytes);
if (std::memcmp(region.bytes.data(), reinterpret_cast<const void*>(address), bytes) != 0) return false;
}
return true;
}
}
bool Driver::pipelineDispatch() {
@@ -66,7 +80,7 @@ bool Driver::pipelineDispatch() {
void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments) {
PerformanceTimer timing("Driver.Dispatch");
const auto address = (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20d) & 0xffu) << 40u);
if (address == SkippedShader() && address != 0) return;
if (SkippedShader(address)) return;
auto registeredShader = RegisteredProgram(*submission.shaders, address, {0});
if (!registeredShader) registeredShader = ReadRawShader(address);
const auto& snapshot = *registeredShader;
@@ -281,6 +295,14 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
fromAhead = false;
++groupStats.keyMismatches;
}
if (fromAhead) {
if (const auto hook = aheadAdoptHook.load(std::memory_order_acquire)) hook();
if (!CapturedCurrent(ahead.captured, snapshot)) {
fromAhead = false;
++dispatchesRejectedAhead;
++groupStats.memoryMismatches;
}
}
phaseTiming.Phase(PhaseKey);
timing.Mark("decode_key_device_setup");
bool readsClear = false;
@@ -414,6 +436,11 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
parked.resolvedAt = std::chrono::steady_clock::now();
++groupStats.resolved;
if (cached) ++groupStats.resolvedHits;
if (!cached && compiledResult != nullptr && !localDevice->HasComputePipeline(*compiledResult)) {
SubmitLibraryPrebuild([localDevice, compiledResult, address] {
try { localDevice->PrebuildComputePipeline(*compiledResult, address); } catch (const std::exception&) {}
});
}
return;
}
if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult);
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp
+129 -4
View File
@@ -2,9 +2,11 @@
#include "ThreadOwned.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
#include "Optimization/ResourceProgram.hpp"
#include <algorithm>
#include <cstdlib>
#include <cstring>
@@ -40,7 +42,25 @@ std::uint64_t Driver::DispatchesAdoptedAhead() const {
return dispatchesAdoptedAhead.load(std::memory_order_relaxed);
}
std::uint64_t Driver::DispatchesRejectedAhead() const {
return dispatchesRejectedAhead.load(std::memory_order_relaxed);
}
void Driver::SetAheadAdoptHook(void (*hook)()) {
aheadAdoptHook.store(hook, std::memory_order_release);
}
namespace {
std::size_t& PreviewedUntil() {
static thread_local std::size_t until = 0;
return until;
}
}
void Driver::clearResolvedAhead() {
PreviewedUntil() = 0;
auto& resolved = resolvedAhead();
if (resolved.empty()) return;
groupCaptureStats().stale += resolved.size();
@@ -107,7 +127,8 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
static const bool allQueues = std::getenv("APS5_GROUP_CAPTURE_ALL_QUEUES") != nullptr;
if (!allQueues && submission.queue == 0) return;
auto& resolved = resolvedAhead();
if (resolved.contains(from) || device.Load() == nullptr) return;
const auto localDevice = device.Load();
if (resolved.contains(from) || localDevice == nullptr) return;
if (submission.queue == 0 && DrawPipeline::Depth() != 0 && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, (submission.commands[from] >> 8u) & 0xffu);
auto& stats = groupCaptureStats();
const auto begin = std::chrono::steady_clock::now();
@@ -126,6 +147,110 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
}
} resolving;
const auto& commands = submission.commands;
if (submission.shaders != nullptr && from >= PreviewedUntil()) {
bool hasFurtherDispatch = false;
std::size_t limit = from < commands.size() ? from + Pm4::PacketWords(commands[from]) : commands.size();
while (limit < commands.size()) {
const auto header = commands[limit];
if (Pm4::FillerPacket(header)) {
++limit;
continue;
}
const auto count = Pm4::PacketWords(header);
if (limit + count > commands.size()) break;
const auto opcode = (header >> 8u) & 0xffu;
if (header == FlipPacketHeader || header == RenderingWaitPacketHeader || opcode == 0x22 || opcode == 0x3f || opcode == 0x3c || opcode == 0x93) break;
if (opcode == 0x15 || opcode == 0x16) hasFurtherDispatch = true;
limit += count;
}
PreviewedUntil() = limit;
if (hasFurtherDispatch) {
static thread_local QueueState* previewSlot = nullptr;
auto& preview = ShaderRecompiler::ThreadOwned(previewSlot);
preview.shader = live.shader;
preview.predication = live.predication;
std::vector<PreparedShaders*> pendingShaders;
std::vector<PreparedShaders*> submittedShaders;
std::size_t scan = from;
while (scan < limit) {
const auto header = commands[scan];
if (Pm4::FillerPacket(header)) {
++scan;
continue;
}
const auto count = Pm4::PacketWords(header);
if (scan + count > limit) break;
const auto packet = std::span(commands).subspan(scan, count);
const auto opcode = (header >> 8u) & 0xffu;
const auto next = scan + count;
if (opcode == 0x15 || opcode == 0x16) {
if (count < (opcode == 0x15 ? 5u : 3u)) break;
if (scan > from) {
const auto low = preview.shader.find(0x20c);
const auto high = preview.shader.find(0x20d);
if (low != preview.shader.end() && high != preview.shader.end()) {
const auto address = (static_cast<std::uint64_t>(low->second) << 8u) | (static_cast<std::uint64_t>(high->second & 0xffu) << 40u);
if (auto registered = RegisteredProgram(*submission.shaders, address, {0}); registered != nullptr && registered->type == 0) {
bool isPending = false;
bool isEmpty = false;
{
std::lock_guard lock(registered->prepared->mutex);
isPending = registered->prepared->pending;
isEmpty = registered->prepared->entries.empty();
}
if (isPending) {
if (std::ranges::find(pendingShaders, registered->prepared.get()) == pendingShaders.end()) {
pendingShaders.push_back(registered->prepared.get());
}
} else if (isEmpty && std::ranges::find(submittedShaders, registered->prepared.get()) == submittedShaders.end()) {
const auto codeOffset = static_cast<std::size_t>((address - registered->codeAddress) / sizeof(std::uint32_t));
if (codeOffset < registered->code.size()) {
try {
auto compute = Graphics::DecodeComputeStageInfo(preview.shader, registered->header);
submittedShaders.push_back(registered->prepared.get());
const auto rsrc2 = preview.shader.find(0x213);
const auto userCount = rsrc2 != preview.shader.end() ? ((rsrc2->second >> 1u) & 0x1fu) : 0u;
std::vector<std::uint32_t> userData(userCount, 0u);
for (std::uint32_t i = 0; i < userCount; ++i) {
if (const auto reg = preview.shader.find(0x240u + i); reg != preview.shader.end()) userData[i] = reg->second;
}
const auto wave = (packet.back() & 0x8000u) != 0 ? 32u : 64u;
SubmitRegistrationPreparation([registered, localDevice, address, codeOffset, wave, compute, userData = std::move(userData), state = preview.shader]() mutable {
std::vector<ShaderRecompiler::MemoryRegion> memory{{registered->codeAddress, std::as_bytes(std::span(registered->code))}};
if (!registered->header.empty()) memory.push_back({registered->headerAddress, registered->header});
ShaderRecompiler::RecompileRequest request{
{ShaderRecompiler::ShaderStage::Compute, address, std::span(registered->code).subspan(codeOffset), registered->headerAddress, registered->header},
{wave, 0, userData, compute, std::nullopt, std::nullopt, memory, DispatchFloatMode(*registered, state)},
localDevice->ComputeTarget(wave),
{0, 0, 0, 128}
};
try { static_cast<void>(InvocationFor(*registered, codeOffset, request)); } catch (const std::exception&) {}
}, registered->prepared.get());
} catch (const std::exception&) {}
}
}
}
}
}
Pm4::NoteDispatchRegisters(packet, preview);
} else if (Pm4::IndirectRegisterOpcode(opcode)) {
const auto found = submission.registerLists.find(scan);
if (count != 5 || found == submission.registerLists.end() || found->second.size() != static_cast<std::size_t>(packet[4]) * 2 || !Pm4::IndirectRegistersValid(found->second)) break;
Pm4::ExecuteIndirectRegisters(packet, found->second, preview);
} else if (!Pm4::AccessesMemory(header) && opcode != 0x3c && opcode != 0x93 && opcode != 0x37 && opcode != 0x42 && opcode != 0x46 && opcode != 0x49 && opcode != 0x58) {
try {
Pm4::Execute(packet, preview);
} catch (const std::exception&) {
break;
}
}
scan = next;
}
for (auto it = pendingShaders.rbegin(); it != pendingShaders.rend(); ++it) {
PrioritizeRegistrationPreparation(**it);
}
}
}
for (std::size_t cursor = from; cursor < commands.size();) {
const auto header = commands[cursor];
if (Pm4::FillerPacket(header)) {
@@ -141,7 +266,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
if (opcode == 0x3c || opcode == 0x93) {
if (!Pm4::WaitSatisfiedUnchecked(packet) && !WrittenByRun(packet, labels)) break;
} else if (opcode == 0x15 || opcode == 0x16) {
if (resolved.contains(cursor)) break;
if (count < (opcode == 0x15 ? 5u : 3u) || resolved.contains(cursor)) break;
if (opcode == 0x16) {
std::uint64_t arguments = 0;
try {
@@ -209,9 +334,9 @@ void Driver::reportGroupCapture(std::uint32_t queue) {
auto& stats = groupCaptureStats();
const auto now = std::chrono::steady_clock::now();
if (now - stats.reported < std::chrono::seconds(10)) return;
AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu memory %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device, %llu dropped because the argument re-read differed\n",
queue, static_cast<unsigned long long>(stats.runs), static_cast<unsigned long long>(stats.resolved), static_cast<unsigned long long>(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs,
static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches),
static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches), static_cast<unsigned long long>(stats.memoryMismatches),
static_cast<unsigned long long>(stats.failures), static_cast<unsigned long long>(stats.stale), static_cast<unsigned long long>(stats.labelOverlaps), static_cast<unsigned long long>(stats.writtenBefore), static_cast<unsigned long long>(stats.deviceRejects), static_cast<unsigned long long>(stats.argumentMismatches));
stats = GroupCaptureStats{};
stats.reported = now;
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp
+1 -1
View File
@@ -518,7 +518,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
for (const auto& patch : userPatches) writablePrograms[patch.program].userData[patch.index] = patch.value;
tail.programs = writesPrograms ? &writablePrograms : &decode->programs;
const auto& programs = *tail.programs;
if (const auto skipped = SkippedShader(); skipped != 0 && std::any_of(programs.begin(), programs.end(), [skipped](const DrawProgram& program) { return program.binary.codeAddress == skipped; })) return drawn();
if (std::any_of(programs.begin(), programs.end(), [](const DrawProgram& program) { return SkippedShader(program.binary.codeAddress); })) return drawn();
const bool emptyIndirectSkip = EmptyIndirectSkip();
auto& cpuIndirectPrograms = CpuIndirectPrograms();
const auto recordDraws = [&](const Pm4::DrawArguments& arguments) { return arguments.count != 0 && arguments.instances != 0 && (!drawParameters.indexed || !drawParameters.indirect->indexLimitKnown || arguments.firstVertexOrIndex < drawParameters.indexCount); };
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp
+1
View File
@@ -183,6 +183,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
Graphics::NormalizeUnusedViewportDepth(product->state, product->pixel);
Graphics::ApplyDepthRemap(context, product->state);
Graphics::ApplyDepthBoundsEmulation(context, product->state, product->pixel);
product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
product->pixel.dualSourceBlend = product->state.dualSourceBlend;
product->pixel.reversedBlend = product->state.reversedBlend;
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp
+4
View File
@@ -482,6 +482,10 @@ bool DrawPipeline::HeldReads(std::uint64_t address, std::size_t bytes) {
return false;
}
std::uint64_t DrawPipeline::HeldReadQueries() {
return heldReadQueries.load(std::memory_order_relaxed);
}
void DrawPipeline::ReleaseRunningReads() {
std::lock_guard lock(mutex);
if (!items.empty()) items.front().reads.clear();
core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp
+29 -15
View File
@@ -19,9 +19,8 @@ void Submit(const Packet* packet, std::uint32_t queue) try {
return;
}
GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet));
recorder->Submit(*packet, queue);
const auto flips = driver.FlipSubmissions();
driver.Submit(packet, queue);
recorder->Submit(*packet, queue, [&] { driver.Submit(packet, queue); });
if (driver.FlipSubmissions() != flips) recorder->FrameBoundary();
} catch (const ProcessShutdown&) {
LibcAwaitExit_nid_postfix();
@@ -41,8 +40,12 @@ void RegisterShader(const Shader* shader) {
}
void SuspendPoint() {
if (auto* const recorder = FrameRecorder::Active()) recorder->Suspend();
DriverDetail::Driver::Get().SuspendPoint();
auto* const recorder = FrameRecorder::Active();
if (recorder == nullptr) {
DriverDetail::Driver::Get().SuspendPoint();
return;
}
recorder->Suspend([] { DriverDetail::Driver::Get().SuspendPoint(); });
}
void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr<IVideoOutput>& output) {
@@ -59,13 +62,13 @@ void AttachWindow(const PresentationWindow& window) {
DriverDetail::Driver::Get().AttachWindow(window);
}
void PresentClear(const PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context) {
DriverDetail::Driver::Get().Present(window, nullptr, opaque, gpuReady, context);
void PresentClear(const PresentationWindow& window, bool opaque, const DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
DriverDetail::Driver::Get().Present(window, nullptr, plane, opaque, gpuReady, context);
}
void PresentBuffer(const PresentationWindow& window, const DisplayBuffer& buffer, void (*gpuReady)(void*), void* context) {
void PresentBuffer(const PresentationWindow& window, const DisplayBuffer& buffer, const DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
if (auto* const recorder = FrameRecorder::Active()) recorder->PresentBuffer(buffer);
DriverDetail::Driver::Get().Present(window, &buffer, true, gpuReady, context);
DriverDetail::Driver::Get().Present(window, &buffer, plane, true, gpuReady, context);
}
void ReleaseWindow(void* window) {
@@ -146,9 +149,12 @@ extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std
extern "C" void AgcDriverSubmitFlip_nid_postfix(const AgcDriver::FlipInfo& info) try {
auto* const recorder = AgcDriver::FrameRecorder::Active();
if (recorder != nullptr) recorder->Flip(info);
AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
if (recorder != nullptr) recorder->FrameBoundary();
if (recorder == nullptr) {
AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info);
return;
}
recorder->Flip(info, [&] { AgcDriver::DriverDetail::Driver::Get().SubmitFlip(info); });
recorder->FrameBoundary();
} catch (const ProcessShutdown&) {
LibcAwaitExit_nid_postfix();
}
@@ -157,6 +163,14 @@ extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix() {
return AgcDriver::DriverDetail::Driver::Get().DispatchesAdoptedAhead();
}
extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix() {
return AgcDriver::DriverDetail::Driver::Get().DispatchesRejectedAhead();
}
extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)()) {
AgcDriver::DriverDetail::Driver::Get().SetAheadAdoptHook(hook);
}
extern "C" void AgcDriverRegisterVideoOutput_nid_postfix(std::uint32_t handle, const std::shared_ptr<AgcDriver::IVideoOutput>& output) {
AgcDriver::RegisterVideoOutput(handle, output);
}
@@ -169,12 +183,12 @@ extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationW
AgcDriver::AttachWindow(window);
}
extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context) {
AgcDriver::PresentClear(window, opaque, gpuReady, context);
extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
AgcDriver::PresentClear(window, opaque, plane, gpuReady, context);
}
extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context) {
AgcDriver::PresentBuffer(window, buffer, gpuReady, context);
extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, const AgcDriver::DisplayBuffer* plane, void (*gpuReady)(void*), void* context) {
AgcDriver::PresentBuffer(window, buffer, plane, gpuReady, context);
}
extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window) {
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp
+3 -7
View File
@@ -11,6 +11,7 @@
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
#include <atomic>
#include <cstdio>
#include <cstdlib>
@@ -29,11 +30,6 @@ bool orderedFills() {
return ordered;
}
bool pipelineDma() {
static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr;
return enabled;
}
bool keepsDrawFinisher(std::uint32_t header, std::uint32_t opcode) {
if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false;
if (Pm4::DrawOpcode(opcode)) return true;
@@ -65,7 +61,6 @@ bool keepsDrawPipeline(std::span<const std::uint32_t> packet, std::uint32_t head
if (packet.size() < 7) return false;
const auto destination = Pm4::DmaDestination(packet);
if (destination == Pm4::DmaDestinationNowhere) return true;
if (!pipelineDma()) return false;
if (destination == Pm4::DmaSelectGds) return true;
const auto bytes = Pm4::DmaBytes(packet);
if (bytes == 0) return true;
@@ -76,8 +71,8 @@ bool keepsDrawPipeline(std::span<const std::uint32_t> packet, std::uint32_t head
}
case 0x3c: case 0x93: {
if (packet.size() < 5) return false;
const std::uint64_t awaited = packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u);
const auto bytes = Pm4::WaitAwaitedBytes(packet);
const std::uint64_t awaited = (packet[2] & (bytes == 8 ? ~7u : ~3u)) | (static_cast<std::uint64_t>(packet[3]) << 32u);
if (DrawPipeline::Queue0().Overlaps(awaited, bytes)) return false;
GuestMemory::CheckRange(reinterpret_cast<const void*>(awaited), bytes, bytes);
return Pm4::WaitSatisfiedUnchecked(packet);
@@ -400,6 +395,7 @@ void Driver::execute(const Submission& submission) {
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
std::lock_guard gpuLock(GuestMemory::GpuMutex());
const auto localDevice = device.Load();
if (localDevice != nullptr) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("flip"));
if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0);
if (localDevice != nullptr) {
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp
+8 -4
View File
@@ -38,11 +38,12 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
std::lock_guard gpuLock(GuestMemory::GpuMutex());
const auto localDevice = device.Load();
recordDeferredLabels(localDevice.get(), submission.queue);
const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
int reason = localDevice != nullptr ? 0 : 1;
if (label.has_value()) {
const auto bytes = label->Bytes();
const auto stamp = ++eventSerial;
if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
if (reason == 0 || reason == 1 || reason == 5) {
@@ -52,6 +53,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
countLabelOutcome(reason);
++immediateLabels;
} else {
if (localDevice != nullptr && submission.queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
if (localDevice != nullptr) Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before interrupt");
++noOpLabels;
}
@@ -249,6 +251,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
recordDeferredLabels(draining.get(), submission.queue);
if (draining != nullptr) {
static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("drain"));
if (unlockedDrain && draining->CanWaitUnlocked()) epoch = draining->SubmitAndEpoch();
else draining->WaitIdle();
}
@@ -292,8 +295,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) {
}
bool Driver::enqueueDmaPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) {
static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr;
if (!enabled || opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false;
if (opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false;
const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u);
constexpr std::size_t immediateLimit = std::size_t{16} << 20u;
constexpr std::size_t gpuStoreLimit = 65536;
@@ -438,10 +440,11 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
std::lock_guard gpuLock(GuestMemory::GpuMutex());
const auto localDevice = device.Load();
recordDeferredLabels(localDevice.get(), queue);
const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0;
int reason = localDevice != nullptr ? 0 : 1;
if (!bytes.empty()) {
const auto stamp = ++eventSerial;
if (localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("label") != 0) workOpen = true;
reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4;
if (reason != 0 && reason != 5 && reason != 6) {
if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
@@ -456,6 +459,7 @@ void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span<c
}
if (endOfPipeInterrupt) {
bool deferred = false;
if (bytes.empty() && localDevice != nullptr && queue == 0 && Graphics::StorageTexture::StoreDrawnLinear("interrupt") != 0) workOpen = true;
if (localDevice != nullptr && (reason == 0 || reason == 5 || reason == 6)) {
deferred = localDevice->AfterRecordedWork([this, queue] { deliverGuestEopInterrupt(queue); }, queue == 0);
if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0);
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp
+22 -9
View File
@@ -70,7 +70,7 @@ void Driver::AttachWindow(const PresentationWindow& window) {
device = std::make_shared<VulkanDevice>(&window);
}
void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context) {
void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buffer, const DisplayBuffer* plane, bool opaque, void (*gpuReady)(void*), void* context) {
if (PerformanceTrace::Configured()) PerformanceTrace::Present(window.width, window.height, device.Load().get());
PerformanceContext timingContext(window.timing.get());
PerformanceTimer timing("Driver.Present");
@@ -93,6 +93,7 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
bool submitted = false;
bool presentable = false;
bool trailing = false;
bool queueUnlocked = false;
double waitedMs = 0;
{
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
@@ -129,10 +130,10 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
presenting->WaitIdle();
timing.Mark("device_idle_wait");
}
submitted = presenting->PresentDisplayBuffer(*buffer);
submitted = presenting->PresentDisplayBuffer(*buffer, plane);
timing.Mark("present_display_buffer");
} else {
submitted = presenting->PresentClear(window.width, window.height, opaque);
submitted = presenting->PresentClear(window.width, window.height, opaque, plane);
timing.Mark("present_clear");
}
if (submitted && syncFlip) {
@@ -141,12 +142,20 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
}
if (submitted && (syncFlip || inFlight != 0)) {
presenting->QueuePresent();
timing.Mark("queue_present");
if (presenting->PresentsOnOwnQueue()) {
queueUnlocked = true;
} else {
presenting->QueuePresent();
timing.Mark("queue_present");
}
trailing = !syncFlip;
submitted = false;
}
}
if (queueUnlocked) {
presenting->QueuePresent();
timing.Mark("queue_present");
}
if (trailing) {
waitedMs += presenting->FinishPresent();
timing.Mark("render_fence_wait");
@@ -155,9 +164,13 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff
waitedMs = presenting->FinishPresent();
timing.Mark("render_fence_wait");
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
std::lock_guard lock(GuestMemory::GpuMutex());
presenting->QueuePresent();
if (presenting->PresentsOnOwnQueue()) {
presenting->QueuePresent();
} else {
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present);
std::lock_guard lock(GuestMemory::GpuMutex());
presenting->QueuePresent();
}
timing.Mark("queue_present");
}
gpuReady(context);
@@ -193,7 +206,7 @@ void Driver::reportPresents(double waitedMs, std::size_t inFlight) {
const auto flips = flipsCounted.load() - previousFlips;
const double perFlip = flips != 0 ? 1.0 / static_cast<double>(flips) : 0.0;
const auto serial = flipSerial.load();
AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked));
AgcDriver::ProfilePrint_nid_no_patch( "[present] %llu presents over 10 s (%llu dropped: no free swapchain image; %zu may trail on the GPU); %s p50 %.2f ms, p90 %.2f ms, max %.2f ms over %zu; per frame, presented or dropped: GPU busy %.1f ms, idle gaps %.1f ms (%llu batches without a completion record); per flip: %.1f batches recorded (%.1f unsignaled at the flip); per present: %.1f batches ahead of the blit, %.1f after it; after the flip packet: last submit %.1f ms, blit submit %.1f ms; in-place reads overwritten before execution: %llu of %llu checked; waits for the display keys' writer outside the GPU mutex: %llu, %.1f ms\n", static_cast<unsigned long long>(presents), static_cast<unsigned long long>(dropped), inFlight, inFlight != 0 ? "inflight_wait" : "render_fence_wait", percentile(0.5), percentile(0.9), waits.empty() ? 0.0 : waits.back(), waits.size(), (counts.gpuBusyMs - previous.gpuBusyMs) * perFrame, (counts.gpuGapMs - previous.gpuGapMs) * perFrame, static_cast<unsigned long long>(counts.gpuUnread - previous.gpuUnread), static_cast<double>(serial - windowSerial) * perFlip, static_cast<double>(flipBatchesUnsignaled.load() - previousUnsignaled) * perFlip, static_cast<double>(counts.batchesAheadOfBlit - previous.batchesAheadOfBlit) * perPresent, static_cast<double>(counts.batchesAfterFlip - previous.batchesAfterFlip) * perPresent, (counts.lastSubmitAfterFlipMs - previous.lastSubmitAfterFlipMs) * perPresent, (counts.blitSubmitAfterFlipMs - previous.blitSubmitAfterFlipMs) * perPresent, static_cast<unsigned long long>(counts.readsOverwritten - previous.readsOverwritten), static_cast<unsigned long long>(counts.readsChecked - previous.readsChecked), static_cast<unsigned long long>(counts.keyWriterWaits - previous.keyWriterWaits), counts.keyWriterWaitMs - previous.keyWriterWaitMs);
previous = counts;
previousFlips = flipsCounted.load();
previousUnsignaled = flipBatchesUnsignaled.load();
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp
+1 -1
View File
@@ -138,7 +138,7 @@ std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
cache.erase(found);
}
const auto bytes = result->code.size() * sizeof(std::uint32_t);
while (!cache.empty() && (cache.size() >= 64 || cacheBytes + bytes > 8 * 1024 * 1024)) {
while (!cache.empty() && (cache.size() >= 4096 || cacheBytes + bytes > 64 * 1024 * 1024)) {
cacheBytes -= cache.back()->code.size() * sizeof(std::uint32_t);
cache.pop_back();
}
core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp
+40 -21
View File
@@ -163,20 +163,20 @@ void FrameRecorder::UnregisterVideoOutput(std::uint32_t handle) {
record(std::move(call), true);
}
void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue) {
void FrameRecorder::Submit(const Packet& packet, std::uint32_t queue, const std::function<void()>& accept) {
auto call = make(FrameCallKind::Submit);
call.queue = queue;
call.address = pointer(packet.addr);
call.size = packet.dw_num;
call.flags = packet.flags;
record(std::move(call), false);
record(std::move(call), false, {}, accept);
}
void FrameRecorder::Flip(const FlipInfo& info) {
void FrameRecorder::Flip(const FlipInfo& info, const std::function<void()>& accept) {
auto call = make(FrameCallKind::Flip);
call.cpuFlip = true;
call.values = {info.handle, flipValue(info.index), info.mode, flipValue(info.argument)};
record(std::move(call), false);
record(std::move(call), false, {}, accept);
}
void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
@@ -198,8 +198,8 @@ void FrameRecorder::ReserveEopFlip(const FlipInfo& info, std::uint32_t token) {
++recordedCalls;
}
void FrameRecorder::Suspend() {
record(make(FrameCallKind::Suspend), false);
void FrameRecorder::Suspend(const std::function<void()>& accept) {
record(make(FrameCallKind::Suspend), false, {}, accept);
}
void FrameRecorder::PresentBuffer(const DisplayBuffer& buffer) {
@@ -236,8 +236,13 @@ void FrameRecorder::writeReferenceFrames() {
}
}
void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
void FrameRecorder::record(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders, const std::function<void()>& accept) {
std::lock_guard lock(mutex);
recordLocked(std::move(call), prologueCall, shaders);
if (accept) accept();
}
void FrameRecorder::recordLocked(FrameCall call, bool prologueCall, std::span<const Shader* const> shaders) {
if (state == State::Waiting) {
if (!prologueCall) return;
call.ordinal = ++ordinal;
@@ -360,10 +365,23 @@ void FrameRecorder::emitShaderDelta(std::span<const Shader* const> shaders) {
void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std::uint64_t> hostPages, std::span<const Range> added) {
if (abandonIfDiskLow()) return;
const auto [accepted, completed] = DriverDetail::Driver::Get().SerialProgress();
std::vector<std::pair<std::uint64_t, std::uint64_t>> blocks;
std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
for (const auto& span : spans) {
GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
blocks.insert(blocks.end(), written.begin(), written.end());
}
auto& driver = DriverDetail::Driver::Get();
auto [accepted, completed] = driver.SerialProgress();
if (completed < accepted && (!added.empty() || !blocks.empty())) {
if (!driver.WaitIdleFor(std::chrono::seconds(10))) {
stop("stopped: the GPU work submitted before a memory delta did not finish within 10 s");
return;
}
std::tie(accepted, completed) = driver.SerialProgress();
}
const auto group = deltaGroups + 1;
bool any = false;
std::vector<std::pair<std::uint64_t, std::uint64_t>> written;
std::vector<std::byte> bytes;
for (const auto& range : added) {
writer->WriteRange(range.address, range.bytes, range.kind, range.protection, range.access);
@@ -397,15 +415,12 @@ void FrameRecorder::writeDelta(std::span<const Span> spans, std::span<const std:
deltaBytes += bytes.size();
any = true;
}
for (const auto& span : spans) {
GuestMemory::CollectWrittenBlocks(span.address, static_cast<std::size_t>(span.bytes), span.since, written);
for (const auto& [begin, end] : written) {
bytes.resize(static_cast<std::size_t>(end - begin));
GuestMemory::ReadCommittedUnsynced(begin, bytes);
writer->WriteDelta(group, completed, begin, bytes);
deltaBytes += bytes.size();
any = true;
}
for (const auto& [begin, end] : blocks) {
bytes.resize(static_cast<std::size_t>(end - begin));
GuestMemory::ReadCommittedUnsynced(begin, bytes);
writer->WriteDelta(group, completed, begin, bytes);
deltaBytes += bytes.size();
any = true;
}
if (!any) return;
deltaGroups = group;
@@ -436,8 +451,13 @@ bool FrameRecorder::abandonIfDiskLow() {
std::error_code error;
const auto space = std::filesystem::space(directory, error);
if (error || space.available >= MinimumFreeBytes) return false;
warnings.push_back("stopped: less than 4 GiB free on the capture volume");
std::fprintf(stderr, "[frame-capture] stopped: less than 4 GiB free on the capture volume\n");
stop("stopped: less than 4 GiB free on the capture volume");
return true;
}
void FrameRecorder::stop(const char* reason) {
warnings.push_back(reason);
std::fprintf(stderr, "[frame-capture] %s\n", reason);
try {
writer->Finish(header());
} catch (const std::exception& failure) {
@@ -445,7 +465,6 @@ bool FrameRecorder::abandonIfDiskLow() {
}
writer.reset();
state = State::Done;
return true;
}
void FrameRecorder::start() {
core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp
+24 -5
View File
@@ -24,6 +24,7 @@
#include <map>
#include <memory>
#include <new>
#include <optional>
#include <stdexcept>
#include <string>
#include <tuple>
@@ -1751,7 +1752,7 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
if (!tracker.watched) return;
bool restored = false;
for (const auto& [begin, end] : ranges) {
if (end <= begin || begin < tracker.base || end - tracker.base > tracker.size || (begin - tracker.base) % WriteBlockBytes != 0 || (end - tracker.base) % WriteBlockBytes != 0) continue;
if (end <= begin || begin < tracker.base || end - tracker.base > tracker.size || (begin - tracker.base) % WritePageBytes != 0 || (end - tracker.base) % WritePageBytes != 0) continue;
const auto bytes = static_cast<std::size_t>(end - begin);
if (tracker.coverage.Covers(begin, bytes)) continue;
GuestArena::GuestArenaCompareWhenShared_nid_no_patch(static_cast<std::uintptr_t>(begin), bytes);
@@ -1770,7 +1771,7 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
#endif
}
bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched, bool importMayWriteGuest) {
if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
if (!staysWatched) {
constexpr std::uint64_t page = 4096;
@@ -1790,7 +1791,11 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
const auto runImport = [&] {
#ifdef __APPLE__
const GuestArena::HostWrite open(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
std::optional<GuestArena::HostWrite> open;
if (importMayWriteGuest) open.emplace(reinterpret_cast<void*>(first), static_cast<std::size_t>(stop - first));
#endif
#ifndef __APPLE__
static_cast<void>(importMayWriteGuest);
#endif
return import();
};
@@ -1820,12 +1825,26 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
hashes[static_cast<std::size_t>((cursor - first) / page)] = HashPage(reinterpret_cast<const std::byte*>(cursor));
}
}
std::erase_if(spans, [&](const auto& span) { return !walkWrites(tracker, span.first, span.second, StampKind::Cpu, true); });
const auto saveBytes = [&](std::uint64_t begin, std::uint64_t end) {
lock.Suspend();
beforeUnwatch(begin, static_cast<std::size_t>(end - begin));
lock.Resume();
};
std::vector<std::pair<std::uint64_t, std::uint64_t>> leaving;
std::erase_if(spans, [&](const auto& span) {
if (walkWrites(tracker, span.first, span.second, StampKind::Cpu, true)) return false;
leaving.push_back(span);
return true;
});
for (const auto& [begin, end] : leaving) saveBytes(begin, end);
if (!runImport()) return false;
const WindowCopy window{first, hashes};
auto unwatchedFrom = first;
for (const auto& [begin, end] : spans) {
if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) continue;
if (!walkWrites(tracker, begin, end, StampKind::Cpu, true, &window)) {
saveBytes(begin, end);
continue;
}
#ifdef _WIN32
GuestArena::GuestArenaCompareResidentWhenShared_nid_no_patch(static_cast<std::uintptr_t>(begin), static_cast<std::size_t>(end - begin));
#endif
core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp
+141 -18
View File
@@ -86,6 +86,7 @@ void PresentationPass::release() noexcept {
for (std::size_t i = 0; i < slots.size(); ++i) {
Release(i);
releaseDumpTarget(slots[i]);
releaseUploadImage(slots[i]);
}
for (const auto& [key, entry] : targets) {
if (entry.pipeline) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, entry.pipeline, nullptr);
@@ -155,22 +156,24 @@ VkFramebuffer PresentationPass::createFramebuffer(VkRenderPass renderPass, VkIma
return framebuffer;
}
const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImageLayout finalLayout) {
const auto key = std::pair{format, finalLayout};
const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImageLayout finalLayout, VkImageLayout loadFrom) {
const auto key = std::tuple{format, finalLayout, loadFrom};
if (const auto found = targets.find(key); found != targets.end()) return found->second;
const bool blended = loadFrom != VK_IMAGE_LAYOUT_UNDEFINED;
VkFormatProperties properties{};
context.formatProperties(context.physical, format, &properties);
Graphics::Require((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) != 0, "presentation pass format is not renderable");
Graphics::Require(!blended || (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0, "presentation pass cannot blend an overlay into this format");
Target created;
try {
VkAttachmentDescription attachment{};
attachment.format = format;
attachment.samples = VK_SAMPLE_COUNT_1_BIT;
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.loadOp = blended ? VK_ATTACHMENT_LOAD_OP_LOAD : VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
attachment.initialLayout = loadFrom;
attachment.finalLayout = finalLayout;
const VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
VkSubpassDescription subpass{};
@@ -179,7 +182,7 @@ const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImag
subpass.pColorAttachments = &reference;
const bool transfer = finalLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
const VkSubpassDependency dependencies[2] = {
{VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0},
{VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, blended ? VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT} : VkAccessFlags{0}, blended ? VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT} : VkAccessFlags{VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT}, 0},
{0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, transfer ? VK_PIPELINE_STAGE_TRANSFER_BIT : VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, transfer ? VK_ACCESS_TRANSFER_READ_BIT : VkAccessFlags{0}, 0},
};
VkRenderPassCreateInfo renderPassInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
@@ -212,7 +215,16 @@ const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImag
VkPipelineMultisampleStateCreateInfo multisample{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO};
multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
VkPipelineColorBlendAttachmentState blendAttachment{};
blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | (blended ? 0u : VK_COLOR_COMPONENT_A_BIT);
if (blended) {
blendAttachment.blendEnable = VK_TRUE;
blendAttachment.srcColorBlendFactor = VK_BLEND_FACTOR_ONE;
blendAttachment.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
blendAttachment.colorBlendOp = VK_BLEND_OP_ADD;
blendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
blendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
blendAttachment.alphaBlendOp = VK_BLEND_OP_ADD;
}
VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
blend.attachmentCount = 1;
blend.pAttachments = &blendAttachment;
@@ -262,7 +274,75 @@ void PresentationPass::Bind(std::size_t index, VkImage image, std::uint32_t widt
context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, 1, &write, 0, nullptr);
}
void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, const std::int32_t (&parameters)[9]) const {
void PresentationPass::releaseUploadImage(Slot& slot) noexcept {
if (slot.uploadImage) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, slot.uploadImage, nullptr);
if (slot.uploadMemory) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, slot.uploadMemory, nullptr);
slot.uploadImage = VK_NULL_HANDLE;
slot.uploadMemory = VK_NULL_HANDLE;
slot.uploadExtent = {};
slot.uploadTexels.reset();
}
void PresentationPass::ensureUploadImage(Slot& slot, VkExtent2D extent) {
if (slot.uploadImage != VK_NULL_HANDLE && slot.uploadExtent.width == extent.width && slot.uploadExtent.height == extent.height) return;
releaseUploadImage(slot);
try {
VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = VK_FORMAT_R32_UINT;
imageInfo.extent = {extent.width, extent.height, 1u};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
Graphics::Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &slot.uploadImage), "vkCreateImage presentation upload");
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, slot.uploadImage, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Graphics::Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &slot.uploadMemory), "vkAllocateMemory presentation upload");
Graphics::Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, slot.uploadImage, slot.uploadMemory, 0), "vkBindImageMemory presentation upload");
slot.uploadTexels = std::make_unique<Graphics::Buffer>(context, static_cast<std::size_t>(extent.width) * extent.height * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
slot.uploadExtent = extent;
} catch (...) {
releaseUploadImage(slot);
throw;
}
}
void PresentationPass::BindTexels(std::size_t index, VkCommandBuffer commands, std::span<const std::byte> texels, std::uint32_t width, std::uint32_t height, std::uint64_t pixelFormat, float whiteNits) {
Graphics::Require(index < slots.size(), "presentation pass slot is out of range");
Graphics::Require(width != 0 && height != 0 && texels.size() == static_cast<std::size_t>(width) * height * 4, "presentation pass texels do not cover their extent");
auto& slot = slots[index];
ensureUploadImage(slot, {width, height});
std::memcpy(slot.uploadTexels->Bytes().data(), texels.data(), texels.size());
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = slot.uploadImage;
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
const auto pipelineBarrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
VkBufferImageCopy copy{};
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
copy.imageExtent = {width, height, 1u};
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, slot.uploadTexels->Handle(), slot.uploadImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &copy);
barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
Bind(index, slot.uploadImage, width, height, pixelFormat, whiteNits);
}
void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (&parameters)[9]) const {
VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
begin.renderPass = entry.renderPass;
begin.framebuffer = framebuffer;
@@ -270,7 +350,6 @@ void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Ta
context.Function<PFN_vkCmdBeginRenderPass>("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, entry.pipeline);
const VkViewport viewport{0.0f, 0.0f, static_cast<float>(extent.width), static_cast<float>(extent.height), 0.0f, 1.0f};
const VkRect2D scissor{{0, 0}, extent};
context.Function<PFN_vkCmdSetViewport>("vkCmdSetViewport")(commands, 0, 1, &viewport);
context.Function<PFN_vkCmdSetScissor>("vkCmdSetScissor")(commands, 0, 1, &scissor);
context.Function<PFN_vkCmdBindDescriptorSets>("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &slot.set, 0, nullptr);
@@ -279,12 +358,13 @@ void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Ta
context.Function<PFN_vkCmdEndRenderPass>("vkCmdEndRenderPass")(commands);
}
void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout) {
Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
auto& slot = slots[index];
const auto& entry = target(format, finalLayout);
const auto view = createView(destination, format, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | (finalLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL ? VK_IMAGE_USAGE_TRANSFER_SRC_BIT : 0u));
VkFramebuffer PresentationPass::destinationFramebuffer(Slot& slot, const Target& entry, VkImage destination, VkFormat format, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
for (const auto layout : {initialLayout, finalLayout}) {
if (layout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
if (layout == VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL) usage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT;
}
const auto view = createView(destination, format, usage);
VkFramebuffer framebuffer = VK_NULL_HANDLE;
try {
framebuffer = createFramebuffer(entry.renderPass, view, extent);
@@ -293,12 +373,35 @@ void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index
throw;
}
slot.transient.emplace_back(view, framebuffer);
return framebuffer;
}
void PresentationPass::RecordPresent(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout finalLayout) {
Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
auto& slot = slots[index];
const auto& entry = target(format, finalLayout);
const auto framebuffer = destinationFramebuffer(slot, entry, destination, format, extent, VK_IMAGE_LAYOUT_UNDEFINED, finalLayout);
const auto rect = ComputeContainRect_nid_postfix(slot.width, slot.height, extent.width, extent.height);
const bool hdr10 = colorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT;
Graphics::Require(colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR || (hdr10 && (slot.mode & ModePq) != 0 && TenBitFormat(format)), "presentation pass presents only PQ display buffers to a 10-bit HDR10 destination");
const std::uint32_t mode = (hdr10 ? slot.mode & ~ModePq : slot.mode) | (TenBitFormat(format) ? ModeKeepTenBit : 0u);
const std::int32_t parameters[9] = {rect.x, rect.y, static_cast<std::int32_t>(rect.width), static_cast<std::int32_t>(rect.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), 1, static_cast<std::int32_t>(mode), std::bit_cast<std::int32_t>(slot.whiteNits)};
draw(commands, slot, entry, framebuffer, extent, parameters);
draw(commands, slot, entry, framebuffer, extent, {{0, 0}, extent}, parameters);
}
void PresentationPass::RecordOverlay(VkCommandBuffer commands, std::size_t index, VkImage destination, VkFormat format, VkColorSpaceKHR colorSpace, VkExtent2D extent, VkImageLayout initialLayout, VkImageLayout finalLayout) {
Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
Graphics::Require(destination != VK_NULL_HANDLE && extent.width != 0 && extent.height != 0, "presentation pass destination is unavailable");
Graphics::Require(initialLayout != VK_IMAGE_LAYOUT_UNDEFINED, "an overlay is blended over a destination whose contents are kept");
auto& slot = slots[index];
Graphics::Require((slot.mode & (ModeTenBit | ModePq)) == 0, "an overlay plane is blended only from an 8-bit SDR display buffer");
Graphics::Require(colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, "an overlay plane is blended only into an SDR destination");
const auto& entry = target(format, finalLayout, initialLayout);
const auto framebuffer = destinationFramebuffer(slot, entry, destination, format, extent, initialLayout, finalLayout);
const auto rect = ComputeContainRect_nid_postfix(slot.width, slot.height, extent.width, extent.height);
const std::int32_t parameters[9] = {rect.x, rect.y, static_cast<std::int32_t>(rect.width), static_cast<std::int32_t>(rect.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), 1, static_cast<std::int32_t>(slot.mode), std::bit_cast<std::int32_t>(slot.whiteNits)};
draw(commands, slot, entry, framebuffer, extent, {{rect.x, rect.y}, {rect.width, rect.height}}, parameters);
}
void PresentationPass::releaseDumpTarget(Slot& slot) noexcept {
@@ -346,18 +449,38 @@ void PresentationPass::ensureDumpTarget(Slot& slot, VkExtent2D extent) {
}
void PresentationPass::RecordDump(VkCommandBuffer commands, std::size_t index, std::uint32_t stride, VkBuffer destination) {
RecordDumpImage(commands, index, stride);
RecordDumpReadback(commands, index, destination);
}
void PresentationPass::RecordDumpImage(VkCommandBuffer commands, std::size_t index, std::uint32_t stride) {
Graphics::Require(index < slots.size() && slots[index].source != VK_NULL_HANDLE, "presentation pass slot has no source");
Graphics::Require(destination != VK_NULL_HANDLE, "presentation dump buffer is unavailable");
auto& slot = slots[index];
const auto extent = DumpExtent(slot.width, slot.height, stride);
ensureDumpTarget(slot, extent);
const auto& entry = target(VK_FORMAT_B8G8R8A8_UNORM, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
const std::int32_t parameters[9] = {0, 0, static_cast<std::int32_t>(extent.width), static_cast<std::int32_t>(extent.height), static_cast<std::int32_t>(slot.width), static_cast<std::int32_t>(slot.height), static_cast<std::int32_t>(stride), static_cast<std::int32_t>(slot.mode | ModeSubsample), std::bit_cast<std::int32_t>(slot.whiteNits)};
draw(commands, slot, entry, slot.dumpFramebuffer, extent, parameters);
draw(commands, slot, entry, slot.dumpFramebuffer, extent, {{0, 0}, extent}, parameters);
}
VkImage PresentationPass::DumpImage(std::size_t index) const {
Graphics::Require(index < slots.size() && slots[index].dumpImage != VK_NULL_HANDLE, "presentation pass slot has no dump image");
return slots[index].dumpImage;
}
VkExtent2D PresentationPass::DumpImageExtent(std::size_t index) const {
Graphics::Require(index < slots.size() && slots[index].dumpImage != VK_NULL_HANDLE, "presentation pass slot has no dump image");
return slots[index].dumpExtent;
}
void PresentationPass::RecordDumpReadback(VkCommandBuffer commands, std::size_t index, VkBuffer destination) {
Graphics::Require(destination != VK_NULL_HANDLE, "presentation dump buffer is unavailable");
const auto image = DumpImage(index);
const auto extent = DumpImageExtent(index);
VkBufferImageCopy copy{};
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
copy.imageExtent = {extent.width, extent.height, 1u};
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, slot.dumpImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, &copy);
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination, 1, &copy);
Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
}
core/libs/prx/libSceAgcDriver/Execution/src/PresentationScaler.cpp
+8 -4
View File
@@ -1,5 +1,6 @@
#include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
#include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
namespace AgcDriver {
@@ -23,7 +24,7 @@ PresentationScaler::~PresentationScaler() {
void PresentationScaler::release() noexcept {
if (sourceImage != VK_NULL_HANDLE) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, sourceImage, nullptr);
if (sourceMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, sourceMemory, nullptr);
if (sourceMemory != VK_NULL_HANDLE) Graphics::ReleaseImageMemory(context, sourceMemory, sourceBytes, sourceMemoryType);
sourceImage = VK_NULL_HANDLE;
sourceMemory = VK_NULL_HANDLE;
sourceWidth = 0;
@@ -42,7 +43,9 @@ void PresentationScaler::EnsureSourceImage(std::uint32_t width, std::uint32_t he
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
VkFormatProperties properties{};
context.formatProperties(context.physical, sourceFormat, &properties);
imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT) != 0 ? VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT : 0u);
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
Graphics::Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &sourceImage), "vkCreateImage presentation source");
@@ -50,8 +53,9 @@ void PresentationScaler::EnsureSourceImage(std::uint32_t width, std::uint32_t he
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, sourceImage, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Graphics::Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &sourceMemory), "vkAllocateMemory presentation source");
allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "presentation source");
sourceMemory = Graphics::AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory presentation source", &sourceMemoryType, true);
sourceBytes = requirements.size;
Graphics::Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, sourceImage, sourceMemory, 0), "vkBindImageMemory presentation source");
sourceWidth = width;
sourceHeight = height;
core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp
+30 -1
View File
@@ -12,6 +12,12 @@
#ifdef _WIN32
#include <windows.h>
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <mach/mach_time.h>
#include <mach/thread_policy.h>
#include <pthread.h>
#include <sys/qos.h>
#else
#include <dlfcn.h>
#include <sched.h>
@@ -42,7 +48,30 @@ void LogOnce(const char* role, const std::string& text) {
std::fprintf(stderr, "[thread-priority] %s: %s\n", role, text.c_str());
}
#ifndef _WIN32
#if defined(__APPLE__)
void Raise(const char* role) {
if (RequestedMode() == ThreadPriorityMode::Realtime) {
mach_timebase_info_data_t timebase{};
mach_timebase_info(&timebase);
const auto ticks = [&](std::uint64_t nanos) {
return static_cast<std::uint32_t>(nanos * timebase.denom / timebase.numer);
};
thread_time_constraint_policy_data_t policy{ticks(1000000), ticks(500000), ticks(1000000), TRUE};
const auto status = thread_policy_set(pthread_mach_thread_np(pthread_self()), THREAD_TIME_CONSTRAINT_POLICY,
reinterpret_cast<thread_policy_t>(&policy), THREAD_TIME_CONSTRAINT_POLICY_COUNT);
if (status != KERN_SUCCESS)
throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=rt: ") + role + " time-constraint policy refused (kern_return " + std::to_string(status) + ")");
LogOnce(role, "Mach time-constraint policy");
return;
}
const int error = pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0);
if (error != 0)
throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=high: ") + role + " user-interactive QoS refused: " + std::strerror(error));
LogOnce(role, "user-interactive QoS");
}
#elif !defined(_WIN32)
struct DBusConnection;
struct DBusMessage;
core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp
+192 -34
View File
@@ -8,6 +8,7 @@
#include "SpirvBackend/SpirvOptimizer.hpp"
#endif
#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Execution/include/FloatFlushProbe.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
@@ -20,9 +21,11 @@
#include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp"
#include "prx/libSceAgcDriver/Execution/include/DrawInputsInPlace.hpp"
#include "prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp"
#include "prx/libSceAgcDriver/Execution/include/TessellationPipelineCache.hpp"
#include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp"
#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp"
#include "prx/libSceAgcDriver/Execution/include/PresentMode.hpp"
#include "prx/libSceAgcDriver/Execution/include/SwapchainImages.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
#include "prx/libSceAgcDriver/Execution/include/PresentationPass.hpp"
#include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
@@ -83,7 +86,7 @@
namespace AgcDriver {
DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding) {
DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> families, std::uint32_t family, bool settles, bool sparseBinding, bool presents) {
DeviceQueuePlan plan;
const auto count = static_cast<std::uint32_t>(families.size());
if (settles) {
@@ -102,6 +105,13 @@ DeviceQueuePlan PlanDeviceQueues(std::span<const VkQueueFamilyProperties> famili
}
}
}
if (presents && family < count) {
if (const auto index = SelectPresentQueue(families[family].queueCount, plan.mainCount)) {
plan.present = true;
plan.presentIndex = *index;
plan.mainCount = *index + 1;
}
}
if (!sparseBinding) return plan;
const auto taken = [&](std::uint32_t index) { return plan.settle && index == plan.settleFamily ? plan.settleIndex + 1 : 0u; };
const auto binds = [&](std::uint32_t index) { return index != family && families[index].queueCount > taken(index) && (families[index].queueFlags & VK_QUEUE_SPARSE_BINDING_BIT) != 0; };
@@ -351,6 +361,7 @@ struct VulkanDevice::State {
VkDriverId driverId = VK_DRIVER_ID_MAX_ENUM;
std::uint32_t pixelFunctionStorageDwords = 4096u;
bool individuallyFreeDescriptorSets = false;
bool tessellationPipelineCache = true;
PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr;
PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr;
PFN_vkCmdWriteBufferMarkerAMD bufferMarker = nullptr;
@@ -359,6 +370,8 @@ struct VulkanDevice::State {
void* markerMapping = nullptr;
BufferMarkerLog markerLog;
bool deviceLostHook = false;
bool deferDestruction = false;
PFN_vkGetDeviceProcAddr contextDeviceProc = nullptr;
VkTimeDomainKHR hostTimeDomain = VK_TIME_DOMAIN_DEVICE_KHR;
VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
std::vector<std::uint32_t> capabilities{1};
@@ -439,6 +452,8 @@ struct VulkanDevice::State {
std::uint32_t settleFamily = 0;
VkCommandPool settlePool = VK_NULL_HANDLE;
std::mutex settleQueueMutex;
VkQueue presentQueue = VK_NULL_HANDLE;
std::mutex presentQueueMutex;
bool memoryBudget = false;
bool depthRangeUnrestricted = false;
bool samplerAnisotropy = false;
@@ -506,6 +521,7 @@ struct VulkanDevice::State {
VkPhysicalDeviceMeshShaderPropertiesEXT meshLimits{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_PROPERTIES_EXT};
std::unique_ptr<PresentationScaler> scaler;
std::unique_ptr<PresentationPass> presentation;
std::unique_ptr<PresentationPass> planePass;
std::unique_ptr<PerformanceOverlay> overlay;
VkFormat surfaceFormat = VK_FORMAT_B8G8R8A8_UNORM;
VkColorSpaceKHR surfaceColorSpace = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR;
@@ -528,6 +544,7 @@ struct VulkanDevice::State {
VkFence fence = VK_NULL_HANDLE;
VkCommandBuffer commands = VK_NULL_HANDLE;
std::shared_ptr<void> kept;
std::shared_ptr<void> keptPlane;
std::unique_ptr<Graphics::Buffer> dumpBuffer;
std::unique_ptr<PresentationScaler> dumpScaler;
std::uint32_t dumpWidth = 0;
@@ -729,7 +746,9 @@ struct VulkanDevice::State {
void RetireSlot(PresentSlot& slot) {
slot.inFlight = false;
if (presentation) presentation->Release(static_cast<std::size_t>(&slot - presentSlots.data()));
if (planePass) planePass->Release(static_cast<std::size_t>(&slot - presentSlots.data()));
slot.kept.reset();
slot.keptPlane.reset();
if (slot.dumpRecorded) {
slot.dumpRecorded = false;
WriteFrameDump(slot);
@@ -806,7 +825,7 @@ struct VulkanDevice::State {
for (std::uint32_t x = 0; x < width; ++x) std::memcpy(pixels.data() + (static_cast<std::size_t>(y) * width + x) * 4, full.data() + (static_cast<std::size_t>(y) * scale * slot.dumpWidth + static_cast<std::size_t>(x) * scale) * 4, 4);
}
static const auto start = std::chrono::steady_clock::now();
std::fprintf(stderr, "[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, slot.dumpWidth, slot.dumpHeight, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
AgcDriver::ProfilePrint_nid_no_patch("[gpu] frame_%03d.bmp: %ux%u display read back on the GPU at %.1f s\n", index, slot.dumpWidth, slot.dumpHeight, std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count());
dumpWriter.Enqueue(index, width, height, 1, false, std::move(pixels));
}
@@ -902,6 +921,7 @@ struct VulkanDevice::State {
detiler.reset();
colorTransfer.reset();
overlay.reset();
planePass.reset();
presentation.reset();
scaler.reset();
hdrScaler.reset();
@@ -933,6 +953,7 @@ struct VulkanDevice::State {
destroyPool(device, pool, nullptr);
}
if (settlePool != VK_NULL_HANDLE) destroyPool(device, settlePool, nullptr);
Graphics::DestroyDeferredObjects(device);
destroyDevice(device, nullptr);
}
if (instance != VK_NULL_HANDLE) {
@@ -1271,10 +1292,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
driverId = driver.driverID;
state->deferDestruction = driver.driverID == VK_DRIVER_ID_MOLTENVK && std::getenv("APS5_NO_DEFERRED_DESTRUCTION") == nullptr;
state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
state->driverId = driver.driverID;
state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
state->fusedExactArithmetic = driver.driverID == VK_DRIVER_ID_MOLTENVK;
state->tessellationPipelineCache = TessellationPipelineCacheUsable(driver.driverID);
state->subgroupControlBarriers = SubgroupControlBarrierUsable(driver.driverID);
state->workgroupScopeFences = WorkgroupScopeFenceUsable(driver.driverID);
}
@@ -1682,7 +1705,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
queueInfo.queueFamilyIndex = family;
queueInfo.queueCount = 1;
queueInfo.pQueuePriorities = &priority;
const float priorities[2]{1.0f, 1.0f};
const float priorities[3]{1.0f, 1.0f, 1.0f};
#ifdef _WIN32
const bool settles = true;
#else
@@ -1729,13 +1752,13 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
// Guest shaders routinely read past descriptor ranges; robust access turns that into zeros
// instead of a GPU fault that loses the device.
enabled.robustBufferAccess = available.robustBufferAccess;
const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE);
const auto plan = PlanDeviceQueues(queues, family, settles, available.sparseBinding == VK_TRUE, window != nullptr);
const bool settleQueue = plan.settle;
const auto settleFamily = plan.settleFamily;
const auto settleIndex = plan.settleIndex;
VkDeviceQueueCreateInfo queueInfos[3]{queueInfo, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}, {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}};
queueInfos[0].queueCount = plan.mainCount;
queueInfos[0].pQueuePriorities = plan.mainCount == 2 ? priorities : &priority;
queueInfos[0].pQueuePriorities = plan.mainCount > 1 ? priorities : &priority;
deviceInfo.queueCreateInfoCount = 1;
const auto addQueue = [&](std::uint32_t queueFamily, std::uint32_t index) {
for (std::uint32_t info = 1; info < deviceInfo.queueCreateInfoCount; ++info) {
@@ -2082,6 +2105,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
check(state->InstanceFunction<PFN_vkCreateDevice>("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice");
state->shaderProfile = std::move(shaderProfile);
if (compileOnly) return;
if (state->deferDestruction) state->contextDeviceProc = Graphics::DeferDestructionToIdle(state->device, state->deviceProc);
if (calibratedTimestamps != nullptr) {
const bool khr = std::strcmp(calibratedTimestamps, VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME) == 0;
const auto domains = reinterpret_cast<PFN_vkGetPhysicalDeviceCalibrateableTimeDomainsKHR>(state->instanceProc(state->instance, khr ? "vkGetPhysicalDeviceCalibrateableTimeDomainsKHR" : "vkGetPhysicalDeviceCalibrateableTimeDomainsEXT"));
@@ -2103,6 +2127,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
}
state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, family, 0, &state->queue);
state->queueFamily = family;
if (plan.present) {
state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, family, plan.presentIndex, &state->presentQueue);
APS5_LOG_OUT("Presentation queue family=%u index=%u", family, plan.presentIndex);
}
if (settleQueue) {
state->settleFamily = settleFamily;
state->DeviceFunction<PFN_vkGetDeviceQueue>("vkGetDeviceQueue")(state->device, settleFamily, settleIndex, &state->settleQueue);
@@ -2193,7 +2221,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
require((surface.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) != 0, "surface does not support color attachment images");
VkSwapchainCreateInfoKHR swapchain{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
swapchain.surface = state->surface;
swapchain.minImageCount = surface.minImageCount;
swapchain.minImageCount = AgcDriver::SwapchainImageCount(surface);
swapchain.imageFormat = state->surfaceFormat;
swapchain.imageColorSpace = state->surfaceColorSpace;
swapchain.imageExtent = state->extent;
@@ -2297,6 +2325,26 @@ bool VulkanDevice::ReplayComputePipeline(const ShaderRecompiler::RecompileResult
return state->computePipelines.try_emplace(key, std::move(objects)).second;
}
bool VulkanDevice::HasComputePipeline(const ShaderRecompiler::RecompileResult& shader) const {
const std::array<Graphics::CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}};
const auto key = ComputePipelineKey::Of(shader, Graphics::PushConstantStages(shaders) != 0);
std::lock_guard pipelines(state->computePipelinesMutex);
return !key.identity.Empty() && (state->computePipelines.contains(key) || state->computePipelinesBuilding.contains(key));
}
void VulkanDevice::PrebuildComputePipeline(const ShaderRecompiler::RecompileResult& shader, std::uint64_t programAddress) {
if (shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) return;
ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
const auto context = graphicsContext();
if (context.descriptorCache == nullptr) return;
const std::array<Graphics::CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}};
const auto pushStages = Graphics::PushConstantStages(shaders);
if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::ComputePushConstantBytes) return;
if (HasComputePipeline(shader)) return;
const auto layouts = Graphics::DescribeSetLayouts(context, shaders);
static_cast<void>(computePipeline(context, shader, layouts.layouts.main, layouts.mainKey, pushStages, programAddress));
}
bool VulkanDevice::ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
if (adoptDevice) GuestMemory::AssertGpuLockHeld("VulkanDevice::ImportGuestMemory");
return Graphics::ImportMappedRanges(graphicsContext(), ranges, generation, adoptDevice);
@@ -2325,6 +2373,7 @@ void VulkanDevice::PrepareForReplacement() {
void VulkanDevice::WaitIdle() {
APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle begin");
static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("wait idle"));
RetirePresents(0);
if (state->recorder) {
// Announced with this call's return address: the [recorder] site table then names the
@@ -2332,6 +2381,8 @@ void VulkanDevice::WaitIdle() {
if (!state->recorder->Idle()) Graphics::Recorder::CountSync(0, __builtin_return_address(0));
state->recorder->Sync();
}
std::unique_lock presentQueueLock(state->presentQueueMutex, std::defer_lock);
if (state->presentQueue != VK_NULL_HANDLE) presentQueueLock.lock();
check(state->DeviceFunction<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle");
Graphics::CheckPixelFunctionLds(graphicsContext());
APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete");
@@ -2429,6 +2480,7 @@ bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) c
int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
static const bool storeAtLabel = Graphics::StorageTexture::StoreAtLabelRequested(std::getenv("APS5_STORE_AT_LABEL"));
if (storeAtLabel) Graphics::StorageTexture::FlushAllPending("label");
if (queue == 0) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("label"));
Graphics::StorageTexture::StoreReadbackBeforeLabel("readback before label");
if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
auto& recorder = *state->recorder;
@@ -3044,7 +3096,7 @@ void VulkanDevice::recreateSwapchain(std::uint32_t width, std::uint32_t height,
require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0 && (surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0 && (surface.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) != 0, "resized surface capabilities are unsupported");
VkSwapchainCreateInfoKHR create{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
create.surface = state->surface;
create.minImageCount = surface.minImageCount;
create.minImageCount = AgcDriver::SwapchainImageCount(surface);
create.imageFormat = state->surfaceFormat;
create.imageColorSpace = state->surfaceColorSpace;
create.imageExtent = {width, height};
@@ -3158,9 +3210,9 @@ bool VulkanDevice::Presentable() const {
return state->extent.width != 0 && state->extent.height != 0;
}
bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque) {
bool VulkanDevice::PresentClear(std::uint32_t width, std::uint32_t height, bool opaque, const DisplayBuffer* plane) {
APS5_LOG_OUT_DEBUG("PresentClear width=%u height=%u opaque=%u", width, height, static_cast<unsigned>(opaque));
return present(width, height, opaque, {});
return present(width, height, opaque, {}, nullptr, nullptr, VK_FILTER_LINEAR, false, false, nullptr, 0, false, plane);
}
void VulkanDevice::PresentPixels(std::uint32_t width, std::uint32_t height, std::span<const std::byte> pixels) {
@@ -3291,6 +3343,56 @@ bool NoResidentPresent() {
return no;
}
// 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown stale (the
// write-back keeps the CPU's bytes for them): Refresh merges them into the image first, as a sampled
// texture of the image would (see cachedTexture). The uncached walk: the collect memo is per worker
// packet, so on this thread a memoized answer could miss a CPU write that landed after a worker's
// walk of the same range.
bool RefreshPresented(Graphics::StorageTexture& resident, const DisplayBuffer& buffer, std::uint32_t layer) {
static const bool noAdopt = std::getenv("APS5_NO_PRESENT_EPOCH") != nullptr;
const auto bytes = DisplayBufferSize(buffer);
if (!noAdopt) GuestMemory::AdoptLatestCollectEpoch();
GuestMemory::CollectWritesUncached(buffer.address, bytes);
if (GuestMemory::UnchangedSince(buffer.address, bytes, resident.GenerationFor(buffer.address, bytes)) && (buffer.dccAddress == 0 || !ResidentKeysMoved(resident))) return false;
if (resident.ImageLayers() > 1) resident.RefreshLayers(layer, 1);
else resident.Refresh();
return true;
}
struct PlaneSource {
std::shared_ptr<Graphics::StorageTexture> resident;
std::uint32_t layer = 0;
std::vector<std::byte> texels;
};
PlaneSource OverlayPlaneSource(const Graphics::Context& context, const DisplayBuffer& buffer) {
require(DisplayTexelBytes(buffer.pixelFormat) == 4u && !DisplayTenBit(buffer.pixelFormat) && !DisplayPq(buffer.pixelFormat), "VideoOut: only an 8-bit SDR overlay plane is composited over the main output");
PlaneSource source;
if (!NoResidentPresent()) {
VkFilter filter = VK_FILTER_LINEAR;
bool pending = false;
bool convert = false;
source.resident = PresentableResident(context, buffer, filter, pending, convert, source.layer);
if (source.resident != nullptr && buffer.dccAddress != 0 && !ResidentServesDisplay(*source.resident, buffer)) source.resident.reset();
if (source.resident != nullptr) {
static_cast<void>(RefreshPresented(*source.resident, buffer, source.layer));
return source;
}
}
const auto bytes = DisplayBufferSize(buffer);
GuestMemory::FlushGpuWrites(buffer.address, bytes);
const auto keys = buffer.dccAddress != 0 ? DisplayBufferKeys(buffer) : Graphics::DccKeys::Uncompressed;
if (keys == Graphics::DccKeys::Uncompressed) {
GuestMemory::CheckRange(reinterpret_cast<const void*>(buffer.address), bytes, buffer.alignment);
source.texels = DisplayBufferTexels(buffer, {reinterpret_cast<const std::byte*>(buffer.address), bytes});
return source;
}
const auto texel = DisplayBufferClearTexel(buffer, keys);
source.texels.resize(static_cast<std::size_t>(buffer.width) * buffer.height * texel.size());
for (std::size_t offset = 0; offset < source.texels.size(); offset += texel.size()) std::memcpy(source.texels.data() + offset, texel.data(), texel.size());
return source;
}
// Debug aid: APS5_DUMP_FRAMES=<n> saves the first n presented display buffers as frame_<index>.bmp,
// read back by the blit's own submission and written after its fence (RetireSlot), or with
// APS5_NO_GPU_DUMP=1 decoded from the tiled guest buffer on the CPU, as before.
@@ -3315,7 +3417,16 @@ FrameDumps& Dumps() {
}
bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer, const DisplayBuffer* plane) {
if (buffer.dccAddress != 0 && buffer.tilingMode == 0) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
const auto before = profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0;
state->recorder->SyncThrough(buffer.dccAddress, DisplayBufferKeyBytes(buffer), true);
if (const auto waited = profile ? Graphics::Recorder::ThreadWaitedMs() - before : 0.0; waited > 0.0) {
++presentCounters.keyWriterWaits;
presentCounters.keyWriterWaitMs += waited;
}
}
static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP"));
if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip");
else Graphics::StorageTexture::FlushDumpSelected("flip");
@@ -3338,7 +3449,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
}
const bool screenshot = takeScreenshot(dumpFrame);
auto shot = screenshot && passthrough ? ReadDisplayBuffer(buffer) : std::vector<std::byte>{};
if (!present(buffer.width, buffer.height, true, pixels, nullptr, nullptr, VK_FILTER_LINEAR, dumpFrame || (screenshot && !passthrough), false, nullptr, 0, passthrough)) {
if (!present(buffer.width, buffer.height, true, pixels, nullptr, nullptr, VK_FILTER_LINEAR, dumpFrame || (screenshot && !passthrough), false, nullptr, 0, passthrough, plane)) {
if (dumpFrame) --dumps.dumped;
if (screenshot) Screenshot::Request();
return false;
@@ -3370,19 +3481,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
} else if (resident == nullptr) {
++unsuitable;
} else {
// 64 KiB blocks the CPU wrote since the image last matched guest memory would be shown
// stale (the write-back keeps the CPU's bytes for them): Refresh merges them into the
// image first, as a sampled texture of the image would (see cachedTexture). The uncached
// walk: the collect memo is per worker packet, so on this thread a memoized answer could
// miss a CPU write that landed after a worker's walk of the same range.
static const bool noAdopt = std::getenv("APS5_NO_PRESENT_EPOCH") != nullptr;
if (!noAdopt) GuestMemory::AdoptLatestCollectEpoch();
GuestMemory::CollectWritesUncached(buffer.address, bytes);
if (!GuestMemory::UnchangedSince(buffer.address, bytes, resident->GenerationFor(buffer.address, bytes)) || (buffer.dccAddress != 0 && ResidentKeysMoved(*resident))) {
if (resident->ImageLayers() > 1) resident->RefreshLayers(layer, 1);
else resident->Refresh();
++refreshedPresents;
}
if (RefreshPresented(*resident, buffer, layer)) ++refreshedPresents;
++residentPresents;
}
}
@@ -3413,7 +3512,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
const auto channel = [&](std::size_t index) { return static_cast<float>(std::to_integer<unsigned>((*cleared)[index])) / 255.0f; };
uniform = {{channel(2), channel(1), channel(0), channel(3)}};
}
if (!present(buffer.width, buffer.height, true, {}, cleared ? nullptr : &buffer, resident, filter, dumpFrame || (screenshot && !cpuScreenshot), convert, cleared ? &uniform : nullptr, layer, passthrough)) {
if (!present(buffer.width, buffer.height, true, {}, cleared ? nullptr : &buffer, resident, filter, dumpFrame || (screenshot && !cpuScreenshot), convert, cleared ? &uniform : nullptr, layer, passthrough, plane)) {
// A dropped frame (swapchain out of date) keeps the dump numbering contiguous.
if (dumpFrame) --dumps.dumped;
if (screenshot) Screenshot::Request();
@@ -3469,7 +3568,7 @@ bool VulkanDevice::AcquireImage() {
return true;
}
bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display, const std::shared_ptr<Graphics::StorageTexture>& resident, VkFilter residentFilter, bool dumpFrame, bool residentConvert, const VkClearColorValue* uniform, std::uint32_t residentLayer, bool passthrough) {
bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaque, std::span<const std::byte> pixels, const DisplayBuffer* display, const std::shared_ptr<Graphics::StorageTexture>& resident, VkFilter residentFilter, bool dumpFrame, bool residentConvert, const VkClearColorValue* uniform, std::uint32_t residentLayer, bool passthrough, const DisplayBuffer* plane) {
PerformanceTimer timing("Vulkan.Present");
APS5_LOG_OUT_DEBUG("present begin width=%u height=%u opaque=%u pixels=%zu", width, height, static_cast<unsigned>(opaque), pixels.size());
require(state->swapchain != VK_NULL_HANDLE, "device has no swapchain");
@@ -3503,6 +3602,12 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
auto& slot = state->presentSlots[state->presentCursor];
require(!slot.inFlight, "presentation slot is still in flight");
const bool overlay = PerformanceOverlay::Visible();
if (plane != nullptr && state->surfaceColorSpace == VK_COLOR_SPACE_HDR10_ST2084_EXT) {
static std::once_flag reported;
std::call_once(reported, [] { std::fprintf(stderr, "[present] the overlay plane is not composited over an HDR10 presentation\n"); });
plane = nullptr;
}
const auto planeSource = plane != nullptr ? OverlayPlaneSource(graphicsContext(), *plane) : PlaneSource{};
if (display != nullptr && resident == nullptr) {
static_cast<void>(DisplayBufferSize(*display));
state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget, DisplayTexelBytes(display->pixelFormat));
@@ -3547,6 +3652,34 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
barrier.image = state->images[index];
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
auto pipelineBarrier = state->DeviceFunction<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
const auto planeSlot = static_cast<std::size_t>(&slot - state->presentSlots.data());
VkImageMemoryBarrier planeBarrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
bool planeBound = false;
const auto bindPlane = [&] {
if (planeBound) return;
planeBound = true;
if (!state->planePass) state->planePass = std::make_unique<PresentationPass>(graphicsContext(), state->presentSlots.size());
if (planeSource.resident == nullptr) {
state->planePass->BindTexels(planeSlot, commands, planeSource.texels, plane->width, plane->height, plane->pixelFormat, whiteNits);
return;
}
planeBarrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
planeBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
planeBarrier.oldLayout = VK_IMAGE_LAYOUT_GENERAL;
planeBarrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
planeBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
planeBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
planeBarrier.image = planeSource.resident->Image();
planeBarrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &planeBarrier);
Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
state->planePass->Bind(planeSlot, planeSource.resident->Image(), plane->width, plane->height, plane->pixelFormat, whiteNits, planeSource.layer);
};
const auto blendPlaneInto = [&](VkImage image, VkFormat format, VkExtent2D extent) {
if (plane == nullptr) return;
bindPlane();
state->planePass->RecordOverlay(commands, planeSlot, image, format, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR, extent, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL);
};
if (!drawn) {
pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
@@ -3594,7 +3727,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
}
if (drawn) {
state->presentation->Bind(slotIndex, resident->Image(), width, height, display->pixelFormat, whiteNits, residentLayer);
state->presentation->RecordPresent(commands, slotIndex, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, overlay ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
state->presentation->RecordPresent(commands, slotIndex, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, overlay || plane != nullptr ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
} else {
scaler->RecordLetterbox(commands, width, height, barrier.image, state->extent.width, state->extent.height);
if (direct) PresentationScaler::RecordBlitFrom(graphicsContext(), commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, residentFilter, barrier.image, state->extent.width, state->extent.height, residentLayer);
@@ -3615,13 +3748,17 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
const auto dumpBytes = static_cast<std::size_t>(slot.dumpWidth) * slot.dumpHeight * 4;
if (!slot.dumpBuffer || slot.dumpBuffer->Bytes().size() != dumpBytes) slot.dumpBuffer = std::make_unique<Graphics::Buffer>(graphicsContext(), dumpBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
if (drawn) {
state->presentation->RecordDump(commands, slotIndex, scale, slot.dumpBuffer->Handle());
state->presentation->RecordDumpImage(commands, slotIndex, scale);
blendPlaneInto(state->presentation->DumpImage(slotIndex), VK_FORMAT_B8G8R8A8_UNORM, state->presentation->DumpImageExtent(slotIndex));
state->presentation->RecordDumpReadback(commands, slotIndex, slot.dumpBuffer->Handle());
} else if (direct) {
if (!slot.dumpScaler) slot.dumpScaler = std::make_unique<PresentationScaler>(graphicsContext(), VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM);
slot.dumpScaler->EnsureSourceImage(slot.dumpWidth, slot.dumpHeight);
slot.dumpScaler->RecordBlitInto(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_FILTER_NEAREST, width, height, residentLayer);
blendPlaneInto(slot.dumpScaler->SourceImage(), VK_FORMAT_B8G8R8A8_UNORM, {slot.dumpWidth, slot.dumpHeight});
slot.dumpScaler->RecordReadback(commands, slot.dumpBuffer->Handle());
} else {
blendPlaneInto(scaler->SourceImage(), scaler->SourceFormat(), {scaler->SourceWidth(), scaler->SourceHeight()});
scaler->RecordReadback(commands, slot.dumpBuffer->Handle());
}
}
@@ -3632,11 +3769,21 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
}
}
if (plane != nullptr) {
bindPlane();
state->planePass->RecordOverlay(commands, planeSlot, barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, overlay ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
if (planeSource.resident != nullptr) {
planeBarrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
planeBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
pipelineBarrier(commands, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &planeBarrier);
Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
}
}
if (overlay) {
if (!state->overlay) state->overlay = std::make_unique<PerformanceOverlay>(graphicsContext(), state->presentSlots.size());
state->overlay->NotePresent(std::chrono::steady_clock::now(), clearOnly ? VkExtent2D{} : VkExtent2D{width, height});
state->overlay->Record(commands, static_cast<std::size_t>(&slot - state->presentSlots.data()), barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
} else if (!drawn) {
state->overlay->Record(commands, static_cast<std::size_t>(&slot - state->presentSlots.data()), barrier.image, state->surfaceFormat, state->surfaceColorSpace, state->extent, drawn || plane != nullptr ? VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
} else if (!drawn && plane == nullptr) {
barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
barrier.dstAccessMask = 0;
barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
@@ -3664,6 +3811,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
// The image stays alive until the slot is retired (storage-cache eviction only drops the
// cache's reference).
slot.kept = resident;
slot.keptPlane = planeSource.resident;
slot.imageIndex = index;
slot.inFlight = true;
slot.shared = shared;
@@ -3679,6 +3827,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
state->queuePending = true;
state->queueIndex = index;
++presentCounters.presents;
if (plane != nullptr) ++presentCounters.planes;
if (auto* frame = PerformanceContext::Current()) {
frame->NoteBlit(batchesAtBlit, lastSubmittedAt, submittedAt);
// The serials are cumulative: only their difference (the batches other queues put
@@ -3780,13 +3929,19 @@ void VulkanDevice::QueuePresent() {
present.swapchainCount = 1;
present.pSwapchains = &state->swapchain;
present.pImageIndices = &index;
const auto presented = state->DeviceFunction<PFN_vkQueuePresentKHR>("vkQueuePresentKHR")(state->queue, &present);
std::unique_lock presentQueueLock(state->presentQueueMutex, std::defer_lock);
if (state->presentQueue != VK_NULL_HANDLE) presentQueueLock.lock();
const auto presented = state->DeviceFunction<PFN_vkQueuePresentKHR>("vkQueuePresentKHR")(state->presentQueue != VK_NULL_HANDLE ? state->presentQueue : state->queue, &present);
if (presented == VK_ERROR_OUT_OF_DATE_KHR || presented == VK_SUBOPTIMAL_KHR) state->extent = {0, 0};
else check(presented, "vkQueuePresentKHR");
timing.Mark("queue_present");
APS5_LOG_OUT_DEBUG("vkQueuePresentKHR queued imageIndex=%u", index);
}
bool VulkanDevice::PresentsOnOwnQueue() const {
return state->presentQueue != VK_NULL_HANDLE;
}
ShaderRecompiler::SpirvTarget VulkanDevice::ComputeTarget(std::uint32_t waveSize) const {
auto target = Target();
if (waveSize == 32u && state->computeWave32) target.subgroupSize = 32u;
@@ -3930,7 +4085,7 @@ Graphics::Context VulkanDevice::buildContext() const {
state->physical,
state->queue,
state->pool,
state->deviceProc,
state->contextDeviceProc != nullptr ? state->contextDeviceProc : state->deviceProc,
state->InstanceFunction<PFN_vkGetPhysicalDeviceFormatProperties>("vkGetPhysicalDeviceFormatProperties"),
state->InstanceFunction<PFN_vkGetPhysicalDeviceImageFormatProperties>("vkGetPhysicalDeviceImageFormatProperties"),
state->memoryProperties,
@@ -3955,6 +4110,7 @@ Graphics::Context VulkanDevice::buildContext() const {
};
context.pipelineCacheSynchronization = state->pipelineCache ? &state->pipelineCache->Synchronization() : nullptr;
context.individuallyFreeDescriptorSets = state->individuallyFreeDescriptorSets;
context.tessellationPipelineCache = state->tessellationPipelineCache;
context.dmaBufImport = state->dmaBufImport;
context.sparseBinding = state->sparseBinding;
context.sparseResidencyBuffer = state->sparseResidencyBuffer;
@@ -4332,7 +4488,7 @@ struct DispatchCounters {
std::array<double, DispatchRows> phaseTotals{};
std::uint64_t profiledDispatches = 0;
IndirectHold indirectHold;
std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0, templateRefreshed = 0, templateSameWords = 0;
std::uint64_t cacheHits = 0, cacheMisses = 0, cacheInvalidated = 0, templateRefreshed = 0, templateSameWords = 0, templateByNumber = 0;
double templateRevalidateMs = 0;
std::uint64_t procLookups = 0, preBarriersRecorded = 0, preBarriersSkipped = 0;
std::chrono::steady_clock::time_point cacheReport = std::chrono::steady_clock::now();
@@ -4975,6 +5131,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
};
if (recorder.HasQueuedKeyStores() && (resources.HoldsLease() || recorder.AnyQueuedKeyStore(touches))) recorder.FlushKeyStores();
if (recorder.HasQueuedStores() && (resources.HoldsLease() || recorder.AnyQueuedStore(touches))) recorder.FlushStores();
static_cast<void>(Graphics::StorageTexture::StoreDrawnLinearTouching("dispatch access", touches, resources.HoldsLease()));
VkAccessFlags covered = 0;
const auto commands = recorder.Commands(&covered);
recordStep(PhaseRecordCommands);
@@ -4993,6 +5150,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
// after an earlier dispatch's reads of the buffers by that dispatch's trailing barrier. A
// recipe compares the two 64-bit hashes first: equal hashes mean the buffers hold the words.
const bool differs = record.dataRefresh == RecordedDispatch::DataRefresh::Words || resources.DataWordsHash() != record.dataWordsHash;
if (timer.profile && record.shader->program != nullptr && record.shader->program->bindings.size() != resources.BindingCount()) ++d.templateByNumber;
if (differs && (record.resourceKeyMatched ? resources.refreshData(commands, *record.shader, &recorder) : resources.RefreshData(commands, *record.shader, &recorder))) {
covered = 0;
++d.templateRefreshed;
@@ -5506,7 +5664,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
if (profile && std::chrono::steady_clock::now() - d.cacheReport > std::chrono::seconds(10)) {
d.cacheReport = std::chrono::steady_clock::now();
AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words; Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
AgcDriver::ProfilePrint_nid_no_patch("[rescache] (cumulative): %llu hits, %llu misses, %llu invalidated, %zu entries (dispatch + draw); template hits: %llu refreshed the data buffers (revalidate %.1f ms), %llu had the same words, %llu paired their data bindings by number (no image table binding of their own); Find calls %llu, Touch calls %llu\n", static_cast<unsigned long long>(d.cacheHits), static_cast<unsigned long long>(d.cacheMisses), static_cast<unsigned long long>(d.cacheInvalidated), state->resourceCache.Size(), static_cast<unsigned long long>(d.templateRefreshed), d.templateRevalidateMs, static_cast<unsigned long long>(d.templateSameWords), static_cast<unsigned long long>(d.templateByNumber), static_cast<unsigned long long>(ResourceCache::Finds()), static_cast<unsigned long long>(ResourceCache::Touches()));
AgcDriver::ProfilePrint_nid_no_patch("[vk] deviceProc lookups inside the device call (cumulative): %llu (%.2f per dispatch); pre-dispatch barriers recorded %llu, skipped %llu; compute pipelines created %llu (%llu under the GPU mutex)\n", static_cast<unsigned long long>(d.procLookups), d.profiledDispatches != 0 ? static_cast<double>(d.procLookups) / static_cast<double>(d.profiledDispatches) : 0.0, static_cast<unsigned long long>(d.preBarriersRecorded), static_cast<unsigned long long>(d.preBarriersSkipped), static_cast<unsigned long long>(computePipelinesCreated.load(std::memory_order_relaxed)), static_cast<unsigned long long>(computePipelinesCreatedLocked.load(std::memory_order_relaxed)));
reportRecipes();
reportDispatchMove();
core/libs/prx/libSceAgcDriver/Execution/src/WorkerSampler.cpp
+15 -5
View File
@@ -163,18 +163,29 @@ struct Sampler {
return frame;
}
static void writeModules(std::FILE* file, const std::map<std::uint64_t, std::string>& loaded) {
for (const auto& [base, name] : loaded) {
const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
}
}
void write() {
std::lock_guard lock(mutex);
std::FILE* file = std::fopen(path.c_str(), "w");
if (file == nullptr) return;
std::fprintf(file, "# %llu samples\n", static_cast<unsigned long long>(samples));
std::map<std::uint64_t, std::string> loaded;
for (const auto& [address, count] : counts) {
HMODULE module = nullptr;
char name[MAX_PATH] = "?";
if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
const char* base = std::strrchr(name, '\\');
if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(count.first), static_cast<unsigned long long>(count.second));
}
writeModules(file, loaded);
std::fclose(file);
if (keepStacks) writeStacks();
}
@@ -195,11 +206,7 @@ struct Sampler {
const double tickUs = 1e6 / static_cast<double>(frequency.QuadPart);
std::fprintf(file, "# samples=%llu depth=%d truncated=%llu suspended_us_mean=%.1f suspended_us_max=%.1f\n", static_cast<unsigned long long>(samples), depth, static_cast<unsigned long long>(stacks.Truncated()),
samples != 0 ? static_cast<double>(suspendedTicks) * tickUs / static_cast<double>(samples) : 0.0, static_cast<double>(longestSuspendedTicks) * tickUs);
for (const auto& [base, name] : loaded) {
const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
}
writeModules(file, loaded);
const auto text = stacks.Format([&](std::uint64_t address) { return names.at(address); });
const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size();
if (std::fclose(file) != 0 || !written) return;
@@ -246,6 +253,7 @@ struct ProcessSampler {
std::FILE* file = std::fopen(path.c_str(), "w");
if (file == nullptr) return;
std::fprintf(file, "# %llu rounds\n", static_cast<unsigned long long>(rounds));
std::map<std::uint64_t, std::string> loaded;
for (auto& [id, thread] : threads) {
std::lock_guard lock(thread.mutex);
std::vector<std::pair<std::uint64_t, std::pair<std::uint64_t, std::uint64_t>>> hot(thread.counts.begin(), thread.counts.end());
@@ -260,9 +268,11 @@ struct ProcessSampler {
char name[MAX_PATH] = "?";
if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
const char* base = std::strrchr(name, '\\');
if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
std::fprintf(file, " %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(hot[i].second.first), static_cast<unsigned long long>(hot[i].second.second));
}
}
Sampler::writeModules(file, loaded);
std::fclose(file);
}
};
core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp
+1
View File
@@ -240,6 +240,7 @@ struct Context {
std::uint32_t imageTableCapacity = 0;
std::uint32_t samplerTableCapacity = 0;
bool individuallyFreeDescriptorSets = false;
bool tessellationPipelineCache = true;
bool imageTableUpdateAfterBind = false;
bool samplerTableUpdateAfterBind = false;
bool geometryShader = false;
core/libs/prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp
+17
View File
@@ -0,0 +1,17 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEFERREDDESTRUCTION_HPP
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include <chrono>
#include <cstddef>
namespace AgcDriver::Graphics {
PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver);
void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now());
void DestroyDeferredObjects(VkDevice device) noexcept;
std::size_t DeferredObjects(VkDevice device);
}
#endif
core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp
+5
View File
@@ -38,6 +38,7 @@ struct HostImport {
// Identity for the life of this import (see HostImportSerial); 0 until first asked for.
std::uint64_t serial = 0;
bool unwatched = false;
bool settleOnDemand = false;
bool dmaBuf = false;
bool joined = false;
std::shared_ptr<void> chunk;
@@ -102,6 +103,7 @@ void ReportHostImports();
// address)` in the import's buffer.
const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes, ImportRequester requester = ImportRequester::Access);
const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
bool WatchedAfterSettle(const Context& context, std::uint64_t address, std::size_t bytes);
void SetHostImportWindow(std::uint64_t bytes);
// Tests: the window of one large registered range (64 MiB) and the import budget (0: the configured one).
void SetHostImportRangeWindow(std::uint64_t bytes);
@@ -133,6 +135,7 @@ bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped&
// made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken
// re-checks with HostImportFor when it binds the import.
bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes);
bool ComparedHostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes);
struct RegisteredPiece {
std::uint64_t address;
std::uint64_t bytes;
@@ -264,6 +267,8 @@ struct AddressCopy {
std::uint64_t end;
std::uint64_t committed;
const char* reason;
bool heap = false;
bool overBudget = false;
};
std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t limit);
core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp
+6 -2
View File
@@ -109,8 +109,10 @@ private:
// stages and every pipeline state field. A configuration whose stage has no module gets a private
// pipeline. A miss builds the pipeline outside the store's mutex (a concurrent build of the same
// key keeps the first inserted) and records its recipe in the pipeline journal. Entries are
// LRU-bounded (APS5_PIPELINE_STORE_ENTRIES, 1024) and only evicted once no recorded draw holds
// them. Debug aid: APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
// LRU-bounded and only evicted once no recorded draw holds them. The bound starts at 1024 and
// doubles up to 4096 for a journal replay that does not fit and while pipelines it evicted keep
// coming back; it never shrinks during a run. APS5_PIPELINE_STORE_ENTRIES fixes it. Debug aid:
// APS5_NO_PIPELINE_CACHE=1 builds one per draw as before.
std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
// `created` reports whether this call built the pipeline (false: served from the store or private).
std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, bool* created = nullptr);
@@ -124,6 +126,8 @@ struct PipelineStoreCounters {
std::uint64_t bound = 0;
};
PipelineStoreCounters PipelineStoreTotals();
// Whether the store may still raise its bound (not fixed by APS5_PIPELINE_STORE_ENTRIES, below 4096).
bool PipelineStoreCanGrow();
// The pipeline store's key of a configuration (empty when the configuration gets a private pipeline).
void BuildPipelineStoreKey(std::vector<std::byte>& key, const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout);
struct DescribedSetLayouts {
core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp
+1
View File
@@ -183,6 +183,7 @@ public:
// Whether the OPEN (unsubmitted) batch writes the range: a wait on such a range must submit it.
bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes) const;
std::uint64_t PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const;
// What SyncThrough(address, bytes) would wait for, for the [hooksync] attribution: the newest
// batch whose noted range overlaps the access (the open batch counts first, with the serial it
// will get), whether that batch's fence has already signaled (its GPU work is done, so the
core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp
+2
View File
@@ -32,6 +32,7 @@ void FlushCachedTextures(VkDevice device);
void ClearCachedTextures(VkDevice device);
std::uint64_t ReleaseTextureCaches(std::uint64_t bytes, bool newImage = false);
std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory);
std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported);
std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes);
std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes = 0);
bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport);
@@ -315,6 +316,7 @@ public:
}
// Whether RefreshData would record anything for `shader` (the per-word compare; verification).
bool DataWordsDiffer(const CompiledShader& shader) const;
std::size_t BindingCount() const { return bindings.size(); }
// Why the fast proof of a Revalidate left the object to the full walk (the [rescache]
// revalidate line's reasons); Count: it did not.
enum class FastFail : std::size_t { NoRecord, Collect, Pending, Evicted, Changed, Keys, ClearedView, StorageKeys, Depth, Replaced, Audit, Count };
core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp
+2
View File
@@ -45,6 +45,7 @@ struct ColorTarget {
ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None;
// DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp).
std::uint64_t dccAddress = 0;
std::uint64_t chainDccAddress = 0;
bool dccAlphaOnMsb = false;
bool dccPipeAligned = false;
std::uint64_t cmaskAddress = 0;
@@ -180,6 +181,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup =
void ApplyPointSize(const State& state, ShaderRecompiler::ShaderVertexStageInfo& vertex);
State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
void ApplyDepthRemap(const Context& context, State& state);
void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
void NormalizeUnusedViewportDepth(State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
std::array<VkSampleLocationEXT, 32> SampleLocations(const State& state);
std::array<std::uint8_t, 32> PixelSampleLocations(const State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel);
core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp
+33
View File
@@ -16,6 +16,7 @@
#include <memory>
#include <mutex>
#include <optional>
#include <functional>
#include <span>
#include <tuple>
#include <utility>
@@ -204,6 +205,7 @@ public:
bool BlockUnits() const { return blockUnits; }
std::uint32_t TrackedUnits() const { return trackedLayers; }
void MarkSliceDirty(std::uint32_t slice, std::uint32_t count = 1);
bool MarkDrawn(std::uint32_t slice, std::uint32_t count, VkRect2D scissor);
void Flush();
bool Dirty() const;
// Stores every pending image overlapping the range, except `except`; returns whether any was.
@@ -220,6 +222,8 @@ public:
static constexpr std::uint64_t SmallStoreBytes = 64 * 1024;
static std::size_t StoreSmallPending(const char* reason);
static void StoreReadbackBeforeLabel(const char* reason);
static std::size_t StoreDrawnLinear(const char* reason);
static std::size_t StoreDrawnLinearTouching(const char* reason, const std::function<bool(std::uint64_t, std::uint64_t)>& touches, bool all);
static bool AnyPending();
static void FlushDumpSelected(const char* reason);
static bool StoreAtFlipRequested(const char* value);
@@ -482,6 +486,27 @@ private:
// (`shadowed` receives those guest ranges, `imported` the pieces written to the import).
std::uint64_t uploadWindows(const HostImport& import, std::span<const std::pair<std::uint64_t, std::uint64_t>> runs, bool discard = false);
std::uint64_t writeBackWindows(std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep, std::uint64_t firstStored, std::uint64_t lastStored, std::vector<ShadowedRange>& shadowed, std::vector<std::pair<std::uint64_t, std::uint64_t>>& imported, Buffer* save = nullptr);
struct DrawnRect {
std::uint32_t left = 0;
std::uint32_t top = 0;
std::uint32_t right = 0;
std::uint32_t bottom = 0;
};
struct DrawnArea {
bool drawn = false;
bool rect = false;
std::uint32_t left = 0;
std::uint32_t top = 0;
std::uint32_t right = 0;
std::uint32_t bottom = 0;
std::vector<DrawnRect> rects;
};
DrawnArea drawnArea() const;
std::vector<std::pair<std::uint64_t, std::uint64_t>> drawnSpans(const DrawnArea& area, std::span<const std::pair<std::uint64_t, std::uint64_t>> keep) const;
std::uint64_t storeDrawnRect(const DrawnArea& area, std::span<const ImportSegment> segments, std::span<const std::pair<std::uint64_t, std::uint64_t>> stores, const std::vector<std::uint32_t>& saveUnits);
void addDrawnRect(DrawnRect rect);
void forgetDrawn();
void noteStore(std::uint64_t note);
// Per-layer validity (thin array surfaces whose layers are 64 KiB-aligned guest slices; every
// other surface is one tracked layer): a layer's generation is the write generation its guest
// bytes were last known to match the image at, and a pending layer holds results (a clear, a
@@ -518,6 +543,7 @@ private:
// Per 64 KiB tracker block of the surface (block 0 holds the base address): the generation of
// the tracked layer it belongs to, for GuestMemory::ChangedBlocks.
void blockGenerations(std::vector<std::uint64_t>& generations) const;
std::uint64_t unitBlockGeneration(std::uint32_t unit, std::uint64_t at) const;
// Another image of exactly this surface under another storage format with results pending
// (block units, uncompressed, still cached), if any: Refresh takes its pending units from its
// image on the device (borrowUnits) instead of having them stored and re-read, and the alias
@@ -602,6 +628,7 @@ private:
std::uint32_t trackedLayers = 1;
std::uint64_t trackedLayerBytes = 0;
std::vector<std::uint64_t> layerGeneration;
std::vector<std::uint64_t> completionKept;
std::vector<bool> layerPending;
std::vector<bool> layerBehind;
bool blockUnits = false;
@@ -641,6 +668,12 @@ private:
VkFormat storageFormat = VK_FORMAT_UNDEFINED;
// Results are on the GPU only (guarded by the pending-write registry lock).
bool dirty = false;
bool drawnListed = false;
bool drawnWhole = false;
bool drawingTarget = false;
bool pendingFromOthers = false;
std::vector<DrawnRect> drawnRects;
std::vector<std::uint64_t> storeNotes;
std::uint64_t version = 0;
std::uint64_t writes = 0;
// `original` holds the guest bytes; false after a GPU-side clear, which never read them.
core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp
+1
View File
@@ -13,6 +13,7 @@ namespace AgcDriver::Graphics {
struct Context;
VkFormat ResolveTextureFormat(std::uint32_t guestFormat);
VkFormat DccClearTextureFormat(std::uint32_t guestFormat);
VkComponentSwizzle TextureComponentChannel(std::uint32_t guestFormat, VkComponentSwizzle component);
std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical);
VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat);
core/libs/prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp
+1 -1
View File
@@ -75,7 +75,7 @@ struct SurfaceGeometry {
};
SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor);
bool LevelsFitAllocation(const GuestTextureResource& surface, std::uint32_t levels);
GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed);
GuestTextureResource ViewedArrayWindow(const GuestTextureResource& viewed, bool keysReadAsStored);
GuestTextureResource ArrayLayerWindow(const GuestTextureResource& viewed, std::uint32_t maxArrayLayers);
}
core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp
+4 -4
View File
@@ -521,9 +521,9 @@ DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t gue
if (const auto known = WaitForKeyWriters(resource, guestBytes)) keys = *known;
else keys = currentDccKeys(resource.dccAddress, DccKeyBytes(guestBytes), memoized);
if (keys == DccKeys::Uncompressed || (keys == DccKeys::ClearSingle && SingleKeysReadable(resource))) return keys;
if (IsConvertedTextureFormat(resource.format) && (keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110)) throw std::runtime_error(std::string("AGC graphics: DCC clear code ") + DccKeysName(keys) + " of converted texture format " + std::to_string(resource.format) + " is not implemented");
if (DccClearTextureFormat(resource.format) == ResolveTextureFormat(resource.format) && IsConvertedTextureFormat(resource.format) && (keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110)) throw std::runtime_error(std::string("AGC graphics: DCC clear code ") + DccKeysName(keys) + " of converted texture format " + std::to_string(resource.format) + " is not implemented");
std::byte probe[16]{};
if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) {
if (!IsDccClear(keys) || !FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) {
static AgcDriver::Mutex reportedMutex;
static std::set<std::pair<std::uint64_t, int>> reported;
std::lock_guard lock(reportedMutex);
@@ -776,7 +776,7 @@ void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std:
// Named for the [hooksync] attribution: the read goes through the flush hook.
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureRead);
if (keys == DccKeys::Uncompressed || keys == DccKeys::ClearSingle) GuestMemory::ReadCommitted(resource.baseAddress, bytes);
else FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes);
else FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, bytes);
if (keys == DccKeys::ClearSingle) ExpandSingleBlocks(bytes, BytesPerElement(resource.format));
}
@@ -823,7 +823,7 @@ namespace {
DccKeys EncodedKeys(const GuestTextureResource& resource, DccKeys keys) {
if (keys == DccKeys::ClearSingle && SingleKeysReadable(resource)) return keys;
std::byte probe[16]{};
if (!IsDccClear(keys) || !FillDccClear(ResolveTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) return DccKeys::Uncompressed;
if (!IsDccClear(keys) || !FillDccClear(DccClearTextureFormat(resource.format), keys, resource.dccAlphaOnMsb, std::span(probe, std::min<std::size_t>(sizeof(probe), BytesPerElement(resource.format))))) return DccKeys::Uncompressed;
return keys;
}
core/libs/prx/libSceAgcDriver/Graphics/src/DeferredDestruction.cpp
+134
View File
@@ -0,0 +1,134 @@
#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
#include <atomic>
#include <cstring>
#include <mutex>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <vector>
namespace AgcDriver::Graphics {
namespace {
constexpr std::size_t DrainObjects = 512;
constexpr auto DrainAge = std::chrono::milliseconds(100);
struct Deferred {
PFN_vkDestroyBuffer destroyBuffer = nullptr;
PFN_vkFreeMemory freeMemory = nullptr;
std::vector<VkBuffer> buffers;
std::vector<VkDeviceMemory> memory;
std::chrono::steady_clock::time_point oldest{};
};
struct Registry {
std::mutex mutex;
std::unordered_map<VkDevice, Deferred> devices;
std::atomic<PFN_vkGetDeviceProcAddr> resolver{nullptr};
};
Registry& registry() {
static auto& state = *new Registry();
return state;
}
void destroyAll(VkDevice device, const Deferred& deferred) noexcept {
for (const auto buffer : deferred.buffers) deferred.destroyBuffer(device, buffer, nullptr);
for (const auto memory : deferred.memory) deferred.freeMemory(device, memory, nullptr);
}
template<typename THandle>
bool defer(VkDevice device, THandle handle, std::vector<THandle> Deferred::*list) noexcept {
try {
auto& state = registry();
std::lock_guard lock(state.mutex);
const auto found = state.devices.find(device);
if (found == state.devices.end()) return false;
auto& deferred = found->second;
if (deferred.buffers.empty() && deferred.memory.empty()) deferred.oldest = std::chrono::steady_clock::now();
(deferred.*list).push_back(handle);
return true;
} catch (...) {
return false;
}
}
VKAPI_ATTR void VKAPI_CALL deferredDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) {
if (buffer == VK_NULL_HANDLE || defer(device, buffer, &Deferred::buffers)) return;
reinterpret_cast<PFN_vkDestroyBuffer>(registry().resolver.load(std::memory_order_acquire)(device, "vkDestroyBuffer"))(device, buffer, allocator);
}
VKAPI_ATTR void VKAPI_CALL deferredFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) {
if (memory == VK_NULL_HANDLE || defer(device, memory, &Deferred::memory)) return;
reinterpret_cast<PFN_vkFreeMemory>(registry().resolver.load(std::memory_order_acquire)(device, "vkFreeMemory"))(device, memory, allocator);
}
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL deferringDeviceProc(VkDevice device, const char* name) {
if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredDestroyBuffer);
if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(&deferredFreeMemory);
return registry().resolver.load(std::memory_order_acquire)(device, name);
}
}
PFN_vkGetDeviceProcAddr DeferDestructionToIdle(VkDevice device, PFN_vkGetDeviceProcAddr resolver) {
if (device == VK_NULL_HANDLE || resolver == nullptr) throw std::invalid_argument("AGC graphics: deferred destruction needs a device and its resolver");
Deferred deferred;
deferred.destroyBuffer = reinterpret_cast<PFN_vkDestroyBuffer>(resolver(device, "vkDestroyBuffer"));
deferred.freeMemory = reinterpret_cast<PFN_vkFreeMemory>(resolver(device, "vkFreeMemory"));
if (deferred.destroyBuffer == nullptr || deferred.freeMemory == nullptr) throw std::runtime_error("AGC graphics: missing Vulkan function: vkDestroyBuffer or vkFreeMemory");
auto& state = registry();
std::lock_guard lock(state.mutex);
const auto current = state.resolver.load(std::memory_order_acquire);
if (current != nullptr && current != resolver) throw std::runtime_error("AGC graphics: deferred destruction found two different device resolvers");
if (!state.devices.emplace(device, std::move(deferred)).second) throw std::runtime_error("AGC graphics: deferred destruction was set up twice for one device");
state.resolver.store(resolver, std::memory_order_release);
return &deferringDeviceProc;
}
void DestroyDeferredAtIdle(const Context& context, std::chrono::steady_clock::time_point now) {
auto& state = registry();
{
std::lock_guard lock(state.mutex);
const auto found = state.devices.find(context.device);
if (found == state.devices.end()) return;
const auto objects = found->second.buffers.size() + found->second.memory.size();
if (objects == 0 || (objects < DrainObjects && now - found->second.oldest < DrainAge)) return;
}
const auto idle = context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue);
if (idle != VK_SUCCESS && idle != VK_ERROR_DEVICE_LOST) throw std::runtime_error("AGC graphics: vkQueueWaitIdle before destroying deferred objects: Vulkan result " + std::to_string(idle));
Deferred taken;
{
std::lock_guard lock(state.mutex);
const auto found = state.devices.find(context.device);
if (found == state.devices.end()) return;
taken.destroyBuffer = found->second.destroyBuffer;
taken.freeMemory = found->second.freeMemory;
taken.buffers.swap(found->second.buffers);
taken.memory.swap(found->second.memory);
}
destroyAll(context.device, taken);
}
void DestroyDeferredObjects(VkDevice device) noexcept {
Deferred taken;
{
auto& state = registry();
std::lock_guard lock(state.mutex);
const auto found = state.devices.find(device);
if (found == state.devices.end()) return;
taken = std::move(found->second);
state.devices.erase(found);
}
destroyAll(device, taken);
}
std::size_t DeferredObjects(VkDevice device) {
auto& state = registry();
std::lock_guard lock(state.mutex);
const auto found = state.devices.find(device);
return found == state.devices.end() ? 0 : found->second.buffers.size() + found->second.memory.size();
}
}
core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp
+36 -18
View File
@@ -719,10 +719,30 @@ public:
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT);
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, storage->Image(), VK_IMAGE_LAYOUT_GENERAL, depthPlane.staging->Handle(), 1, &colorRegion);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
if (target.stencilAddress == 0) {
VkImageMemoryBarrier discard{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
discard.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
discard.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
discard.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
discard.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
discard.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
discard.image = image;
discard.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
const auto barrier = context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier");
barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &discard);
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &depthRegion);
VkImageMemoryBarrier toGeneral = discard;
toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toGeneral.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral);
} else {
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, depthPlane.staging->Handle(), image, VK_IMAGE_LAYOUT_GENERAL, 1, &depthRegion);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
}
if (batch) batch->SubmitAndWait();
else Recorder::CountBarriers(Recorder::CommandClass::Copy, 3);
else Recorder::CountBarriers(Recorder::CommandClass::Copy, target.stencilAddress == 0 ? 4 : 3);
}
std::shared_ptr<StorageTexture> Mirror(const GuestTextureResource& resource) {
@@ -1227,8 +1247,10 @@ public:
continue;
}
surface = {};
const GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
if (held.generation != 0 && held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation)) {
GuestLayer current{resource.baseAddress + geometry.GuestLayerOffset(layer), geometry.layerBytes, resource.tileMode, geometry.thick, GuestMemory::CollectWrites(resource.baseAddress + geometry.GuestLayerOffset(layer), static_cast<std::size_t>(geometry.layerBytes))};
current.uncommitted = current.generation == 0 && GuestMemory::CommittedRanges(current.address, static_cast<std::size_t>(current.bytes)).empty();
const bool unchanged = current.uncommitted ? held.uncommitted : held.generation != 0 && GuestMemory::UnchangedSince(current.address, static_cast<std::size_t>(current.bytes), held.generation);
if (held.address == current.address && held.bytes == current.bytes && held.tileMode == current.tileMode && held.thick == current.thick && !Recorder::SnapshotWriteOverlaps(current.address, static_cast<std::size_t>(current.bytes)) && unchanged) {
held.generation = current.generation;
continue;
}
@@ -1266,6 +1288,7 @@ private:
TextureTileMode tileMode = TextureTileMode::kLinear;
bool thick = false;
std::uint64_t generation = 0;
bool uncommitted = false;
};
VkImage image = VK_NULL_HANDLE;
@@ -1970,20 +1993,15 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target,
const bool created = surface == nullptr;
GuestAllocations::Lease creationLease;
if (created) {
for (const auto& surface : surfaces()) {
if (!surface->active->load(std::memory_order_acquire) || !surface->samplingActive || surface->context.device != context.device) continue;
const auto bytes = target.samples == 1u ? DepthSliceBytes(target.extent, target.format == VK_FORMAT_D16_UNORM ? 2u : 4u) : MultisampledDepthBytes_nid_no_patch(target.extent, target.format == VK_FORMAT_D16_UNORM ? 2u : 4u, target.samples);
Require(target.address <= std::numeric_limits<std::uint64_t>::max() - bytes && target.htileAddress <= std::numeric_limits<std::uint64_t>::max() - HtileMetadataBytes(target.extent), "native depth attachment range overflows");
for (const auto& range : surface->sampling.initial) {
Require(target.address >= range.address + range.bytes || range.address >= target.address + bytes, "native multisample depth attachment layout or metadata changed");
if (target.stencilAddress != 0u) {
const auto stencilBytes = target.samples == 1u ? DepthSliceBytes(target.extent, 1u) : MultisampledDepthBytes_nid_no_patch(target.extent, 1u, target.samples);
Require(target.stencilAddress <= std::numeric_limits<std::uint64_t>::max() - stencilBytes && (target.stencilAddress >= range.address + range.bytes || range.address >= target.stencilAddress + stencilBytes), "native multisample stencil attachment aliases a resident depth surface");
}
if (target.htileAddress != 0u) Require(target.htileAddress >= range.address + range.bytes || range.address >= target.htileAddress + HtileMetadataBytes(target.extent), "native multisample depth attachment layout or metadata changed");
}
}
const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
const auto depthBytes = target.samples == 1u ? DepthSliceBytes(target.extent, d16 ? 2u : 4u) : MultisampledDepthBytes_nid_no_patch(target.extent, d16 ? 2u : 4u, target.samples);
const auto stencilBytes = target.stencilAddress == 0u ? 0u : target.samples == 1u ? DepthSliceBytes(target.extent, 1u) : MultisampledDepthBytes_nid_no_patch(target.extent, 1u, target.samples);
const auto metadataBytes = target.htileAddress == 0u ? 0u : HtileMetadataBytes(target.extent);
Require(target.address <= std::numeric_limits<std::uint64_t>::max() - depthBytes && target.stencilAddress <= std::numeric_limits<std::uint64_t>::max() - stencilBytes && target.htileAddress <= std::numeric_limits<std::uint64_t>::max() - metadataBytes, "native depth attachment range overflows");
lock.unlock();
RequireNoResidentDepthRead_nid_no_patch(target.address, static_cast<std::size_t>(depthBytes));
if (stencilBytes != 0u) RequireNoResidentDepthRead_nid_no_patch(target.stencilAddress, static_cast<std::size_t>(stencilBytes));
if (metadataBytes != 0u) RequireNoResidentDepthRead_nid_no_patch(target.htileAddress, static_cast<std::size_t>(metadataBytes));
auto sampling = target.samples == 1u ? DepthSamplingState{} : prepareDepthSampling(context, target);
creationLease = target.samples == 1u || !sampling.refusal.empty() ? GuestAllocations::Lease{} : GuestAllocations::GuestAllocationsAcquireAll_nid_postfix();
lock.lock();
core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp
+19 -16
View File
@@ -1921,7 +1921,7 @@ struct Kept {
// The completion side of a recorded draw: the kept objects, the record check, the lease outcome,
// the GPU write notes, the copied-buffer write-back (listed in DrawCopiedWriters or run as a
// completion action) and the targets marked dirty.
void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>& resources, std::shared_ptr<Pipeline> pipeline, std::shared_ptr<Framebuffer> framebuffer, DrawInputs& inputs, std::vector<std::shared_ptr<StorageTexture>> targets, std::vector<std::pair<std::uint32_t, std::uint32_t>> slices, VkRect2D scissor, std::unique_ptr<DeviceBuffer> scratch, std::function<void()> checkRecords, bool listed, bool completion, const DrawOutcome& outcome) {
auto kept = std::make_shared<Kept>();
kept->resources = resources;
kept->pipeline = std::move(pipeline);
@@ -1961,14 +1961,15 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
if (resources->Reusable()) recorder.OnComplete([resources] { resources->CheckFaults(); });
else recorder.OnComplete([resources] { resources->WriteBackBuffers(); });
}
std::vector<bool> storeNow(residents.size(), false);
for (std::size_t index = 0; index < residents.size(); ++index) {
if (residents[index] == nullptr) continue;
if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
if (residents[index]->Descriptor().tileMode == TextureTileMode::kLinear) storeNow[index] = !residents[index]->MarkDrawn(index < slices.size() ? slices[index].first : 0u, index < slices.size() ? slices[index].second : 0u, scissor) || !residents[index]->Cached();
else if (index < slices.size()) residents[index]->MarkSliceDirty(slices[index].first, slices[index].second);
else residents[index]->MarkDirty();
}
for (const auto& resident : residents) {
if (resident == nullptr || resident->Descriptor().tileMode != TextureTileMode::kLinear) continue;
StorageTexture::FlushPending(resident->Descriptor().baseAddress, static_cast<std::size_t>(resident->GuestBytes()), nullptr, "linear target");
for (std::size_t index = 0; index < residents.size(); ++index) {
if (storeNow[index]) StorageTexture::FlushPending(residents[index]->Descriptor().baseAddress, static_cast<std::size_t>(residents[index]->GuestBytes()), nullptr, "linear target");
}
}
@@ -2195,7 +2196,6 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
recorder->FlushStores();
outcome.storesFlushed = true;
}
const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
const bool capture = CaptureInputsEnabled();
const bool meshIndirect = state.stages.mesh && args != nullptr;
const bool addressBased = resources.HoldsLease();
@@ -2217,6 +2217,8 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
}
for (const auto image : record.multisampleImages) passAttachments.emplace_back(image, true);
const bool unknownMemory = addressBased || resources.UsesAddressTable() || resources.HasCopiedWrites();
static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return std::any_of(passReads.begin(), passReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }) || resources.WritesOverlap(begin, static_cast<std::size_t>(end - begin)); }, unknownMemory));
const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved);
static const bool addressReadPasses = std::getenv("APS5_NO_ADDRESS_READ_PASSES") == nullptr;
const bool readOnlyBda = addressReadPasses && resources.BdaReadOnly();
outcome.addressDraw = addressBased;
@@ -2361,7 +2363,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
}
recorder->LeaveRenderPassOpen(passKey, drawTiming, fenced, passAccess, std::move(storeProxies));
timer.phase(PhaseRecord);
keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(record.targetSlices), state.scissor, std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome);
timer.phase(PhaseKeep);
if (record.waited) {
// The wait the dispatch path makes for such work (source 3, "address-based"): the
@@ -2952,6 +2954,14 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
report(outcome.waited ? (outcome.reason == SyncLease ? " recorded then waited (lease)" : " recorded then waited (copied writes)") : completion ? " recorded with completion" : " recorded");
return;
}
const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
const auto bytes = static_cast<std::size_t>(end - begin);
if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
if (!gpuIndirect) return false;
return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
};
static_cast<void>(StorageTexture::StoreDrawnLinearTouching("draw access", [&](std::uint64_t begin, std::uint64_t end) { return touches(begin, end) || std::any_of(inputs.inPlaceReads.begin(), inputs.inPlaceReads.end(), [&](const auto& read) { return read.first < end && begin < read.second; }); }, resources->HoldsLease() || resources->UsesAddressTable() || resources->HasCopiedWrites()));
APS5_LOG_CHARS_OUT_DEBUG("Creating CommandBatch");
std::optional<CommandBatch> batch;
if (!recorded) {
@@ -2961,16 +2971,9 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
}
using CommandClass = Recorder::CommandClass;
const auto countBarrier = [&](std::uint32_t count = 1) { if (recorded) Recorder::CountBarriers(CommandClass::Draw, count); };
const bool gpuIndirect = args != nullptr && indirect.path == IndirectDrawPath::Gpu;
// A queued DCC key store over memory the draw writes or reads in place (unknown for an
// address-based build), or over its GPU-side records, lands before it, as before a dispatch
// (VulkanDevice::dispatch).
const auto touches = [&](std::uint64_t begin, std::uint64_t end) {
const auto bytes = static_cast<std::size_t>(end - begin);
if (resources->WritesOverlap(begin, bytes) || resources->ReadsOverlap(begin, bytes)) return true;
if (!gpuIndirect) return false;
return (begin < args->arguments + args->RangeBytes() && args->arguments < end) || (args->countIndirect && begin < args->countAddress + 4 && args->countAddress < end);
};
if (recorded && recorder->HasQueuedKeyStores() && (resources->HoldsLease() || recorder->AnyQueuedKeyStore(touches))) recorder->FlushKeyStores();
// A queued label store over such memory likewise (Recorder::RecordStore).
if (recorded && recorder->HasQueuedStores() && (resources->HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores();
@@ -3137,7 +3140,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
APS5_LOG_CHARS_OUT_DEBUG("SubmitAndWait");
timer.phase(PhaseRecord);
if (recorded) {
keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), std::move(scratch), std::move(checkRecords), listed, completion, outcome);
keepRecordedDraw(*recorder, resources, pipeline, framebuffer, inputs, owners, [&] { std::vector<std::pair<std::uint32_t, std::uint32_t>> slices; for (const auto& binding : targets) slices.emplace_back(binding.color.arraySlice, binding.color.arrayLastSlice != 0 ? binding.color.layers : 1u); return slices; }(), state.scissor, std::move(scratch), std::move(checkRecords), listed, completion, outcome);
timer.phase(PhaseKeep);
if (outcome.waited) {
// The wait the dispatch path makes for such work (source 3, "address-based"): the
@@ -3279,7 +3282,7 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con
timer.phase(PhaseSetup);
if (state.colors[index].fmaskAddress != 0) GuardOneFragmentFmask(context, state.colors[index]);
auto stored = recipe.targets[index].lock();
if (stored == nullptr || !StorageImageCached(context, stored.get()) || !StorageImageServesKeys(*stored, state.colors[index].dccAddress)) return miss(DrawRecipeMiss::TargetGone);
if (stored == nullptr || !StorageImageCached(context, stored.get()) || !StorageImageServesKeys(*stored, SurfaceForTarget(state.colors[index]).dccAddress)) return miss(DrawRecipeMiss::TargetGone);
auto resident = refreshResidentTarget(context, state, state.colors[index], outcome, profile, [&] {
if (state.colors[index].arrayLastSlice != 0) stored->RefreshLayers(state.colors[index].arraySlice, 1);
else stored->Refresh();
core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp
+263 -23
View File
@@ -69,6 +69,7 @@ struct ImageMirror {
std::uint64_t bytes = 0;
bool writable = false;
bool heap = false;
bool joinedHeap = false;
bool sparse = false;
std::uint64_t reserved = 0;
std::vector<std::pair<std::uint64_t, std::uint64_t>> committed;
@@ -285,6 +286,7 @@ struct HostImports {
bool allocationFailed = false;
std::uint8_t reportNoted = 0;
std::map<std::uint64_t, std::uint64_t> spaceUses;
std::uint64_t budgetRefusals = 0;
};
HostImports& Imports() {
@@ -868,6 +870,7 @@ bool importsStale(const Context& context, const HostImports& state);
constexpr std::uint64_t settleFloorBytes = 0x100000;
#ifdef _WIN32
constexpr std::uint64_t settleBlockBytes = 0x10000;
constexpr std::uint64_t settlePageBytes = 0x1000;
constexpr auto settlePollPeriod = std::chrono::microseconds(100);
std::atomic<bool> deferredSettles{std::getenv("APS5_NO_DEFERRED_SETTLES") == nullptr};
#endif
@@ -1094,8 +1097,22 @@ void reapSettleBatches(const Context& context, HostImports& state) {
}
}
bool comparedImportShares(const HostImports& state, std::uint64_t own, std::uint64_t begin, std::uint64_t end) {
const auto shares = [&](const HostImport& entry) { return entry.unwatched && entry.memoryBase < end && begin < entry.memoryBase + entry.pinnedBytes; };
for (const auto& [base, entry] : state.imports) {
if (base != own && shares(entry)) return true;
}
for (const auto& [device, list] : state.parked) {
for (const auto& retired : list) {
if (shares(retired->node.mapped())) return true;
}
}
return false;
}
void rewatchSettledImports(HostImports& state) {
if (state.settleReady.empty()) return;
std::vector<std::uint64_t> bases;
std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
for (const auto base : state.settleReady) {
const auto settle = state.pendingSettles.find(base);
@@ -1103,19 +1120,23 @@ void rewatchSettledImports(HostImports& state) {
state.pendingSettles.erase(settle);
const auto import = state.imports.find(base);
if (import == state.imports.end() || !import->second.unwatched) continue;
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> span{{{base, base + import->second.bytes}}};
const auto first = base & ~(settleBlockBytes - 1);
const auto stop = (base + import->second.bytes + settleBlockBytes - 1) & ~(settleBlockBytes - 1);
if ((first != base || stop != base + import->second.bytes) && comparedImportShares(state, base, first, stop)) continue;
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> span{{{first, stop}}};
if (StorageTexture::AnyLiveOverlaps(span)) {
++state.settlesKeptCompared;
continue;
}
ranges.push_back(span.front());
bases.push_back(base);
ranges.emplace_back(base & ~(settlePageBytes - 1), (base + import->second.bytes + settlePageBytes - 1) & ~(settlePageBytes - 1));
}
state.settleReady.clear();
if (ranges.empty()) return;
GuestMemory::Rewatch(ranges);
for (const auto& [begin, end] : ranges) {
const auto import = state.imports.find(begin);
if (import == state.imports.end() || !GuestMemory::Watched(begin, static_cast<std::size_t>(end - begin))) continue;
for (const auto base : bases) {
const auto import = state.imports.find(base);
if (import == state.imports.end() || !GuestMemory::Watched(base, static_cast<std::size_t>(import->second.bytes))) continue;
import->second.unwatched = false;
++state.settlesRewatched;
}
@@ -1130,6 +1151,49 @@ void settleUnwatchedImports(const Context& context, HostImports& state) {
rewatchSettledImports(state);
submitSettleBatch(context, state);
}
void settleImportsNow(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end) {
if (context.settleQueue == VK_NULL_HANDLE || context.settleQueueMutex == nullptr || !GuestMemory::GpuMutex().HeldByThisThread()) return;
std::vector<std::uint64_t> needed;
auto it = state.imports.lower_bound(begin);
if (it != state.imports.begin()) --it;
for (; it != state.imports.end() && it->first < end; ++it) {
auto& entry = it->second;
if (entry.base + entry.bytes <= begin || !entry.unwatched) continue;
auto settle = state.pendingSettles.find(it->first);
if (settle == state.pendingSettles.end()) {
if (!entry.settleOnDemand || !deferredSettles.load(std::memory_order_relaxed)) continue;
entry.settleOnDemand = false;
settle = state.pendingSettles.insert_or_assign(it->first, PendingSettle{}).first;
}
if (settle->second.phase == SettlePhase::Wanted) {
std::erase(state.settleWanted, it->first);
state.settleWanted.insert(state.settleWanted.begin(), it->first);
}
needed.push_back(it->first);
}
if (needed.empty()) return;
auto& deviceSettle = settleFor(context, state);
const auto wanted = [&] {
return std::any_of(needed.begin(), needed.end(), [&](std::uint64_t base) {
const auto settle = state.pendingSettles.find(base);
return settle != state.pendingSettles.end() && settle->second.phase == SettlePhase::Wanted;
});
};
while (wanted()) {
if (std::all_of(deviceSettle.batches.begin(), deviceSettle.batches.end(), [](const SettleBatch& batch) { return batch.busy; })) {
for (const auto& batch : deviceSettle.batches) waitSettleBatch(deviceSettle, context.device, batch);
reapSettleBatches(context, state);
}
submitSettleBatch(context, state);
}
for (const auto base : needed) {
const auto settle = state.pendingSettles.find(base);
if (settle != state.pendingSettles.end() && settle->second.phase == SettlePhase::Submitted && settle->second.device == context.device) waitSettleBatch(deviceSettle, context.device, deviceSettle.batches[settle->second.batch]);
}
reapSettleBatches(context, state);
rewatchSettledImports(state);
}
#endif
void decideImportWatch(const Context& context, HostImports& state) {
@@ -1429,8 +1493,32 @@ bool evictingRefusalCurrent(const HostImports& state, const EvictingRefusal& ref
return refusal.epoch == state.epoch && refusal.liveBytes == state.liveBytes && budget <= refusal.budget && refusal.window == importWindow.load(std::memory_order_relaxed) && state.useCounter < refusal.ready && (state.buildFloor <= refusal.floor || state.useCounter < refusal.readyHeld);
}
void noteRetiredUnwatched(std::vector<std::pair<std::uint64_t, std::uint64_t>>& spans, const HostImport& entry, const GuestAllocations::Lease& lease) {
if (!entry.unwatched) return;
for (const auto& range : entry.ranges) {
if (leasedRangeOwner(lease, range->address) != range) continue;
const auto begin = std::max(range->address, entry.base);
const auto end = std::min(range->address + range->bytes, entry.base + entry.bytes);
if (begin < end) spans.emplace_back(begin, end);
}
}
void rewatchRetired(const HostImports& state, std::vector<std::pair<std::uint64_t, std::uint64_t>> spans) {
std::erase_if(spans, [&](const auto& span) {
auto it = state.imports.upper_bound(span.first);
if (it != state.imports.begin()) --it;
for (; it != state.imports.end() && it->first < span.second; ++it) {
if (it->second.unwatched && span.first < it->first + it->second.bytes) return true;
}
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> one{{span}};
return StorageTexture::AnyLiveOverlaps(one);
});
if (!spans.empty()) GuestMemory::Rewatch(spans);
}
std::uint64_t retireLeastRecent(const Context& context, HostImports& state, const GuestAllocations::Lease& lease, std::uint64_t wanted) {
std::uint64_t freed = 0;
std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
while (freed < wanted) {
auto oldest = state.imports.end();
for (auto it = state.imports.begin(); it != state.imports.end(); ++it) {
@@ -1438,8 +1526,10 @@ std::uint64_t retireLeastRecent(const Context& context, HostImports& state, cons
}
if (oldest == state.imports.end()) break;
freed += oldest->second.pinnedBytes;
noteRetiredUnwatched(unwatched, oldest->second, lease);
retireImport(context.device, state, oldest, lease);
}
rewatchRetired(state, std::move(unwatched));
return freed;
}
@@ -1631,6 +1721,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
++counts.refused;
counts.refusedBytes += lease[index]->bytes;
++counts.remembered;
++state.budgetRefusals;
return nullptr;
}
const auto bounds = importRunBounds(lease, address, largest, budget);
@@ -1796,6 +1887,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
if (evict) state.evictingRefusals.insert_or_assign(std::pair{base, bytes}, EvictingRefusal{base, bytes, GuestAllocations::Mapped(run.ranges.begin(), run.ranges.end()), state.epoch, state.liveBytes, budget, room, importWindow.load(std::memory_order_relaxed), state.buildFloor, ready, readyHeld, joined});
if (stepDown && containing != run.ranges.end() && room >= (*containing)->bytes) return importAllocation(context, state, address, lease, requester, end, evict, std::min(bytes / 2, room));
if (containing != run.ranges.end() && run.ranges.size() > 1) remember((*containing)->address, (*containing)->bytes, GuestAllocations::Mapped{*containing});
++state.budgetRefusals;
return nullptr;
}
for (auto it = first; it != state.imports.end() && it->first < base + bytes;) {
@@ -1884,7 +1976,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
GuestMemory::ImportWatched(base, bytes, [&] {
step = createBackingImport(context, entry, backing, result);
return step == nullptr;
}, watchedDmaBuf);
}, watchedDmaBuf, false);
}
if (step == nullptr && entry.dmaBuf && bytes >= settleFloorBytes && context.settleQueue != VK_NULL_HANDLE && context.settleQueueMutex != nullptr) {
static const bool traceSettle = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
@@ -1948,8 +2040,10 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
entry.joined = joined;
entry.lastUse = ++state.useCounter;
#ifdef _WIN32
if (deferredSettles.load(std::memory_order_relaxed) && entry.alias == nullptr && state.watchPrivateImports && writable && base % settleBlockBytes == 0 && bytes % settleBlockBytes == 0) {
if (entry.unwatched) {
if (deferredSettles.load(std::memory_order_relaxed) && entry.alias == nullptr && state.watchPrivateImports && writable) {
if (base % settleBlockBytes != 0 || bytes % settleBlockBytes != 0) {
entry.settleOnDemand = entry.unwatched;
} else if (entry.unwatched) {
state.pendingSettles.insert_or_assign(base, PendingSettle{});
state.settleWanted.push_back(base);
} else if (settled && !GuestMemory::Watched(base, static_cast<std::size_t>(bytes))) {
@@ -2020,6 +2114,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
std::erase_if(state.sparseRunEnds, [&](const auto& entry) { return !starts(entry.first); });
std::erase_if(state.sparseRunJoins, [&](const auto& entry) { return !starts(entry.first); });
const auto budget = effectiveBudget(context, state);
std::vector<std::pair<std::uint64_t, std::uint64_t>> unwatched;
for (auto it = state.imports.begin(); it != state.imports.end();) {
if (sameRange(it->second, lease, budget)) {
if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes);
@@ -2029,9 +2124,11 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
if (trace && leasedRangeOwner(lease, it->first, it->first + it->second.bytes) != nullptr) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast<unsigned long long>(it->first), static_cast<unsigned long long>(it->second.bytes));
const auto next = std::next(it);
noteRetiredUnwatched(unwatched, it->second, lease);
retireImport(context.device, state, it, lease);
it = next;
}
rewatchRetired(state, std::move(unwatched));
for (auto it = state.failed.begin(); it != state.failed.end();) it = sameRange(it->second, lease, budget) ? std::next(it) : state.failed.erase(it);
for (auto it = state.failedJoins.begin(); it != state.failedJoins.end();) it = sameRun(it->second, joinedRun(lease, it->second.base, it->second.base + it->second.bytes)) ? std::next(it) : state.failedJoins.erase(it);
for (auto it = state.failedBlocks.begin(); it != state.failedBlocks.end();) it = sameRun(it->second, exactRun(lease, it->first.first, it->first.second)) ? std::next(it) : state.failedBlocks.erase(it);
@@ -2202,6 +2299,39 @@ void unlistMirror(ImageMirrors& state, ImageMirror& mirror) {
mirror.listed = false;
}
struct JoinedHeapRange {
std::vector<std::weak_ptr<const GuestAllocations::Range>> members;
std::shared_ptr<const GuestAllocations::Range> range;
};
struct JoinedHeapRanges {
std::mutex mutex;
std::map<std::uint64_t, JoinedHeapRange> entries;
};
JoinedHeapRanges& JoinedHeaps() {
static JoinedHeapRanges ranges;
return ranges;
}
bool releaseExpiredJoinedHeaps() {
std::vector<std::shared_ptr<const GuestAllocations::Range>> released;
{
auto& joined = JoinedHeaps();
std::lock_guard lock(joined.mutex);
for (auto it = joined.entries.begin(); it != joined.entries.end();) {
const auto& members = it->second.members;
if (std::none_of(members.begin(), members.end(), [](const auto& member) { return member.expired(); })) {
++it;
continue;
}
released.push_back(std::move(it->second.range));
it = joined.entries.erase(it);
}
}
return !released.empty();
}
std::uint64_t blockBytes(const GuestRuns& runs) {
constexpr std::uint64_t block = ResidentBuffer::BlockBytes;
std::uint64_t blocks = 0;
@@ -2708,10 +2838,15 @@ bool importsNow(const Context& context, HostImports& state, const GuestAllocatio
return context.hostImportAlignment != 0 && (importsStale(context, state) || importAllocation(context, state, address, lease, ImportRequester::Build, 0, false) != nullptr);
}
const char* heapCopyReason(bool overBudget, bool watched) {
if (overBudget) return watched ? "import over the host import budget, heap mirror refused" : "import over the host import budget, heap not fully write-watched";
return watched ? "heap mirror refused" : "heap not fully write-watched";
}
// The mirror for a leased image range: the existing one, or a new one when none exists or the
// registered Range object changed (a protection change). Null when the range cannot be mirrored. An
// existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every
std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0) {
std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::shared_ptr<const GuestAllocations::Range>& range, std::vector<RefreshBlock>& blocks, bool heap, std::uint64_t begin = 0, std::uint64_t bytes = 0, bool joinedHeap = false) {
auto selected = *range;
if (bytes != 0) {
Require(begin >= range->address && bytes <= range->bytes && begin - range->address <= range->bytes - bytes, "heap mirror selection exceeds its registered range");
@@ -2732,7 +2867,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
}
if (state.failed.contains(selected.address)) return nullptr;
const auto found = state.entries.find(selected.address);
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == selected.bytes && found->second->writable == range->writable && found->second->heap == heap && found->second->joinedHeap == joinedHeap) mirror = found->second;
}
if (mirror != nullptr) {
if (!mirror->sparse) {
@@ -2755,6 +2890,7 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
mirror->sparse = sparse;
mirror->writable = range->writable;
mirror->heap = heap;
mirror->joinedHeap = joinedHeap;
mirror->range = range;
GuestRuns backed;
if (heap) {
@@ -2887,9 +3023,11 @@ std::shared_ptr<ImageMirror> findMirror(const Context& context, std::uint64_t be
// Drops mirrors whose registered range is gone (a later build made a new one for the replacement).
void sweepMirrors() {
const bool released = releaseExpiredJoinedHeaps();
const auto space = Spaces().current.load();
auto& state = Mirrors();
std::lock_guard lock(state.mutex);
if (released) ++state.version;
if (space != nullptr && state.pinnedSpace == space->serial && state.pinnedVersion == state.version) return;
++state.sweeps;
bool pinned = space != nullptr;
@@ -2913,6 +3051,14 @@ void sweepMirrors() {
void releaseImportedMirrors(const Context& context, std::span<const std::pair<std::uint64_t, std::uint64_t>> imported) {
if (imported.empty()) return;
std::vector<std::pair<std::uint64_t, std::uint64_t>> coverage(imported.begin(), imported.end());
std::sort(coverage.begin(), coverage.end());
std::size_t count = 0;
for (const auto& interval : coverage) {
if (count != 0 && interval.first <= coverage[count - 1].second) coverage[count - 1].second = std::max(coverage[count - 1].second, interval.second);
else coverage[count++] = interval;
}
coverage.resize(count);
std::vector<std::shared_ptr<ImageMirror>> released;
auto& state = Mirrors();
std::lock_guard lock(state.mutex);
@@ -2926,6 +3072,13 @@ void releaseImportedMirrors(const Context& context, std::span<const std::pair<st
++it;
continue;
}
if (mirror->joinedHeap) {
const auto covered = std::upper_bound(coverage.begin(), coverage.end(), mirror->base, [](std::uint64_t address, const auto& interval) { return address < interval.first; });
if (covered == coverage.begin() || std::prev(covered)->second < mirror->base + mirror->reserved) {
++it;
continue;
}
}
++state.heapImported;
state.heapImportedBytes += mirror->accounted;
unlistMirror(state, *mirror);
@@ -3235,7 +3388,8 @@ std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t l
std::string message = text;
for (std::size_t index = 0; index < copies.size() && index < 8; ++index) {
const auto& copy = copies[index];
std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast<unsigned long long>(copy.begin), static_cast<unsigned long long>(copy.end - copy.begin), copy.committed / 1048576.0, copy.reason);
const char* reason = copy.heap ? heapCopyReason(copy.overBudget, GuestMemory::Watched(copy.begin, copy.end - copy.begin)) : copy.reason;
std::snprintf(text, sizeof(text), " 0x%llx+0x%llx (%.1f MiB committed, %s)", static_cast<unsigned long long>(copy.begin), static_cast<unsigned long long>(copy.end - copy.begin), copy.committed / 1048576.0, reason);
message += text;
}
return message;
@@ -3716,6 +3870,22 @@ const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t ad
return findImport(state, address, address + bytes);
}
bool WatchedAfterSettle(const Context& context, std::uint64_t address, std::size_t bytes) {
#ifdef _WIN32
if (context.hostImportAlignment != 0 && bytes != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address && GuestMemory::GpuMutex().HeldByThisThread()) {
ensurePinWaiter();
auto& state = Imports();
std::lock_guard lock(state.mutex);
if (importsStale(context, state)) {
const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
refreshImports(context, state, lease);
}
settleImportsNow(context, state, address, address + bytes);
}
#endif
return GuestMemory::Watched(address, bytes);
}
bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
if (context.hostImportAlignment == 0) return true;
ensurePinWaiter();
@@ -3780,6 +3950,21 @@ bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t
return state.device == context.device && peekImport(state, address, address + bytes) != nullptr;
}
bool ComparedHostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) {
#ifdef _WIN32
static_cast<void>(context);
static_cast<void>(address);
static_cast<void>(bytes);
return false;
#else
if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
auto& state = Imports();
std::lock_guard lock(state.mutex);
const auto* import = state.device == context.device ? peekImport(state, address, address + bytes) : nullptr;
return import != nullptr && import->unwatched;
#endif
}
GuestBufferMemory::GuestBufferMemory(const Context& context) : context(context) {}
bool GuestBufferMemory::WritesOverlap(std::uint64_t address, std::size_t bytes) const {
@@ -3925,16 +4110,24 @@ void GuestBufferMemory::AcquireRegistered() {
// The import lock is taken per range: a mirror's preparation may wait for recorded work, which
// must not happen under it.
regions.reserve(lease.size());
static const std::uint64_t copyLimit = [] {
const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
}();
const auto copyOverride = addressCopyLimitOverride.load(std::memory_order_relaxed);
const auto cap = copyOverride != 0 ? copyOverride : copyLimit;
std::vector<AddressCopy> copies;
std::vector<std::pair<std::uint64_t, std::uint64_t>> imported;
const HostImport* covering = nullptr;
std::uint64_t coveredBegin = 0;
std::uint64_t coveredEnd = 0;
std::uint64_t copiedBytes = 0;
for (const auto& range : lease) {
if (!range->readable || !range->gpu) continue;
validate(range->address, range->bytes);
Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr};
const HostImport* entry = nullptr;
bool overBudget = false;
std::vector<ImportPiece> pieces;
const auto limit = importMaxBytes.load(std::memory_order_relaxed);
if (importable && covering != nullptr && range->address >= coveredBegin && range->address + range->bytes <= coveredEnd) {
@@ -3946,7 +4139,9 @@ void GuestBufferMemory::AcquireRegistered() {
} else if (importable) {
auto& state = Imports();
std::lock_guard lock(state.mutex);
const auto refusals = state.budgetRefusals;
entry = importAllocation(context, state, range->address, lease, ImportRequester::Build, 0, false);
overBudget = state.budgetRefusals != refusals;
covering = entry;
if (entry != nullptr) {
coveredBegin = entry->base;
@@ -4012,17 +4207,36 @@ void GuestBufferMemory::AcquireRegistered() {
committed = 0;
for (const auto& [first, last] : copied.backed) committed += last - first;
}
const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena";
copies.push_back({copied.begin, copied.end, committed, reason});
if (overBudget && copiedBytes + committed > cap) {
auto& state = Imports();
std::lock_guard lock(state.mutex);
entry = importAllocation(context, state, range->address, lease, ImportRequester::Build, 0, true);
if (entry != nullptr) {
covering = entry;
coveredBegin = entry->base;
coveredEnd = entry->base + entry->bytes;
}
static auto lastReport = std::chrono::steady_clock::now() - std::chrono::seconds(60);
if (entry != nullptr && std::chrono::steady_clock::now() - lastReport > std::chrono::seconds(10)) {
lastReport = std::chrono::steady_clock::now();
std::fprintf(stderr, "[gpu] host import budget: 0x%llx+0x%llx imported by evicting idle imports, as copying it would pass APS5_ADDRESS_COPY_MAX_MIB\n", static_cast<unsigned long long>(range->address), static_cast<unsigned long long>(range->bytes));
}
}
if (entry != nullptr) {
regions.pop_back();
region.hostBacked = true;
region.direct = entry;
regions.push_back(std::move(region));
imported.emplace_back(range->address, range->address + range->bytes);
continue;
}
copiedBytes += committed;
const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : nullptr;
copies.push_back({copied.begin, copied.end, committed, reason, reason == nullptr, overBudget});
}
releaseImportedMirrors(context, imported);
regionsSorted = sortedLookup;
static const std::uint64_t copyLimit = [] {
const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB");
return (value ? std::strtoull(value, nullptr, 10) : 64ull) << 20u;
}();
const auto limit = addressCopyLimitOverride.load(std::memory_order_relaxed);
if (const auto overflow = AddressCopyOverflow(std::move(copies), limit != 0 ? limit : copyLimit); !overflow.empty()) {
if (const auto overflow = AddressCopyOverflow(std::move(copies), cap); !overflow.empty()) {
std::fprintf(stderr, "FATAL: %s\n", overflow.c_str());
std::fflush(stderr);
std::abort();
@@ -4270,15 +4484,40 @@ bool GuestBufferMemory::sparseHeapCandidate(const Region& region) {
return region.sparseHeap && region.sparse && region.mirror == nullptr && region.direct == nullptr && (region.hostBacked || region.writable) && mirrorsEnabled() && sparseMirrorsEnabled();
}
std::shared_ptr<const GuestAllocations::Range> joinedHeapRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) {
const auto run = crossedRun(lease, begin, end);
if (run.ranges.size() < 2 || std::any_of(run.ranges.begin(), run.ranges.end(), [](const auto& range) { return !range->releasable; })) return nullptr;
releaseExpiredJoinedHeaps();
auto& joined = JoinedHeaps();
std::lock_guard lock(joined.mutex);
auto& entry = joined.entries[run.base];
bool same = entry.range != nullptr && entry.members.size() == run.ranges.size();
for (std::size_t index = 0; same && index < run.ranges.size(); ++index) same = sameRange(entry.members[index], run.ranges[index]);
if (!same) {
const auto& first = *run.ranges.front();
entry.members.assign(run.ranges.begin(), run.ranges.end());
entry.range = std::make_shared<const GuestAllocations::Range>(GuestAllocations::Range{run.base, static_cast<std::size_t>(run.bytes), true, first.writable, first.allocationAddress, first.allocationBytes, true, first.gpu, first.backingSerial});
}
return entry.range;
}
bool GuestBufferMemory::bindSparseMirror(Region& region, const GuestAllocations::Lease& lease) {
const auto backedEnd = region.backed.empty() ? region.begin : region.backed.back().second;
const auto neededEnd = std::min(region.end, std::max(backedEnd, std::max(region.lastView, region.begin) + 4u));
const auto* range = containingRange(lease, region.begin, neededEnd);
if (range == nullptr || !range->releasable) return false;
std::shared_ptr<const GuestAllocations::Range> owner;
bool joined = false;
if (const auto* range = containingRange(lease, region.begin, neededEnd); range != nullptr) {
if (!range->releasable) return false;
owner = leasedRangeOwner(lease, range->address);
} else {
joined = true;
owner = joinedHeapRange(lease, region.begin, neededEnd);
if (owner == nullptr) return false;
}
std::vector<RefreshBlock> blocks;
std::shared_ptr<ImageMirror> mirror;
try {
mirror = acquireMirror(context, leasedRangeOwner(lease, range->address), blocks, true);
mirror = acquireMirror(context, owner, blocks, true, 0, 0, joined);
} catch (const std::runtime_error& error) {
static std::atomic<int> reported{0};
if (reported.fetch_add(1, std::memory_order_relaxed) < 4) std::fprintf(stderr, "[gpu] %s; copying the sparse heap instead\n", error.what());
@@ -4462,6 +4701,7 @@ VkBufferUsageFlags gpuCopyUsage(bool addressable) {
// window takes those elements there. APS5_ATOMIC_STAGE_MAX_KIB (1024) bounds the atomic one.
constexpr std::uint64_t SweptShadowMax = std::uint64_t{16} << 20u;
constexpr std::uint64_t DiscreteWrittenShadowMax = std::uint64_t{8} << 20u;
constexpr std::uint64_t DiscreteSweptShadowMax = std::uint64_t{256} << 20u;
bool writtenShadowEnabled() {
static const bool disabled = std::getenv("APS5_NO_WRITTEN_SHADOW") != nullptr;
@@ -4604,7 +4844,7 @@ bool GuestBufferMemory::stagingEligible(const Region& region, bool addressable)
const auto bytes = region.end - region.begin;
if (!WritesOverlap(region.begin, static_cast<std::size_t>(bytes))) return false;
if (region.atomic && atomicStagingEnabled() && bytes <= atomicStageMax()) return true;
return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(context.discreteGpu), SweptShadowMax) : writtenShadowMax(context.discreteGpu));
return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(context.discreteGpu), context.discreteGpu ? DiscreteSweptShadowMax : SweptShadowMax) : writtenShadowMax(context.discreteGpu));
}
bool GuestBufferMemory::overlayCandidate(const Region& inside, bool readOnly) const {
core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp
+152 -42
View File
@@ -4,6 +4,7 @@
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
@@ -331,7 +332,9 @@ public:
colorFill = MultisampledColorFill(use);
fromTexture = false;
dccRanges.clear();
dccCopies.clear();
fmaskRange.reset();
fmaskCopy = {};
metadataValid = std::make_shared<std::atomic<bool>>(true);
const auto initial = initialDccClear(use);
recordCommands(context, [&](VkCommandBuffer commands, Recorder* recorder) {
@@ -500,6 +503,7 @@ public:
Context context;
std::shared_ptr<std::atomic<bool>> active = std::make_shared<std::atomic<bool>>(true);
std::shared_ptr<std::atomic<bool>> hostOnly = std::make_shared<std::atomic<bool>>(true);
std::shared_ptr<std::atomic<bool>> metadataValid;
ColorTarget target;
VkImage image = VK_NULL_HANDLE;
@@ -533,9 +537,35 @@ private:
std::unique_ptr<FirstSampleCopy> firstSampleCopy;
std::map<std::array<std::uint32_t, 4>, std::shared_ptr<Texture>> firstSampleTextures;
std::vector<GuestMemory::UnchangedQuery> dccRanges;
struct DccCopy {
bool compared = false;
std::vector<std::byte> bytes;
std::uint64_t stamp = 0;
std::array<std::uint64_t, 2> checked{};
};
mutable std::vector<DccCopy> dccCopies;
mutable DccCopy fmaskCopy;
std::optional<std::uint32_t> dccFill;
std::optional<std::uint32_t> cmaskFill;
bool compared(std::size_t index) const {
return index < dccCopies.size() && dccCopies[index].compared;
}
static DccCopy comparedCopy(const GuestMemory::UnchangedQuery& range, std::vector<std::byte> bytes) {
return {true, std::move(bytes), GuestMemory::NewestStamp(range.address, range.bytes), {}};
}
static bool copyCurrent(const GuestMemory::UnchangedQuery& range, DccCopy& copy) {
if (GuestMemory::NewestStamp(range.address, range.bytes) != copy.stamp) return false;
const auto epoch = GuestMemory::CollectEpoch();
const std::array<std::uint64_t, 2> seen{epoch, GuestMemory::ForgetSerial()};
if (epoch != 0 && (seen[1] & 1u) == 0 && copy.checked == seen) return true;
if (!GuestMemory::EqualsCommittedUnsynced(range.address, copy.bytes)) return false;
copy.checked = seen;
return true;
}
bool absorbDccFill(const ColorTarget& use, VkClearColorValue& clear) {
const auto dcc = std::exchange(dccFill, std::nullopt);
const auto cmaskPattern = std::exchange(cmaskFill, std::nullopt);
@@ -546,19 +576,30 @@ private:
if (dccRanges.size() > 2) {
const auto& range = dccRanges[2];
auto* recorder = Recorder::Active();
const bool written = !GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || (recorder != nullptr && recorder->PendingWriteOverlaps(range.address, range.bytes));
const bool unchanged = compared(2) ? copyCurrent(range, dccCopies[2]) : GuestMemory::UnchangedSince(range.address, range.bytes, range.generation);
const bool written = !unchanged || (recorder != nullptr && recorder->PendingWriteOverlaps(range.address, range.bytes));
const auto landed = LandedUniformMetadataByte(range.address, range.bytes);
if (!landed || (written && (!cmaskPattern || *cmaskPattern != *landed * 0x01010101u))) return false;
cmaskKeys = *landed;
}
std::vector<GuestMemory::UnchangedQuery> metadata(dccRanges.begin() + 1, dccRanges.end());
for (auto& range : metadata) {
if (!GuestMemory::Watched(range.address, range.bytes)) return false;
std::vector<bool> comparedMetadata(metadata.size(), false);
for (std::size_t index = 0; index < metadata.size(); ++index) {
auto& range = metadata[index];
if (!GuestMemory::Watched(range.address, range.bytes)) {
if (!compared(index + 1) && !ComparedHostImportCovers(context, range.address, range.bytes)) return false;
comparedMetadata[index] = true;
range.generation = 0;
continue;
}
range.generation = collectBlocks(range.address, range.bytes);
if (range.generation == 0) return false;
}
clear = MultisampledDccClear(use, std::vector<std::byte>(dccRanges[1].bytes, std::byte{*keys}), std::vector<std::byte>(use.cmaskBytes, std::byte{cmaskKeys}));
std::array<std::vector<std::byte>, 2> landedBytes{std::vector<std::byte>(dccRanges[1].bytes, std::byte{*keys}), std::vector<std::byte>(use.cmaskBytes, std::byte{cmaskKeys})};
clear = MultisampledDccClear(use, landedBytes[0], landedBytes[1]);
std::copy(metadata.begin(), metadata.end(), dccRanges.begin() + 1);
dccCopies.resize(dccRanges.size());
for (std::size_t index = 0; index < metadata.size(); ++index) dccCopies[index + 1] = comparedMetadata[index] ? comparedCopy(dccRanges[index + 1], std::move(landedBytes[index])) : DccCopy{};
return true;
}
@@ -572,11 +613,19 @@ private:
dccRanges.push_back({use.address, static_cast<std::size_t>(colorBytes), 0});
dccRanges.push_back({use.dccAddress, dccBytes, 0});
if (use.cmaskAddress != 0) dccRanges.push_back({use.cmaskAddress, use.cmaskBytes, 0});
for (auto& range : dccRanges) {
dccCopies.assign(dccRanges.size(), {});
std::vector<GuestMemory::UnchangedQuery> watched;
for (std::size_t index = 0; index < dccRanges.size(); ++index) {
auto& range = dccRanges[index];
GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 256);
if (!WatchedAfterSettle(context, range.address, range.bytes) && ComparedHostImportCovers(context, range.address, range.bytes)) {
dccCopies[index].compared = true;
continue;
}
Require(GuestMemory::Watched(range.address, range.bytes), "initial multisampled DCC contents require watched guest memory");
range.generation = collectBlocks(range.address, range.bytes);
Require(range.generation != 0, "initial multisampled DCC contents have no write generation");
watched.push_back(range);
}
for (std::size_t first = 0; first < dccRanges.size(); ++first) {
for (std::size_t second = first + 1; second < dccRanges.size(); ++second) {
@@ -591,17 +640,28 @@ private:
fmaskRange = GuestMemory::UnchangedQuery{use.fmaskAddress, static_cast<std::size_t>(DescribeSurface(resource).guestBytes), 0};
auto& range = *fmaskRange;
GuestMemory::CheckRange(reinterpret_cast<const void*>(range.address), range.bytes, 65536u);
Require(GuestMemory::Watched(range.address, range.bytes), "native FMASK contents require watched guest memory");
range.generation = GuestMemory::CollectWritesUncached(range.address, range.bytes);
Require(range.generation != 0, "native FMASK contents have no write generation");
fmaskCopy.compared = !WatchedAfterSettle(context, range.address, range.bytes) && ComparedHostImportCovers(context, range.address, range.bytes);
Require(fmaskCopy.compared || GuestMemory::Watched(range.address, range.bytes), "native FMASK contents require watched guest memory");
if (!fmaskCopy.compared) {
range.generation = GuestMemory::CollectWritesUncached(range.address, range.bytes);
Require(range.generation != 0, "native FMASK contents have no write generation");
}
for (const auto& other : dccRanges) Require(range.address >= other.address + other.bytes || other.address >= range.address + range.bytes, "native FMASK overlaps its color or metadata planes");
}
std::vector<std::byte> dcc(dccRanges[1].bytes);
GuestMemory::Read(use.dccAddress, dcc);
std::vector<std::byte> cmask(use.cmaskBytes);
if (!cmask.empty()) GuestMemory::Read(use.cmaskAddress, cmask);
const auto clear = MultisampledDccClear(use, dcc, cmask);
Require(GuestMemory::UnchangedSinceAll(dccRanges) && (!fmaskRange || GuestMemory::UnchangedSinceAll(std::span(&*fmaskRange, 1u))), "initial multisampled DCC contents changed during import");
std::array<std::vector<std::byte>, 3> bytes{};
for (std::size_t index = 0; index < dccRanges.size(); ++index) {
if (index == 0 && !compared(0)) continue;
bytes[index].resize(dccRanges[index].bytes);
GuestMemory::Read(dccRanges[index].address, bytes[index]);
}
std::vector<std::byte> fmaskBytes(fmaskCopy.compared ? fmaskRange->bytes : 0u);
if (fmaskCopy.compared) GuestMemory::Read(fmaskRange->address, fmaskBytes);
const auto clear = MultisampledDccClear(use, bytes[1], bytes[2]);
Require((watched.empty() || GuestMemory::UnchangedSinceAll(watched)) && (!fmaskRange || fmaskCopy.compared || GuestMemory::UnchangedSinceAll(std::span(&*fmaskRange, 1u))), "initial multisampled DCC contents changed during import");
for (std::size_t index = 0; index < dccRanges.size(); ++index) {
if (compared(index)) dccCopies[index] = comparedCopy(dccRanges[index], std::move(bytes[index]));
}
if (fmaskCopy.compared) fmaskCopy = comparedCopy(*fmaskRange, std::move(fmaskBytes));
return clear;
}
@@ -621,19 +681,28 @@ private:
Require(metadataValid->load(std::memory_order_acquire), "multisampled DCC metadata was released while its color target remains resident");
static_cast<void>(MultisampledDccBytes(use));
Require(use.address == target.address && use.dccAddress == target.dccAddress && use.dccControl == target.dccControl && use.dccAlphaOnMsb == target.dccAlphaOnMsb && use.dccPipeAligned == target.dccPipeAligned && use.cmaskAddress == target.cmaskAddress && use.cmaskBytes == target.cmaskBytes && use.tileMode == target.tileMode && use.elementBytes == target.elementBytes && use.pipeBankXor == target.pipeBankXor, "multisampled DCC target layout or metadata changed");
const auto ranges = std::span(dccRanges).first(metadataRefilled ? 1u : dccRanges.size());
for (const auto& range : ranges) requireDccRange(GuestMemory::Watched(range.address, range.bytes), range, "multisampled DCC target reuse requires watched guest memory");
for (const auto& range : ranges) {
GuestMemory::CollectWritesUncached(range.address, range.bytes);
requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
const auto ranges = metadataRefilled ? std::size_t{1} : dccRanges.size();
for (std::size_t index = 0; index < ranges; ++index) requireDccRange(compared(index) || GuestMemory::Watched(dccRanges[index].address, dccRanges[index].bytes), dccRanges[index], "multisampled DCC target reuse requires watched guest memory");
for (std::size_t index = 0; index < ranges; ++index) {
const auto& range = dccRanges[index];
if (compared(index)) {
requireDccRange(copyCurrent(range, dccCopies[index]), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
} else {
GuestMemory::CollectWritesUncached(range.address, range.bytes);
requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident multisampled DCC target or its metadata are not implemented");
}
if (auto* recorder = Recorder::Active(); recorder != nullptr) requireDccRange(!recorder->PendingWriteOverlaps(range.address, range.bytes), range, "pending writes to a resident multisampled DCC target or its metadata are not implemented");
}
if (fmaskRange) {
Require(use.fmaskAddress == target.fmaskAddress && use.fmaskPipeBankXor == target.fmaskPipeBankXor, "native FMASK target layout changed");
const auto& range = *fmaskRange;
requireDccRange(GuestMemory::Watched(range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
GuestMemory::CollectWritesUncached(range.address, range.bytes);
requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident native FMASK are not implemented");
requireDccRange(fmaskCopy.compared || WatchedAfterSettle(context, range.address, range.bytes), range, "native FMASK sampling requires watched guest memory");
if (fmaskCopy.compared) {
requireDccRange(copyCurrent(range, fmaskCopy), range, "writes to a resident native FMASK are not implemented");
} else {
GuestMemory::CollectWritesUncached(range.address, range.bytes);
requireDccRange(GuestMemory::UnchangedSince(range.address, range.bytes, range.generation) || !GuestMemory::StoredOver(range.address, range.bytes, range.generation), range, "writes to a resident native FMASK are not implemented");
}
if (auto* recorder = Recorder::Active(); recorder != nullptr) requireDccRange(!recorder->PendingWriteOverlaps(range.address, range.bytes), range, "pending writes to a resident native FMASK are not implemented");
}
}
@@ -660,6 +729,7 @@ struct GuardedRange {
std::uint32_t samples;
std::shared_ptr<std::atomic<bool>> active;
std::shared_ptr<std::atomic<bool>> metadataValid;
std::shared_ptr<std::atomic<bool>> hostOnly;
bool fmask = false;
};
@@ -734,7 +804,7 @@ void retireUnmapped(std::uintptr_t address, std::size_t bytes) {
anyGuarded().store(!guarded().empty(), std::memory_order_release);
}
void guard(const Context& context, const ColorTarget& target, const std::shared_ptr<std::atomic<bool>>& active, const std::shared_ptr<std::atomic<bool>>& metadataValid) {
void guard(const Context& context, const ColorTarget& target, const std::shared_ptr<std::atomic<bool>>& active, const std::shared_ptr<std::atomic<bool>>& metadataValid, const std::shared_ptr<std::atomic<bool>>& hostOnly) {
auto bytes = std::max<std::uint64_t>(target.bytes * target.samples, static_cast<std::uint64_t>(target.extent.width) * target.extent.height * target.elementBytes * target.samples);
if (MultisampledBlockLog2(target.tileMode) == 16u && target.tileMode != ColorTileMode::Standard64KB && std::has_single_bit(target.elementBytes) && std::has_single_bit(target.samples)) {
const auto log2Samples = std::countr_zero(target.samples);
@@ -746,11 +816,11 @@ void guard(const Context& context, const ColorTarget& target, const std::shared_
bytes = align(target.extent.width, blockWidth) * align(target.extent.height, blockHeight) * target.elementBytes * target.samples;
}
std::unique_lock lock(guardedMutex());
guarded().push_back({context.device, target.address, bytes, target.samples, active});
guarded().push_back({context.device, target.address, bytes, target.samples, active, nullptr, hostOnly});
if (target.dccAddress != 0) {
guarded().push_back({context.device, target.dccAddress, MultisampledDccBytes(target), target.samples, active, metadataValid});
if (target.cmaskAddress != 0) guarded().push_back({context.device, target.cmaskAddress, target.cmaskBytes, target.samples, active, metadataValid});
if (target.samples == 8u && target.fmaskAddress != 0u && target.cmaskAddress != 0u) guarded().push_back({context.device, target.fmaskAddress, DescribeSurface(fmaskResource(target)).guestBytes, target.samples, active, metadataValid});
guarded().push_back({context.device, target.dccAddress, MultisampledDccBytes(target), target.samples, active, metadataValid, hostOnly});
if (target.cmaskAddress != 0) guarded().push_back({context.device, target.cmaskAddress, target.cmaskBytes, target.samples, active, metadataValid, hostOnly});
if (target.samples == 8u && target.fmaskAddress != 0u && target.cmaskAddress != 0u) guarded().push_back({context.device, target.fmaskAddress, DescribeSurface(fmaskResource(target)).guestBytes, target.samples, active, metadataValid, hostOnly});
}
anyGuarded().store(true, std::memory_order_release);
InstallResidentImageReadGuard_nid_no_patch();
@@ -770,6 +840,13 @@ void discardRetiredTargets() {
std::erase_if(targets(), [](const auto& target) { return !target->active->load(std::memory_order_acquire); });
}
bool holdHostOnly(const MultisampleTarget& target) {
std::unique_lock lock(guardedMutex());
if (!target.active->load(std::memory_order_acquire)) return false;
target.hostOnly->store(true, std::memory_order_release);
return true;
}
MultisampleTarget& acquire(const Context& context, const ColorTarget& requested) {
discardRetiredTargets();
for (const auto& target : targets()) {
@@ -780,14 +857,21 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& requested)
if (recreated(target->target, use)) {
{
std::unique_lock lock(guardedMutex());
if (!target->active->load(std::memory_order_acquire)) continue;
std::erase_if(guarded(), [&](const auto& range) { return range.active == target->active; });
anyGuarded().store(!guarded().empty(), std::memory_order_release);
}
target->Recreate(use);
guard(context, use, target->active, target->metadataValid);
target->hostOnly->store(true, std::memory_order_release);
guard(context, use, target->active, target->metadataValid, target->hostOnly);
target->ApplyFastClear(use);
return *target;
}
{
std::unique_lock lock(guardedMutex());
if (!target->active->load(std::memory_order_acquire)) continue;
target->hostOnly->store(true, std::memory_order_release);
}
if (target->fromTexture && use.cmaskAddress != 0) {
target->fromTexture = false;
target->cmask = MultisampledCmask(use.cmaskAddress, use.cmaskBytes);
@@ -803,14 +887,14 @@ MultisampleTarget& acquire(const Context& context, const ColorTarget& requested)
target->Keep();
targets().push_back(std::move(target));
auto& created = *targets().back();
guard(context, use, created.active, created.metadataValid);
guard(context, use, created.active, created.metadataValid, created.hostOnly);
if (use.cmaskFastClear && use.dccAddress == 0) created.cmask.SeedCode(CurrentUniformMetadataByte(use.cmaskAddress, static_cast<std::size_t>(use.cmaskBytes)));
created.ApplyFastClear(use);
created.Keep();
return created;
}
MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format, const char* access) {
MultisampleTarget& createFilled(const Context& context, const GuestTextureResource& resource, VkFormat format, const char* access, bool written) {
const auto refuse = [&](const std::string& reason) {
char text[192];
std::snprintf(text, sizeof(text), "AGC graphics: %s the %ux%u multisampled texture 0x%llx of %u samples, which no draw rendered to and ", access, resource.width, resource.height, static_cast<unsigned long long>(resource.baseAddress), resource.samples);
@@ -877,7 +961,8 @@ MultisampleTarget& createFilled(const Context& context, const GuestTextureResour
target->Keep();
targets().push_back(std::move(target));
auto& created = *targets().back();
guard(context, use, created.active, created.metadataValid);
created.hostOnly->store(written, std::memory_order_release);
guard(context, use, created.active, created.metadataValid, created.hostOnly);
created.colorFill.NoteFill(use.address, static_cast<std::size_t>(bytes), value);
created.fromTexture = true;
created.htileAddress = resource.dccAddress;
@@ -1177,7 +1262,7 @@ std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const
}
if (other == nullptr) {
Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches");
return createFilled(context, resource, format, "sampling").Sampled(components, viewType);
return createFilled(context, resource, format, "sampling", false).Sampled(components, viewType);
}
char text[320];
std::snprintf(text, sizeof(text), "AGC graphics: sampling multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1200,7 +1285,7 @@ std::shared_ptr<Texture> MultisampleTargetFirstSample(const Context& context, co
}
if (other == nullptr) {
Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches");
return createFilled(context, resource, format, "sampling").FirstSample(components);
return createFilled(context, resource, format, "sampling", false).FirstSample(components);
}
char text[320];
std::snprintf(text, sizeof(text), "AGC graphics: reading sample 0 of multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1221,12 +1306,15 @@ VkImageView MultisampleTargetStorageView(const Context& context, const GuestText
if (!target->active->load(std::memory_order_acquire)) continue;
if (target->context.device != context.device || target->target.surfaceAddress != resource.baseAddress) continue;
const auto& rendered = target->target;
if (sameStorageForm(context, rendered.format, format) && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) return target->Storage(StorageFormatForGuest(context, resource.format));
if (sameStorageForm(context, rendered.format, format) && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) {
if (!holdHostOnly(*target)) continue;
return target->Storage(StorageFormatForGuest(context, resource.format));
}
other = target.get();
}
if (other == nullptr) {
Require(Recorder::Active() != nullptr, "a multisampled color target is only written by recorded dispatches");
return createFilled(context, resource, format, "writing").Storage(StorageFormatForGuest(context, resource.format));
return createFilled(context, resource, format, "writing", true).Storage(StorageFormatForGuest(context, resource.format));
}
char text[320];
std::snprintf(text, sizeof(text), "AGC graphics: storage image access to multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
@@ -1266,10 +1354,13 @@ void ResolveMultisampleTarget(const Context& context, const ColorTarget& source,
bool MultisampledColorFillAbsorbs(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
if (bytes == 0) return false;
std::lock_guard lock(targetsMutex());
return std::any_of(targets().begin(), targets().end(), [&](const auto& target) {
bool absorbed = false;
for (const auto& target : targets()) {
VkClearColorValue clear{};
return target->active->load(std::memory_order_acquire) && target->target.address == address && MultisampledColorBytes(target->target) == bytes && MultisampledFillClearColor(target->target, pattern, clear);
});
if (!target->active->load(std::memory_order_acquire) || target->target.address != address || MultisampledColorBytes(target->target) != bytes || !MultisampledFillClearColor(target->target, pattern, clear)) continue;
absorbed = holdHostOnly(*target) || absorbed;
}
return absorbed;
}
void NoteColorMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
@@ -1327,7 +1418,7 @@ void GuardOneFragmentFmask(const Context& context, const ColorTarget& target) {
const auto bytes = std::max<std::uint64_t>((bits / 8u + 0xffffu) & ~0xffffull, 0x10000u);
std::unique_lock lock(guardedMutex());
const bool known = std::ranges::any_of(guarded(), [&](const auto& range) { return range.fmask && range.device == context.device && range.address == target.fmaskAddress && range.bytes == bytes; });
if (!known) guarded().push_back({context.device, target.fmaskAddress, bytes, target.coverageSamples, std::make_shared<std::atomic<bool>>(true), nullptr, true});
if (!known) guarded().push_back({context.device, target.fmaskAddress, bytes, target.coverageSamples, std::make_shared<std::atomic<bool>>(true), nullptr, std::make_shared<std::atomic<bool>>(true), true});
anyGuarded().store(true, std::memory_order_release);
lock.unlock();
InstallResidentImageReadGuard_nid_no_patch();
@@ -1340,9 +1431,8 @@ void RequireNoMultisampleTarget(std::uint64_t address, std::size_t bytes, const
void RequireNoMultisampleColorTarget(std::uint64_t address, std::size_t bytes, const char* access) {
if (bytes == 0 || !anyGuarded().load(std::memory_order_acquire)) return;
std::shared_lock lock(guardedMutex());
for (const auto& range : guarded()) {
if (address >= range.address + range.bytes || range.address >= address + bytes) continue;
const auto overlaps = [&](const GuardedRange& range) { return address < range.address + range.bytes && range.address < address + bytes; };
const auto refuse = [&](const GuardedRange& range) {
char text[256];
if (range.fmask) {
std::snprintf(text, sizeof(text), "AGC graphics: %s of 0x%llx+0x%zx overlaps the FMASK 0x%llx+0x%llx of a one-fragment color target of %u samples drawn on the host, which does not write it: guest access to it is not implemented", access, static_cast<unsigned long long>(address), bytes, static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes), range.samples);
@@ -1350,7 +1440,27 @@ void RequireNoMultisampleColorTarget(std::uint64_t address, std::size_t bytes, c
}
std::snprintf(text, sizeof(text), "AGC graphics: %s of 0x%llx+0x%zx overlaps multisampled color target 0x%llx+0x%llx (%u samples), whose samples are only on the host: guest access to it is not implemented", access, static_cast<unsigned long long>(address), bytes, static_cast<unsigned long long>(range.address), static_cast<unsigned long long>(range.bytes), range.samples);
throw std::runtime_error(text);
};
{
std::shared_lock lock(guardedMutex());
bool retirable = false;
for (const auto& range : guarded()) {
if (!overlaps(range)) continue;
if (range.hostOnly->load(std::memory_order_acquire)) refuse(range);
retirable = true;
}
if (!retirable) return;
}
std::unique_lock lock(guardedMutex());
std::vector<std::shared_ptr<std::atomic<bool>>> retired;
for (const auto& range : guarded()) {
if (!overlaps(range)) continue;
if (range.hostOnly->load(std::memory_order_acquire)) refuse(range);
retired.push_back(range.active);
}
for (const auto& active : retired) active->store(false, std::memory_order_release);
std::erase_if(guarded(), [&](const auto& range) { return std::find(retired.begin(), retired.end(), range.active) != retired.end(); });
anyGuarded().store(!guarded().empty(), std::memory_order_release);
}
}
core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp
+70 -11
View File
@@ -321,7 +321,7 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
using Stage = ShaderRecompiler::ShaderStage;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; }) || FragmentInterlock(shaders)) return false;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || (state.stages.tessellation && !context.tessellationPipelineCache) || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; }) || FragmentInterlock(shaders)) return false;
keys = {};
for (const auto& shader : shaders) {
const auto identity = shader.program->Identity();
@@ -723,20 +723,21 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
prebuilt = true;
return;
}
const VkPipelineCache pipelineCache = state.stages.tessellation && !context.tessellationPipelineCache ? VK_NULL_HANDLE : context.pipelineCache;
if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, false, _modules);
else {
auto pipelineCacheLock = context.LockPipelineCacheForUse();
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
}
if (alternateAlpha) {
blend.pAttachments = secondChannels.data();
auto pipelineCacheLock = context.LockPipelineCacheForUse();
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &secondChannel), "vkCreateGraphicsPipelines");
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &secondChannel), "vkCreateGraphicsPipelines");
}
if (state.depthBiasPerFace) {
raster.cullMode = VK_CULL_MODE_FRONT_BIT;
auto pipelineCacheLock = context.LockPipelineCacheForUse();
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines");
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines");
}
for (const auto module : _modules) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
_modules.clear();
@@ -1126,6 +1127,14 @@ struct PipelineStore {
std::uint64_t evicted = 0;
PipelineStoreCounters totals;
std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now();
std::size_t capacity = 0;
bool fixedCapacity = false;
std::list<std::uint64_t> evictedOrder;
std::unordered_map<std::uint64_t, std::list<std::uint64_t>::iterator> evictedKeys;
std::size_t evictionsSinceCheck = 0;
std::size_t returnsSinceCheck = 0;
static constexpr std::size_t BaseEntries = 1024;
static constexpr std::size_t MaximumEntries = 4096;
};
// Never destroyed: the pipelines belong to a device that may already be gone when statics die, and
@@ -1154,9 +1163,41 @@ std::list<PipelineStore::Entry>::iterator abandon(PipelineStore& store, std::lis
return store.entries.erase(it);
}
std::size_t storeBound() {
static const auto bound = static_cast<std::size_t>(std::max<std::uint64_t>(1u, PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 1024u)));
return bound;
std::size_t storeBound(PipelineStore& store) {
if (store.capacity == 0) {
const auto set = PipelineSwitch("APS5_PIPELINE_STORE_ENTRIES", 0u);
store.fixedCapacity = set != 0;
store.capacity = store.fixedCapacity ? static_cast<std::size_t>(set) : PipelineStore::BaseEntries;
}
return store.capacity;
}
void noteStoreEviction(PipelineStore& store, std::uint64_t hash) {
if (store.fixedCapacity) return;
if (const auto found = store.evictedKeys.find(hash); found != store.evictedKeys.end()) store.evictedOrder.erase(found->second);
store.evictedOrder.push_front(hash);
store.evictedKeys[hash] = store.evictedOrder.begin();
while (store.evictedOrder.size() > store.capacity) {
store.evictedKeys.erase(store.evictedOrder.back());
store.evictedOrder.pop_back();
}
if (++store.evictionsSinceCheck < store.capacity / 32u) return;
const bool churning = store.returnsSinceCheck * 2u > store.evictionsSinceCheck;
store.evictionsSinceCheck = 0;
store.returnsSinceCheck = 0;
if (!churning || store.capacity >= PipelineStore::MaximumEntries) return;
const auto previous = store.capacity;
store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
std::fprintf(stderr, "[pipecache] bound %zu -> %zu pipelines: pipelines it evicted keep coming back\n", previous, store.capacity);
}
void noteStoreReturn(PipelineStore& store, std::uint64_t hash) {
if (store.fixedCapacity) return;
const auto found = store.evictedKeys.find(hash);
if (found == store.evictedKeys.end()) return;
store.evictedOrder.erase(found->second);
store.evictedKeys.erase(found);
++store.returnsSinceCheck;
}
void reportPipelines(PipelineStore& store) {
@@ -1228,12 +1269,12 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
}
++store.misses;
++store.totals.misses;
noteStoreReturn(store, hash);
}
// Built outside the mutex: a draw that finds its pipeline resident is not held behind another
// thread's build (the journal replay builds here too), and a device's driver compiles in
// parallel when several threads build at once.
auto pipeline = std::make_shared<Pipeline>(context, state, vertexInput, resources, shaders, attachmentLayout);
const auto bound = storeBound();
const bool replaying = PipelineJournalReplaying();
{
std::lock_guard lock(store.mutex);
@@ -1251,7 +1292,10 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
}
// A replayed pipeline enters at the least recently used end and never displaces a draw's:
// the title's own pipelines evict it first, and a full store takes none.
if (replaying && store.entries.size() >= bound) return pipeline;
if (replaying && store.entries.size() >= storeBound(store)) {
if (store.fixedCapacity || store.capacity >= PipelineStore::MaximumEntries) return pipeline;
store.capacity = std::min(store.capacity * 2u, PipelineStore::MaximumEntries);
}
if (replaying) {
store.entries.push_front({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
store.index[hash] = store.entries.begin();
@@ -1259,12 +1303,14 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
store.entries.push_back({context.device, context.bufferPool, context.bufferPool != nullptr, hash, std::vector<std::byte>(key), pipeline});
store.index[hash] = std::prev(store.entries.end());
}
while (store.entries.size() > bound) {
while (store.entries.size() > storeBound(store)) {
// Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence).
const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; });
if (victim == store.entries.end()) break;
const auto evictedHash = victim->hash;
store.index.erase(victim->hash);
store.entries.erase(victim);
noteStoreEviction(store, evictedHash);
++store.evicted;
++store.totals.evicted;
}
@@ -1283,10 +1329,16 @@ PipelineStoreCounters PipelineStoreTotals() {
std::lock_guard lock(store.mutex);
auto totals = store.totals;
totals.cached = store.entries.size();
totals.bound = storeBound();
totals.bound = storeBound(store);
return totals;
}
bool PipelineStoreCanGrow() {
auto& store = Pipelines();
std::lock_guard lock(store.mutex);
return storeBound(store) < PipelineStore::MaximumEntries && !store.fixedCapacity;
}
void ClearCachedPipelines(VkDevice device) {
auto& store = Pipelines();
std::lock_guard lock(store.mutex);
@@ -1298,6 +1350,13 @@ void ClearCachedPipelines(VkDevice device) {
store.index.erase(it->hash);
it = store.entries.erase(it);
}
if (store.entries.empty() && !store.fixedCapacity) {
store.capacity = 0;
store.evictedOrder.clear();
store.evictedKeys.clear();
store.evictionsSinceCheck = 0;
store.returnsSinceCheck = 0;
}
ClearPipelineLibraries(device);
ReleasePixelFunctionLds(device);
}
core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp
+6 -3
View File
@@ -38,7 +38,7 @@ static_assert(sizeof(VkStencilOpState) == 28, "VkStencilOpState changed: update
static_assert(sizeof(VkVertexInputBindingDescription) == 12, "VkVertexInputBindingDescription changed: update the recipe codec");
static_assert(sizeof(VkVertexInputAttributeDescription) == 16, "VkVertexInputAttributeDescription changed: update the recipe codec");
#if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__)
static_assert(sizeof(State) == 832, "State changed: update CanonicalPipelineState and the recipe codec");
static_assert(sizeof(State) == 840, "State changed: update CanonicalPipelineState and the recipe codec");
static_assert(sizeof(ShaderStages) == 112, "ShaderStages changed: update the recipe codec");
static_assert(sizeof(ShaderRecompiler::MeshConfiguration) == 40, "MeshConfiguration changed: update the recipe codec");
static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "TessellationConfiguration changed: update the recipe codec");
@@ -1346,10 +1346,13 @@ public:
private:
void run() {
#ifdef _WIN32
if (!SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL)) std::fprintf(stderr, "[pipeline-journal] replay thread priority not lowered (error %lu)\n", GetLastError());
#endif
std::call_once(loadOnce, [&] {
try {
auto journal = Journal::Instance().Load();
const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 1024u);
const auto limit = configured("APS5_PIPELINE_JOURNAL_ENTRIES", 4096u);
if (journal.entries.size() > limit) journal.entries.erase(journal.entries.begin(), journal.entries.end() - static_cast<std::ptrdiff_t>(limit));
loaded = std::move(journal);
} catch (const std::exception& error) {
@@ -1435,7 +1438,7 @@ private:
return;
}
const auto store = PipelineStoreTotals();
if (store.cached >= store.bound) {
if (store.cached >= store.bound && !PipelineStoreCanGrow()) {
counters.skipped.fetch_add(1, std::memory_order_relaxed);
return;
}
core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp
+3 -2
View File
@@ -185,7 +185,8 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
std::vector<VkPipeline> libraries;
for (std::size_t part = 0; part < flags.size(); ++part) {
if (part == 0 && mesh) continue;
const auto key = Combined(*partKeys[part], *contextKeys[part]);
auto key = Combined(*partKeys[part], *contextKeys[part]);
key.push_back(static_cast<std::byte>(optimize));
auto& cache = device.parts[part];
if (const auto found = cache.find(key); found != cache.end()) {
libraries.push_back(found->second);
@@ -195,7 +196,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
VkGraphicsPipelineLibraryCreateInfoEXT library{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, flags[part] == VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT ? &rendering : &shaderRendering};
library.flags = flags[part];
VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library};
create.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT;
create.flags = optimize ? VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT : VK_PIPELINE_CREATE_LIBRARY_BIT_KHR;
create.pDynamicState = info.pDynamicState;
create.layout = partLayouts[part];
switch (flags[part]) {
core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp
+22
View File
@@ -8,6 +8,7 @@
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
#include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h"
#include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h"
#include "prx/libSceAgcDriver/Graphics/shaders/SampleDumps_spv.h"
@@ -3134,6 +3135,26 @@ std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t b
return 0;
}
std::uint64_t Recorder::PendingWriterSerial(std::uint64_t address, std::size_t bytes, std::span<const std::uint64_t> exceptNotes) const {
if (exceptNotes.empty()) return PendingWriterSerial(address, bytes);
if (bytes == 0) return 0;
const auto end = address + bytes;
const auto writes = [&](const Batch& batch) {
for (std::size_t index = 0; index < batch.writes.size(); ++index) {
const auto& [begin, finish] = batch.writes[index];
if (address >= finish || begin >= end) continue;
if (index < batch.writeNotes.size() && std::find(exceptNotes.begin(), exceptNotes.end(), batch.writeNotes[index]) != exceptNotes.end()) continue;
return true;
}
return readOverlap(batch, address, end, ReadKind::AddressBased).has_value();
};
if (open != nullptr && writes(*open)) return OpenSerial();
for (auto it = inFlight.rbegin(); it != inFlight.rend(); ++it) {
if (writes(**it)) return (*it)->serial;
}
return 0;
}
bool Recorder::signaled(const Batch& batch) const {
return batch.submitted && fenceStatus(batch.fence) == VK_SUCCESS;
}
@@ -3659,6 +3680,7 @@ void Recorder::Submit() {
submission.pSignalSemaphores = &timeline;
}
if (hold != VK_NULL_HANDLE || timeline != VK_NULL_HANDLE) submission.pNext = &timelineInfo;
DestroyDeferredAtIdle(context);
const auto submitStart = std::chrono::steady_clock::now();
const auto result = function(queueSubmit, "vkQueueSubmit")(context.queue, 1, &submission, batch->fence);
if (result == VK_ERROR_DEVICE_LOST) {
core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp
+2
View File
@@ -1,5 +1,6 @@
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
@@ -381,6 +382,7 @@ void CommandBatch::Submit() {
submission.commandBufferCount = 1;
submission.pCommandBuffers = &commands;
timing.Mark("command_end");
DestroyDeferredAtIdle(context);
Check(context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submission, fence), "vkQueueSubmit graphics");
pending = true;
submitted = true;
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp
+37 -27
View File
@@ -66,7 +66,7 @@ VkComponentSwizzle ComponentSwizzleFor(std::uint8_t dstSel) {
}
VkComponentMapping ViewComponents(const GuestTextureResource& resource) {
if (IsConvertedTextureFormat(resource.format) && !IsRepackedTextureFormat(resource.format)) return {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A};
if (IsConvertedTextureFormat(resource.format) && !IsRepackedTextureFormat(resource.format) && resource.format != static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Format11_11_10Float)) return {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A};
const auto channel = [&](std::uint8_t dstSel) { return TextureComponentChannel(resource.format, ComponentSwizzleFor(dstSel)); };
return {channel(resource.dstSelX), channel(resource.dstSelY), channel(resource.dstSelZ), channel(resource.dstSelW)};
}
@@ -624,10 +624,7 @@ std::uint64_t storageBudget(const Context& context, std::uint64_t required = 0)
cached.device = context.physical;
cached.read = now;
cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
cached.ceiling = cached.budget;
if (const auto heap = largestDeviceLocalHeap(context.memory); heap && reported.heapBudget[*heap] != 0) {
cached.ceiling = std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(TextureCacheBudget(context.memory), reported.heapBudget[*heap] / 4u));
}
cached.ceiling = StorageTextureCeiling(context.memory, &reported);
};
if (cached.device != context.physical || now - cached.read >= std::chrono::seconds(1)) read();
const auto needed = std::min(required, cached.ceiling);
@@ -752,7 +749,14 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
normalized.format = unorm16;
return cachedTexture(context, words, normalized, components, guestBytes, depthCompare);
}
if (const auto window = ViewedArrayWindow(resource); window.baseAddress != resource.baseAddress) {
const auto viewedWindow = [&] {
const auto window = ViewedArrayWindow(resource, false);
if (window.baseAddress != resource.baseAddress || resource.dccAddress == 0) return window;
if (ViewedArrayWindow(resource, true).baseAddress == resource.baseAddress) return window;
if (TextureClearKeys(resource, guestBytes != 0 ? guestBytes : DescribeSurface(resource).guestBytes) != DccKeys::Uncompressed) return window;
return ViewedArrayWindow(resource, true);
};
if (const auto window = viewedWindow(); window.baseAddress != resource.baseAddress) {
static std::atomic<std::uint64_t> windows{0};
const auto count = windows.fetch_add(1) + 1;
if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] sampled array view 0x%llx (%ux%u format %u, dimension %d, slices %u..%u) served from its viewed slices at 0x%llx; %llu such views\n", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<int>(resource.dimension), resource.baseArray, resource.depthOrLastArray, static_cast<unsigned long long>(window.baseAddress), static_cast<unsigned long long>(count));
@@ -799,7 +803,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
keys = scanKeys();
if (*keys == source->FilledKeys()) {
std::array<std::byte, 16> probe{};
if (!FillDccClear(ResolveTextureFormat(resource.format), *keys, resource.dccAlphaOnMsb, probe)) {
if (!FillDccClear(DccClearTextureFormat(resource.format), *keys, resource.dccAlphaOnMsb, probe)) {
char text[256];
std::snprintf(text, sizeof(text), "AGC graphics: sampled texture 0x%llx (%ux%u format %u, dcc 0x%llx) reads %s DCC keys filled over its pending image, a clear value the format has no encoding for", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<unsigned long long>(resource.dccAddress), DccKeysName(*keys));
throw std::runtime_error(text);
@@ -1185,9 +1189,9 @@ std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std
const auto viewed = ArrayLayerWindow(requested, context.limits.maxImageArrayLayers);
if (viewed.baseAddress != requested.baseAddress) guestBytes = 0;
Require(viewed.dimension != TextureDimension::k2DMsaa, "storage image access to a multisampled texture is not implemented");
RequireNoMultisampleTarget(viewed.baseAddress, guestBytes != 0 ? static_cast<std::size_t>(guestBytes) : 1u, "a storage image access");
if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
const auto viewedBytes = guestBytes != 0 ? guestBytes : DescribeSurface(viewed).guestBytes;
RequireNoMultisampleTarget(viewed.baseAddress, static_cast<std::size_t>(viewedBytes), "a storage image access");
if (auto depth = DepthSurfaceStorage(context, words, viewed, mip)) return depth;
if (viewedBytes >= (1ull << 30u)) {
static std::atomic<std::uint64_t> large{0};
const auto count = large.fetch_add(1) + 1;
@@ -1283,14 +1287,19 @@ std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory)
return std::max<std::uint64_t>(2048ull << 20u, heap ? memory.memoryHeaps[*heap].size / 4u : 0u);
}
std::uint64_t StorageTextureCeiling(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported) {
const auto heap = largestDeviceLocalHeap(memory);
if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, reported->heapBudget[*heap] / 2u));
}
std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes) {
const auto heap = largestDeviceLocalHeap(memory);
if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory);
const auto budget = reported->heapBudget[*heap];
const auto usage = reported->heapUsage[*heap];
const auto reserved = usage - std::min<std::uint64_t>(usage, sampledBytes + storageBytes) + sampledBytes + budget / 8u;
const auto cap = std::min<std::uint64_t>(memory.memoryHeaps[*heap].size / 2u, budget / 2u);
return std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(cap, budget > reserved ? budget - reserved : 0u));
return std::min<std::uint64_t>(StorageTextureCeiling(memory, reported), std::max<std::uint64_t>(512ull << 20u, budget > reserved ? budget - reserved : 0u));
}
std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes, std::uint64_t storageBytes) {
@@ -2075,9 +2084,10 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
const bool classified = element < binding.bufferWritten.size() && element < binding.bufferRead.size() && element < binding.bufferAtomic.size();
allocations[index].writeOnly = classified && allocations[index].guest && allocations[index].written && binding.bufferWritten[element] && !read && !atomic;
const auto& push = shader.program->pushConstants;
if (!push.empty()) {
const auto pushDataBytes = shader.program->stencilReferences.empty() ? push.size() : shader.program->stencilReferenceDword * sizeof(std::uint32_t);
if (pushDataBytes != 0u) {
const auto position = shader.program->memoryOffsetDword * 4u + element;
Require(position < push.size(), "guest buffer offset lies outside the shader's push constants");
Require(position < pushDataBytes, "guest buffer offset lies outside the shader's push constants");
allocations[index].pushByte = static_cast<std::int32_t>(shader.pushConstantOffset + position);
} else {
allocations[index].dataByte = shader.program->memoryOffsetDword * 4u + element;
@@ -2445,7 +2455,7 @@ void ShaderResources::buildComplete() {
for (auto element = first; element < last; ++element) {
const auto index = binding.imageAllocations[element];
if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED});
else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
}
if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount);
@@ -4384,7 +4394,8 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer");
written = written || allWritten;
const std::uint64_t recordBytes = descriptor.Stride() != 0 ? descriptor.Stride() : extent;
const bool swept = read && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize;
constexpr std::uint64_t WidestBufferStoreBytes = 16;
const bool swept = (read || (context.discreteGpu && descriptor.Stride() > WidestBufferStoreBytes)) && dispatchThreads != 0 && recordBytes != 0 && dispatchThreads * recordBytes * 2u >= byteSize;
if (written) guestMemory.AddWritable(address, size, atomic, swept);
else {
guestMemory.AddReadable(address, size);
@@ -4492,12 +4503,11 @@ std::uint64_t ShaderResources::DataWordsHash(const CompiledShader& shader) {
bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const {
Require(shader.program != nullptr, "missing compiled shader");
const auto& program = *shader.program;
if (program.bindings.size() != bindings.size()) return true;
for (std::size_t index = 0; index < program.bindings.size(); ++index) {
const auto& binding = program.bindings[index];
for (const auto& binding : program.bindings) {
if (!DataRole(binding.role)) continue;
if (bindings[index].allocations.size() != 1) return true;
const auto& allocation = allocations[bindings[index].allocations.front()];
const auto held = std::ranges::find_if(bindings, [&](const Binding& candidate) { return candidate.layout.binding == binding.binding; });
if (held == bindings.end() || held->allocations.size() != 1) return true;
const auto& allocation = allocations[held->allocations.front()];
if (allocation.dataWords.size() != binding.guestDescriptor.size() || !std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) return true;
}
return false;
@@ -4512,23 +4522,23 @@ bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader
bool ShaderResources::refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) {
Require(shader.program != nullptr, "missing compiled shader");
const auto& program = *shader.program;
Require(program.bindings.size() == bindings.size(), "template bindings disagree with the shader");
bool recorded = false;
// One [gputime] range of class TemplateDataRefresh around the updates, begun at the first one
// (a refresh that finds every word equal records nothing and times nothing).
auto timing = Recorder::NoTiming;
std::uint64_t refreshedBytes = 0;
for (std::size_t index = 0; index < program.bindings.size(); ++index) {
const auto& binding = program.bindings[index];
for (const auto& binding : program.bindings) {
if (!DataRole(binding.role)) continue;
Require(bindings[index].allocations.size() == 1, "data binding without its buffer");
auto& allocation = allocations[bindings[index].allocations.front()];
const auto held = std::ranges::find_if(bindings, [&](const Binding& candidate) { return candidate.layout.binding == binding.binding; });
Require(held != bindings.end(), "template bindings disagree with the shader");
Require(held->allocations.size() == 1, "data binding without its buffer");
auto& allocation = allocations[held->allocations.front()];
const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t);
Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words");
if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue;
if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, binding.guestDescriptor);
if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh);
writeDataWords(commands, bindings[index].allocations.front(), binding.guestDescriptor);
writeDataWords(commands, held->allocations.front(), binding.guestDescriptor);
allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end());
refreshedBytes += size;
recorded = true;
@@ -5227,7 +5237,7 @@ void ShaderResources::resolveImageTables() {
storageFirstLayer.push_back(firstLayer);
storageWritten.push_back(true);
storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]);
storageAtomic64.push_back(false);
storageAtomic64.push_back(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]);
describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress, true});
bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u;
continue;
core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp
+82 -9
View File
@@ -6,6 +6,7 @@
#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libc/include/General.hpp"
#include <algorithm>
@@ -314,13 +315,33 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
auto fixed = compare == 0 || compare == 7 ? 0u : face.compareMask;
std::uint32_t invertMask = 0;
std::uint32_t inverted = 0;
std::uint32_t logicMask = 0;
const auto logic = [](std::uint32_t op) { return op == 10 || op == 11 || op == 12 || op == 15; };
const auto convert = [&](std::uint32_t op, bool testFailed) {
if (face.writeMask == 0) return VK_STENCIL_OP_KEEP;
char detail[112];
if (opValExport && ((op == 4 && !opValReference) || op == 5 || op == 6 || op == 8 || op == 9)) {
if (opValExport && ((op == 4 && !opValReference) || (op >= 5 && op != 7))) {
std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operation %u with a pixel shader that exports the stencil operation value is unsupported", op);
throw std::runtime_error(detail);
}
if (logic(op)) {
const auto bits = (op == 11 || op == 12 ? opValue : ~opValue) & face.writeMask;
if (bits == 0) return VK_STENCIL_OP_KEEP;
if (logicMask != 0 && logicMask != bits) {
std::snprintf(detail, sizeof(detail), "AGC graphics: stencil logic operations on write masks 0x%02x and 0x%02x are unsupported", logicMask, bits);
throw std::runtime_error(detail);
}
logicMask = bits;
if (op != 11) return op == 10 ? VK_STENCIL_OP_ZERO : VK_STENCIL_OP_INVERT;
if (testValExport) throw std::runtime_error("AGC graphics: stencil OR is unsupported with a pixel shader that exports the stencil test value");
if ((~reference & fixed & bits) != 0) {
std::snprintf(detail, sizeof(detail), "AGC graphics: stencil OR of 0x%02x against test value 0x%02x (masks 0x%02x/0x%02x) is unsupported", opValue, test, face.compareMask, face.writeMask);
throw std::runtime_error(detail);
}
reference |= bits;
fixed |= bits;
return VK_STENCIL_OP_REPLACE;
}
switch (op) {
case 0: return VK_STENCIL_OP_KEEP;
case 1: return VK_STENCIL_OP_ZERO;
@@ -377,6 +398,15 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint
if (inverted != active) throw std::runtime_error("AGC graphics: a stencil replacement drawn as an inversion beside another stencil operation is unsupported");
face.writeMask = invertMask;
}
if (logicMask != 0) {
const auto other = [&](std::uint32_t op) { return op != 0 && !logic(op); };
if ((other(ops & 0xfu) || other((ops >> 4u) & 0xfu) || other((ops >> 8u) & 0xfu)) && logicMask != face.writeMask) {
char detail[128];
std::snprintf(detail, sizeof(detail), "AGC graphics: stencil operations 0x%03x combine a logic operation on write mask 0x%02x with write mask 0x%02x", ops & 0xfffu, logicMask, face.writeMask);
throw std::runtime_error(detail);
}
face.writeMask = logicMask;
}
face.reference = reference;
Require(!opValReference || compare == 0 || compare == 7 || face.compareMask == 0, "a stencil compare against the test value is unsupported with a pixel shader that exports the stencil operation value");
return face;
@@ -953,12 +983,17 @@ struct DecodedColorFormat {
const char* PackedColorTargetName(ShaderRecompiler::ColorExportPacking packing) {
if (packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4) return "4_4_4_4 unorm";
if (packing == ShaderRecompiler::ColorExportPacking::Uint) return "unsigned integer";
return packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "11_11_10 float" : "10_11_11 unorm";
}
bool WordPacked(ShaderRecompiler::ColorExportPacking packing) {
return packing == ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 || packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4;
}
// CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list
// components from the least significant bits, as Vulkan's non-packed formats do.
DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
DecodedColorFormat DecodeColorFormatRaw(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
constexpr std::uint32_t unorm = 0, snorm = 1, sint = 5, srgb = 6, floating = 7;
constexpr std::uint32_t uint = 4;
const auto fail = [&]() -> DecodedColorFormat {
@@ -1326,6 +1361,27 @@ ReversedBlend TranslateReversedBlend(std::uint32_t blendControl, std::uint8_t co
return {blend, routing->code};
}
void ApplyDepthBoundsEmulation(const Context& context, State& state, const ShaderRecompiler::ShaderPixelStageInfo& pixel) {
if (!state.depthBoundsTest || !state.depth || context.depthBounds) return;
ValidateDepthBounds(context, state);
Require(!context.depthRangeUnrestricted, "the depth bounds test without the depthBounds feature is emulated only where stored depth stays in [0, 1]");
Require(state.depth->format == VK_FORMAT_D32_SFLOAT || state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT, "the depth bounds test on a normalized depth target needs the depthBounds feature");
Require(!state.depthTest && !state.depthWrite, "the depth bounds test with depth testing needs the depthBounds feature");
Require(!pixel.posZ && !pixel.depthExportEnable, "the depth bounds test from a pixel program that reads or exports depth needs the depthBounds feature");
const auto keepsOnFail = [](const VkStencilOpState& face) { return (face.writeMask & 0xffu) == 0 || (face.failOp == VK_STENCIL_OP_KEEP && face.depthFailOp == VK_STENCIL_OP_KEEP); };
Require(!state.stencilTest || (keepsOnFail(state.stencilFront) && keepsOnFail(state.stencilBack)), "the depth bounds test with a stencil test that writes on a failed test needs the depthBounds feature");
const bool upper = state.minDepthBounds <= 0.0f;
Require(upper || state.maxDepthBounds >= 1.0f, "the depth bounds test with both bounds inside (0, 1) needs the depthBounds feature");
const float bound = upper ? state.maxDepthBounds : state.minDepthBounds;
state.depthBoundsTest = false;
state.depthTest = true;
state.depthCompare = upper ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
state.depthBias = false;
state.depthBiasPerFace = false;
state.viewport.minDepth = bound;
state.viewport.maxDepth = bound;
}
void ApplyDepthRemap(const Context& context, State& state) {
const auto& viewport = state.viewport;
const bool inRange = viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1;
@@ -1533,6 +1589,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
if (slotExport == 7 && ((read(cx, 0x31c + slot * 0xfu) >> 8u) & 7u) != 4u) throw std::runtime_error("AGC graphics: color export format 7 (UINT16_ABGR) into a target that is not unsigned integer is unsupported");
color.exportIndex = index;
color.uintExport = slotExport == 7;
if (color.uintExport && color.packing == ShaderRecompiler::ColorExportPacking::Uint) color.packing = ShaderRecompiler::ColorExportPacking::None;
if (color.samples == 1 && result.samples > 1 && result.colors.empty() && !result.depth && (read(cx, 0x292) & 1u) == 0) {
const auto sampleInputs = read(cx, 0x1b3) & read(cx, 0x1b4) & (ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage));
Require(sampleInputs == 0 && (read(cx, 0x203) & 0x100u) == 0, "a one-sample color target under a multisampled rasterizer without MSAA_ENABLE is unsupported with a sample index or coverage input, or a sample-mask export");
@@ -1690,7 +1747,8 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
if ((weighted(state.colorBlendOp) && (constantColor(state.srcColorBlendFactor) || constantColor(state.dstColorBlendFactor))) || (weighted(state.alphaBlendOp) && (constantColor(state.srcAlphaBlendFactor) || constantColor(state.dstAlphaBlendFactor)))) throw std::runtime_error("AGC graphics: blending into an 8_8 color target with the alternate component order with a constant color factor, which Vulkan applies to the second channel with the green blend constant instead of the alpha one, is not implemented");
}
}
if (color.packing != ShaderRecompiler::ColorExportPacking::None) {
if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) state.blendEnable = VK_FALSE;
if (WordPacked(color.packing)) {
const std::string packed = std::string(color.packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "an " : "a ") + PackedColorTargetName(color.packing);
const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into " + packed + " target is unsupported");
@@ -1715,6 +1773,10 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
}
}
}
if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) {
const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
if (slotExport != 1 && slotExport != 2 && slotExport != 3 && slotExport != 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into an unsigned integer target is unsupported");
}
if (state.blendEnable && !reversed && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) {
const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; };
Require(addition(state.colorBlendOp) && addition(state.alphaBlendOp), "dual-source blending with a MIN or MAX operation is unsupported");
@@ -1912,11 +1974,17 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) {
surface.dstSelY = 5;
surface.dstSelZ = 6;
surface.dstSelW = 7;
surface.dccAddress = color.dccAddress;
surface.dccAddress = color.dccAddress != 0 ? color.dccAddress : color.chainDccAddress;
surface.dccAlphaOnMsb = color.dccAlphaOnMsb;
surface.dccPipeAligned = color.dccPipeAligned;
return surface;
}
DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, std::uint32_t swap) {
auto decoded = DecodeColorFormatRaw(format, number, swap);
if (number == 4u && decoded.packing == ShaderRecompiler::ColorExportPacking::None) decoded.packing = ShaderRecompiler::ColorExportPacking::Uint;
return decoded;
}
ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allowMultisampleDcc) {
const auto stride = slot * 0xfu;
@@ -2025,7 +2093,7 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
color.format = decoded.format;
color.componentMapping = decoded.componentMapping;
color.packing = decoded.packing;
if (color.packing != ShaderRecompiler::ColorExportPacking::None && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
if (WordPacked(color.packing) && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 4_4_4_4 unorm color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Float11_11_10 || (info & 0x00040000u) != 0, "rounding (ROUND_MODE 0) 11_11_10 float color targets are unsupported");
@@ -2081,9 +2149,10 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
Require(color.fmaskAddress != 0, "a compressed one-fragment color target without an FMASK address is unsupported");
}
if ((info & 0x10000000u) != 0) {
const auto dccHigh = find(cx, 0x3a8 + slot);
const auto dccBase = ((dccHigh == cx.end() ? 0ull : static_cast<std::uint64_t>(dccHigh->second & 0xffu)) << 40u) | (static_cast<std::uint64_t>(read(cx, 0x325 + stride)) << 8u);
if (maxMip == 0) {
const auto dccHigh = find(cx, 0x3a8 + slot);
color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast<std::uint64_t>(dccHigh->second & 0xffu)) << 40u) | (static_cast<std::uint64_t>(read(cx, 0x325 + stride)) << 8u);
color.dccAddress = dccBase;
color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap);
color.dccPipeAligned = ((attrib3 >> 30u) & 1u) != 0;
if (color.samples > 1) {
@@ -2091,11 +2160,15 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
color.dccControl = read(cx, 0x31e + stride);
if (!allowMultisampleDcc) Require((color.dccControl & 0x3ff80u) == 0, "multisampled DCC color transforms or lossy modes are not implemented");
}
} else if (!volume && color.arrayLastSlice == 0 && slice == 0) {
color.chainDccAddress = dccBase;
color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap);
color.dccPipeAligned = ((attrib3 >> 30u) & 1u) != 0;
} else {
static bool reported = false;
if (!reported) {
reported = true;
std::fprintf(stderr, "[gpu] DCC keys of mipmapped color targets are ignored\n");
std::fprintf(stderr, "[gpu] DCC keys of mipmapped array and 3D color targets are ignored\n");
}
}
}
@@ -2192,7 +2265,7 @@ std::optional<ColorMetadataPass> DecodeColorMetadataPass(const QueueState& queue
Require(destination.samples == 1, "CB resolve into a multisampled target is unsupported");
Require(destination.cmaskAddress == 0, "CB resolve into a fast-clear color target is unsupported (the resolve does not update its CMASK)");
Require(source.format == destination.format && source.extent.width == destination.extent.width && source.extent.height == destination.extent.height, "CB resolve between different formats or extents is unsupported");
if (source.packing != ShaderRecompiler::ColorExportPacking::None) throw std::runtime_error("AGC graphics: CB resolves of " + std::string(PackedColorTargetName(source.packing)) + " color targets are unsupported");
if (WordPacked(source.packing)) throw std::runtime_error("AGC graphics: CB resolves of " + std::string(PackedColorTargetName(source.packing)) + " color targets are unsupported");
requireCovered(destination);
pass.source = source;
pass.targets.push_back(destination);
Loaded 100 of 1026 files, more files were not shown because too many files have changed in this diff. Show more