Mirror-bot/AnyPS5
1
0 Fork 0
mirror of https://github.com/boykopovar/AnyPS5.git synced 2026-10-28 00:35:47 +00:00
Code Issues Projects Releases Activity

fix(job-manager): merge main and reconcile scheduler debt

Browse Source
This commit is contained in:
Felipebmaitan committed 2026-10-23 22:48:31 -03:00
2 parents dca749aaa0 db6af9e0fd
commit b7640ae5a3
410 files changed
+22401 -2504
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/DISCUSSION_TEMPLATE/game-bring-up.yml
+1 -1
View File
@@ -6,7 +6,7 @@ body:
One thread per title, for every platform: search this category for the title ID first. If a thread exists, reply there, also when your OS or GPU differs from the one it reports.
Name the discussion `Game name (title ID)`, for example `DOOM Shareware (PPSA99666)`, without the platform.
Bug reports and implementation work belong in issues and pull requests; link them here as the blocker.
Per-game progress is also tracked on the [blocker map](https://alxcbtw.github.io/anyps5-roadmap/), built from public GitHub data. Only share material you have the right to distribute.
Only share material you have the right to distribute.
- type: checkboxes
id: searched
attributes:
.github/workflows/build.yml
+1 -1
View File
@@ -33,7 +33,7 @@ permissions:
concurrency:
group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.event_name != 'push' }}
env:
CONFIGURE: >-
README.md
+1 -1
View File
@@ -28,7 +28,7 @@ The [shader recompiler](core/shader/recompiler/Recompiler.cpp) successfully prod
## Compatibility
Per-game progress (stage reached, open blockers, libraries and GPU coverage) is tracked on the [blocker map](https://alxcbtw.github.io/anyps5-roadmap/), a community project built from public GitHub data ([source](https://github.com/AlxcBTW/anyps5-roadmap)). Test reports go in the [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussions, one thread per title.
Per-game status and test reports live in the [Game bring-up](https://github.com/boykopovar/AnyPS5/discussions/categories/game-bring-up) discussions, one thread per title.
## Input mapping
core/Decoder/Png/src/Png.cpp
+33
View File
@@ -48,6 +48,37 @@ bool isChunkType(const std::uint8_t* bytes, const char* type) {
return std::equal(bytes, bytes + 4, type);
}
std::optional<std::uint16_t> readGrayscaleTransparencyKey(std::span<const std::uint8_t> png) {
std::size_t offset = SIGNATURE.size();
if (png.size() < offset || !std::equal(SIGNATURE.begin(), SIGNATURE.end(), png.begin())) return std::nullopt;
std::optional<std::uint16_t> key;
while (png.size() - offset >= CHUNK_OVERHEAD) {
const std::uint32_t length = readBigEndian32(&png[offset]);
const std::uint8_t* type = &png[offset + 4];
if (length > png.size() - offset - CHUNK_OVERHEAD) return std::nullopt;
if (isChunkType(type, "tRNS")) {
if (length != 2) return std::nullopt;
key = static_cast<std::uint16_t>(png[offset + 8] << 8 | png[offset + 9]);
}
if (isChunkType(type, "IDAT") || isChunkType(type, "IEND")) return key;
offset += CHUNK_OVERHEAD + length;
}
return std::nullopt;
}
void repairGrayscaleTransparency(const Header& header, std::span<const std::uint8_t> png, Image& image) {
if (header.colorType != ColorType::Grayscale || !header.hasTransparency) return;
if (header.bitDepth != 1 && header.bitDepth != 2 && header.bitDepth != 4) return;
const std::optional<std::uint16_t> key = readGrayscaleTransparencyKey(png);
if (!key.has_value()) return;
const std::uint16_t mask = static_cast<std::uint16_t>((1u << header.bitDepth) - 1);
if ((*key & 0xFF) == (*key & mask)) return;
const std::uint8_t transparent = static_cast<std::uint8_t>((*key & mask) * (255u / mask));
for (std::size_t offset = 3; offset < image.pixels.size(); offset += 4) {
image.pixels[offset] = static_cast<std::uint8_t>(image.pixels[offset - 3] == transparent ? 0 : 255);
}
}
bool isValidFormat(std::uint8_t bitDepth, std::uint8_t colorType) {
switch (colorType) {
case 0:
@@ -158,6 +189,8 @@ std::optional<Image> Decode(std::span<const std::uint8_t> png) {
Image image{static_cast<std::uint32_t>(width), static_cast<std::uint32_t>(height), {}};
image.pixels.assign(decoded.get(), decoded.get() + static_cast<std::size_t>(image.width) * image.height * 4);
const std::optional<Header> header = ParseHeader(png);
if (header.has_value()) repairGrayscaleTransparency(*header, png, image);
return image;
}
core/Decoder/Png/tests/Png.cpp
+172
View File
@@ -3,6 +3,7 @@
#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <optional>
#include <stdexcept>
#include <vector>
@@ -50,6 +51,17 @@ static const std::uint8_t GRAY1[] = {
0x42, 0x60, 0x82,
};
static const std::uint8_t GRAY1_TRNS[] = {
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x59, 0x42,
0x27, 0x00, 0x00, 0x00, 0x02, 0x74, 0x52, 0x4E, 0x53, 0x00, 0x02, 0x98, 0x9D, 0xAC, 0x14, 0x00,
0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, 0x54, 0x78, 0xDA, 0x63, 0x70, 0x00, 0x00, 0x00, 0x42, 0x00,
0x41, 0x84, 0xBF, 0x8E, 0x62, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60,
0x82,
};
static const std::uint8_t PNG_SIGNATURE[] = {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A};
static void AppendBigEndian32(std::vector<std::uint8_t>& bytes, std::uint32_t value) {
for (int shift = 24; shift >= 0; shift -= 8) bytes.push_back(static_cast<std::uint8_t>(value >> shift));
}
@@ -64,6 +76,130 @@ static std::vector<std::uint8_t> HeaderOnly(std::uint32_t width, std::uint32_t h
return bytes;
}
static std::uint32_t Crc32(std::span<const std::uint8_t> bytes) {
std::uint32_t crc = 0xFFFFFFFFu;
for (const std::uint8_t byte : bytes) {
crc ^= byte;
for (int bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xEDB88320u : 0u);
}
return ~crc;
}
static std::uint32_t Adler32(std::span<const std::uint8_t> bytes) {
std::uint32_t low = 1;
std::uint32_t high = 0;
for (const std::uint8_t byte : bytes) {
low = (low + byte) % 65521;
high = (high + low) % 65521;
}
return high << 16 | low;
}
static void AppendTestChunk(std::vector<std::uint8_t>& png, const char* type, std::span<const std::uint8_t> data) {
AppendBigEndian32(png, static_cast<std::uint32_t>(data.size()));
const std::size_t start = png.size();
png.insert(png.end(), type, type + 4);
png.insert(png.end(), data.begin(), data.end());
AppendBigEndian32(png, Crc32(std::span<const std::uint8_t>(png.data() + start, png.size() - start)));
}
static std::vector<std::uint8_t> ZlibStore(std::span<const std::uint8_t> bytes) {
Require(bytes.size() <= 0xFFFF);
const std::uint16_t length = static_cast<std::uint16_t>(bytes.size());
const std::uint16_t inverse = static_cast<std::uint16_t>(~length);
std::vector<std::uint8_t> output{0x78, 0x01, 0x01, static_cast<std::uint8_t>(length), static_cast<std::uint8_t>(length >> 8),
static_cast<std::uint8_t>(inverse), static_cast<std::uint8_t>(inverse >> 8)};
output.insert(output.end(), bytes.begin(), bytes.end());
AppendBigEndian32(output, Adler32(bytes));
return output;
}
static std::vector<std::uint8_t> GrayLowBitPng(std::uint8_t bitDepth, std::span<const std::uint8_t> samples,
std::optional<std::uint16_t> transparency,
std::optional<std::uint16_t> precedingTransparency = std::nullopt) {
Require(bitDepth == 1 || bitDepth == 2 || bitDepth == 4);
std::vector<std::uint8_t> rows(1 + (samples.size() * bitDepth + 7) / 8, 0);
std::size_t bit = 0;
for (const std::uint8_t sample : samples) {
Require(sample < (1u << bitDepth));
rows[1 + bit / 8] |= static_cast<std::uint8_t>(sample << (8 - bitDepth - bit % 8));
bit += bitDepth;
}
std::vector<std::uint8_t> ihdr;
AppendBigEndian32(ihdr, static_cast<std::uint32_t>(samples.size()));
AppendBigEndian32(ihdr, 1);
ihdr.insert(ihdr.end(), {bitDepth, 0, 0, 0, 0});
std::vector<std::uint8_t> png(PNG_SIGNATURE, PNG_SIGNATURE + sizeof(PNG_SIGNATURE));
AppendTestChunk(png, "IHDR", ihdr);
if (precedingTransparency.has_value()) {
const std::uint8_t key[2] = {static_cast<std::uint8_t>(*precedingTransparency >> 8), static_cast<std::uint8_t>(*precedingTransparency)};
AppendTestChunk(png, "tRNS", key);
}
if (transparency.has_value()) {
const std::uint8_t key[2] = {static_cast<std::uint8_t>(*transparency >> 8), static_cast<std::uint8_t>(*transparency)};
AppendTestChunk(png, "tRNS", key);
}
AppendTestChunk(png, "IDAT", ZlibStore(rows));
AppendTestChunk(png, "IEND", std::span<const std::uint8_t>());
return png;
}
static std::uint8_t GrayScale(std::uint8_t bitDepth) {
if (bitDepth == 1) return 255;
if (bitDepth == 2) return 85;
return 17;
}
static void RequireGrayTransparency(std::uint8_t bitDepth, std::span<const std::uint8_t> samples, std::uint16_t key,
std::uint8_t transparentSample, std::optional<std::uint16_t> precedingKey = std::nullopt) {
const std::vector<std::uint8_t> png = GrayLowBitPng(bitDepth, samples, key, precedingKey);
const std::vector<std::uint8_t> original = png;
const auto header = Decoder::Png::ParseHeader(png);
Require(header.has_value());
Require(header->width == static_cast<std::uint32_t>(samples.size()) && header->height == 1 && header->bitDepth == bitDepth);
Require(header->colorType == Decoder::Png::ColorType::Grayscale && header->hasTransparency);
const auto image = Decoder::Png::Decode(png);
Require(image.has_value() && image->pixels.size() == samples.size() * 4);
const std::uint8_t scale = GrayScale(bitDepth);
for (std::size_t i = 0; i < samples.size(); ++i) {
const std::uint8_t gray = static_cast<std::uint8_t>(samples[i] * scale);
Require(image->pixels[i * 4] == gray && image->pixels[i * 4 + 1] == gray && image->pixels[i * 4 + 2] == gray);
Require(image->pixels[i * 4 + 3] == (samples[i] == transparentSample ? 0 : 255));
}
Require(png == original);
}
static void RequireGrayOpaque(std::uint8_t bitDepth, std::span<const std::uint8_t> samples) {
const std::vector<std::uint8_t> png = GrayLowBitPng(bitDepth, samples, std::nullopt);
const std::vector<std::uint8_t> original = png;
const auto header = Decoder::Png::ParseHeader(png);
Require(header.has_value() && !header->hasTransparency);
const auto image = Decoder::Png::Decode(png);
Require(image.has_value() && image->pixels.size() == samples.size() * 4);
const std::uint8_t scale = GrayScale(bitDepth);
for (std::size_t i = 0; i < samples.size(); ++i) {
const std::uint8_t gray = static_cast<std::uint8_t>(samples[i] * scale);
Require(image->pixels[i * 4] == gray && image->pixels[i * 4 + 1] == gray && image->pixels[i * 4 + 2] == gray);
Require(image->pixels[i * 4 + 3] == 255);
}
Require(png == original);
}
static void RequireTruncatedTransparency() {
const std::uint8_t samples[] = {0, 1};
const std::vector<std::uint8_t> png = GrayLowBitPng(1, samples, 0x0002);
const std::size_t transparencyOffset = sizeof(PNG_SIGNATURE) + 25;
const std::vector<std::uint8_t> truncated(png.begin(), png.begin() + transparencyOffset + 9);
const std::vector<std::uint8_t> original = truncated;
const auto header = Decoder::Png::ParseHeader(truncated);
Require(header.has_value() && !header->hasTransparency);
(void)Decoder::Png::Decode(truncated);
Require(truncated == original);
}
static void RequireRoundTrip(std::uint32_t channels, Decoder::Png::ColorType colorType) {
constexpr std::uint32_t width = 20;
constexpr std::uint32_t height = 10;
@@ -176,6 +312,42 @@ int main() {
const std::uint8_t levels[8] = {0, 255, 0, 255, 255, 0, 0, 255};
for (int i = 0; i < 8; ++i) Require(gray1->pixels[i * 4] == levels[i] && gray1->pixels[i * 4 + 3] == 255);
const auto gray1TrnsHeader = Decoder::Png::ParseHeader(GRAY1_TRNS);
Require(gray1TrnsHeader.has_value());
Require(gray1TrnsHeader->width == 2 && gray1TrnsHeader->height == 1 && gray1TrnsHeader->bitDepth == 1);
Require(gray1TrnsHeader->colorType == Decoder::Png::ColorType::Grayscale && gray1TrnsHeader->hasTransparency);
const auto gray1Trns = Decoder::Png::Decode(GRAY1_TRNS);
Require(gray1Trns.has_value());
Require(gray1Trns->pixels == std::vector<std::uint8_t>({0, 0, 0, 0, 255, 255, 255, 255}));
const std::uint8_t gray1Samples[] = {0, 1, 0, 1, 1, 0, 0, 1};
RequireGrayTransparency(1, gray1Samples, 0x0000, 0);
RequireGrayTransparency(1, gray1Samples, 0x0002, 0);
RequireGrayTransparency(1, gray1Samples, 0x0100, 0);
RequireGrayTransparency(1, gray1Samples, 0xFF02, 0);
RequireGrayTransparency(1, gray1Samples, 0xFFFF, 1);
RequireGrayTransparency(1, gray1Samples, 0x0001, 1, 0x0002);
RequireGrayOpaque(1, gray1Samples);
const std::uint8_t gray2Samples[] = {0, 1, 2, 3};
RequireGrayTransparency(2, gray2Samples, 0x0001, 1);
RequireGrayTransparency(2, gray2Samples, 0x0005, 1);
RequireGrayTransparency(2, gray2Samples, 0x0100, 0);
RequireGrayTransparency(2, gray2Samples, 0x0101, 1);
RequireGrayTransparency(2, gray2Samples, 0xFF05, 1);
RequireGrayTransparency(2, gray2Samples, 0xFFFF, 3);
RequireGrayOpaque(2, gray2Samples);
const std::uint8_t gray4Samples[] = {0, 3, 10, 15, 1};
RequireGrayTransparency(4, gray4Samples, 0x0003, 3);
RequireGrayTransparency(4, gray4Samples, 0x0013, 3);
RequireGrayTransparency(4, gray4Samples, 0x0100, 0);
RequireGrayTransparency(4, gray4Samples, 0x0103, 3);
RequireGrayTransparency(4, gray4Samples, 0xFF13, 3);
RequireGrayTransparency(4, gray4Samples, 0xFFFF, 15);
RequireGrayOpaque(4, gray4Samples);
RequireTruncatedTransparency();
const auto interlaced = Decoder::Png::ParseHeader(HeaderOnly(3, 4, 8, 6, 1));
Require(interlaced.has_value() && interlaced->interlaced && !interlaced->hasTransparency);
Require(!Decoder::Png::ParseHeader(HeaderOnly(0, 4, 8, 6, 0)).has_value());
core/libs/CMakeLists.txt
+31
View File
@@ -407,6 +407,16 @@ if(BUILD_TESTING)
target_compile_options(crash_text_tests PRIVATE -fasynchronous-unwind-tables)
endif()
add_test(NAME crash_text COMMAND crash_text_tests)
add_test_executable(guest_kernel_sys_pselect_tests tests/GuestKernelSysPselect.cpp)
target_include_directories(guest_kernel_sys_pselect_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_pselect_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_pselect_tests)
add_test(NAME guest_kernel_sys_pselect COMMAND guest_kernel_sys_pselect_tests)
set_tests_properties(guest_kernel_sys_pselect PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_kernel_sys_pselect PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
if(WIN32)
add_test_executable(windows_crash_text_tests tests/WindowsCrashText.cpp)
@@ -470,6 +480,16 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpParty>")
endif()
add_test_executable(guest_np_ps4_imports_tests tests/GuestNpPs4Imports.cpp)
target_include_directories(guest_np_ps4_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_np_ps4_imports_tests PRIVATE libSceNpMatching2 libSceNpSignaling libSceNpTrophy libc)
configure_windows_unwind(guest_np_ps4_imports_tests)
add_test(NAME guest_np_ps4_imports COMMAND guest_np_ps4_imports_tests)
if(WIN32)
set_tests_properties(guest_np_ps4_imports PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpMatching2>")
endif()
add_test_executable(guest_np_common_tests tests/GuestNpCommon.cpp)
target_include_directories(guest_np_common_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_np_common_tests PRIVATE libSceNpCommon libkernel libc)
@@ -483,6 +503,17 @@ if(BUILD_TESTING)
endif()
endforeach()
add_test_executable(guest_np_common_runtime_tests tests/GuestNpCommonRuntime.cpp)
target_include_directories(guest_np_common_runtime_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_np_common_runtime_tests PRIVATE libSceNpCommon libkernel libc)
configure_windows_unwind(guest_np_common_runtime_tests)
add_test(NAME guest_np_common_runtime COMMAND guest_np_common_runtime_tests)
set_tests_properties(guest_np_common_runtime PROPERTIES TIMEOUT 20)
if(WIN32)
set_tests_properties(guest_np_common_runtime PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceNpCommon>")
endif()
add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp)
target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libSceVideoRecordingP libc)
core/libs/SceTypes.hpp
+7
View File
@@ -655,6 +655,13 @@ struct AudioOutPortState {
std::uint64_t activeState;
};
struct AudioOutSystemState {
float loudness;
std::uint8_t reserved8[4];
std::uint64_t reserved[3];
};
static_assert(sizeof(AudioOutSystemState) == 32);
struct Audio3dOpenParameters {
std::uint64_t size_this;
std::uint32_t granularity;
core/libs/prx/libSceAgc/Acb/src/Sync.cpp
+1 -4
View File
@@ -44,10 +44,7 @@ std::uint32_t APS5_VABI sceAgcAcbQueueEndOfShaderActionGetSize() {
return 32;
}
std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address) {
const std::uint8_t operation = action != 0 ? 7u : 6u;
const std::uint8_t signalType = clientCode & 1u;
const std::uint8_t waitForMailbox = (clientCode >> 1u) & 1u;
std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation) {
return Agc::Command::WriteMemSemaphore(buf, address, waitForMailbox, signalType, operation, __func__);
}
core/libs/prx/libSceAgc/CMakeLists.txt
+8 -8
View File
@@ -157,7 +157,7 @@ add_test_executable(agc_register_command_tests EXCLUDE_FROM_ALL
Patch/src/IndirectRegisters.cpp
)
target_include_directories(agc_register_command_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_register_command_tests PRIVATE libc)
target_link_libraries(agc_register_command_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_register_command_tests)
if(BUILD_TESTING)
set_target_properties(agc_register_command_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -176,7 +176,7 @@ add_test_executable(agc_cb_cond_write_tests EXCLUDE_FROM_ALL
Cb/src/Branch.cpp
)
target_include_directories(agc_cb_cond_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_cb_cond_write_tests PRIVATE libc)
target_link_libraries(agc_cb_cond_write_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_cb_cond_write_tests)
if(BUILD_TESTING)
set_target_properties(agc_cb_cond_write_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -220,7 +220,7 @@ add_test_executable(agc_branch_patch_tests EXCLUDE_FROM_ALL
Patch/src/Branch.cpp
)
target_include_directories(agc_branch_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_branch_patch_tests PRIVATE libc)
target_link_libraries(agc_branch_patch_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_branch_patch_tests)
if(BUILD_TESTING)
set_target_properties(agc_branch_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -240,7 +240,7 @@ add_test_executable(agc_predication_tests EXCLUDE_FROM_ALL
Predication/src/Predication.cpp
)
target_include_directories(agc_predication_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_predication_tests PRIVATE libc)
target_link_libraries(agc_predication_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_predication_tests)
if(BUILD_TESTING)
set_target_properties(agc_predication_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -370,7 +370,7 @@ add_test_executable(agc_dma_data_tests EXCLUDE_FROM_ALL
DcbFlow/src/Memory.cpp
)
target_include_directories(agc_dma_data_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_dma_data_tests PRIVATE libc)
target_link_libraries(agc_dma_data_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_dma_data_tests)
if(BUILD_TESTING)
set_target_properties(agc_dma_data_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -390,7 +390,7 @@ add_test_executable(agc_write_data_patch_tests EXCLUDE_FROM_ALL
Patch/src/WaitRegMem.cpp
)
target_include_directories(agc_write_data_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_write_data_patch_tests PRIVATE libc)
target_link_libraries(agc_write_data_patch_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_write_data_patch_tests)
if(BUILD_TESTING)
set_target_properties(agc_write_data_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -410,7 +410,7 @@ add_test_executable(agc_prime_utcl2_tests EXCLUDE_FROM_ALL
DcbState/src/Display.cpp
)
target_include_directories(agc_prime_utcl2_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_prime_utcl2_tests PRIVATE libc)
target_link_libraries(agc_prime_utcl2_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_prime_utcl2_tests)
if(BUILD_TESTING)
set_target_properties(agc_prime_utcl2_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
@@ -431,7 +431,7 @@ add_test_executable(agc_cond_exec_patch_tests EXCLUDE_FROM_ALL
Patch/src/CondExec.cpp
)
target_include_directories(agc_cond_exec_patch_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_cond_exec_patch_tests PRIVATE libc)
target_link_libraries(agc_cond_exec_patch_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_cond_exec_patch_tests)
if(BUILD_TESTING)
set_target_properties(agc_cond_exec_patch_tests PROPERTIES EXCLUDE_FROM_ALL FALSE)
core/libs/prx/libSceAgc/Command/src/Packet.cpp
+2
View File
@@ -1,3 +1,4 @@
#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
#include "prx/libSceAgc/Command/include/Packet.hpp"
#include <algorithm>
@@ -67,6 +68,7 @@ void Reserve(CommandBuffer* buffer, std::uint32_t count, const char* function) {
std::uint32_t* Allocate(CommandBuffer* buffer, std::uint32_t count, const char* function) {
Reserve(buffer, count, function);
auto* result = buffer->cursor_up;
AgcDriverPrepareCommandWrite_nid_no_patch(reinterpret_cast<std::uint64_t>(result), static_cast<std::size_t>(count) * sizeof(std::uint32_t));
buffer->cursor_up += count;
return result;
}
core/libs/prx/libSceAgc/DcbFlow/src/Sync.cpp
+1 -4
View File
@@ -52,10 +52,7 @@ std::uint32_t APS5_VABI sceAgcDcbStallCommandBufferParserGetSize() {
return 8;
}
std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, std::uint8_t action, std::uint8_t clientCode, const volatile void* address) {
const std::uint8_t operation = action != 0 ? 7u : 6u;
const std::uint8_t signalType = clientCode & 1u;
const std::uint8_t waitForMailbox = (clientCode >> 1u) & 1u;
std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation) {
return Agc::Command::WriteMemSemaphore(buf, address, waitForMailbox, signalType, operation, __func__);
}
core/libs/prx/libSceAgc/tests/Semaphore.cpp
+38
View File
@@ -15,6 +15,8 @@
extern "C" int APS5_VABI sceAgcSetAmmSemaphoreMemory(void* memory, std::uint64_t sizeInBytes);
extern "C" int APS5_VABI sceAgcGetSemaphoreLabel(std::uint32_t index, void** label);
extern "C" std::uint32_t* APS5_VABI sceAgcCbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint32_t operation, std::uint32_t modifier, std::uint32_t value);
extern "C" std::uint32_t* APS5_VABI sceAgcAcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation);
extern "C" std::uint32_t* APS5_VABI sceAgcDcbMemSemaphore(CommandBuffer* buf, const volatile void* address, std::uint8_t waitForMailbox, std::uint8_t signalType, std::uint8_t operation);
namespace {
@@ -180,6 +182,40 @@ void testMemSemaphore() {
expectFailure([&] { sceAgcCbMemSemaphore(nullptr, label, 0, 0, 0); });
}
using QueueMemSemaphore = std::uint32_t* APS5_VABI (*)(CommandBuffer*, const volatile void*, std::uint8_t, std::uint8_t, std::uint8_t);
void testQueueMemSemaphore(QueueMemSemaphore write) {
const auto* semaphore = reinterpret_cast<const volatile void*>(0x102f221d98ull);
{
std::array<std::uint32_t, 16> words{};
words.fill(0xdeadbeefu);
CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
const auto* packet = write(&buffer, semaphore, 0, 1, 6);
check(packet == words.data(), "packet address mismatch");
check(packet[0] == 0xc0023900u, "MEM_SEMAPHORE header opcode mismatch");
check(packet[1] == 0x2f221d98u, "MEM_SEMAPHORE address low mismatch");
check(packet[2] == 0x10u, "MEM_SEMAPHORE address high mismatch");
check(packet[3] == 0xc0100000u, "MEM_SEMAPHORE signal control word mismatch");
check(buffer.cursor_up == words.data() + 4, "cursor did not advance by 4 words");
check(words[4] == 0xdeadbeefu, "words past packet modified");
}
{
std::array<std::uint32_t, 16> words{};
words.fill(0xdeadbeefu);
CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
const auto* packet = write(&buffer, semaphore, 1, 0, 7);
check(packet[3] == 0xe0010000u, "MEM_SEMAPHORE wait control word mismatch");
}
std::array<std::uint32_t, 16> words{};
CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0};
expectFailure([&] { write(&buffer, reinterpret_cast<const volatile void*>(1ull), 0, 1, 6); });
expectFailure([&] { write(&buffer, semaphore, 0, 1, 5); });
expectFailure([&] { write(&buffer, semaphore, 2, 0, 7); });
}
}
int main() {
@@ -191,6 +227,8 @@ int main() {
testRegistration();
testLabels();
testMemSemaphore();
testQueueMemSemaphore(sceAgcAcbMemSemaphore);
testQueueMemSemaphore(sceAgcDcbMemSemaphore);
testLabelRejections();
testRepeatedRegistration();
storage.reset();
core/libs/prx/libSceAgcDriver/CMakeLists.txt
+133 -2
View File
@@ -225,6 +225,22 @@ foreach(agcSampleOrderVariant IN ITEMS "R8;comp;-DTEXEL_FORMAT=r8ui" "R16;comp;-
)
list(APPEND agcSampleOrderShaderHeaders ${agcSampleOrderShaderHeader})
endforeach()
foreach(agcFirstSampleClass IN ITEMS Float Uint Sint)
string(TOUPPER "${agcFirstSampleClass}" agcFirstSampleSymbol)
set(agcFirstSampleShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/FirstSample.frag)
set(agcFirstSampleShaderSpv ${agcTextureDetileGeneratedDir}/FirstSample${agcFirstSampleClass}.spv)
set(agcFirstSampleShaderHeader ${agcTextureDetileGeneratedDir}/FirstSample${agcFirstSampleClass}_spv.h)
add_custom_command(
OUTPUT "${agcFirstSampleShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -D${agcFirstSampleSymbol}_SAMPLES=1 -o "${agcFirstSampleShaderSpv}" "${agcFirstSampleShaderSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcFirstSampleShaderSpv} -DOUTPUT=${agcFirstSampleShaderHeader}
-DSYMBOL=FIRST_SAMPLE_${agcFirstSampleSymbol}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcFirstSampleShaderSource}" glslang-standalone
VERBATIM
)
list(APPEND agcPresentationShaderHeaders ${agcFirstSampleShaderHeader})
endforeach()
foreach(agcOverlayStage IN ITEMS Vert Frag)
string(TOLOWER "${agcOverlayStage}" agcOverlayExtension)
string(TOUPPER "${agcOverlayStage}" agcOverlaySymbol)
@@ -323,6 +339,7 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Draw.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DepthSurface.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/MultisampleTarget.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/FirstSampleCopy.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetLayout.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp
@@ -714,6 +731,17 @@ target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel l
set_target_properties(agc_shared_write_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shared_write_tests)
if(BUILD_TESTING AND WIN32)
add_test_executable(agc_page_state_tests tests/PageStates.cpp)
target_include_directories(agc_page_state_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_page_state_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(agc_page_state_tests)
add_test(NAME agc_page_states COMMAND agc_page_state_tests)
add_test(NAME agc_page_state_metadata COMMAND agc_page_state_tests --metadata)
set_tests_properties(agc_page_states agc_page_state_metadata PROPERTIES
TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(agc_locks_tests EXCLUDE_FROM_ALL tests/Locks.cpp)
target_include_directories(agc_locks_tests PRIVATE
${LIBS_INCLUDE_DIR}
@@ -748,6 +776,19 @@ if(BUILD_TESTING)
endif()
endif()
if(BUILD_TESTING AND NOT WIN32)
add_test_executable(agc_collect_memo_generation_tests
tests/CollectMemoGeneration.cpp
Execution/src/GuestMemory.cpp
Execution/src/PerformanceTraceThread.cpp
)
target_include_directories(agc_collect_memo_generation_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_collect_memo_generation_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_collect_memo_generation_tests PROPERTIES CXX_EXTENSIONS OFF)
add_test(NAME agc_collect_memo_generation COMMAND agc_collect_memo_generation_tests)
set_tests_properties(agc_collect_memo_generation PROPERTIES TIMEOUT 30)
endif()
add_test_executable(agc_thread_priority_tests EXCLUDE_FROM_ALL
tests/ThreadPriority.cpp
Execution/src/ThreadPriority.cpp
@@ -1446,6 +1487,7 @@ target_include_directories(agc_driver_recorder_tests PRIVATE
)
target_link_libraries(agc_driver_recorder_tests PRIVATE
libSceAgc
libkernel
libSceAgcDriver
SDL2-static
@@ -1465,8 +1507,20 @@ if(BUILD_TESTING)
set_tests_properties(agc_recorder_device_selection agc_gpu_timing_complete agc_gpu_timing_incomplete PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_command_memory_reuse COMMAND agc_driver_recorder_tests --command-reuse-only)
set_tests_properties(agc_command_memory_reuse PROPERTIES SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_command_memory_reuse PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only)
add_test(NAME agc_pipeline_cache_file COMMAND agc_driver_recorder_tests --pipeline-cache-file-only)
add_test(NAME agc_pipeline_cache_header COMMAND agc_driver_recorder_tests --pipeline-cache-header-only)
set_tests_properties(agc_pipeline_cache_header PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_pipeline_cache_header PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_pipeline_library COMMAND agc_driver_recorder_tests --pipeline-library-only)
add_test(NAME agc_pipeline_library_off COMMAND agc_driver_recorder_tests --pipeline-library-only)
set_tests_properties(agc_pipeline_library agc_pipeline_library_off PROPERTIES SKIP_RETURN_CODE 77)
@@ -1489,6 +1543,13 @@ if(BUILD_TESTING)
set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77)
add_test(NAME agc_import_settle_deferred COMMAND agc_driver_recorder_tests --deferred-settle-only)
add_test(NAME agc_import_settle_deferred_other_family COMMAND agc_driver_recorder_tests --deferred-settle-other-family-only)
add_test(NAME agc_import_settle_write_order COMMAND agc_driver_recorder_tests --settle-write-order-only)
add_test(NAME agc_import_settle_write_order_other_family COMMAND agc_driver_recorder_tests --settle-write-order-other-family-only)
add_test(NAME agc_import_settle_wait_failure COMMAND agc_driver_recorder_tests --settle-wait-failure-only)
set_tests_properties(agc_import_settle_write_order agc_import_settle_write_order_other_family agc_import_settle_wait_failure PROPERTIES
SKIP_RETURN_CODE 77 TIMEOUT 30
FAIL_REGULAR_EXPRESSION "SYNC-HAZARD-|VUID-"
ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
add_test(NAME agc_import_settle_first_use COMMAND agc_driver_recorder_tests --settle-first-use-only)
add_test(NAME agc_import_settle_earlier_import COMMAND agc_driver_recorder_tests --settle-earlier-import-only)
add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only)
@@ -1700,10 +1761,12 @@ if(BUILD_TESTING)
set_tests_properties(agc_shadowed_key_stores PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
add_test(NAME agc_sample_dumps COMMAND agc_driver_recorder_tests --sample-dumps-only)
set_tests_properties(agc_sample_dumps PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
add_test(NAME agc_uncounted_passes COMMAND agc_driver_recorder_tests --uncounted-passes-only)
set_tests_properties(agc_uncounted_passes PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
add_test(NAME agc_draw_descriptors COMMAND agc_driver_recorder_tests --draw-descriptors-only)
set_tests_properties(agc_draw_descriptors PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60)
if(WIN32)
set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
set_tests_properties(agc_pending_key_stores agc_shadowed_key_stores agc_sample_dumps agc_uncounted_passes agc_draw_descriptors PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_host_import_lifetime COMMAND agc_driver_recorder_tests --host-import-lifetime-only)
@@ -1903,6 +1966,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
tests/GuestSamplerResource.cpp
tests/TextureDetiler.cpp
tests/DepthSurfaceReuse.cpp
tests/LastRecordIndices.cpp
)
if(BUILD_TESTING)
@@ -2070,6 +2134,13 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_address_passes_targets_split PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_target_properties(agc_driver_rcp_hardware_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_driver_rcp_hardware_exact COMMAND agc_driver_rcp_hardware_tests)
set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT "APS5_EXACT_RCP=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60)
if(WIN32)
set_tests_properties(agc_driver_rcp_hardware_exact PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
foreach(agcExecutionTestName IN LISTS ANYPS5_AGC_EXECUTION_TESTS)
if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/execution/${agcExecutionTestName}.cpp)
@@ -2209,6 +2280,12 @@ target_link_libraries(agc_driver_job_manager_compute_tests PRIVATE libSceJobMana
target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_unorm4444_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_unorm4444_partial_write_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_unorm4444_multisample_partial_write_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_color1555_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_color565_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -2361,7 +2438,7 @@ if(BUILD_TESTING)
set_tests_properties(agc_shader_bindless agc_shader_selected_records_image_limit agc_shader_selected_records_pair_limit agc_shader_selected_records_direct_image_limit agc_shader_selected_records_direct_pair_limit agc_shader_srt_read_patching agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off agc_shader_entry_lane_proof_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(agcTest IN ITEMS agc_vertex_input_tests agc_mesh_output_budget_tests agc_shared_write_tests agc_locks_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_ir_reclaim_tests agc_vector_window_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vertex_input_validation_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_barrier_batch_tests agc_shader_disk_cache_tests agc_screenshot_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests agc_formatted_buffer_byte_stores_tests ${agcExecutionTests} agc_fma_mix_once_tests agc_f16_single_convert_tests)
if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests")
if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests" OR agcTest STREQUAL "agc_driver_pipeline_optimized_readback_tests" OR agcTest STREQUAL "agc_driver_rcp_hardware_tests")
continue()
endif()
string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
@@ -2394,6 +2471,12 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_draw_unsupported_buffer_type_skip_refused PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_period_copy_no_hle COMMAND agc_driver_period_copy_tests)
set_tests_properties(agc_driver_period_copy_no_hle PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_COPY_HLE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_period_copy_no_hle PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_sparse_indirect_indices_unreadable COMMAND agc_driver_sparse_indirect_indices_tests --unreadable)
set_tests_properties(agc_driver_sparse_indirect_indices agc_driver_sparse_indirect_indices_unreadable PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
@@ -2425,6 +2508,16 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_draw_pipeline_trace_disabled agc_driver_draw_finisher_trace agc_driver_draw_finisher_trace_disabled PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_copied_writer_label_order_drain COMMAND agc_driver_copied_writer_label_order_tests)
set_tests_properties(agc_driver_copied_writer_label_order_drain PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAIN_COMPLETION_LABELS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 30)
if(WIN32)
set_tests_properties(agc_driver_copied_writer_label_order_drain PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_lazy_resolved_abi_switch_set COMMAND agc_driver_lazy_resolved_abi_tests)
set_tests_properties(agc_driver_lazy_resolved_abi_switch_set PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_LAZY_SHADER_PREPARE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_lazy_resolved_abi_switch_set PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_driver_queued_raw_shader PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=8" TIMEOUT 30)
set(agcDrawPipelineTests agc_driver_draw_write_ranges agc_driver_draw_unsupported_buffer_type agc_driver_bda_span_reads agc_driver_copied_writer_copy agc_driver_image_table_reuse agc_driver_depth_storage_reuse agc_driver_staged_clear_sampled_read agc_driver_storage_write_back_overhang agc_driver_write_data_written_ahead agc_driver_gated_dispatch_capture agc_driver_buffer_load_store agc_driver_pipelined_dma agc_driver_pipelined_fill agc_driver_pipelined_predication_read agc_driver_pipelined_predicated_label_order agc_driver_pipelined_metadata_pass agc_driver_draw_cache_reuse agc_driver_draw_cache_relocation agc_driver_draw_data_hit_user_words agc_driver_finisher_label_order agc_driver_same_queue_label_wait agc_driver_unimported_label_order)
foreach(agcTestName IN LISTS agcDrawPipelineTests)
@@ -2760,6 +2853,15 @@ if(BUILD_TESTING)
set_tests_properties(agc_sampled_cache_device_selection agc_driver_sampled_cache_device_selection PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(imageRecoveryMode IN ITEMS sampled storage srgb repack)
add_test(NAME agc_driver_recover_${imageRecoveryMode}_allocation COMMAND agc_driver_sampled_cache_budget_tests recover-${imageRecoveryMode})
set_tests_properties(agc_driver_recover_${imageRecoveryMode}_allocation PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30)
if(WIN32)
set_tests_properties(agc_driver_recover_${imageRecoveryMode}_allocation PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
set_property(TEST agc_driver_recover_srgb_allocation APPEND PROPERTY ENVIRONMENT_MODIFICATION "APS5_SRGB_SHADER_DECODE=set:copy")
add_test(NAME agc_driver_resource_cache_storage_admission COMMAND agc_driver_sampled_cache_budget_tests resource-admission)
add_test(NAME agc_driver_resource_cache_oversized_storage_admission COMMAND agc_driver_sampled_cache_budget_tests resource-oversized)
add_test(NAME agc_driver_resource_cache_storage_clear COMMAND agc_driver_sampled_cache_budget_tests resource-clear)
@@ -2839,6 +2941,12 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77)
add_test(NAME agc_driver_draw_cache_reuse_verify COMMAND agc_driver_draw_cache_reuse_tests)
set_tests_properties(agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_DATA_HITS=1" SKIP_RETURN_CODE 77)
add_test(NAME agc_driver_draw_cache_reuse_entry_verify COMMAND agc_driver_draw_cache_reuse_tests)
set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_ENTRIES=1" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
if(WIN32)
set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_draw_cache_reuse_inline COMMAND agc_driver_draw_cache_reuse_tests)
set_tests_properties(agc_driver_draw_cache_reuse_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0;APS5_VERIFY_DRAW_RECIPE=1;APS5_VERIFY_REGISTER_KEYS=1" SKIP_RETURN_CODE 77)
set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_inline PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:")
@@ -2955,6 +3063,16 @@ if(BUILD_TESTING)
if(WIN32)
set_tests_properties(agc_driver_shutdown_threads_joined agc_driver_shutdown_threads_queued_prebuilds agc_driver_shutdown_threads_compare_helpers agc_driver_shutdown_threads_draw_committer agc_driver_shutdown_threads_shader_cache_writer agc_driver_shutdown_threads_late_shader_cache agc_driver_shutdown_threads_async_log_held agc_driver_shutdown_threads_exit_after_start PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_prepared_shaders_abi_deferred_concurrent_lazy COMMAND agc_driver_prepared_shaders_tests --abi-deferred-concurrent)
set_tests_properties(agc_driver_prepared_shaders_abi_deferred_concurrent_lazy PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FAILURE_MEMO=1;APS5_LAZY_SHADER_PREPARE=1;APS5_SUSPEND_DRAIN=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_prepared_shaders_abi_deferred_concurrent_lazy PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_prepared_shaders_abi_deferred_compute_lazy COMMAND agc_driver_prepared_shaders_tests --abi-deferred-compute)
set_tests_properties(agc_driver_prepared_shaders_abi_deferred_compute_lazy PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FAILURE_MEMO=1;APS5_LAZY_SHADER_PREPARE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_prepared_shaders_abi_deferred_compute_lazy PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_compute_user_data_calls_registered COMMAND agc_driver_compute_user_data_calls_tests --registered)
set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
@@ -3170,12 +3288,15 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_covered_centroid_state PROPERTIES TIMEOUT 30)
add_test(NAME agc_driver_sample_location_grid_fallback COMMAND agc_driver_graphics_tests --sample-location-grid-fallback)
set_tests_properties(agc_driver_sample_location_grid_fallback PROPERTIES TIMEOUT 30)
add_test(NAME agc_driver_unobserved_provoking_vertex COMMAND agc_driver_graphics_tests --unobserved-provoking-vertex)
set_tests_properties(agc_driver_unobserved_provoking_vertex PROPERTIES TIMEOUT 30)
add_test(NAME agc_driver_sample_order_planning COMMAND agc_driver_graphics_tests --sample-order)
set_tests_properties(agc_driver_sample_order_planning PROPERTIES TIMEOUT 30)
if(WIN32)
set_tests_properties(agc_compute_descriptor_layout_flags PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_driver_covered_centroid_state PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_driver_sample_location_grid_fallback PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_driver_unobserved_provoking_vertex PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_driver_sample_order_planning PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_compute_pipeline_capture COMMAND agc_compute_pipeline_capture_tests)
@@ -3631,6 +3752,16 @@ if(BUILD_TESTING)
endif()
endif()
if(BUILD_TESTING)
add_test_executable(agc_flat_scratch_tests tests/FlatScratch.cpp)
target_link_libraries(agc_flat_scratch_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(agc_flat_scratch_tests)
add_test(NAME agc_flat_scratch COMMAND agc_flat_scratch_tests)
set_tests_properties(agc_flat_scratch PROPERTIES TIMEOUT 30)
if(WIN32)
set_tests_properties(agc_flat_scratch PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_constant_select_tests EXCLUDE_FROM_ALL tests/ConstantSelect.cpp)
target_link_libraries(agc_constant_select_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_constant_select_tests PROPERTIES CXX_EXTENSIONS OFF)
core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp
+2
View File
@@ -3,6 +3,7 @@
#include <cstdint>
#include <span>
#include <cstddef>
#include "SceTypes.hpp"
#include "SceShaders.hpp"
@@ -12,6 +13,7 @@ void Submit(const Packet* packet, std::uint32_t queue);
}
extern "C" void AgcDriverPrepareCommandWrite_nid_no_patch(std::uint64_t address, std::size_t bytes);
extern "C" void AgcDriverWaitIdle_nid_postfix();
extern "C" void AgcDriverRaiseWorkerThreadPriority_nid_postfix(const char* role);
extern "C" void AgcDriverLockVulkanLoader_nid_postfix();
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp
+12 -1
View File
@@ -74,6 +74,17 @@ public:
void ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
void SettleShaderPreparations(std::span<const Shader* const> stages);
[[nodiscard]] std::size_t PreparedEntries(const Shader* shader);
struct DeferredAbiUseCounts {
std::uint64_t firstUses;
std::uint64_t found;
std::uint64_t own;
std::uint64_t entries;
std::uint64_t unmatchedEntries;
};
[[nodiscard]] static DeferredAbiUseCounts DeferredAbiUses();
[[nodiscard]] std::vector<std::array<std::uint64_t, 2>> PreparedEntryHashes(const Shader* shader);
[[nodiscard]] std::array<std::uint64_t, 4> PrebuiltLibraries();
static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
static void tagVariantRuns(DispatchVariant& variant, std::span<const ShaderRecompiler::MemoryRegion> regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span<const std::shared_ptr<const ShaderRecompiler::SrtGuestWords>> supplied = {});
@@ -174,7 +185,7 @@ private:
void lookupDraw(const Submission& submission, const std::shared_ptr<VulkanDevice>& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<DrawProgram>& programs, const std::vector<ShaderRecompiler::ProgramRole>& roles, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr<DrawEntry>& entry, std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, const std::shared_ptr<DrawEntry>& dataEntry, DrawDataCandidates& dataCandidates, std::vector<StageDataHit>& stageData);
std::shared_ptr<const ShaderRecompiler::RecompileResult> materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector<DrawProgram>& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, std::vector<ShaderRecompiler::MemoryRegion>& memory, const std::vector<ShaderRecompiler::LinkedProgram>& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr<VulkanDevice>& localDevice, ShaderMemory& shaderMemory, std::vector<StageCapture>& stageCaptures, std::vector<bool>& recompiled, bool drawHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<std::vector<ShaderRecompiler::MemoryRegion>>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array<double, DrawDriverPhaseCount>& phaseMs, std::string& rejected, const std::optional<ShaderRecompiler::MeshBarycentricLayout>& meshBarycentrics);
static void recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector<Graphics::DecodeRead>& decodeReads);
void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry = false);
void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<StageDataHit>& stageData, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry = false);
static bool drawPrecheck();
std::optional<DrawVerdict> precheckDraw(const QueueState& queue, const Submission& submission, std::span<const std::uint32_t> packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect);
static std::uint32_t drawUserWord(const DrawProgram& program, std::int32_t sgpr);
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp
+10
View File
@@ -127,6 +127,8 @@ struct PreparedShaderState {
struct Entry {
std::size_t codeOffset;
std::shared_ptr<const ShaderRecompiler::SourceHandle> handle;
bool fromDeferredAbi = false;
bool matched = false;
};
struct Rectangle {
std::uint64_t vertexId;
@@ -153,6 +155,14 @@ struct PreparedShaderState {
bool deferred = false;
};
struct PreparedShaders : PreparedShaderState {
struct DeferredAbi {
std::vector<std::uint64_t> key;
std::vector<std::pair<std::uint32_t, std::uint32_t>> context;
std::vector<std::pair<std::uint32_t, std::uint32_t>> primitive;
std::weak_ptr<VulkanDevice> device;
};
std::vector<DeferredAbi> deferredAbis;
bool deferredPreparing = false;
std::mutex mutex;
std::condition_variable settled;
std::exception_ptr failure;
core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp
+1
View File
@@ -16,6 +16,7 @@ public:
ShaderPreparationTransaction& operator=(const ShaderPreparationTransaction&) = delete;
PreparedShaderState& Edit(const ShaderSnapshot& snapshot);
const PreparedShaderState& Read(const ShaderSnapshot& snapshot) const;
bool Tracks(const ShaderSnapshot& snapshot) const;
void Hold();
void Commit();
core/libs/prx/libSceAgcDriver/Execution/include/SubgroupBarrier.hpp
+4
View File
@@ -9,6 +9,10 @@ inline bool SubgroupControlBarrierUsable(VkDriverId driver) {
return driver != VK_DRIVER_ID_AMD_PROPRIETARY;
}
inline bool WorkgroupScopeFenceUsable(VkDriverId driver) {
return driver != VK_DRIVER_ID_AMD_PROPRIETARY;
}
}
#endif
core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp
+6 -4
View File
@@ -33,6 +33,7 @@ struct ComputePipelineObjects;
struct RecipeHit;
// The record tail shared by an ordinary dispatch and a recipe hit (VulkanDevice::recordDispatch).
struct RecordedDispatch;
struct DispatchCompletion;
// The module, layout and pipeline a compute dispatch binds (VulkanDevice::createComputePipeline).
struct ComputePipelineObjects;
@@ -110,10 +111,10 @@ public:
// Records a store of `bytes` at a guest address behind the recorded work, so the value appears
// once that work completed; the batch is submitted by the queue worker's flush rules (or at once
// with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording
// queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept
// as a completion action behind pending write-backs, else why the CPU must write it:
// 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not
// imported, 4 unsuitable size or alignment.
// queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when stored
// by a completion behind pending CPU write-backs or completion stores, else why the CPU must
// write it: 1 nothing recorded (the write is already ordered), 2 write-backs pending (only with
// APS5_DRAIN_COMPLETION_LABELS=1) and 3 memory not imported, 4 unsuitable size or alignment.
// `reapFirst` retires finished batches before the checks; a caller recording a group of labels
// under one lock passes it for the first label only.
int WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true);
@@ -372,6 +373,7 @@ private:
Graphics::Context buildContext() const;
// Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups.
IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipe, const std::array<std::uint32_t, 3>& threadGroup = {}, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear = std::nullopt);
IndirectOutcome recordCompletedDispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear, DispatchCompletion& completion);
// The stage-A pre-sync over `surfaces` (see PrepareDispatch): the serial waited for, 0 none.
std::uint64_t presync(std::span<const Graphics::PresyncSurface> surfaces);
std::shared_ptr<ComputePipelineObjects> computePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span<const std::uint32_t> layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress);
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp
+3 -3
View File
@@ -828,7 +828,7 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) {
return lastData[shapeKey % lastData.size()].exchange(data, std::memory_order_relaxed) == data;
};
const bool shapeNeedsEntry = shapeRefresh && registerKey && shapeKey != 0 && drawDataHits() && !drawParameters.indirect && dataEntry != nullptr && dataStagesCompiled != 0 && refreshSeenTwice();
cacheDrawStages(stageEntries, drawHit, drawParameters, indirectCpu, programs, stageCaptures, vertexInfos, decodeReads, verifyHit, matched, fresh, drawKey, registerKey, decode, phaseTiming, reused, entry, shapeKey, shapeNeedsEntry);
cacheDrawStages(stageEntries, drawHit, drawParameters, indirectCpu, programs, stageCaptures, vertexInfos, decodeReads, verifyHit, matched, stageData, fresh, drawKey, registerKey, decode, phaseTiming, reused, entry, shapeKey, shapeNeedsEntry);
timing.Mark("shader_compile_and_link");
for (const auto& reads : decodeReads) {
@@ -875,8 +875,8 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) {
require(programResults[0]->pushConstants.size() == pushBytes && programResults[0]->parameterExports == exports, "pairing the mesh program with its pixel shader changed the push constants or the parameter exports the pixel shader was built for");
}
require(programResults[0]->meshBarycentrics == meshBarycentrics, "the mesh program was compiled for a different barycentric layout than its pixel shader");
} else if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
} else if (!graphics.rectList && !programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
require(graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
const auto geometryLimits = localDevice->GeometryLimits();
if (geometryLimits || !localDevice->Target().tessellation) {
results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), geometryLimits));
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp
+13 -9
View File
@@ -152,7 +152,7 @@ void Driver::recordStageDataPositions(DispatchVariant& variant, const ShaderReco
variant.dataSlots.resize(kept);
}
void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry) {
void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional<Graphics::IndirectDrawPath>& indirectCpu, const std::vector<DrawProgram>& programs, const std::vector<StageCapture>& stageCaptures, const std::vector<std::optional<ShaderRecompiler::ShaderVertexStageInfo>>& vertexInfos, const std::vector<std::vector<Graphics::DecodeRead>>& decodeReads, bool verifyHit, const std::vector<std::shared_ptr<DispatchVariant>>& matched, const std::vector<StageDataHit>& stageData, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr<const DrawDecode>& decode, DrawPhaseTiming& phaseTiming, const std::vector<bool>& reused, const std::shared_ptr<DrawEntry>& entry, std::uint64_t shapeKey, bool shapeNeedsEntry) {
const bool cacheable = useDrawEntries && !drawHit && !(drawParameters.indirect && indirectCpu);
const bool admitted = cacheable && (verifyHit || admitDrawKey(drawKey, registerKey && drawDataHits() ? shapeKey : 0));
const bool replaceShape = cacheable && !admitted && shapeNeedsEntry;
@@ -192,17 +192,17 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
++differingRunsChanged;
}
}
const auto sameContent = [](const DispatchVariant& a, const DispatchVariant& b) {
const auto flatten = [](const DispatchVariant& source) {
const auto sameContent = [](const DispatchVariant& a, const std::vector<std::uint32_t>& words, const DispatchVariant& b) {
const auto flatten = [](const DispatchVariant& source, const std::vector<std::uint32_t>& values) {
std::vector<std::pair<std::uint64_t, std::uint32_t>> flat;
std::size_t word = 0;
for (const auto& [begin, end] : source.runs) {
for (auto address = begin; address < end && word < source.words.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, source.words[word++]);
for (auto address = begin; address < end && word < values.size(); address += sizeof(std::uint32_t)) flat.emplace_back(address, values[word++]);
}
std::sort(flat.begin(), flat.end());
return flat;
};
return (a.runs == b.runs && a.words == b.words) || flatten(a) == flatten(b);
return (a.runs == b.runs && words == b.words) || flatten(a, words) == flatten(b, b.words);
};
const auto sameForwardedWords = [](const ShaderRecompiler::RecompileResult& a, const ShaderRecompiler::RecompileResult& b, std::uint32_t hidden) {
if (a.pushConstants.size() != b.pushConstants.size() || a.shaderDataUserWords != b.shaderDataUserWords) return false;
@@ -211,10 +211,14 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
}
return true;
};
if (verifyHit && matched[i] != nullptr && (!sameContent(*matched[i], *variant) || !sameForwardedWords(*matched[i]->compiled, *variant->compiled, programs[i].hiddenUserWords))) {
++mismatches;
static std::atomic<std::uint64_t> reports{0};
if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast<unsigned long long>(programs[i].binary.codeAddress), matched[i]->runs.size(), matched[i]->words.size(), variant->runs.size(), variant->words.size());
if (verifyHit && matched[i] != nullptr) {
const auto& words = stageData[i].compiled != nullptr ? stageData[i].words : matched[i]->words;
const auto& compiled = stageData[i].compiled != nullptr ? *stageData[i].compiled : *matched[i]->compiled;
if (!sameContent(*matched[i], words, *variant) || !sameForwardedWords(compiled, *variant->compiled, programs[i].hiddenUserWords)) {
++mismatches;
static std::atomic<std::uint64_t> reports{0};
if (reports.fetch_add(1) < 20) std::fprintf(stderr, "[draw-cache] verify: stage %zu (program 0x%llx) of a hit captured differently: %zu runs / %zu words matched, %zu / %zu fresh\n", i, static_cast<unsigned long long>(programs[i].binary.codeAddress), matched[i]->runs.size(), words.size(), variant->runs.size(), variant->words.size());
}
}
if (insertCompare()) {
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DrawCache);
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp
+1
View File
@@ -185,6 +185,7 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
product->pixel.dualSourceBlend = product->state.dualSourceBlend;
product->pixel.reversedBlend = product->state.reversedBlend;
product->pixel.targetMerge = Graphics::ExportTargetMerge(product->state);
product->pixel.alphaToCoverage = product->state.alphaToCoverage;
product->pixel.quadPixelMask = Graphics::QuadPixelMask(product->state);
if (product->state.sampleLocationGrid) {
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp
+6
View File
@@ -79,6 +79,12 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
for (const auto value : pixel.targetExportPacking) mix(static_cast<std::uint64_t>(value));
mix(pixel.dualSourceBlend);
mix(pixel.reversedBlend);
mix(pixel.targetMerge.target);
mix(pixel.targetMerge.mask);
mix(pixel.targetMerge.samples);
for (const auto word : pixel.targetMerge.descriptor) mix(word);
mix(pixel.targetMerge.depthStencilWrites);
mix(pixel.targetMerge.depthStencilTests);
mix(pixel.alphaToCoverage);
mix(static_cast<std::uint64_t>(pixel.rasterPrimitive));
mix(pixel.ditheredAlphaToCoverage);
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp
+3 -3
View File
@@ -228,7 +228,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
const auto& s = a.state;
const auto& t = b.state;
const auto sameColor = [](const Graphics::ColorTarget& x, const Graphics::ColorTarget& y) {
return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot && x.exportIndex == y.exportIndex;
return x.address == y.address && x.extent.width == y.extent.width && x.extent.height == y.extent.height && x.format == y.format && x.bytes == y.bytes && x.componentMapping == y.componentMapping && x.tileMode == y.tileMode && x.elementBytes == y.elementBytes && x.dccAddress == y.dccAddress && x.dccAlphaOnMsb == y.dccAlphaOnMsb && x.slot == y.slot && x.exportIndex == y.exportIndex && x.partialWriteMask == y.partialWriteMask;
};
const auto sameBlend = [](const VkPipelineColorBlendAttachmentState& x, const VkPipelineColorBlendAttachmentState& y) {
return x.blendEnable == y.blendEnable && x.srcColorBlendFactor == y.srcColorBlendFactor && x.dstColorBlendFactor == y.dstColorBlendFactor && x.colorBlendOp == y.colorBlendOp && x.srcAlphaBlendFactor == y.srcAlphaBlendFactor && x.dstAlphaBlendFactor == y.dstAlphaBlendFactor && x.alphaBlendOp == y.alphaBlendOp && x.colorWriteMask == y.colorWriteMask;
@@ -262,8 +262,8 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
const auto& y = b.programs[i];
if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset || x.tailContinuation != y.tailContinuation) return false;
}
if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.reversedBlend != q.reversedBlend || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive || p.ditheredAlphaToCoverage != q.ditheredAlphaToCoverage || p.quadPixelMask != q.quadPixelMask || p.quadSampleMask != q.quadSampleMask || p.iteratesSamples != q.iteratesSamples || p.positionLocation != q.positionLocation || p.sampleLocations != q.sampleLocations || p.sampleOrder != q.sampleOrder) return false;
if (s.samples != t.samples || s.mergedCoverage != t.mergedCoverage || s.truncatedCoverage != t.truncatedCoverage || s.quadSampleMask != t.quadSampleMask || s.pixelEarlyZ != t.pixelEarlyZ) return false;
if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.reversedBlend != q.reversedBlend || p.targetMerge != q.targetMerge || p.alphaToCoverage != q.alphaToCoverage || p.rasterPrimitive != q.rasterPrimitive || p.ditheredAlphaToCoverage != q.ditheredAlphaToCoverage || p.quadPixelMask != q.quadPixelMask || p.quadSampleMask != q.quadSampleMask || p.iteratesSamples != q.iteratesSamples || p.positionLocation != q.positionLocation || p.sampleLocations != q.sampleLocations || p.sampleOrder != q.sampleOrder) return false;
if (s.samples != t.samples || s.mergedCoverage != t.mergedCoverage || s.truncatedCoverage != t.truncatedCoverage || s.quadSampleMask != t.quadSampleMask || s.pixelEarlyZ != t.pixelEarlyZ || s.pixelKill != t.pixelKill) return false;
return true;
}
core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp
+6
View File
@@ -1,3 +1,4 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Screenshot.hpp"
@@ -160,3 +161,8 @@ extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Sh
} catch (const ProcessShutdown&) {
LibcAwaitExit_nid_postfix();
}
extern "C" void AgcDriverPrepareCommandWrite_nid_no_patch(std::uint64_t address, std::size_t bytes) {
if (AgcDriver::Graphics::Recorder::SnapshotWriteOverlaps(address, bytes))
AgcDriver::GuestMemory::FlushGpuWrites(address, bytes);
}
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp
+4
View File
@@ -14,6 +14,7 @@
#include "prx/libc/include/GuestAllocations.hpp"
#include "ShaderDiskCache.hpp"
#include "prx/libc/include/GuestArena.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include <atomic>
#include <cstdint>
#include <cstdio>
@@ -87,6 +88,9 @@ void MapDriverMemory() {
auto* slot = reinterpret_cast<std::uint64_t*>(FsTableAddress);
slot[0] = (FsRoutineTableAddress & 0xffffffffffffull) | 16ull << 48u;
slot[1] = FsTableDword3 << 32u | FsRoutineRecords;
#ifndef _WIN32
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(mapped, DriverMemoryBytes);
#endif
GuestAllocations::Mutation mutation;
mutation.Add(mapped, DriverMemoryBytes, true, true, true);
}
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp
+2 -2
View File
@@ -18,14 +18,14 @@ bool Driver::matchesCopyKernel(std::span<const std::uint32_t> code, const std::v
static const bool enabled = std::getenv("APS5_NO_COPY_HLE") == nullptr;
if (!enabled || userData.size() != 12 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1 || !compute.groupIdEnable[0]) return false;
static constexpr std::size_t copyKernelWords = 38;
static constexpr std::uint64_t copyKernelHash = 0x6ec00fe8aa95f99aull;
static constexpr std::array<std::uint64_t, 2> copyKernelHashes{0x6ec00fe8aa95f99aull, 0xc771c42e4d069d62ull};
if (code.size() < copyKernelWords || code[0] != 0xbfa00002u || code[copyKernelWords - 1] != 0xbf810000u) return false;
std::uint64_t hash = 0xcbf29ce484222325ull;
for (const auto word : code.first(copyKernelWords)) {
hash ^= word;
hash *= 0x100000001b3ull;
}
if (hash != copyKernelHash) return false;
if (std::find(copyKernelHashes.begin(), copyKernelHashes.end(), hash) == copyKernelHashes.end()) return false;
const auto stride = [&](std::size_t word) { return (userData[word] >> 16u) & 0x3fffu; };
const auto swizzled = [&](std::size_t word) { return ((userData[word] >> 31u) & 1u) != 0; };
return userData[3] == 0x00014004u && userData[7] == 0x00014004u && userData[11] == 0x0004dfacu && stride(1) == 4 && stride(5) == 4 && stride(9) == 16 && !swizzled(1) && !swizzled(5) && !swizzled(9) && userData[10] >= 1;
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp
+2
View File
@@ -303,8 +303,10 @@ void Driver::execute(const Submission& submission) {
continue;
}
}
timing.Mark("pipeline_prepare");
if (pipelined && DrawPipeline::Queue0().Busy() && !keepsDrawPipeline(packet, header, opcode)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, header == FlipPacketHeader ? 0xffu : opcode);
if (pipelined && predicated && DrawPipeline::Queue0().Busy() && DrawPipeline::Queue0().Overlaps(queue.predication.address, queue.predication.operation == 3 ? 8 : 4)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode);
timing.Mark("pipeline_drain");
std::shared_lock deviceUse(deviceReplacement, std::defer_lock);
if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock();
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp
+1
View File
@@ -282,6 +282,7 @@ void Driver::executeRewindTail(const Submission& stalled) {
checkStopping();
noteAwaitingTitle(stalled.queue, reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
stall.Check("REWIND at", reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
if (stalled.queue == 0 && completionsPending()) reapCompletionLabels();
PollSleep();
}
}
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp
+248 -9
View File
@@ -1,9 +1,11 @@
#include "prx/libc/include/general/LogMacros.hpp"
#include "ControlFlow/GraphBuilder.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "CacheKey.hpp"
@@ -220,6 +222,24 @@ bool CommittedAbi(const ShaderSnapshot& snapshot, std::span<const std::uint64_t>
return std::ranges::any_of(snapshot.prepared->registeredAbis, [&](const auto& abi) { return std::ranges::equal(abi, key); });
}
std::optional<std::vector<std::weak_ptr<const ShaderSnapshot>>> CommittedGraphicsAbi(const ShaderSnapshot& owner, std::span<const std::uint64_t> key, const std::shared_ptr<const ShaderRegistry>& registry) {
std::unique_lock lock(owner.prepared->mutex);
AwaitRegisteredPreparation(*owner.prepared, lock);
const auto& abis = owner.prepared->graphicsAbis;
const auto found = std::ranges::find_if(abis, [&](const auto& abi) { return std::ranges::equal(abi.key, key) && abi.registry.lock() == registry; });
if (found == abis.end()) return std::nullopt;
return found->stages;
}
bool CommittedLink(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType) {
if (primitiveType == 7 || primitiveType == 17) return false;
std::unique_lock lock(front.prepared->mutex);
AwaitRegisteredPreparation(*front.prepared, lock);
const auto& fragments = front.prepared->fragments;
if (front.prepared->rectangleRequested || std::ranges::any_of(fragments, [](const auto& entry) { return entry.expired(); })) return false;
return fragment == nullptr || std::ranges::any_of(fragments, [&](const auto& entry) { return entry.lock() == fragment; });
}
}
bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset) {
@@ -561,6 +581,10 @@ const PreparedShaderState& ShaderPreparationTransaction::Read(const ShaderSnapsh
return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, false, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared;
}
bool ShaderPreparationTransaction::Tracks(const ShaderSnapshot& snapshot) const {
return root->changes.contains(snapshot.prepared.get());
}
void ShaderPreparationTransaction::Hold() {
require(!committed, "shader preparation transaction is already committed");
root->Hold();
@@ -801,6 +825,65 @@ void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const st
namespace {
bool LazyShaderPreparation() {
static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr;
return lazy;
}
using PreparedAtUse = std::optional<std::vector<const ShaderRecompiler::SourceHandle*>>;
PreparedAtUse PrepareDeferredAbis(const ShaderSnapshot& snapshot, std::unique_lock<std::mutex>& lock);
struct DeferredAbiUseCounters {
std::atomic<std::uint64_t> firstUses{0};
std::atomic<std::uint64_t> found{0};
std::atomic<std::uint64_t> own{0};
std::atomic<std::uint64_t> entries{0};
std::atomic<std::uint64_t> matchedEntries{0};
std::atomic<std::int64_t> lastReport{0};
};
DeferredAbiUseCounters& DeferredAbiCounters() {
static DeferredAbiUseCounters counters;
return counters;
}
void ReportDeferredAbiUses() {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (!profile) return;
auto& counters = DeferredAbiCounters();
const auto now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
auto previous = counters.lastReport.load(std::memory_order_relaxed);
if ((previous != 0 && now - previous < 60000) || !counters.lastReport.compare_exchange_strong(previous, now, std::memory_order_relaxed)) return;
const auto entries = counters.entries.load(std::memory_order_relaxed);
const auto matched = counters.matchedEntries.load(std::memory_order_relaxed);
AgcDriver::ProfilePrint_nid_no_patch("[shaders] deferred ABIs: %llu prepared at first use, %llu found their artifact there, %llu prepared their own; %llu of the %llu entries prepared from them never matched a use\n", static_cast<unsigned long long>(counters.firstUses.load(std::memory_order_relaxed)), static_cast<unsigned long long>(counters.found.load(std::memory_order_relaxed)), static_cast<unsigned long long>(counters.own.load(std::memory_order_relaxed)), static_cast<unsigned long long>(entries - std::min(entries, matched)), static_cast<unsigned long long>(entries));
}
void NoteMatchedEntry(PreparedShaders::Entry& entry, const PreparedAtUse& preparedAtUse) {
auto& counters = DeferredAbiCounters();
bool changed = false;
if (entry.fromDeferredAbi && !entry.matched) {
entry.matched = true;
counters.matchedEntries.fetch_add(1, std::memory_order_relaxed);
changed = true;
}
if (preparedAtUse) {
counters.firstUses.fetch_add(1, std::memory_order_relaxed);
if (std::ranges::find(*preparedAtUse, entry.handle.get()) != preparedAtUse->end()) counters.found.fetch_add(1, std::memory_order_relaxed);
changed = true;
}
if (changed) ReportDeferredAbiUses();
}
void NoteOwnPreparation(const PreparedAtUse& preparedAtUse) {
if (!preparedAtUse) return;
auto& counters = DeferredAbiCounters();
counters.firstUses.fetch_add(1, std::memory_order_relaxed);
counters.own.fetch_add(1, std::memory_order_relaxed);
ReportDeferredAbiUses();
}
void ReportPreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) {
if (request.context.convertedSamples) {
APS5_LOG_OUT_DEBUG("Shader 0x%llx samples a converted image; preparing its converted-sample variant at use", static_cast<unsigned long long>(request.shader.codeAddress));
@@ -836,9 +919,15 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const Shad
ShaderRecompiler::BuildPreparedShaderKey(request, key);
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle;
PreparedAtUse preparedAtUse;
if (!snapshot.prepared->deferredAbis.empty() || snapshot.prepared->deferredPreparing) preparedAtUse = PrepareDeferredAbis(snapshot, lock);
for (auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) {
NoteMatchedEntry(entry, preparedAtUse);
return entry.handle;
}
}
NoteOwnPreparation(preparedAtUse);
ReportPreparedAtUse(snapshot, request);
auto handle = PrepareShaderWithDiagnostics(request);
snapshot.prepared->entries.push_back({codeOffset, handle});
@@ -989,12 +1078,18 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s
struct PreparedKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
ShaderRecompiler::BuildPreparedShaderKey(source, key);
for (const auto& entry : snapshot.prepared->entries) {
PreparedAtUse preparedAtUse;
if (!snapshot.prepared->deferredAbis.empty() || snapshot.prepared->deferredPreparing) preparedAtUse = PrepareDeferredAbis(snapshot, lock);
for (auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset != codeOffset) continue;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle);
if (!linked.code.empty() && !std::ranges::equal(invocationRequest.shader.code, linked.code)) continue;
if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation);
if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) {
NoteMatchedEntry(entry, preparedAtUse);
return std::move(*invocation);
}
}
NoteOwnPreparation(preparedAtUse);
ReportPreparedAtUse(snapshot, source);
auto handle = PrepareShaderWithDiagnostics(source);
invocationRequest = source;
@@ -1281,6 +1376,60 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
return plan != nullptr ? PrepareRegistered(*plan) : std::vector<PreparedShaders::Entry>{};
}
PreparedAtUse PrepareDeferredAbis(const ShaderSnapshot& snapshot, std::unique_lock<std::mutex>& lock) {
auto& prepared = *snapshot.prepared;
if (prepared.deferredPreparing) {
prepared.settled.wait(lock, [&] { return !prepared.deferredPreparing; });
return std::nullopt;
}
if (prepared.deferredAbis.empty()) return std::nullopt;
std::vector<PreparedShaders::DeferredAbi> deferred;
deferred.swap(prepared.deferredAbis);
prepared.deferredPreparing = true;
struct Preparing {
PreparedShaders& prepared;
std::unique_lock<std::mutex>& lock;
~Preparing() {
if (!lock.owns_lock()) lock.lock();
prepared.deferredPreparing = false;
prepared.settled.notify_all();
}
} preparing{prepared, lock};
lock.unlock();
std::vector<PreparedShaders::Entry> entries;
for (const auto& abi : deferred) {
const auto device = abi.device.lock();
if (device == nullptr || snapshot.registeredState == nullptr) continue;
QueueState state{};
state.shader = snapshot.registeredState->shader;
state.context = snapshot.registeredState->context;
state.userConfig = snapshot.registeredState->userConfig;
for (const auto& [offset, value] : abi.context) state.context[offset] = value;
for (const auto& [offset, value] : abi.primitive) state.userConfig[offset] = value;
try {
const auto plan = PlanRegistered(snapshot, *device, state, false);
if (plan == nullptr) continue;
plan->prebuildDevice = device;
auto made = PrepareRegistered(*plan);
std::ranges::move(made, std::back_inserter(entries));
} catch (const std::exception& error) {
APS5_LOG_ERR("Shader 0x%llx could not be prepared at its first use for its resolved ABI (%s); preparing it for the use instead", static_cast<unsigned long long>(snapshot.codeAddress), error.what());
continue;
}
}
lock.lock();
std::vector<const ShaderRecompiler::SourceHandle*> added;
for (auto& entry : entries) {
const auto duplicate = std::ranges::any_of(prepared.entries, [&](const auto& existing) { return existing.codeOffset == entry.codeOffset && existing.handle->artifact == entry.handle->artifact; });
if (duplicate) continue;
entry.fromDeferredAbi = true;
added.push_back(entry.handle.get());
prepared.entries.push_back(std::move(entry));
}
DeferredAbiCounters().entries.fetch_add(added.size(), std::memory_order_relaxed);
return added;
}
void DeferPipelineStage(PreparedShaderState& prepared, const ShaderSnapshot& snapshot, const char* reason) {
if (!prepared.deferred) APS5_LOG_ERR("Shader 0x%llx could not be prepared when the title created its pipeline (%s); preparing it when it is used", static_cast<unsigned long long>(snapshot.codeAddress), reason);
prepared.deferred = true;
@@ -1749,14 +1898,23 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
auto& key = HostThreadLocal<std::vector<std::uint64_t>, GraphicsAbiKeyStorage>();
BuildRegisteredAbiKey(state, *localDevice, key);
const auto primitiveType = state.userConfig.find(0x242u);
for (const auto& abi : transaction.Read(*owner).graphicsAbis) {
if (abi.key != key || abi.registry.lock() != registry) continue;
for (const auto& stage : abi.stages) {
const auto resolveStages = [&](const std::vector<std::weak_ptr<const ShaderSnapshot>>& stages) {
for (const auto& stage : stages) {
const auto snapshot = stage.lock();
require(snapshot != nullptr, "prepared graphics ABI lost a registered stage");
if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true);
}
transaction.Commit();
};
if (!transaction.Tracks(*owner)) {
if (const auto committed = CommittedGraphicsAbi(*owner, key, registry)) {
resolveStages(*committed);
return;
}
}
for (const auto& abi : transaction.Read(*owner).graphicsAbis) {
if (abi.key != key || abi.registry.lock() != registry) continue;
resolveStages(abi.stages);
return;
}
DrawDecode decoded{};
@@ -1792,6 +1950,55 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
transaction.Commit();
}
std::size_t Driver::PreparedEntries(const Shader* shader) {
std::shared_ptr<const ShaderSnapshot> snapshot;
{
std::lock_guard lock(mutex);
if (shaders == nullptr) return 0;
snapshot = RegisteredHeader(*shaders, shader);
}
if (snapshot == nullptr) return 0;
std::unique_lock lock(snapshot->prepared->mutex);
AwaitRegisteredPreparation(*snapshot->prepared, lock);
return snapshot->prepared->entries.size();
}
Driver::DeferredAbiUseCounts Driver::DeferredAbiUses() {
auto& counters = DeferredAbiCounters();
const auto entries = counters.entries.load(std::memory_order_relaxed);
const auto matched = counters.matchedEntries.load(std::memory_order_relaxed);
return {counters.firstUses.load(std::memory_order_relaxed), counters.found.load(std::memory_order_relaxed), counters.own.load(std::memory_order_relaxed), entries, entries - std::min(entries, matched)};
}
std::vector<std::array<std::uint64_t, 2>> Driver::PreparedEntryHashes(const Shader* shader) {
std::shared_ptr<const ShaderSnapshot> snapshot;
{
std::lock_guard lock(mutex);
if (shaders == nullptr) return {};
snapshot = RegisteredHeader(*shaders, shader);
}
if (snapshot == nullptr) return {};
std::unique_lock lock(snapshot->prepared->mutex);
AwaitRegisteredPreparation(*snapshot->prepared, lock);
std::vector<std::array<std::uint64_t, 2>> hashes;
for (const auto& entry : snapshot->prepared->entries) {
if (entry.handle == nullptr || entry.handle->artifact == nullptr) continue;
const auto hash = entry.handle->artifact->artifact.spirv.ContentHash();
std::uint64_t key = 0xcbf29ce484222325ull;
for (const auto word : entry.handle->staticKey) key = (key ^ word) * 0x100000001b3ull;
hashes.push_back({hash.low, hash.high ^ key});
}
std::ranges::sort(hashes);
return hashes;
}
std::array<std::uint64_t, 4> Driver::PrebuiltLibraries() {
if (auto* pool = RegistrationPreparePool::ExistingPrebuild()) pool->AwaitSubmitted();
const auto localDevice = device.Load();
if (localDevice == nullptr) return {};
return Graphics::PipelineLibraryCountersOf(localDevice->Device()).built;
}
void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
BindTitleThread();
PerformanceContext timingContext(FrameTiming::Preparation());
@@ -1829,6 +2036,31 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
struct RegisteredAbiKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, RegisteredAbiKeyStorage>();
BuildRegisteredAbiKey(state, *localDevice, key);
if (LazyShaderPreparation()) {
const auto& resolved = transaction.Read(*snapshot).registeredAbis;
if (std::ranges::find(resolved, key) != resolved.end()) {
transaction.Commit();
return;
}
std::optional<std::string> invalid;
try {
static_cast<void>(PlanRegistered(*snapshot, *localDevice, state, false));
} catch (const std::exception& error) {
invalid = error.what();
}
auto& prepared = transaction.Edit(*snapshot);
if (invalid) DeferPipelineStage(prepared, *snapshot, invalid->c_str());
prepared.registeredAbis.push_back(key);
prepared.deferred = true;
transaction.Commit();
if (invalid) return;
PreparedShaders::DeferredAbi deferred{key, {}, {}, localDevice};
for (const auto reg : context) deferred.context.emplace_back(reg.offset, reg.value);
for (const auto reg : primitive) deferred.primitive.emplace_back(reg.offset, reg.value);
std::lock_guard lock(snapshot->prepared->mutex);
snapshot->prepared->deferredAbis.push_back(std::move(deferred));
return;
}
std::optional<std::string> deferral;
const auto prepare = [&] {
try {
@@ -1842,6 +2074,10 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
}
};
const bool committedAbi = CommittedAbi(*snapshot, key);
if (committedAbi && !transaction.Tracks(*snapshot)) {
transaction.Commit();
return;
}
auto entries = committedAbi ? std::vector<PreparedShaders::Entry>{} : prepare();
const auto& registeredAbis = transaction.Read(*snapshot).registeredAbis;
if (std::ranges::find(registeredAbis, key) != registeredAbis.end()) {
@@ -1927,6 +2163,10 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<
return;
}
}
if (!transaction.Tracks(front) && CommittedLink(front, fragment, primitiveType)) {
transaction.Commit();
return;
}
std::vector<std::shared_ptr<const ShaderSnapshot>> fragments;
{
const auto& current = transaction.Read(front);
@@ -2069,8 +2309,7 @@ void Driver::RegisterShader(const Shader* shader) {
registered.context = snapshot.registeredState->context;
registered.userConfig = snapshot.registeredState->userConfig;
std::shared_ptr<RegisteredPreparation> preparation;
static const bool lazy = std::getenv("APS5_LAZY_SHADER_PREPARE") != nullptr;
if (lazy) {
if (LazyShaderPreparation()) {
static_cast<void>(PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred));
snapshot.prepared->deferred = true;
} else if (AsyncRegistrationPrepare()) {
core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp
+97 -22
View File
@@ -22,6 +22,7 @@
#include <cstring>
#include <limits>
#include <memory>
#include <new>
#include <stdexcept>
#include <string>
#include <tuple>
@@ -372,18 +373,38 @@ struct PageSpan {
std::size_t size = 0;
// Per page: access flags, zero while unknown.
std::uint8_t* pages = nullptr;
static constexpr std::size_t ChunkEntries = 65536;
struct alignas(8) Chunk { std::array<std::uint8_t, ChunkEntries> entries{}; };
std::atomic<Chunk*>* chunks = nullptr;
std::uint8_t* chunkData(std::size_t index, bool create) const {
auto* chunk = chunks[index].load(std::memory_order_acquire);
if (chunk == nullptr && create) {
auto fresh = std::unique_ptr<Chunk>(new (std::nothrow) Chunk{});
if (fresh != nullptr && chunks[index].compare_exchange_strong(chunk, fresh.get(), std::memory_order_acq_rel, std::memory_order_acquire)) chunk = fresh.release();
}
return chunk == nullptr ? nullptr : chunk->entries.data();
}
std::uint8_t* entry(std::size_t index, bool create) const {
if (chunks == nullptr) return pages + index;
auto* data = chunkData(index / ChunkEntries, create);
return data == nullptr ? nullptr : data + index % ChunkEntries;
}
// Reserves and commits the table; untouched table pages cost nothing until a guest page is recorded.
bool allocate() {
if (size == 0) return false;
const auto count = (size / PageBytes + 8) & ~static_cast<std::size_t>(7);
#ifdef _WIN32
pages = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, count, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
const auto chunkCount = (count + ChunkEntries - 1) / ChunkEntries;
chunks = new (std::nothrow) std::atomic<Chunk*>[chunkCount]{};
if (chunks == nullptr) size = 0;
return chunks != nullptr;
#else
pages = static_cast<std::uint8_t*>(std::calloc(count, 1));
#endif
if (pages == nullptr) size = 0;
return pages != nullptr;
#endif
}
bool covers(std::uintptr_t address) const {
@@ -391,11 +412,23 @@ struct PageSpan {
}
std::uint8_t load(std::uintptr_t address) const {
return std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[(address - base) / PageBytes])).load(std::memory_order_relaxed);
auto* value = entry((address - base) / PageBytes, false);
return value == nullptr ? 0 : std::atomic_ref<std::uint8_t>(*value).load(std::memory_order_relaxed);
}
void store(std::uintptr_t address, std::uint8_t value) {
std::atomic_ref<std::uint8_t>(pages[(address - base) / PageBytes]).store(value, std::memory_order_relaxed);
if (auto* at = entry((address - base) / PageBytes, value != 0 && value != PageInaccessible)) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
}
void store(std::uintptr_t first, std::uintptr_t last, std::uint8_t value) {
auto index = (first - base) / PageBytes;
const auto stop = (last - 1 - base) / PageBytes + 1;
while (index < stop) {
const auto next = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
if (auto* at = entry(index, value != 0 && value != PageInaccessible)) {
for (; index < next; ++index, ++at) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
} else index = next;
}
}
// End of the run of pages equal to `value` that starts at `address`, at most `limit`.
@@ -404,11 +437,17 @@ struct PageSpan {
const auto stop = std::min<std::uintptr_t>((std::min(limit, base + size) - base + PageBytes - 1) / PageBytes, size / PageBytes);
const std::uint64_t wide = 0x0101010101010101ull * value;
while (index < stop) {
if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(const_cast<std::uint8_t*>(pages + index))).load(std::memory_order_relaxed) == wide) {
auto* at = entry(index, false);
if (at == nullptr) {
if (value != 0) break;
index = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
continue;
}
if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(at)).load(std::memory_order_relaxed) == wide) {
index += 8;
continue;
}
if (std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[index])).load(std::memory_order_relaxed) != value) break;
if (std::atomic_ref<std::uint8_t>(*at).load(std::memory_order_relaxed) != value) break;
++index;
}
return std::min(limit, base + index * PageBytes);
@@ -418,7 +457,7 @@ struct PageSpan {
if (size == 0 || bytes == 0 || address >= base + size || address + bytes <= base) return;
const auto first = (std::max(address, base) - base) / PageBytes;
const auto last = (std::min(address + bytes, base + size) - 1 - base) / PageBytes;
for (auto page = first; page <= last; ++page) std::atomic_ref<std::uint8_t>(pages[page]).store(0, std::memory_order_relaxed);
store(base + first * PageBytes, base + (last + 1) * PageBytes, 0);
}
};
@@ -449,7 +488,7 @@ struct PageStates {
const std::uint8_t value = readable ? static_cast<std::uint8_t>(PageReadable | (writable ? PageWritable : 0u)) : PageInaccessible;
const auto first = std::max(base, span->base);
const auto last = std::min(regionEnd, span->base + span->size);
for (auto at = first; at < last; at += PageBytes) span->store(at, value);
span->store(first, last, value);
if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation || accessSerial.load(std::memory_order_acquire) != access) span->forget(first, static_cast<std::size_t>(last - first));
}
}
@@ -878,6 +917,7 @@ struct WriteTracker {
// bumps, or an image refreshed after a stamp would keep failing UnchangedSince until the next
// walk of its range and re-upload at every use of the epoch.
std::atomic<std::uint32_t> generation{1};
std::atomic<std::uint32_t> stable{1};
std::vector<void*> pages;
// Collect memo: page-rounded ranges walked during the current epoch (see BumpCollectEpoch). A
// collect fully inside one skips GetWriteWatch, whose cost is proportional to the range's pages,
@@ -1229,16 +1269,51 @@ WriteTracker& Tracker() {
// Takes the tracker mutex; under APS5_PROFILE_DRAW an acquisition that found it held is counted
// ('tracker waits N / M' in [guestmem]), which says whether walking outside the lock would pay.
std::unique_lock<AgcDriver::Mutex> lockTracker(WriteTracker& tracker) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (!profile) return std::unique_lock(tracker.mutex);
std::unique_lock lock(tracker.mutex, std::try_to_lock);
if (!lock.owns_lock()) {
trackerWaits.fetch_add(1, std::memory_order_relaxed);
class TrackerLock {
public:
explicit TrackerLock(WriteTracker& tracker) : tracker(tracker), lock(tracker.mutex, std::defer_lock) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (!profile) {
lock.lock();
return;
}
if (!lock.try_lock()) {
trackerWaits.fetch_add(1, std::memory_order_relaxed);
lock.lock();
}
trackerAcquisitions.fetch_add(1, std::memory_order_relaxed);
}
~TrackerLock() {
if (lock.owns_lock()) publish();
}
TrackerLock(const TrackerLock&) = delete;
TrackerLock& operator=(const TrackerLock&) = delete;
void Unlock() {
publish();
lock.unlock();
}
void Suspend() {
lock.unlock();
}
void Resume() {
lock.lock();
}
trackerAcquisitions.fetch_add(1, std::memory_order_relaxed);
return lock;
private:
void publish() {
const auto current = tracker.generation.load(std::memory_order_relaxed);
if (tracker.stable.load(std::memory_order_relaxed) != current) tracker.stable.store(current, std::memory_order_release);
}
WriteTracker& tracker;
std::unique_lock<AgcDriver::Mutex> lock;
};
TrackerLock lockTracker(WriteTracker& tracker) {
return TrackerLock(tracker);
}
#ifdef _WIN32
@@ -1411,7 +1486,7 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
const auto& map = epochCollectMap(epoch);
if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && found->second.mapping == mapping && stop <= found->second.end) {
collectMemoHits.fetch_add(1, std::memory_order_relaxed);
return tracker.generation.load(std::memory_order_relaxed);
return tracker.stable.load(std::memory_order_acquire);
}
}
for (const auto& entry : threadCollectMemo().entries) {
@@ -1420,7 +1495,7 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
// overlapping collect) were written before this caller reads, and an older value would
// make every later UnchangedSince fail until the epoch ends.
collectMemoHits.fetch_add(1, std::memory_order_relaxed);
return tracker.generation.load(std::memory_order_relaxed);
return tracker.stable.load(std::memory_order_acquire);
}
}
}
@@ -1609,7 +1684,7 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
auto lock = lockTracker(tracker);
tracker.initialize();
if (!tracker.watched) {
lock.unlock();
lock.Unlock();
return import();
}
constexpr std::uint64_t page = 4096;
@@ -1789,9 +1864,9 @@ static std::uint64_t markWritten(std::uint64_t address, std::size_t bytes, bool
const bool collected = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs);
#endif
if (!collected) {
lock.unlock();
lock.Suspend();
beforeUnwatch(blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
lock.lock();
lock.Resume();
unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
tracker.noteUnstampedStore(address, address + bytes);
return 0;
core/libs/prx/libSceAgcDriver/Execution/src/SubmissionCapture.cpp
+5 -1
View File
@@ -101,8 +101,12 @@ SubmissionCapture::SubmissionCapture(const std::filesystem::path& directory) : d
SubmissionCapture* SubmissionCapture::Active() {
static SubmissionCapture* const capture = [] {
#ifdef _WIN32
const wchar_t* directory = _wgetenv(L"APS5_DUMP_SUBMITS");
#else
const char* directory = std::getenv("APS5_DUMP_SUBMITS");
return directory != nullptr && *directory != '\0' ? new SubmissionCapture(directory) : nullptr;
#endif
return directory != nullptr && *directory != 0 ? new SubmissionCapture(directory) : nullptr;
}();
return capture;
}
core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp
+104 -6
View File
@@ -29,6 +29,7 @@
#include "prx/libSceAgcDriver/Execution/include/Screenshot.hpp"
#include "prx/libkernel/AppMetadata/include/AppMetadata.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
#include "prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp"
@@ -168,6 +169,12 @@ void CreateComputePipeline(VkDevice device, PFN_vkCreateComputePipelines create,
PerformanceCounters::CountPipeline();
}
#ifdef _WIN32
const wchar_t* ComputePipelineDumpDirectory() { return _wgetenv(L"APS5_DUMP_COMPUTE_PIPELINES"); }
#else
const char* ComputePipelineDumpDirectory() { return std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); }
#endif
bool CaptureComputePipeline(std::size_t words) {
const char* minimum = std::getenv("APS5_DUMP_COMPUTE_MIN_WORDS");
if (minimum == nullptr) return true;
@@ -341,6 +348,8 @@ struct VulkanDevice::State {
VkDeviceSize uploadSize = 0;
VkPhysicalDeviceProperties properties{};
std::string driverInfo;
VkDriverId driverId = VK_DRIVER_ID_MAX_ENUM;
std::uint32_t pixelFunctionStorageDwords = 4096u;
bool individuallyFreeDescriptorSets = false;
PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr;
PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr;
@@ -379,6 +388,7 @@ struct VulkanDevice::State {
bool earlyFragmentSampleMaskTestBeforeSampleCounting = false;
bool robustBufferAccess2 = false;
bool subgroupControlBarriers = true;
bool workgroupScopeFences = true;
// VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in
// graphics stages, and compute workgroups wider than a wave).
bool descriptorIndexing = false;
@@ -1261,9 +1271,22 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
driverId = driver.driverID;
state->driverInfo = std::string(driver.driverName) + " " + driver.driverInfo;
state->driverId = driver.driverID;
state->individuallyFreeDescriptorSets = driver.driverID == VK_DRIVER_ID_MESA_LLVMPIPE;
state->fusedExactArithmetic = driver.driverID == VK_DRIVER_ID_MOLTENVK;
state->subgroupControlBarriers = SubgroupControlBarrierUsable(driver.driverID);
state->workgroupScopeFences = WorkgroupScopeFenceUsable(driver.driverID);
}
if (driverId == VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_NV_SHADER_SM_BUILTINS_EXTENSION_NAME)) {
VkPhysicalDeviceShaderSMBuiltinsPropertiesNV smBuiltins{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SM_BUILTINS_PROPERTIES_NV};
VkPhysicalDeviceProperties2 smProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &smBuiltins};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &smProperties);
const auto subgroup = state->subgroup.subgroupSize;
if (smBuiltins.shaderSMCount != 0u && smBuiltins.shaderWarpsPerSM != 0u && subgroup != 0u) {
const auto invocations = static_cast<std::uint64_t>(smBuiltins.shaderSMCount) * smBuiltins.shaderWarpsPerSM * subgroup;
const auto budgetDwords = (4ull << 30) / (invocations * 4ull);
if (budgetDwords < state->pixelFunctionStorageDwords) state->pixelFunctionStorageDwords = static_cast<std::uint32_t>(budgetDwords);
}
}
auto byteFeatures = QueryBdaByteFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
auto bdaFeatures = QueryBdaFeatures(selected, state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2"), availableExtensions);
@@ -1499,7 +1522,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
// Guest memory is host memory: importing it lets address-based shaders use it in place instead of
// copying every registered allocation per draw.
if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) {
VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
VkPhysicalDeviceMaintenance3Properties maintenance3{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES};
VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES, &maintenance3};
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties};
VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
@@ -1511,6 +1535,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
#endif
state->drawInputsInPlace = std::getenv("APS5_NO_DRAW_INPUTS_IN_PLACE") == nullptr && (std::getenv("APS5_DRAW_INPUTS_IN_PLACE") != nullptr || DrawInputsInPlaceUsable(driverProperties.driverID, windows));
if (!compileOnly) Graphics::SetHostImportWindow(8ull << 20u);
if (!compileOnly) Graphics::SetHostImportMaxBytes(maintenance3.maxMemoryAllocationSize);
#ifdef _WIN32
// Imported pages are locked; locking is bounded by the process working-set minimum.
if (const char* value = std::getenv("APS5_WORKING_SET_MIB"); value != nullptr && !compileOnly) {
@@ -2282,6 +2307,7 @@ void VulkanDevice::WaitIdle() {
state->recorder->Sync();
}
check(state->DeviceFunction<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(state->device), "vkDeviceWaitIdle");
Graphics::CheckPixelFunctionLds(graphicsContext());
APS5_LOG_CHARS_OUT_DEBUG("VulkanDevice::WaitIdle complete");
}
@@ -2395,10 +2421,37 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
// stores of a run against each other and against the work before and after, the queued DCC
// key stores included).
const auto recordStore = [&] { recorder.RecordStore(import->buffer, Graphics::HostImportOffset(*import, address), bytes, address); };
// Only a write-back that can land over these bytes (a listed copied writer of the range, as
// FillBuffer tests) puts the label behind the completions; any other pending completion
// stores elsewhere. Debug aid: APS5_LABEL_GATE_ALL=1 gates on any completion, as before (a
// completion label there registered its own completion, so every later label followed).
// On the console every store a queue made before a label is in memory once the label is
// visible. A listed copied writer (a dispatch or draw whose written buffer the device holds as a
// CPU copy) is written back by the CPU when its batch is reaped, after the GPU already ran any
// store recorded here, so a title polling the label could rewrite the written region and have
// the late write-back land over its bytes (a title that rebuilds its command buffer there sees
// a stale packet). While one is listed, a label of 8 bytes or less (every RELEASE_MEM, the fence
// WRITE_DATAs) is therefore not recorded on the GPU at all: a completion action of the open (or
// newest in-flight) batch stores it when that batch is reaped. Completions run in push order and
// batches finish front-first, so it lands after every earlier write-back, whatever range that
// one wrote. The GPU store plus completion re-store below would not do: its GPU store is
// visible before the write-back, and its re-store runs only over a write-back that overlapped
// the label. While such a completion store is pending (PendingCompletionStores: no end-of-pipe
// delivery and no write-back counts there), later small labels follow it even after every
// writer was written back, or a GPU store would become visible before the earlier completion
// store lands (#1458), or land under it at the reap when both are at one address. Larger stores
// (DUMP_CONST_RAM, COPY_DATA and DMA_DATA payloads that later GPU work reads through the import)
// stay on the GPU and follow only a pending completion store over their own range. Readers of a
// range stored this way sync through its batch first (the label is a pending write of its batch
// and counted in PendingCompletionLabels). Debug aid: APS5_DRAIN_COMPLETION_LABELS=1 drains
// (reason 2) instead and stores the label on the CPU.
const bool listed = !state->copiedWriters->empty() || !Graphics::DrawCopiedWriters()->empty();
if (drain) {
if (listed) return 2;
} else if (const bool storesPending = Graphics::Recorder::PendingCompletionStores() != 0; bytes.size() <= 8 ? (listed || storesPending) : (storesPending && recorder.CompletionLabelIn(address, bytes.size()))) {
recorder.AfterCompletions(address, bytes, stamp, queue, false);
return 5;
}
// A larger store over a listed copied writer's range (as FillBuffer tests it) goes behind the
// completions too, below; any other one stores on the GPU. Debug aid: APS5_LABEL_GATE_ALL=1
// gates on any completion, as before (a completion label there registered its own completion,
// so every later label followed).
static const bool gateAll = std::getenv("APS5_LABEL_GATE_ALL") != nullptr;
if (gateAll ? recorder.HasCompletions() : state->CopiedWriterOverlaps(address, bytes.size())) {
// A copied buffer's write-back is still to run: the store must land after it. It is stored
@@ -3650,6 +3703,7 @@ double VulkanDevice::RetirePresents(std::size_t keepInFlight) {
const auto status = state->DeviceFunction<PFN_vkGetFenceStatus>("vkGetFenceStatus");
auto inFlight = static_cast<std::size_t>(std::count_if(slots.begin(), slots.end(), [](const auto& slot) { return slot.inFlight; }));
double waitedMs = 0;
bool retired = false;
// Oldest first from the cursor; the fences of one queue signal in submission order, so the
// first unsignaled one ends the scan, and the ones beyond the bound are the oldest.
for (std::size_t i = 0; i < slots.size() && inFlight != 0; ++i) {
@@ -3668,9 +3722,11 @@ double VulkanDevice::RetirePresents(std::size_t keepInFlight) {
}
const bool dump = slot.dumpRecorded;
state->RetireSlot(slot);
retired = true;
if (dump) timing.Mark("dump");
--inFlight;
}
if (retired) Graphics::CheckPixelFunctionLds(graphicsContext());
return waitedMs;
}
@@ -3777,6 +3833,9 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
target.registrationModules = state->graphicsPipelineLibrary;
target.robustBufferAccess2 = state->robustBufferAccess2;
target.waveLdsMemoryBarriers = !state->subgroupControlBarriers;
target.largePixelFunctionLdsBuffer = state->driverId == VK_DRIVER_ID_NVIDIA_PROPRIETARY;
target.pixelFunctionStorageDwords = state->pixelFunctionStorageDwords;
target.deviceScopeLdsMemoryBarriers = !state->workgroupScopeFences;
target.srgbDecodeFormats = state->srgbDecodeFormats & ~state->srgbCopyFormats;
target.vertexSubgroups = (state->subgroup.supportedStages & VK_SHADER_STAGE_VERTEX_BIT) != 0;
if (state->meshShader) {
@@ -4499,6 +4558,24 @@ struct RecordedDispatch {
std::array<std::uint32_t, 3> threadLimit{};
};
struct DispatchCompletion {
const Graphics::Context& context;
const std::array<Graphics::CompiledShader, 1>& shaders;
VkShaderStageFlags pushStages;
const std::array<std::byte, Graphics::PipelinePushConstantBytes>& pushBytes;
const std::shared_ptr<ComputePipelineObjects>& objects;
const std::shared_ptr<Graphics::ShaderResources>& resources;
const ResourceCache::Key& contentKey;
bool cacheable;
bool fromCache;
double revalidateMs;
const std::shared_ptr<Graphics::ShaderResources::DrawBindings>& overlay;
std::uint64_t lookupsBefore;
DispatchTimer& timer;
PerformanceTimer& timing;
char* groupsText;
};
VkDevice VulkanDevice::Device() const {
return state->device;
}
@@ -5072,7 +5149,7 @@ std::shared_ptr<ComputePipelineObjects> VulkanDevice::createComputePipeline(cons
const Graphics::PipelineSpecialization specialization(shader);
const auto subgroupSize = computeSubgroupSize(shader);
const auto stageFlags = computeStageFlags(shader);
if (const char* directory = std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); directory != nullptr && *directory != '\0' && ComputePipelineCapture::OptionalEnabled() && CaptureComputePipeline(shader.spirv.size())) {
if (const auto* directory = ComputePipelineDumpDirectory(); directory != nullptr && *directory != 0 && !state->probingFloatControls && ComputePipelineCapture::OptionalEnabled() && CaptureComputePipeline(shader.spirv.size())) {
ComputePipelineCapture capture;
capture.spirv = shader.spirv.Copy();
ComputeDescriptorSetLayout resourcesLayout;
@@ -5417,6 +5494,27 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
timing.Mark("pipeline_create");
timer.phase(PhasePipeline);
}
DispatchCompletion completion{context, shaders, pushStages, pushBytes, objects, resources, contentKey, cacheable, fromCache, revalidateMs, overlay, lookupsBefore, timer, timing, groupsText};
return recordCompletedDispatch(shader, x, y, z, arguments, programAddress, recipeOut, threadGroup, htileClear, completion);
}
VulkanDevice::IndirectOutcome VulkanDevice::recordCompletedDispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, std::shared_ptr<const Recipe>* recipeOut, const std::array<std::uint32_t, 3>& threadGroup, std::optional<std::pair<std::uint64_t, std::size_t>> htileClear, DispatchCompletion& completion) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
const auto& context = completion.context;
const auto& shaders = completion.shaders;
const auto pushStages = completion.pushStages;
const auto& pushBytes = completion.pushBytes;
const auto& objects = completion.objects;
const auto& resources = completion.resources;
const auto& contentKey = completion.contentKey;
const bool cacheable = completion.cacheable;
const bool fromCache = completion.fromCache;
const double revalidateMs = completion.revalidateMs;
const auto& overlay = completion.overlay;
const auto lookupsBefore = completion.lookupsBefore;
auto& timer = completion.timer;
auto& timing = completion.timing;
auto* groupsText = completion.groupsText;
// Recorded into the device's open batch: the CPU moves on to the next command while the GPU
// works; the batch is waited for where the guest expects results (labels, flips, CPU reads).
auto& recorder = *state->recorder;
core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp
+9 -4
View File
@@ -66,13 +66,14 @@ public:
// The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large.
static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties);
static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties);
VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr);
VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr, bool newImage = false);
static VkDeviceSize ImageBudget(VkDeviceSize deviceBudget);
VkDeviceMemory TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr);
VkDeviceMemory TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType = std::nullopt, std::uint32_t* allocatedType = nullptr, bool newImage = false);
void PutImageMemory(VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept;
std::size_t RetainedImages();
std::optional<BufferAllocation> Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
void Put(const BufferAllocation& allocation) noexcept;
VkDeviceSize ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now);
static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom);
static VkDeviceSize SlabBlockBytes(std::size_t capacity);
SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable);
@@ -143,7 +144,7 @@ private:
// The retained-slot bound of each tier (APS5_BUFFER_POOL_SLOTS, default `defaultSlots`), read once.
static std::size_t MaxSlots();
std::size_t maxSlots;
VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes);
VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes, bool newImage);
VkDevice device;
PFN_vkUnmapMemory unmap;
PFN_vkDestroyBuffer destroyBuffer;
@@ -168,6 +169,10 @@ private:
std::uint64_t imageEvictions = 0;
std::list<Slot*> imageOrder;
std::uint64_t clock = 0;
std::optional<std::chrono::steady_clock::time_point> idleRead;
std::uint64_t idleClock = 0;
std::uint64_t reclaimBefore = 0;
std::uint64_t scannedIdleClock = 0;
AgcDriver::Mutex slabMutex;
std::unordered_map<std::uint64_t, Slab> slabs;
std::unordered_map<VkDeviceMemory, std::unique_ptr<SlabBlock>> slabBlocks;
@@ -182,7 +187,7 @@ private:
};
std::shared_ptr<BufferPool> GetBufferPool(const Context& context);
VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType = nullptr);
VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType = nullptr, bool newImage = false);
void ReleaseImageMemory(const Context& context, VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept;
}
core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp
+1 -1
View File
@@ -194,7 +194,7 @@ struct Context {
std::uint32_t settleFamily = 0;
std::mutex* settleQueueMutex = nullptr;
PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr;
VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes) = nullptr;
VkDeviceSize (*cacheReclaim)(VkDeviceSize bytes, bool newImage) = nullptr;
RenderCache* renderCache = nullptr;
DrawQueue* drawQueue = nullptr;
GraphicsPipelineCache* graphicsPipelines = nullptr;
core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp
+11
View File
@@ -18,6 +18,17 @@ struct HostImport;
// built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under
// APS5_NO_DRAW_RECIPE=1).
void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots = {}, std::shared_ptr<const DrawRecipe>* recipe = nullptr);
struct PixelFunctionLdsLayouts {
VkDescriptorSetLayout empty = VK_NULL_HANDLE;
VkDescriptorSetLayout buffer = VK_NULL_HANDLE;
};
PixelFunctionLdsLayouts PixelFunctionLdsSetLayouts(const Context& context);
void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, std::uint32_t subgroupSize);
void BindPixelFunctionLds(const Context& context, VkCommandBuffer commands, VkPipelineLayout layout);
void CheckPixelFunctionLds(const Context& context);
void ReleasePixelFunctionLds(VkDevice device);
std::optional<std::string> KnownValidationFailure(const Context& context, std::span<const CompiledShader> shaders, const State& state);
std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span<const VkImageView> targetViews);
core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp
+40
View File
@@ -0,0 +1,40 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_FIRSTSAMPLECOPY_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_FIRSTSAMPLECOPY_HPP
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
namespace AgcDriver::Graphics {
class Recorder;
class FirstSampleCopy {
public:
FirstSampleCopy(const Context& context, VkFormat format, VkExtent2D extent);
~FirstSampleCopy();
FirstSampleCopy(const FirstSampleCopy&) = delete;
FirstSampleCopy& operator=(const FirstSampleCopy&) = delete;
void Record(Recorder& recorder, VkImageView source, VkImageLayout layout);
VkImage Image() const { return image; }
private:
void release() noexcept;
Context context;
VkExtent2D extent;
VkImage image = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
VkImageView view = VK_NULL_HANDLE;
VkSampler sampler = VK_NULL_HANDLE;
VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
VkPipelineLayout pipelineLayout = VK_NULL_HANDLE;
VkRenderPass renderPass = VK_NULL_HANDLE;
VkFramebuffer framebuffer = VK_NULL_HANDLE;
VkShaderModule vertex = VK_NULL_HANDLE;
VkShaderModule fragment = VK_NULL_HANDLE;
VkPipeline pipeline = VK_NULL_HANDLE;
};
}
#endif
core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp
+2
View File
@@ -113,6 +113,7 @@ void SetHostImportDeferredSettles(bool enabled);
// The larger of 6 GiB and half the physical memory, at most the largest host heap less the larger of 2 GiB and half
// that heap: the other half stays for the buffers, mirrors and textures that share it.
std::uint64_t HostImportBudget(const VkPhysicalDeviceMemoryProperties& memory, std::uint64_t physicalBytes);
void SetHostImportMaxBytes(std::uint64_t bytes);
std::uint64_t HostImportLiveBytes();
std::uint64_t HostImportBudgetRemaining(const Context& context);
struct HostImportSettleCounts {
@@ -508,6 +509,7 @@ private:
bool prepared = false;
bool uploaded = false;
bool committed = false;
mutable std::uint64_t directWritesGeneration = 0;
};
}
core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp
+1
View File
@@ -97,6 +97,7 @@ float EffectiveMinLod(const GuestTextureResource& resource);
bool IsNullTextureResource(std::span<const std::uint32_t> words);
GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words);
std::array<std::uint32_t, 8> EncodeTextureResource(const GuestTextureResource& resource);
bool DescriptorSingleLevel(std::span<const std::uint32_t> words);
bool FirstLayerDescriptor(std::span<const std::uint32_t> words, std::array<std::uint32_t, 8>& narrowed);
bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, TextureDimension dimension);
core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp
+1
View File
@@ -67,6 +67,7 @@ bool TakeMultisampledSingleClear(std::span<const std::uint32_t> code, std::span<
bool TakeMultisampledClear(MultisampledCmask& cmask, MultisampledColorFill& colorFill, const ColorTarget& use, VkClearColorValue& clear);
SampleOrderTarget MultisampleTargetOrderTarget(const Context& context, const ColorTarget& target);
std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const GuestTextureResource& resource, VkComponentMapping components, VkImageViewType viewType = VK_IMAGE_VIEW_TYPE_2D);
std::shared_ptr<Texture> MultisampleTargetFirstSample(const Context& context, const GuestTextureResource& resource, VkComponentMapping components);
VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target, VkImage* image = nullptr);
VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target, std::shared_ptr<void>& owner, VkImage* image = nullptr);
void KeepMultisampleTarget_nid_no_patch(const std::shared_ptr<void>& owner);
core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp
+2
View File
@@ -87,6 +87,8 @@ private:
VkPipeline secondChannel = VK_NULL_HANDLE;
std::size_t attachments = 0;
std::size_t colorAttachments = 0;
std::vector<bool> boundAttachments;
std::uint32_t boundColorAttachments = 0;
bool depthBounds = false;
bool depthBias = false;
bool libraries = false;
core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp
+5
View File
@@ -214,6 +214,7 @@ public:
// Submits and waits for every batch, running completions in order.
void Sync();
void CountSamples();
bool CountingSamples() const { return countingSamples; }
bool RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span<const std::uint32_t, 7> rules);
std::uint64_t SamplesTotal();
bool DumpSamples(VkDeviceAddress target, std::uint64_t address = 0);
@@ -221,6 +222,7 @@ public:
bool QueuesSampleDumps() const;
void PrepareSampleSlot();
void EndPassSamples();
void RecordUncounted(const std::function<void(VkCommandBuffer)>& record);
struct SampleDumpStatistics {
std::uint64_t queued;
std::uint64_t queuedInPass;
@@ -403,6 +405,7 @@ public:
static std::uint64_t PublishGeneration();
// labels waiting in completion actions (the CPU must reap their batch for them to land);
static std::uint64_t PendingCompletionLabels();
static std::uint64_t PendingCompletionStores();
// write-back completions (OnComplete) of unfinished batches: a reaper retires them when every
// worker sleeps, as for the labels above;
static std::uint64_t PendingWriteBackCompletions();
@@ -579,6 +582,7 @@ private:
VkQueryPool queries = VK_NULL_HANDLE;
VkQueryPool samples = VK_NULL_HANDLE;
std::shared_ptr<void> samplePool;
std::vector<VkQueryPool> endedSamples;
bool sampleActive = false;
bool samplesDrawn = false;
VkQueryPool segments = VK_NULL_HANDLE;
@@ -600,6 +604,7 @@ private:
// Labels stored by this batch's completion actions (see AfterCompletions); subtracted from
// the lock-free pending count when the batch finishes, whether or not its completions ran.
std::uint32_t completionLabelCount = 0;
std::uint32_t completionStoreCount = 0;
// Write-back completions (OnComplete), likewise.
std::uint32_t writeBackCompletionCount = 0;
// The ranges of the completion labels also stored on the GPU, and whether each entered the
core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp
+10 -1
View File
@@ -30,16 +30,19 @@ class Recorder;
void FlushCachedTextures(VkDevice device);
void ClearCachedTextures(VkDevice device);
std::uint64_t ReleaseTextureCaches(std::uint64_t bytes);
std::uint64_t ReleaseTextureCaches(std::uint64_t bytes, bool newImage = false);
std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory);
std::uint64_t StorageTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t sampledBytes, std::uint64_t storageBytes);
std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes);
bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport);
std::optional<std::uint32_t> LargestDeviceLocalHeap(const VkPhysicalDeviceMemoryProperties& memory);
std::uint64_t StorageTextureCacheBudget(const Context& context);
inline constexpr std::uint64_t SpareSnapshotPresents = 16;
struct TextureCacheUse {
std::size_t sampledEntries = 0;
std::size_t spareEntries = 0;
std::uint64_t sampledBytes = 0;
std::uint64_t spareBytes = 0;
std::uint64_t storageBytes = 0;
};
TextureCacheUse TextureCacheUsage();
@@ -47,9 +50,11 @@ void OverrideCacheBudgetForTest(std::uint64_t bytes);
std::uint64_t SampledTextureCacheBudget(const Context& context);
std::shared_ptr<Texture> CachedSampledTexture(const Context& context, std::span<const std::uint32_t> words, VkImageViewType multisampleViewType = VK_IMAGE_VIEW_TYPE_2D);
bool SampledTexturesShareEntry(std::span<const std::uint32_t> first, std::span<const std::uint32_t> second);
bool AnyPendingCpuWrites(const Context& context);
bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std::size_t bytes);
bool SpirvStoresThroughAddresses(std::span<const std::uint32_t> words);
bool ShaderStoresThroughAddresses(const ShaderRecompiler::SharedSpirv& spirv);
void LastRecordIndices(std::span<const void* const> logged, std::span<const void* const> objects, std::vector<std::size_t>& indices);
// The cached storage image of a surface (render targets use it as their resident image); brought up
// to date with guest memory before it is returned.
@@ -148,6 +153,7 @@ struct ImageTableReuseCounts {
std::uint64_t writesMade = 0;
};
ImageTableReuseCounts ImageTableReuses();
std::uint64_t SampledLookupKeyScans();
struct PresyncSurface {
std::uint64_t address;
@@ -475,6 +481,7 @@ private:
std::array<std::uint32_t, 8> words{};
bool depthCompare = false;
VkImageViewType multisampleViewType = VK_IMAGE_VIEW_TYPE_2D;
bool firstSample = false;
std::shared_ptr<Texture> texture;
};
PreviousSampled previousSampled;
@@ -704,6 +711,7 @@ public:
// meanwhile by another worker stays).
void Remove(const Key& key, const ShaderResources* object = nullptr);
void RemoveImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted = nullptr);
bool TryRemoveImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted = nullptr);
// Moves the entry to the front of the LRU (a recipe hit uses its template without Find, and a
// hot template must not age out under the recipes that depend on it); false when no entry.
bool Touch(const Key& key);
@@ -726,6 +734,7 @@ private:
void noteEvicted(const Key& key);
void noteReturn(const Key& key);
void noteUse(const Key& key);
void removeImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted);
struct KeyHash {
std::size_t operator()(const Key& key) const noexcept {
constexpr std::uint64_t Prime = 0x9e3779b97f4a7c15ull;
core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp
+7
View File
@@ -70,6 +70,7 @@ struct ColorTarget {
std::uint32_t dccControl = 0;
std::uint64_t fmaskAddress = 0;
std::uint32_t fmaskPipeBankXor = 0;
std::uint16_t partialWriteMask = 0;
};
struct DepthTarget {
@@ -162,6 +163,7 @@ struct State {
float pointSizeMax = 0.0f;
std::array<std::uint16_t, 4> quadSampleMask{0xffffu, 0xffffu, 0xffffu, 0xffffu};
bool pixelEarlyZ = false;
bool pixelKill = false;
};
// PA_CL_VS_OUT_CNTL bits that place position exports (clip and cull distances, the misc and the two
@@ -186,6 +188,11 @@ bool SampleMaskFollowsOrder(const State& state, const SampleOrder& order);
std::array<std::uint16_t, 4> QuadSampleMasks(const State& state, const SampleOrder& order);
std::array<std::uint8_t, 8> ExportMappings(const State& state);
std::array<ShaderRecompiler::ColorExportPacking, 8> ExportPackings(const State& state);
std::uint16_t PackedWriteMask(std::uint8_t componentMapping, std::uint32_t exportedMask);
GuestTextureResource TargetMergeSurface(const ColorTarget& color);
bool DepthStencilWrites(const State& state);
bool DepthStencilTests(const State& state);
ShaderRecompiler::TargetMerge ExportTargetMerge(const State& state);
ShaderRecompiler::RasterPrimitive RasterPrimitiveOf(const State& state);
bool PartialSampleMask(const State& state);
bool UniformSampleMask(const State& state);
core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp
+3 -1
View File
@@ -11,6 +11,7 @@
#include <atomic>
#include <chrono>
#include <cstddef>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
@@ -47,7 +48,7 @@ struct OwnedImage {
OwnedImage(const OwnedImage&) = delete;
OwnedImage& operator=(const OwnedImage&) = delete;
~OwnedImage();
void Allocate(const Context& device, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation);
void Allocate(const Context& device, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, bool newImage = false);
Context context;
VkImage image;
VkDeviceMemory memory;
@@ -230,6 +231,7 @@ public:
// any of the [begin, end) ranges: one acquisition of the registry's mutex.
static bool AnyPendingOverlaps(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
static bool AnyLiveOverlaps(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
static std::uint64_t StampKeepingNewerResults(std::uint64_t address, std::size_t bytes, std::uint64_t recorded, const std::function<std::uint64_t()>& stamp);
// Moves with every change of the pending registry (an image marked dirty, stored, flushed or
// dropped; after the store for a flush), bumped under the registry's mutex with the change, so
// a reader that loads it before and after its own reads knows the registry it consulted stayed
core/libs/prx/libSceAgcDriver/Graphics/shaders/FirstSample.frag
+16
View File
@@ -0,0 +1,16 @@
#version 450
#if defined(UINT_SAMPLES)
layout(set = 0, binding = 0) uniform usampler2DMS source;
layout(location = 0) out uvec4 color;
#elif defined(SINT_SAMPLES)
layout(set = 0, binding = 0) uniform isampler2DMS source;
layout(location = 0) out ivec4 color;
#else
layout(set = 0, binding = 0) uniform sampler2DMS source;
layout(location = 0) out vec4 color;
#endif
void main() {
color = texelFetch(source, ivec2(gl_FragCoord.xy), 0);
}
core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp
+4
View File
@@ -283,6 +283,10 @@ void BdaResources::CheckFault() const {
message << "shader stage " << report.stage << " called address 0x" << std::hex << report.address << " at program counter 0x" << report.instruction << ", which no linked callee of that call matches";
throw std::runtime_error(message.str());
}
if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::WindowLaneSpill) {
message << "shader stage " << report.stage << ": a relative write stored " << report.bytes << " in lane " << (report.address >> 32u) << " of vector window slot " << (report.address & 0xffffffffu) << ", which holds a scalar spilled with v_writelane_b32 that a later v_readlane_b32 was replaced with";
throw std::runtime_error(message.str());
}
message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast<std::uint32_t>(report.reason);
if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) {
const auto bytes = table->Bytes();
core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp
+59 -7
View File
@@ -57,7 +57,7 @@ BufferPool::~BufferPool() {
for (auto& [memory, block] : slabBlocks) freeBlock(*block);
}
VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType, std::uint32_t* allocatedType) {
VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType, std::uint32_t* allocatedType, bool newImage) {
if (allocatedType != nullptr) *allocatedType = allocation.memoryTypeIndex;
VkDeviceMemory memory = VK_NULL_HANDLE;
auto result = allocateMemory(device, &allocation, nullptr, &memory);
@@ -75,7 +75,7 @@ VkDeviceMemory BufferPool::AllocateMemory(const VkMemoryAllocateInfo& allocation
}
}
if (exhausted(result)) {
if (cacheReclaim != nullptr) released = cacheReclaim(ReclaimTarget(allocation.allocationSize));
if (cacheReclaim != nullptr) released = cacheReclaim(ReclaimTarget(allocation.allocationSize), newImage);
if (released != 0) {
reclaimed += trim();
memory = VK_NULL_HANDLE;
@@ -146,7 +146,7 @@ bool BufferPool::heapOverBudget() {
return heapOver;
}
VkDeviceMemory BufferPool::TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType, std::uint32_t* allocatedType) {
VkDeviceMemory BufferPool::TakeImageMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional<std::uint32_t> hostType, std::uint32_t* allocatedType, bool newImage) {
const auto type = allocation.memoryTypeIndex;
if (imagePooled(allocation.allocationSize, type)) {
std::lock_guard lock(mutex);
@@ -168,7 +168,7 @@ VkDeviceMemory BufferPool::TakeImageMemory(const VkMemoryAllocateInfo& allocatio
}
++imageMisses;
}
return AllocateMemory(allocation, operation, hostType, allocatedType);
return AllocateMemory(allocation, operation, hostType, allocatedType, newImage);
}
void BufferPool::PutImageMemory(VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept {
@@ -217,7 +217,9 @@ std::size_t BufferPool::RetainedImages() {
}
VkDeviceSize BufferPool::ReclaimTarget(VkDeviceSize bytes) {
return bytes + std::max<VkDeviceSize>(bytes, VkDeviceSize{256} << 20u);
const auto reserve = std::max<VkDeviceSize>(bytes, VkDeviceSize{256} << 20u);
const auto limit = std::numeric_limits<VkDeviceSize>::max();
return bytes > limit - reserve ? limit : bytes + reserve;
}
bool BufferPool::SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom) {
@@ -454,6 +456,56 @@ bool BufferPool::evictOldestImage(std::vector<BufferAllocation>& evicted) {
return true;
}
VkDeviceSize BufferPool::ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now) {
if (bytes == 0) return 0;
const auto heaps = std::min<std::uint32_t>(memoryProperties.memoryHeapCount, VK_MAX_MEMORY_HEAPS);
if (std::count_if(memoryProperties.memoryHeaps, memoryProperties.memoryHeaps + heaps, [](const auto& heap) { return (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0; }) != 1) return 0;
std::vector<BufferAllocation> evicted;
{
std::lock_guard lock(mutex);
if (!idleRead) {
idleRead = now;
idleClock = clock;
return 0;
}
if (now < *idleRead) return 0;
if (now - *idleRead >= std::chrono::seconds(1)) {
reclaimBefore = idleClock;
idleClock = clock;
idleRead = now;
}
if (reclaimBefore <= scannedIdleClock) return 0;
VkDeviceSize selected = 0;
while (selected < bytes) {
auto oldest = deviceTier.free.end();
for (auto it = deviceTier.free.begin(); it != deviceTier.free.end(); ++it) {
if (it->second.empty()) continue;
const auto& slot = it->second.front();
if (slot.allocation.buffer == VK_NULL_HANDLE || slot.allocation.slab || slot.lastUse > reclaimBefore) continue;
if (oldest == deviceTier.free.end() || slot.lastUse < oldest->second.front().lastUse) oldest = it;
}
if (oldest == deviceTier.free.end()) {
scannedIdleClock = reclaimBefore;
break;
}
if (evicted.empty()) evicted.reserve(deviceTier.slots);
const auto allocation = oldest->second.front().allocation;
evicted.push_back(allocation);
selected += allocation.allocationBytes;
deviceTier.retainedBytes -= allocation.allocationBytes;
oldest->second.pop_front();
if (oldest->second.empty()) deviceTier.free.erase(oldest);
--deviceTier.slots;
++deviceTier.evictions;
}
}
VkDeviceSize released = 0;
for (const auto& allocation : evicted) released += destroy(allocation);
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (profile && released != 0) AgcDriver::ProfilePrint_nid_no_patch("[bufferpool] idle device reclaim: %llu bytes released for %llu bytes of storage-cache pressure\n", static_cast<unsigned long long>(released), static_cast<unsigned long long>(bytes));
return released;
}
std::size_t BufferPool::MaxSlots() {
// APS5_BUFFER_POOL_SLOTS=<n> bounds the retained allocations; 64 is the capacity the pool had
// before least-recently-used retention, for A/B runs.
@@ -533,8 +585,8 @@ std::shared_ptr<BufferPool> GetBufferPool(const Context& context) {
return context.bufferPool;
}
VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType) {
return GetBufferPool(context)->TakeImageMemory(allocation, operation, context.RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex), allocatedType);
VkDeviceMemory AllocateImageMemory(const Context& context, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, std::uint32_t* allocatedType, bool newImage) {
return GetBufferPool(context)->TakeImageMemory(allocation, operation, context.RefusedDeviceMemoryFallback(mask, allocation.memoryTypeIndex), allocatedType, newImage);
}
void ReleaseImageMemory(const Context& context, VkDeviceMemory memory, VkDeviceSize bytes, std::uint32_t memoryType) noexcept {
core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp
+7 -2
View File
@@ -665,7 +665,10 @@ bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span<std:
return true;
}
Layout layout{};
if (!IsDccClear(keys) || keys == DccKeys::ClearRegister || !LayoutFor(format, layout)) return false;
if (!IsDccClear(keys) || keys == DccKeys::ClearRegister || !LayoutFor(format, layout)) {
std::fill(bytes.begin(), bytes.end(), std::byte{0});
return false;
}
const bool color = keys == DccKeys::Clear1110 || keys == DccKeys::Clear1111;
const bool alpha = keys == DccKeys::Clear0001 || keys == DccKeys::Clear1111;
const int alphaChannel = layout.channels == 3 ? -1 : alphaOnMsb ? static_cast<int>(layout.channels) - 1 : 0;
@@ -680,7 +683,9 @@ bool FillDccClear(VkFormat format, DccKeys keys, bool alphaOnMsb, std::span<std:
if (((value >> i) & 1u) != 0) element[bit / 8u] |= static_cast<std::byte>(1u << (bit % 8u));
}
}
for (std::size_t offset = 0; offset + element.size() <= bytes.size(); offset += element.size()) std::memcpy(bytes.data() + offset, element.data(), element.size());
std::size_t offset = 0;
for (; offset + element.size() <= bytes.size(); offset += element.size()) std::memcpy(bytes.data() + offset, element.data(), element.size());
std::fill(bytes.begin() + offset, bytes.end(), std::byte{0});
return true;
}
core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp
+3 -7
View File
@@ -1395,6 +1395,7 @@ class LayeredDepthStorage {
public:
LayeredDepthStorage(const Context& context, const GuestTextureResource& resource, const DepthTarget& base, std::uint32_t bytesPerTexel, std::uint32_t firstLayer, std::uint32_t layers) : context(context), base(base), extent(base.extent), bytesPerTexel(bytesPerTexel), firstLayer(firstLayer), layers(layers) {
const auto bytes = DescribeSurface(resource).guestBytes;
layerPitchBytes = DescribeSurface(resource).GuestLayerOffset(1);
StorageTexture::FlushPending(resource.baseAddress, static_cast<std::size_t>(bytes), nullptr, "storage image creation", PublishScope::None);
storage = StorageTexture::Create(context, *context.detiler, resource, 0);
Require(storage->ImageLayers() == layers, "a layered depth storage view's image does not hold its layers");
@@ -1448,6 +1449,7 @@ public:
const std::uint32_t bytesPerTexel;
const std::uint32_t firstLayer;
const std::uint32_t layers;
std::uint64_t layerPitchBytes = 0;
std::shared_ptr<StorageTexture> storage;
private:
@@ -2481,7 +2483,7 @@ std::vector<LayeredSlice> layeredSlices(const Context& context, const LayeredDep
std::vector<LayeredSlice> slices;
const auto& list = surfaces();
for (std::uint32_t layer = storage.firstLayer; layer < storage.layers; ++layer) {
const auto address = base.address + static_cast<std::uint64_t>(layer) * DepthSliceBytes(base.extent, storage.bytesPerTexel);
const auto address = base.address + static_cast<std::uint64_t>(layer) * storage.layerPitchBytes;
const auto slice = std::find_if(list.rbegin(), list.rend(), [&](const auto& surface) { return surface->context.device == context.device && !surface->retired && surface->target.address == address && surface->target.extent.width == base.extent.width && surface->target.extent.height == base.extent.height && surface->target.format == base.format && surface->target.samples == 1; });
if (slice != list.rend()) slices.push_back({slice->get(), layer});
}
@@ -2494,12 +2496,6 @@ std::shared_ptr<StorageTexture> layeredDepthStorage(const Context& context, Dept
const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
const auto bytesPerTexel = d16 ? 2u : 4u;
surface.RequireStorable(resource);
const auto geometry = DescribeSurface(resource);
if (layers > 1 && geometry.GuestLayerOffset(1) != DepthSliceBytes(target.extent, bytesPerTexel)) {
char text[256];
std::snprintf(text, sizeof(text), "AGC graphics: storage image access to the depth planes of depth surface 0x%llx with a layer pitch of 0x%llx bytes, not the depth slice size 0x%llx, is not implemented", static_cast<unsigned long long>(target.address), static_cast<unsigned long long>(geometry.GuestLayerOffset(1)), static_cast<unsigned long long>(DepthSliceBytes(target.extent, bytesPerTexel)));
throw std::runtime_error(text);
}
auto& entry = layeredStorages()[{context.device, resource.baseAddress, resource.width, resource.height, layers, resource.format, static_cast<int>(resource.tileMode), static_cast<int>(resource.dimension), resource.baseArray}];
if (entry == nullptr) entry = std::make_unique<LayeredDepthStorage>(context, resource, target, bytesPerTexel, resource.baseArray, layers);
else static_cast<void>(entry->storage->Refresh());
core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp
+209 -2
View File
@@ -20,6 +20,7 @@
#include "prx/libc/include/General.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "Optimization/include/Optimization/ShaderStageInputInfo.hpp"
#include "SpirvBackend/SpirvAnalysis.hpp"
#include "ThreadOwned.hpp"
#include "prx/libc/include/HostThreadLocal.hpp"
#include <algorithm>
@@ -42,6 +43,184 @@
namespace AgcDriver::Graphics {
namespace {
struct PixelFunctionLdsPool {
Context context{};
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
VkDescriptorPool descriptorPool = VK_NULL_HANDLE;
VkDescriptorSet set = VK_NULL_HANDLE;
VkDescriptorSetLayout empty = VK_NULL_HANDLE;
VkDescriptorSetLayout bufferLayout = VK_NULL_HANDLE;
std::uint32_t* header = nullptr;
std::uint32_t subgroupSize = 0;
std::uint32_t slots = 0;
bool announced = false;
};
std::mutex& PixelLdsMutex() {
static std::mutex mutex;
return mutex;
}
std::unordered_map<VkDevice, PixelFunctionLdsPool>& PixelLdsPools() {
static std::unordered_map<VkDevice, PixelFunctionLdsPool> pools;
return pools;
}
void DestroyPixelFunctionLds(PixelFunctionLdsPool& pool) {
if (pool.header != nullptr) pool.context.Function<PFN_vkUnmapMemory>("vkUnmapMemory")(pool.context.device, pool.memory);
if (pool.descriptorPool != VK_NULL_HANDLE) pool.context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(pool.context.device, pool.descriptorPool, nullptr);
if (pool.buffer != VK_NULL_HANDLE) pool.context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(pool.context.device, pool.buffer, nullptr);
if (pool.memory != VK_NULL_HANDLE) pool.context.Function<PFN_vkFreeMemory>("vkFreeMemory")(pool.context.device, pool.memory, nullptr);
if (pool.bufferLayout != VK_NULL_HANDLE) pool.context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(pool.context.device, pool.bufferLayout, nullptr);
if (pool.empty != VK_NULL_HANDLE) pool.context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(pool.context.device, pool.empty, nullptr);
pool.header = nullptr;
pool.descriptorPool = VK_NULL_HANDLE;
pool.set = VK_NULL_HANDLE;
pool.buffer = VK_NULL_HANDLE;
pool.memory = VK_NULL_HANDLE;
pool.bufferLayout = VK_NULL_HANDLE;
pool.empty = VK_NULL_HANDLE;
}
}
PixelFunctionLdsLayouts PixelFunctionLdsSetLayouts(const Context& context) {
std::lock_guard lock(PixelLdsMutex());
auto& pool = PixelLdsPools()[context.device];
if (pool.bufferLayout == VK_NULL_HANDLE) {
pool.context = context;
VkDescriptorSetLayoutCreateInfo emptyInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
Check(context.Function<PFN_vkCreateDescriptorSetLayout>("vkCreateDescriptorSetLayout")(context.device, &emptyInfo, nullptr, &pool.empty), "vkCreateDescriptorSetLayout pixel function LDS empty");
const VkDescriptorSetLayoutBinding binding{0u, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u, VK_SHADER_STAGE_FRAGMENT_BIT, nullptr};
VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
info.bindingCount = 1u;
info.pBindings = &binding;
Check(context.Function<PFN_vkCreateDescriptorSetLayout>("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &pool.bufferLayout), "vkCreateDescriptorSetLayout pixel function LDS");
}
return {pool.empty, pool.bufferLayout};
}
void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, std::uint32_t subgroupSize) {
const auto operations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT;
Require((context.subgroup.supportedStages & VK_SHADER_STAGE_FRAGMENT_BIT) != 0 && (context.subgroup.supportedOperations & operations) == operations, "pixel function LDS needs fragment subgroup elect, ballot and broadcast");
Require(subgroupSize != 0u && subgroupSize == context.subgroup.subgroupSize, "pixel function LDS subgroup size does not match the device");
std::lock_guard lock(PixelLdsMutex());
auto& pool = PixelLdsPools()[context.device];
Require(pool.bufferLayout != VK_NULL_HANDLE, "pixel function LDS descriptor set layout was not created");
if (pool.buffer == VK_NULL_HANDLE) {
const auto stride = static_cast<std::uint64_t>(subgroupSize) * ShaderRecompiler::FunctionLdsDwordLimit * 4ull;
const auto budget = std::min<std::uint64_t>(context.limits.maxStorageBufferRange, 2ull << 30);
Require(budget > 16ull + stride, "pixel function LDS does not fit in the device storage buffer range");
pool.slots = static_cast<std::uint32_t>((budget - 16ull) / stride);
pool.subgroupSize = subgroupSize;
const auto bytes = 16ull + stride * pool.slots;
VkBufferCreateInfo bufferInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bufferInfo.size = bytes;
bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
Check(context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &bufferInfo, nullptr, &pool.buffer), "vkCreateBuffer pixel function LDS");
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, pool.buffer, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
constexpr auto hostCoherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
constexpr auto deviceLocalHost = hostCoherent | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
std::optional<std::uint32_t> deviceLocalType;
std::optional<std::uint32_t> hostOnlyType;
for (std::uint32_t i = 0; i < context.memory.memoryTypeCount; ++i) {
if ((requirements.memoryTypeBits & (1u << i)) == 0) continue;
const auto flags = context.memory.memoryTypes[i].propertyFlags;
const auto heap = context.memory.memoryTypes[i].heapIndex;
if (!deviceLocalType && (flags & deviceLocalHost) == deviceLocalHost && context.memory.memoryHeaps[heap].size >= requirements.size) {
deviceLocalType = i;
}
if (!hostOnlyType && (flags & hostCoherent) == hostCoherent && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) == 0) {
hostOnlyType = i;
}
}
const auto hostType = hostOnlyType.value_or(context.MemoryType(requirements.memoryTypeBits, hostCoherent));
allocation.memoryTypeIndex = deviceLocalType.value_or(hostType);
auto allocated = context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &pool.memory);
if (allocated != VK_SUCCESS && deviceLocalType && allocation.memoryTypeIndex == *deviceLocalType && hostType != *deviceLocalType) {
allocation.memoryTypeIndex = hostType;
allocated = context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &pool.memory);
}
if (allocated != VK_SUCCESS) {
context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, pool.buffer, nullptr);
pool.buffer = VK_NULL_HANDLE;
pool.memory = VK_NULL_HANDLE;
Check(allocated, "vkAllocateMemory pixel function LDS");
}
Check(context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, pool.buffer, pool.memory, 0), "vkBindBufferMemory pixel function LDS");
Check(context.Function<PFN_vkMapMemory>("vkMapMemory")(context.device, pool.memory, 0, 16, 0, reinterpret_cast<void**>(&pool.header)), "vkMapMemory pixel function LDS");
pool.header[0] = 0u;
pool.header[1] = 0u;
pool.header[2] = pool.slots;
const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u};
VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
poolInfo.maxSets = 1u;
poolInfo.poolSizeCount = 1u;
poolInfo.pPoolSizes = &size;
Check(context.Function<PFN_vkCreateDescriptorPool>("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool.descriptorPool), "vkCreateDescriptorPool pixel function LDS");
VkDescriptorSetAllocateInfo sets{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
sets.descriptorPool = pool.descriptorPool;
sets.descriptorSetCount = 1u;
sets.pSetLayouts = &pool.bufferLayout;
Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &sets, &pool.set), "vkAllocateDescriptorSets pixel function LDS");
const VkDescriptorBufferInfo buffer{pool.buffer, 0, bytes};
VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
write.dstSet = pool.set;
write.dstBinding = 0u;
write.descriptorCount = 1u;
write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
write.pBufferInfo = &buffer;
context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, 1u, &write, 0u, nullptr);
std::fprintf(stderr, "[gpu] pixel function LDS pool: %u subgroups, %llu MiB, stride %u lanes\n", pool.slots, static_cast<unsigned long long>(bytes >> 20u), subgroupSize);
pool.announced = true;
}
Require(pool.subgroupSize == subgroupSize, "pixel function LDS subgroup size does not match the pool");
const VkMemoryBarrier beforeFill{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT};
context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &beforeFill, 0, nullptr, 0, nullptr);
context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, pool.buffer, 0, 4, 0u);
const VkMemoryBarrier afterFill{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT};
context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 1, &afterFill, 0, nullptr, 0, nullptr);
}
std::uint32_t PixelFunctionLdsSubgroup(std::span<const CompiledShader> shaders) {
std::uint32_t subgroup = 0u;
for (const auto& shader : shaders) {
if (shader.program != nullptr && shader.program->functionLdsBuffer) subgroup = shader.program->hostSubgroupSize;
}
return subgroup;
}
void BindPixelFunctionLds(const Context& context, VkCommandBuffer commands, VkPipelineLayout layout) {
std::lock_guard lock(PixelLdsMutex());
const auto& pool = PixelLdsPools().at(context.device);
Require(pool.set != VK_NULL_HANDLE, "pixel function LDS buffer was not prepared");
context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::PixelFunctionLdsDescriptorSet, 1, &pool.set, 0, nullptr);
}
void CheckPixelFunctionLds(const Context& context) {
std::lock_guard lock(PixelLdsMutex());
const auto found = PixelLdsPools().find(context.device);
if (found == PixelLdsPools().end() || found->second.header == nullptr) return;
const auto exhausted = found->second.header[1] != 0u;
found->second.header[1] = 0u;
if (exhausted) throw std::runtime_error("AGC graphics: pixel function LDS subgroup slots exhausted");
}
void ReleasePixelFunctionLds(VkDevice device) {
std::lock_guard lock(PixelLdsMutex());
const auto found = PixelLdsPools().find(device);
if (found == PixelLdsPools().end()) return;
DestroyPixelFunctionLds(found->second);
PixelLdsPools().erase(found);
}
std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span<const VkImageView> targetViews) {
std::uint64_t key = 14695981039346656037ull;
const auto mix = [&](std::uint64_t value) {
@@ -53,6 +232,9 @@ std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std:
mix(state.blends.size());
for (const auto& color : state.colors) mix(color.exportIndex);
}
for (const auto& color : state.colors) {
if (color.partialWriteMask != 0u) mix(0x6d65726765000000ull | color.exportIndex);
}
mix(state.renderExtent.width);
mix(state.renderExtent.height);
mix(state.samples);
@@ -1143,6 +1325,10 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
APS5_LOG_CHARS_OUT_DEBUG("Index buffer range OK");
Require(!draw.indexed || !state.hasColorTarget || draw.indexAddress + indexBytes <= state.color.address || state.color.address + state.color.bytes <= draw.indexAddress, "index buffer aliases the render target");
if (draw.indexed) RequireNoMultisampleTarget(draw.indexAddress, static_cast<std::size_t>(indexBytes), "an index buffer read");
if (state.pixelKill && !state.pixelEarlyZ && std::ranges::any_of(state.colors, [](const ColorTarget& color) { return color.partialWriteMask != 0; })) {
const auto* recorder = Recorder::Active();
Require(recorder == nullptr || !recorder->CountingSamples(), "a partial write of a packed color target merges inside the pixel shader after early depth and stencil tests, which a shader that kills pixels under late tests cannot use while occlusion queries count samples: the early tests would count the pixels it kills");
}
APS5_LOG_CHARS_OUT_DEBUG("ValidateShaders");
if (recipe != nullptr) {
inputs.fragmentOutputs = recipe->fragmentOutputs;
@@ -2021,7 +2207,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
}
const PassAccess passAccess{passReads, passWrites, passImages, passAttachments, unknownMemory, unknownWrites, passWriteOnly};
const bool fenced = state.depth.has_value() || !record.proxies.empty();
const auto start = recorder->StartDrawPass(passKey, capture || readsTarget || gpuIndirect || meshIndirect, fenced, passAccess);
const auto start = recorder->StartDrawPass(passKey, capture || readsTarget || gpuIndirect || meshIndirect || PixelFunctionLdsSubgroup(shaders) != 0u, fenced, passAccess);
for (const auto& target : record.multisampleTargets) KeepMultisampleTarget_nid_no_patch(target);
const bool continued = start.continued;
outcome.passContinued = continued;
@@ -2068,6 +2254,8 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
countBarrier(1);
}
const auto pixelLdsSubgroup = PixelFunctionLdsSubgroup(shaders);
if (pixelLdsSubgroup != 0u) PreparePixelFunctionLds(context, commands, pixelLdsSubgroup);
if (meshIndirect) {
meshArguments = recordMeshArguments(context, commands, recorder, true, state, draw, *record.indirect, countBarrier, inputs.restartTable);
argumentBuffer = meshArguments->Handle();
@@ -2093,6 +2281,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
} else {
resources.Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout());
}
if (std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; })) BindPixelFunctionLds(context, commands, record.pipeline->Layout());
APS5_LOG_CHARS_OUT_DEBUG("Resources bound");
pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty());
APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded");
@@ -2318,6 +2507,10 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara
if (color.samples > 1) {
binding.multisampled = true;
timer.phase(PhaseSetup);
if (color.partialWriteMask != 0) {
static_cast<void>(MultisampleTargetView(context, color));
static_cast<void>(MultisampleTargetStorageView(context, TargetMergeSurface(color)));
}
targetViews.push_back(MultisampleTargetView(context, color, binding.multisampleOwner, &binding.multisampleImage));
timer.phase(PhaseReadTarget);
continue;
@@ -2549,7 +2742,18 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara
const bool multisampled = prepared.multisampled = std::any_of(targets.begin(), targets.end(), [](const TargetBinding& binding) { return binding.multisampled; });
Require(lean || !multisampled, "multisampled draws are only rendered as recorded draws");
for (const auto& binding : targets) {
if (binding.multisampled) Require(std::find(resources->MultisampleSources().begin(), resources->MultisampleSources().end(), binding.color.surfaceAddress) == resources->MultisampleSources().end(), "sampling a multisampled color target the draw renders to is not implemented");
if (!binding.multisampled) continue;
const auto& sources = resources->MultisampleSources();
const auto reads = std::count(sources.begin(), sources.end(), binding.color.surfaceAddress);
Require(reads == 0 || (reads == 1 && binding.color.partialWriteMask != 0), "sampling a multisampled color target the draw renders to is not implemented");
Require(binding.color.partialWriteMask == 0 || (reads == 1 && MultisampleTargetStorable(context, binding.color)), "the partial write of a multisampled packed color target does not store into the target's image");
if (binding.color.partialWriteMask != 0) static_cast<void>(MultisampleTargetStorageView(context, TargetMergeSurface(binding.color)));
}
for (const auto& binding : targets) {
if (binding.color.partialWriteMask == 0 || binding.multisampled) continue;
Require(lean && binding.resident != nullptr && !binding.proxied, "a partial write of a packed color target is merged into its resident image by a recorded draw only");
const auto images = resources->StorageImages();
Require(std::ranges::any_of(images, [&](const std::pair<VkImage, bool>& image) { return image.first == binding.resident->Image() && image.second; }), "the partial write of a packed color target does not store into the target's resident image");
}
if (!lean && !resolved.moved.empty()) {
resolved.resources = std::make_shared<ShaderResources>(context, shaders, state.color, static_cast<std::uint32_t>(state.colors.size()), draw.indexAddress, static_cast<std::size_t>(indexBytes), snapshots);
@@ -2766,6 +2970,8 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
countBarrier();
}
const auto pixelLdsSubgroup = PixelFunctionLdsSubgroup(shaders);
if (pixelLdsSubgroup != 0u) PreparePixelFunctionLds(context, commands, pixelLdsSubgroup);
std::shared_ptr<Buffer> meshArguments;
if (state.stages.mesh && args != nullptr) {
meshArguments = recordMeshArguments(context, commands, recorder, recorded, state, draw, indirect, countBarrier, inputs.restartTable);
@@ -2807,6 +3013,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D
pipeline->Begin(commands, *framebuffer, state.renderExtent, state);
APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK");
resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->Layout());
if (pixelLdsSubgroup != 0u) BindPixelFunctionLds(context, commands, pipeline->Layout());
APS5_LOG_CHARS_OUT_DEBUG("Resources bound");
pushDrawConstants(*pipeline, commands, state, draw, shaders, *resources, nullptr, 0, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty());
APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded");
core/libs/prx/libSceAgcDriver/Graphics/src/FirstSampleCopy.cpp
+252
View File
@@ -0,0 +1,252 @@
#include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libSceAgcDriver/Graphics/shaders/FirstSampleFloat_spv.h"
#include "prx/libSceAgcDriver/Graphics/shaders/FirstSampleSint_spv.h"
#include "prx/libSceAgcDriver/Graphics/shaders/FirstSampleUint_spv.h"
#include "prx/libSceAgcDriver/Graphics/shaders/PresentationPassVert_spv.h"
#include <memory>
#include <span>
namespace AgcDriver::Graphics {
namespace {
bool sintFormat(VkFormat format) {
switch (format) {
case VK_FORMAT_R8_SINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_B8G8R8A8_SINT:
case VK_FORMAT_A8B8G8R8_SINT_PACK32: case VK_FORMAT_A2B10G10R10_SINT_PACK32: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16G16_SINT:
case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R32_SINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32B32A32_SINT:
return true;
default: return false;
}
}
bool uintFormat(VkFormat format) {
switch (format) {
case VK_FORMAT_R8_UINT: case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8B8A8_UINT: case VK_FORMAT_B8G8R8A8_UINT:
case VK_FORMAT_A8B8G8R8_UINT_PACK32: case VK_FORMAT_A2R10G10B10_UINT_PACK32: case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_R16_UINT:
case VK_FORMAT_R16G16_UINT: case VK_FORMAT_R16G16B16A16_UINT: case VK_FORMAT_R32_UINT: case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32B32A32_UINT:
return true;
default: return false;
}
}
std::span<const std::uint32_t> fragmentCode(VkFormat format) {
if (sintFormat(format)) return FIRST_SAMPLE_SINT_SPV;
if (uintFormat(format)) return FIRST_SAMPLE_UINT_SPV;
return FIRST_SAMPLE_FLOAT_SPV;
}
struct DescriptorPool {
DescriptorPool(const Context& context, VkDescriptorPool pool) : context(context), pool(pool) {}
~DescriptorPool() { context.Function<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool")(context.device, pool, nullptr); }
DescriptorPool(const DescriptorPool&) = delete;
DescriptorPool& operator=(const DescriptorPool&) = delete;
Context context;
VkDescriptorPool pool;
};
}
FirstSampleCopy::FirstSampleCopy(const Context& context, VkFormat format, VkExtent2D extent) : context(context), extent(extent) {
try {
VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = format;
imageInfo.extent = {extent.width, extent.height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage first sample");
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, image, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory first sample");
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory first sample");
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
viewInfo.image = image;
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
viewInfo.format = format;
viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView first sample");
VkSamplerCreateInfo samplerInfo{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
samplerInfo.magFilter = VK_FILTER_NEAREST;
samplerInfo.minFilter = VK_FILTER_NEAREST;
samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
Check(context.Function<PFN_vkCreateSampler>("vkCreateSampler")(context.device, &samplerInfo, nullptr, &sampler), "vkCreateSampler first sample");
VkDescriptorSetLayoutBinding binding{};
binding.binding = 0;
binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
binding.descriptorCount = 1;
binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT;
binding.pImmutableSamplers = &sampler;
VkDescriptorSetLayoutCreateInfo descriptorInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
descriptorInfo.bindingCount = 1;
descriptorInfo.pBindings = &binding;
Check(context.Function<PFN_vkCreateDescriptorSetLayout>("vkCreateDescriptorSetLayout")(context.device, &descriptorInfo, nullptr, &descriptorLayout), "vkCreateDescriptorSetLayout first sample");
VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
layoutInfo.setLayoutCount = 1;
layoutInfo.pSetLayouts = &descriptorLayout;
Check(context.Function<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &pipelineLayout), "vkCreatePipelineLayout first sample");
VkAttachmentDescription attachment{};
attachment.format = format;
attachment.samples = VK_SAMPLE_COUNT_1_BIT;
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
attachment.finalLayout = VK_IMAGE_LAYOUT_GENERAL;
const VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_GENERAL};
VkSubpassDescription subpass{};
subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
subpass.colorAttachmentCount = 1;
subpass.pColorAttachments = &reference;
const VkSubpassDependency dependencies[2] = {
{VK_SUBPASS_EXTERNAL, 0, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, 0},
{0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, 0},
};
VkRenderPassCreateInfo renderPassInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
renderPassInfo.attachmentCount = 1;
renderPassInfo.pAttachments = &attachment;
renderPassInfo.subpassCount = 1;
renderPassInfo.pSubpasses = &subpass;
renderPassInfo.dependencyCount = 2;
renderPassInfo.pDependencies = dependencies;
Check(context.Function<PFN_vkCreateRenderPass>("vkCreateRenderPass")(context.device, &renderPassInfo, nullptr, &renderPass), "vkCreateRenderPass first sample");
VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO};
framebufferInfo.renderPass = renderPass;
framebufferInfo.attachmentCount = 1;
framebufferInfo.pAttachments = &view;
framebufferInfo.width = extent.width;
framebufferInfo.height = extent.height;
framebufferInfo.layers = 1;
Check(context.Function<PFN_vkCreateFramebuffer>("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer first sample");
vertex = CreateShaderModule(context, PRESENTATION_PASS_VERT_SPV, sizeof(PRESENTATION_PASS_VERT_SPV), "vkCreateShaderModule first sample vertex");
const auto code = fragmentCode(format);
fragment = CreateShaderModule(context, code.data(), code.size_bytes(), "vkCreateShaderModule first sample fragment");
VkPipelineShaderStageCreateInfo stages[2]{{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}, {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}};
stages[0].stage = VK_SHADER_STAGE_VERTEX_BIT;
stages[0].module = vertex;
stages[0].pName = "main";
stages[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT;
stages[1].module = fragment;
stages[1].pName = "main";
VkPipelineVertexInputStateCreateInfo vertexInput{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO};
VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO};
assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
const VkViewport viewportRect{0.0f, 0.0f, static_cast<float>(extent.width), static_cast<float>(extent.height), 0.0f, 1.0f};
const VkRect2D scissor{{0, 0}, extent};
VkPipelineViewportStateCreateInfo viewport{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO};
viewport.viewportCount = 1;
viewport.pViewports = &viewportRect;
viewport.scissorCount = 1;
viewport.pScissors = &scissor;
VkPipelineRasterizationStateCreateInfo rasterization{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO};
rasterization.polygonMode = VK_POLYGON_MODE_FILL;
rasterization.cullMode = VK_CULL_MODE_NONE;
rasterization.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
rasterization.lineWidth = 1.0f;
VkPipelineMultisampleStateCreateInfo multisample{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO};
multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
VkPipelineColorBlendAttachmentState blendAttachment{};
blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
blend.attachmentCount = 1;
blend.pAttachments = &blendAttachment;
VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO};
pipelineInfo.stageCount = 2;
pipelineInfo.pStages = stages;
pipelineInfo.pVertexInputState = &vertexInput;
pipelineInfo.pInputAssemblyState = &assembly;
pipelineInfo.pViewportState = &viewport;
pipelineInfo.pRasterizationState = &rasterization;
pipelineInfo.pMultisampleState = &multisample;
pipelineInfo.pColorBlendState = &blend;
pipelineInfo.layout = pipelineLayout;
pipelineInfo.renderPass = renderPass;
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines first sample");
} catch (...) {
release();
throw;
}
}
FirstSampleCopy::~FirstSampleCopy() {
release();
}
void FirstSampleCopy::release() noexcept {
if (pipeline) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, pipeline, nullptr);
if (fragment) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, fragment, nullptr);
if (vertex) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, vertex, nullptr);
if (framebuffer) context.Function<PFN_vkDestroyFramebuffer>("vkDestroyFramebuffer")(context.device, framebuffer, nullptr);
if (renderPass) context.Function<PFN_vkDestroyRenderPass>("vkDestroyRenderPass")(context.device, renderPass, nullptr);
if (pipelineLayout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr);
if (descriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr);
if (sampler) context.Function<PFN_vkDestroySampler>("vkDestroySampler")(context.device, sampler, nullptr);
if (view) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, view, nullptr);
if (image) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, image, nullptr);
if (memory) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, memory, nullptr);
pipeline = VK_NULL_HANDLE;
fragment = vertex = VK_NULL_HANDLE;
framebuffer = VK_NULL_HANDLE;
renderPass = VK_NULL_HANDLE;
pipelineLayout = VK_NULL_HANDLE;
descriptorLayout = VK_NULL_HANDLE;
sampler = VK_NULL_HANDLE;
view = VK_NULL_HANDLE;
image = VK_NULL_HANDLE;
memory = VK_NULL_HANDLE;
}
void FirstSampleCopy::Record(Recorder& recorder, VkImageView source, VkImageLayout layout) {
const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1};
VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
poolInfo.maxSets = 1;
poolInfo.poolSizeCount = 1;
poolInfo.pPoolSizes = &size;
VkDescriptorPool created = VK_NULL_HANDLE;
Check(context.Function<PFN_vkCreateDescriptorPool>("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &created), "vkCreateDescriptorPool first sample");
const auto pool = std::make_shared<DescriptorPool>(context, created);
VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
allocation.descriptorPool = pool->pool;
allocation.descriptorSetCount = 1;
allocation.pSetLayouts = &descriptorLayout;
VkDescriptorSet set = VK_NULL_HANDLE;
Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets first sample");
const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, source, layout};
VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
write.dstSet = set;
write.dstBinding = 0;
write.descriptorCount = 1;
write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
write.pImageInfo = &imageInfo;
context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, 1, &write, 0, nullptr);
recorder.Keep(pool);
recorder.RecordUncounted([&](VkCommandBuffer commands) {
VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
begin.renderPass = renderPass;
begin.framebuffer = framebuffer;
begin.renderArea = {{0, 0}, extent};
context.Resolved(&DeviceFunctions::cmdBeginRenderPass, "vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
context.Function<PFN_vkCmdBindDescriptorSets>("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, 0, nullptr);
context.Function<PFN_vkCmdDraw>("vkCmdDraw")(commands, 3, 1, 0, 0);
context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands);
});
}
}
core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp
+61 -31
View File
@@ -294,7 +294,7 @@ HostImports& Imports() {
#ifdef _WIN32
void waitSettleBatch(const ImportSettle& settle, VkDevice device, const SettleBatch& batch) {
if (batch.busy && settle.waitForFences != nullptr) static_cast<void>(settle.waitForFences(device, 1, &batch.fence, VK_TRUE, 10'000'000'000ull));
if (batch.busy && settle.waitForFences != nullptr) Check(settle.waitForFences(device, 1, &batch.fence, VK_TRUE, 60'000'000'000ull), "vkWaitForFences (settle batch)");
}
bool settleBatchRunning(const ImportSettle& settle, VkDevice device, const SettleBatch& batch) {
@@ -988,7 +988,7 @@ const char* prepareSettleBatch(const Context& context, ImportSettle& settle, Set
}
if (batch.commands == VK_NULL_HANDLE) {
VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
allocate.commandPool = context.settlePool;
allocate.commandPool = context.pool;
allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocate.commandBufferCount = 1;
if ((result = context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocate, &batch.commands)) != VK_SUCCESS) {
@@ -1038,9 +1038,7 @@ void submitSettleBatch(const Context& context, HostImports& state) {
begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if ((result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(batch.commands, &begin)) != VK_SUCCESS) return fail("vkBeginCommandBuffer (settle batch)");
const auto copy = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer");
const bool otherFamily = context.settleFamily != context.queueFamily;
const auto createBuffer = otherFamily ? context.Function<PFN_vkCreateBuffer>("vkCreateBuffer") : nullptr;
const auto bindBufferMemory = otherFamily ? context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory") : nullptr;
RecordMemoryBarrier(context, batch.commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
std::size_t consumed = 0;
for (; consumed < state.settleWanted.size() && batch.bases.size() < settleBatchCopies; ++consumed) {
const auto base = state.settleWanted[consumed];
@@ -1051,38 +1049,19 @@ void submitSettleBatch(const Context& context, HostImports& state) {
state.pendingSettles.erase(settle);
continue;
}
VkBuffer source = import->second.buffer;
if (otherFamily) {
VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT;
VkBufferCreateInfo aliasInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
aliasInfo.size = std::min<VkDeviceSize>(import->second.bytes, 65536);
aliasInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
VkBuffer alias = VK_NULL_HANDLE;
if (createBuffer(context.device, &aliasInfo, nullptr, &alias) != VK_SUCCESS) {
state.pendingSettles.erase(settle);
continue;
}
if (bindBufferMemory(context.device, alias, import->second.memory, 0) != VK_SUCCESS) {
destroyBuffer(context.device, alias, nullptr);
state.pendingSettles.erase(settle);
continue;
}
batch.aliases.push_back(alias);
source = alias;
}
const VkBufferCopy region{0, (VkDeviceSize{index} * settleBatchCopies + batch.bases.size()) * 4u, 4};
copy(batch.commands, source, deviceSettle.batchTarget, 1, &region);
copy(batch.commands, import->second.buffer, deviceSettle.batchTarget, 1, &region);
settle->second = PendingSettle{SettlePhase::Submitted, index, context.device};
batch.bases.push_back(base);
}
state.settleWanted.erase(state.settleWanted.begin(), state.settleWanted.begin() + static_cast<std::ptrdiff_t>(consumed));
RecordMemoryBarrier(context, batch.commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_WRITE_BIT);
if ((result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(batch.commands)) != VK_SUCCESS) return fail("vkEndCommandBuffer (settle batch)");
if (batch.bases.empty()) return;
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit.commandBufferCount = 1;
submit.pCommandBuffers = &batch.commands;
if ((result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.settleQueue, 1, &submit, batch.fence)) != VK_SUCCESS) return fail("vkQueueSubmit (settle batch)");
if ((result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submit, batch.fence)) != VK_SUCCESS) return fail("vkQueueSubmit (settle batch)");
batch.busy = true;
state.settlesSubmitted += batch.bases.size();
}
@@ -1547,6 +1526,7 @@ std::string budgetSource(const Context& context, const HostImports& state, std::
}
std::atomic<std::uint64_t> rangeImportWindow{64ull << 20u};
std::atomic<std::uint64_t> importMaxBytes{0};
// The import of an access to a registered range larger than the range window: the cells the access touches, grown to
// the union of the range's imports it overlaps, so accesses across neighbouring cell edges share one import instead of
@@ -1654,6 +1634,32 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
return importRunAt(context, state, std::move(run), false, address, lease, end, evict, requester);
}
struct ImportPiece {
std::uint64_t begin;
std::uint64_t end;
const HostImport* entry;
};
std::vector<ImportPiece> importPieces(const Context& context, HostImports& state, const GuestAllocations::Range& range, const GuestAllocations::Lease& lease) {
const auto limit = importMaxBytes.load(std::memory_order_relaxed);
const auto window = rangeImportWindow.load(std::memory_order_relaxed);
const auto alignment = context.hostImportAlignment;
if (limit < 2 * window || window == 0 || alignment == 0 || window % alignment != 0 || range.address % alignment != 0) return {};
const auto step = (limit / window - 1) * window;
const auto end = range.address + range.bytes;
std::vector<ImportPiece> pieces;
for (auto cursor = range.address; cursor < end;) {
const auto wanted = end - cursor < step + window ? end : cursor + step;
const auto* entry = importAllocation(context, state, cursor, lease, ImportRequester::Build, wanted, false);
if (entry == nullptr || entry->base > cursor || entry->bytes > limit) return {};
const auto to = std::min(end, entry->base + entry->bytes);
if (to <= cursor) return {};
pieces.push_back({cursor, to, entry});
cursor = to;
}
return pieces;
}
const HostImport* importJoined(const Context& context, HostImports& state, std::uint64_t begin, std::uint64_t end, const GuestAllocations::Lease& lease) {
if (const auto* low = peekImport(state, begin, begin + 1); low != nullptr && low->joined) begin = low->base;
if (const auto* high = peekImport(state, end - 1, end); high != nullptr && high->joined) end = high->base + high->bytes;
@@ -1683,6 +1689,7 @@ const HostImport* importRunAt(const Context& context, HostImports& state, Import
return &found->second;
}
if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0) return nullptr;
if (const auto limit = importMaxBytes.load(std::memory_order_relaxed); limit != 0 && bytes > limit) return nullptr;
if (const auto failed = failures.find(base); !stepDown && failed != failures.end()) {
if (sameRun(failed->second, run)) return nullptr;
failures.erase(failed);
@@ -3501,6 +3508,10 @@ void SetHostImportRangeWindow(std::uint64_t bytes) {
rangeImportWindow.store(bytes, std::memory_order_relaxed);
}
void SetHostImportMaxBytes(std::uint64_t bytes) {
importMaxBytes.store(bytes, std::memory_order_relaxed);
}
void SetHostImportBudget(std::uint64_t bytes) {
budgetOverride.store(bytes, std::memory_order_relaxed);
auto& state = Imports();
@@ -3864,8 +3875,14 @@ void GuestBufferMemory::AcquireRegistered() {
validate(range->address, range->bytes);
Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr};
const HostImport* entry = nullptr;
std::vector<ImportPiece> pieces;
const auto limit = importMaxBytes.load(std::memory_order_relaxed);
if (importable && covering != nullptr && range->address >= coveredBegin && range->address + range->bytes <= coveredEnd) {
entry = covering;
} else if (importable && limit != 0 && range->bytes > limit) {
auto& state = Imports();
std::lock_guard lock(state.mutex);
pieces = importPieces(context, state, *range, lease);
} else if (importable) {
auto& state = Imports();
std::lock_guard lock(state.mutex);
@@ -3877,6 +3894,16 @@ void GuestBufferMemory::AcquireRegistered() {
}
}
lap(timing.importsUs, at);
if (!pieces.empty()) {
for (const auto& piece : pieces) {
Region part{piece.begin, piece.end, range->writable, {}, nullptr};
part.hostBacked = true;
part.direct = piece.entry;
regions.push_back(std::move(part));
}
imported.push_back(range->address);
continue;
}
if (entry != nullptr) {
region.hostBacked = true;
// Reused by Upload while no import was dropped since (see importsEpoch).
@@ -4713,7 +4740,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) {
void GuestBufferMemory::completeSpaceWrites() const {
auto* recorder = Recorder::Active();
if (recorder == nullptr || Recorder::InCompletion()) return;
if (recorder == nullptr || Recorder::InCompletion() || !AnyPendingCpuWrites(context)) return;
const auto& base = space->base;
std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
std::vector<std::size_t> candidates;
@@ -5492,7 +5519,10 @@ void GuestBufferMemory::MarkDirectWrites() const {
const auto* found = owner(begin, end);
if (found == nullptr) continue;
const auto& region = *found;
if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin));
if (region.direct != nullptr || (region.gpuCopy && region.copiedBack)) {
const auto generation = GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin));
if (generation != 0) directWritesGeneration = std::max(directWritesGeneration, generation);
}
}
}
@@ -5584,7 +5614,7 @@ void GuestBufferMemory::WriteBack() {
if (region.direct == nullptr && !(region.gpuCopy && (region.copiedBack || region.deviceLocal)) && !(region.mirror != nullptr && !region.mirror->writable)) Recorder::NoteWrittenBack(begin, static_cast<std::size_t>(end - begin));
if (region.direct != nullptr) {
// The GPU wrote imported guest memory directly; page write watching did not see it.
GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin));
StorageTexture::StampKeepingNewerResults(begin, static_cast<std::size_t>(end - begin), directWritesGeneration, [&] { return GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin)); });
continue;
}
if (region.mirror != nullptr) {
@@ -5628,7 +5658,7 @@ void GuestBufferMemory::WriteBack() {
if (region.copiedBack) {
// The GPU copied the written sub-range back into the import; page write watching
// did not see it.
GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin));
StorageTexture::StampKeepingNewerResults(begin, static_cast<std::size_t>(end - begin), directWritesGeneration, [&] { return GuestMemory::MarkPossiblyWritten(begin, static_cast<std::size_t>(end - begin)); });
continue;
}
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp
+44 -3
View File
@@ -6,6 +6,7 @@
#include "RdnaDecoder/RdnaDescriptorFormat.hpp"
#include <algorithm>
#include <array>
#include <bit>
#include <optional>
#include <stdexcept>
#include <string>
@@ -179,12 +180,15 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
Require(baseLevel <= lastLevel, "guest texture descriptor has a base mip level past its last mip level");
const bool fmask = ShaderRecompiler::IsFmaskTextureFormat(static_cast<ShaderRecompiler::IrBufferFormat>(format));
const auto fmaskLayout = ShaderRecompiler::FmaskLayoutOf(static_cast<ShaderRecompiler::IrBufferFormat>(format));
if (fmask) {
Require(format == static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Fmask32_S8_F8), "guest FMASK format is not implemented");
Require(fmaskLayout.has_value(), "guest FMASK format is not implemented");
Require(!writeCompress && (!metaCompress || metaAddr != 0u), "FMASK CMASK compression is not implemented without readable metadata");
Require(!metaCompress || metaPipeAligned, "FMASK CMASK metadata must be pipe aligned");
Require(!metaCompress || format == static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Fmask32_S8_F8), "FMASK CMASK compression is implemented only for 8-sample, 8-fragment FMASK");
Require((typeRaw == 9u || typeRaw == 13u) && tileModeRaw == 24u && baseLevel == 0u && lastLevel == 0u && maxMip == 0u, "FMASK reads require a single-mip 2D or 2D-array Z_64KB_X surface");
for (const auto selector : {dstSelX, dstSelY, dstSelZ, dstSelW}) Require(selector == 0u || selector == 1u || selector == 4u, "FMASK destination channel selector is not implemented");
const bool identity = format != static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Fmask32_S8_F8) && dstSelX == 4u && dstSelY == 5u && dstSelZ == 6u && dstSelW == 7u;
for (const auto selector : {dstSelX, dstSelY, dstSelZ, dstSelW}) Require(identity || selector == 0u || selector == 1u || selector == 4u, "FMASK destination channel selector is not implemented");
}
const auto describe = [&] { return " (format " + std::to_string(format) + ", " + std::to_string(width) + "x" + std::to_string(height) + "x" + std::to_string(depth + 1) + ", type " + std::to_string(typeRaw) + ", tile " + std::to_string(tileModeRaw) + ", mips " + std::to_string(baseLevel) + ".." + std::to_string(lastLevel) + " of " + std::to_string(maxMip + 1) + ")"; };
@@ -265,7 +269,7 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
result.lastLevel = lastLevel;
result.tileMode = tileMode;
result.dimension = dimension;
result.format = fmask ? static_cast<std::uint32_t>(ShaderRecompiler::IrBufferFormat::Format32UInt) : format;
result.format = !fmask ? format : static_cast<std::uint32_t>(fmaskLayout->storageBits == 8u ? ShaderRecompiler::IrBufferFormat::Format8UInt : fmaskLayout->storageBits == 16u ? ShaderRecompiler::IrBufferFormat::Format16UInt : ShaderRecompiler::IrBufferFormat::Format32UInt);
result.dstSelX = static_cast<std::uint8_t>(dstSelX);
result.dstSelY = static_cast<std::uint8_t>(dstSelY);
result.dstSelZ = static_cast<std::uint8_t>(dstSelZ);
@@ -289,6 +293,43 @@ GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words)
return result;
}
std::array<std::uint32_t, 8> EncodeTextureResource(const GuestTextureResource& resource) {
const bool multisampled = resource.dimension == TextureDimension::k2DMsaa;
Require((resource.dimension == TextureDimension::k2D && resource.samples == 1u) || (multisampled && resource.samples >= 2u && resource.samples <= 16u && std::has_single_bit(resource.samples)), "only 2D and multisampled 2D surfaces have a texture descriptor encoding");
Require(resource.mipCount == 1u && resource.allocatedMipCount <= 1u && resource.baseLevel == 0u && resource.lastLevel == 0u && resource.depthOrLastArray == 0u && resource.baseArray == 0u && resource.dccAddress == 0u && resource.minLod == 0u && !resource.iterate256 && resource.bcSwizzle == 0u, "only single-level, single-layer surfaces without metadata have a texture descriptor encoding");
Require(resource.width >= 1u && resource.width <= 0x4000u && resource.height >= 1u && resource.height <= 0x4000u && resource.format < 0x200u, "the surface's extent or format has no texture descriptor encoding");
const auto xorMask = resource.tileMode == TextureTileMode::kS4KBX || resource.tileMode == TextureTileMode::kD4KBX ? 0xfffull : 0xffffull;
Require(XorSwizzleMode(resource.tileMode) != 0 ? (resource.baseAddress & xorMask) == 0 && (resource.pipeBankXor & ~xorMask) == 0 : resource.pipeBankXor == 0, "the surface's pipe and bank XOR has no texture descriptor encoding");
const auto base = (resource.baseAddress | resource.pipeBankXor) >> 8u;
Require(base != 0 && (resource.baseAddress & 0xffu) == 0 && base <= 0xffffffffffull, "the surface's address has no texture descriptor encoding");
std::uint32_t tileMode = 0;
switch (resource.tileMode) {
case TextureTileMode::kLinear: tileMode = 0x00; break;
case TextureTileMode::kStandard256B: tileMode = 0x01; break;
case TextureTileMode::kD256B: tileMode = 0x02; break;
case TextureTileMode::kStandard4KB: tileMode = 0x05; break;
case TextureTileMode::kD4KB: tileMode = 0x06; break;
case TextureTileMode::kStandard64KB: tileMode = 0x09; break;
case TextureTileMode::kD64KB: tileMode = 0x0a; break;
case TextureTileMode::kS64KBT: tileMode = 0x11; break;
case TextureTileMode::kD64KBT: tileMode = 0x12; break;
case TextureTileMode::kS4KBX: tileMode = 0x15; break;
case TextureTileMode::kD4KBX: tileMode = 0x16; break;
case TextureTileMode::kZ64KBX: tileMode = 0x18; break;
case TextureTileMode::kS64KBX: tileMode = 0x19; break;
case TextureTileMode::kD64KBX: tileMode = 0x1a; break;
case TextureTileMode::kR64KBX: tileMode = 0x1b; break;
}
const auto sampleLevel = static_cast<std::uint32_t>(std::countr_zero(resource.samples));
std::array<std::uint32_t, 8> words{};
words[0] = static_cast<std::uint32_t>(base);
words[1] = static_cast<std::uint32_t>(base >> 32u) | (resource.format << 20u) | (((resource.width - 1u) & 3u) << 30u);
words[2] = ((resource.width - 1u) >> 2u) | ((resource.height - 1u) << 14u);
words[3] = resource.dstSelX | (resource.dstSelY << 3u) | (resource.dstSelZ << 6u) | (resource.dstSelW << 9u) | (sampleLevel << 16u) | (tileMode << 20u) | ((multisampled ? 14u : 9u) << 28u);
words[5] = sampleLevel << 4u;
return words;
}
float EffectiveMinLod(const GuestTextureResource& resource) {
if (resource.minLod <= resource.baseLevel * 256u) return 0.0f;
return std::min(static_cast<float>(resource.minLod) / 256.0f, static_cast<float>(resource.lastLevel));
core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp
+42 -1
View File
@@ -2,6 +2,7 @@
#include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
@@ -426,6 +427,19 @@ public:
return {shared_from_this(), texture.get()};
}
std::shared_ptr<Texture> FirstSample(VkComponentMapping components) {
const auto source = Sampled({});
if (firstSampleCopy == nullptr) firstSampleCopy = std::make_unique<FirstSampleCopy>(context, target.format, target.extent);
firstSampleCopy->Record(*Recorder::Active(), source->View(), source->Layout());
const std::array<std::uint32_t, 4> key{static_cast<std::uint32_t>(components.r), static_cast<std::uint32_t>(components.g), static_cast<std::uint32_t>(components.b), static_cast<std::uint32_t>(components.a)};
auto& texture = firstSampleTextures[key];
if (texture == nullptr) {
texture = std::make_shared<Texture>(context, firstSampleCopy->Image(), target.format, VK_IMAGE_ASPECT_COLOR_BIT, components);
texture->MarkRefreshedPerUse();
}
return {shared_from_this(), texture.get()};
}
VkImageView View(VkFormat format) {
if (format == target.format) return view;
Require(twinFormat != VK_FORMAT_UNDEFINED && format == twinFormat, "a multisampled color target is only viewed in its own format or its sRGB twin");
@@ -516,6 +530,8 @@ private:
std::map<std::array<std::uint32_t, 5>, std::unique_ptr<Texture>> fmaskTextures;
std::map<std::array<std::uint32_t, 5>, std::shared_ptr<Texture>> textures;
std::vector<std::shared_ptr<Texture>> retiredTextures;
std::unique_ptr<FirstSampleCopy> firstSampleCopy;
std::map<std::array<std::uint32_t, 4>, std::shared_ptr<Texture>> firstSampleTextures;
std::vector<GuestMemory::UnchangedQuery> dccRanges;
std::optional<std::uint32_t> dccFill;
std::optional<std::uint32_t> cmaskFill;
@@ -626,6 +642,8 @@ private:
fmaskTextures.clear();
textures.clear();
retiredTextures.clear();
firstSampleTextures.clear();
firstSampleCopy.reset();
for (const auto& [format, resolvePass] : resolvePasses) context.Function<PFN_vkDestroyRenderPass>("vkDestroyRenderPass")(context.device, resolvePass, nullptr);
if (orderView) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, orderView, nullptr);
if (twinView) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, twinView, nullptr);
@@ -1015,7 +1033,7 @@ VkClearColorValue MultisampledDccClear(const ColorTarget& target, std::span<cons
Require(dcc.size() == MultisampledDccBytes(target), "multisampled DCC metadata snapshot size mismatch");
Require(std::ranges::all_of(dcc, [&](std::byte key) { return key == dcc.front(); }), "mixed multisampled DCC metadata is not implemented");
Require(target.cmaskAddress == 0 ? cmask.empty() && target.cmaskBytes == 0 : cmask.size() == target.cmaskBytes && target.cmaskBytes == CmaskBytes(target.extent.width, target.extent.height), "multisampled DCC CMASK snapshot size mismatch");
if (!cmask.empty()) {
if (!cmask.empty() && target.cmaskFastClear) {
const auto code = std::to_integer<std::uint32_t>(cmask.front());
const auto expanded = target.samples == 2 ? 0xddu : target.samples == 4 ? 0xeeu : 0xffu;
Require(std::ranges::all_of(cmask, [&](std::byte key) { return key == cmask.front(); }) && (code == 0xccu || code == expanded), "multisampled DCC CMASK compression or register clear is not implemented");
@@ -1159,6 +1177,29 @@ std::shared_ptr<Texture> MultisampleTargetTexture(const Context& context, const
throw std::runtime_error(text);
}
std::shared_ptr<Texture> MultisampleTargetFirstSample(const Context& context, const GuestTextureResource& resource, VkComponentMapping components) {
std::lock_guard lock(targetsMutex());
discardRetiredTargets();
const auto format = ResolveTextureFormat(resource.format);
const MultisampleTarget* other = nullptr;
for (const auto& target : targets()) {
if (!target->active->load(std::memory_order_acquire)) continue;
if (target->context.device != context.device || target->target.surfaceAddress != resource.baseAddress) continue;
const auto& rendered = target->target;
const bool sameShape = rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0;
if (sameShape && rendered.format == format) return target->FirstSample(components);
if (sameShape && redBlueSwapped(rendered.format, format)) return target->FirstSample(swapRedBlue(components));
other = target.get();
}
if (other == nullptr) {
Require(Recorder::Active() != nullptr, "a multisampled texture is only sampled by recorded draws and dispatches");
return createFilled(context, resource, format, "sampling").FirstSample(components);
}
char text[320];
std::snprintf(text, sizeof(text), "AGC graphics: reading sample 0 of multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast<unsigned long long>(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast<int>(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast<int>(format));
throw std::runtime_error(text);
}
void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget& target) {
std::lock_guard lock(targetsMutex());
acquire(context, target);
core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp
+96 -11
View File
@@ -2,6 +2,7 @@
#include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
@@ -124,6 +125,70 @@ void ValidateProvokingVertex(const Context& context, const State& state, std::sp
}
return;
}
struct Decorations {
std::unordered_map<std::uint64_t, std::uint32_t> builtIns;
std::vector<std::uint64_t> perVertexValues;
std::vector<std::uint64_t> perPrimitiveValues;
};
const auto readDecorations = [](const ShaderRecompiler::RecompileResult& program, Decorations& decorations) {
const auto words = program.spirv.Words();
if (words.size() < 5) return false;
constexpr std::uint32_t NoMember = 0xffffffffu;
const auto key = [](std::uint32_t target, std::uint32_t member) { return (static_cast<std::uint64_t>(target) << 32u) | member; };
for (std::size_t cursor = 5; cursor < words.size();) {
const auto count = words[cursor] >> 16u;
if (count == 0 || count > words.size() - cursor) return false;
const auto op = static_cast<spv::Op>(words[cursor] & 0xffffu);
const std::size_t decoration = op == spv::OpDecorate ? 2 : op == spv::OpMemberDecorate ? 3 : 0;
if (decoration != 0 && count > decoration) {
const auto target = key(words[cursor + 1], op == spv::OpMemberDecorate ? words[cursor + 2] : NoMember);
if (words[cursor + decoration] == spv::DecorationFlat || words[cursor + decoration] == spv::DecorationPerVertexKHR) decorations.perVertexValues.push_back(target);
if (words[cursor + decoration] == spv::DecorationPerPrimitiveEXT) decorations.perPrimitiveValues.push_back(target);
if (words[cursor + decoration] == spv::DecorationBuiltIn && count > decoration + 1) decorations.builtIns[target] = words[cursor + decoration + 1];
}
cursor += count;
}
return true;
};
const auto readsProvokingVertex = [&](const ShaderRecompiler::RecompileResult& program) {
if (std::any_of(program.fragmentParameters.begin(), program.fragmentParameters.end(), [](const ShaderRecompiler::FragmentParameter& parameter) {
return parameter.flat || parameter.perVertex || parameter.custom || parameter.flatP10;
})) return true;
Decorations decorations;
if (!readDecorations(program, decorations)) return true;
const auto& builtIns = decorations.builtIns;
return std::any_of(decorations.perVertexValues.begin(), decorations.perVertexValues.end(), [&](std::uint64_t target) {
const auto builtIn = builtIns.find(target);
if (builtIn == builtIns.end()) return true;
switch (builtIn->second) {
case spv::BuiltInFragCoord:
case spv::BuiltInPointCoord:
case spv::BuiltInFrontFacing:
case spv::BuiltInSampleId:
case spv::BuiltInSamplePosition:
case spv::BuiltInSampleMask:
case spv::BuiltInHelperInvocation:
case spv::BuiltInSubgroupSize:
case spv::BuiltInSubgroupLocalInvocationId:
return false;
default:
return true;
}
});
};
const auto writesProvokingVertexBuiltIn = [&](const ShaderRecompiler::RecompileResult& program) {
Decorations decorations;
if (!readDecorations(program, decorations)) return true;
return std::any_of(decorations.builtIns.begin(), decorations.builtIns.end(), [&](const auto& builtIn) {
return (builtIn.second == spv::BuiltInLayer || builtIn.second == spv::BuiltInViewportIndex) &&
std::find(decorations.perPrimitiveValues.begin(), decorations.perPrimitiveValues.end(), builtIn.first) == decorations.perPrimitiveValues.end();
});
};
const bool observed = std::any_of(shaders.begin(), shaders.end(), [&](const CompiledShader& shader) {
if (shader.stage == ShaderRecompiler::ShaderStage::Fragment) return shader.program == nullptr || readsProvokingVertex(*shader.program);
return shader.program != nullptr && writesProvokingVertexBuiltIn(*shader.program);
});
if (!context.provokingVertexLast && !observed) return;
Require(context.provokingVertexLast, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
const bool meshPrimitives = state.stages.path == ShaderPath::Geometry && state.stages.mesh;
Require(!state.stages.tessellation && (meshPrimitives || (state.stages.path == ShaderPath::Vertex && !state.stages.mesh)), "last provoking vertex is unsupported for generated primitive pipelines");
@@ -167,8 +232,10 @@ bool ShadersWriteMemory(std::span<const CompiledShader> shaders) {
if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && any(binding.imageWritten, binding.count)) return true;
break;
case ShaderRecompiler::DescriptorRole::Gds:
case ShaderRecompiler::DescriptorRole::BdaPagetable:
return true;
case ShaderRecompiler::DescriptorRole::BdaPagetable:
if (shader.program->bdaWrites) return true;
break;
case ShaderRecompiler::DescriptorRole::GuestSamplers:
case ShaderRecompiler::DescriptorRole::FaultBuffer:
case ShaderRecompiler::DescriptorRole::FlattenedSrt:
@@ -213,7 +280,7 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders) {
using Stage = ShaderRecompiler::ShaderStage;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state)) return false;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; })) return false;
keys = {};
for (const auto& shader : shaders) {
const auto identity = shader.program->Identity();
@@ -288,6 +355,7 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State&
for (const auto& color : state.colors) {
appendKey(keys.renderPass, color.format);
appendKey(keys.renderPass, color.exportIndex);
appendKey(keys.renderPass, color.partialWriteMask != 0u);
}
appendKey(keys.renderPass, state.depth.has_value());
if (state.depth) appendKey(keys.renderPass, state.depth->format);
@@ -330,6 +398,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span<const CompiledShader> shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) {
PerformanceTimer timing("Vulkan.GraphicsPipeline");
for (const auto& color : state.colors) boundAttachments.push_back(color.partialWriteMask == 0u);
if (state.depth) boundAttachments.push_back(true);
sampleLocationsEnabled = ValidateSampleLocations(context, state);
const auto host = HostSampleLocationsFor(context, state);
sampleLocationGrid = host.grid;
@@ -339,7 +409,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
this->context.bufferPool.reset();
Require(state.blends.size() == (state.colors.empty() ? 0u : state.colors.back().exportIndex + 1u) && state.colors.size() <= state.blends.size(), "blend states do not match decoded color state");
Require(state.blends.size() <= context.limits.maxColorAttachments, "color targets exceed device attachment limits");
Require(state.hasColorTarget || (context.limits.framebufferNoAttachmentsSampleCounts & static_cast<VkSampleCountFlagBits>(state.samples)) != 0, "device does not support rendering without attachments at the draw's sample count");
const bool boundColor = std::ranges::any_of(state.colors, [](const ColorTarget& color) { return color.partialWriteMask == 0u; });
Require((state.hasColorTarget && (boundColor || state.colors.empty() || state.depth.has_value())) || (context.limits.framebufferNoAttachmentsSampleCounts & static_cast<VkSampleCountFlagBits>(state.samples)) != 0, "device does not support rendering without attachments at the draw's sample count");
Require(state.samples == 1 || !state.depth.has_value() || (context.limits.framebufferDepthSampleCounts & static_cast<VkSampleCountFlagBits>(state.samples)) != 0, "device does not support depth targets with the draw's sample count");
Require(!depthBounds || context.depthBounds, "device does not support the depth bounds test");
Require(!depthBias || state.depthBiasClamp == 0.0f || context.depthBiasClamp, "device does not support depth bias clamping");
@@ -397,12 +468,14 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
}
// A descriptor set layout with the same bindings as this one is compatible with the pipeline
// layout, so later draws bind their own ShaderResources' set under it.
const std::array<VkDescriptorSetLayout, 2> setLayouts{resources.main, resources.fragment};
const bool pixelLds = std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; });
const auto pixelLdsLayouts = pixelLds ? PixelFunctionLdsSetLayouts(context) : PixelFunctionLdsLayouts{};
const std::array<VkDescriptorSetLayout, 3> setLayouts{resources.main, pixelLds && !resources.split ? pixelLdsLayouts.empty : resources.fragment, pixelLdsLayouts.buffer};
const VkPushConstantRange push{pushStages, 0, PipelinePushSlotBytes};
const std::array<VkPushConstantRange, 2> slots{{{pushStagesOf(context), 0, PipelinePushSlotBytes}, {VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes}}};
VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
layoutInfo.flags = resources.split ? VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT : 0;
layoutInfo.setLayoutCount = resources.split ? 2u : 1u;
layoutInfo.flags = pixelLds || !resources.split ? 0 : VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT;
layoutInfo.setLayoutCount = pixelLds ? 3u : resources.split ? 2u : 1u;
layoutInfo.pSetLayouts = setLayouts.data();
layoutInfo.pushConstantRangeCount = dynamicRendering ? static_cast<std::uint32_t>(slots.size()) : pushStages != 0 ? 1 : 0;
layoutInfo.pPushConstantRanges = dynamicRendering ? slots.data() : pushStages != 0 ? &push : nullptr;
@@ -410,6 +483,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
std::vector<VkAttachmentDescription> colors;
std::vector<VkAttachmentReference> references(state.blends.size(), VkAttachmentReference{VK_ATTACHMENT_UNUSED, attachmentLayout});
for (std::uint32_t index = 0; index < state.colors.size(); ++index) {
if (state.colors[index].partialWriteMask != 0u) continue;
VkAttachmentDescription color{};
color.format = state.colors[index].format;
color.samples = state.mergedCoverage || state.truncatedCoverage ? VK_SAMPLE_COUNT_1_BIT : static_cast<VkSampleCountFlagBits>(state.samples);
@@ -419,9 +493,10 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
color.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
color.initialLayout = attachmentLayout;
color.finalLayout = attachmentLayout;
references.at(state.colors[index].exportIndex) = {static_cast<std::uint32_t>(colors.size()), attachmentLayout};
colors.push_back(color);
references.at(state.colors[index].exportIndex) = {index, attachmentLayout};
}
boundColorAttachments = static_cast<std::uint32_t>(colors.size());
VkSubpassDescription subpass{};
subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
subpass.colorAttachmentCount = static_cast<std::uint32_t>(references.size());
@@ -441,7 +516,9 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
subpass.pDepthStencilAttachment = &depthReference;
}
std::vector<VkFormat> renderingColors(state.blends.size(), VK_FORMAT_UNDEFINED);
for (const auto& color : state.colors) renderingColors.at(color.exportIndex) = color.format;
for (const auto& color : state.colors) {
if (color.partialWriteMask == 0u) renderingColors.at(color.exportIndex) = color.format;
}
VkPipelineRenderingCreateInfoKHR rendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR};
rendering.colorAttachmentCount = static_cast<std::uint32_t>(renderingColors.size());
rendering.pColorAttachmentFormats = renderingColors.empty() ? nullptr : renderingColors.data();
@@ -458,7 +535,9 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
if (dynamicRendering) {
depthFormat = state.depth ? HostDepthFormat(context, state.depth->format) : VK_FORMAT_UNDEFINED;
colorSlots.assign(state.blends.size(), ~0u);
for (std::uint32_t index = 0; index < state.colors.size(); ++index) colorSlots.at(state.colors[index].exportIndex) = index;
for (std::uint32_t index = 0; index < state.colors.size(); ++index) {
if (state.colors[index].partialWriteMask == 0u) colorSlots.at(state.colors[index].exportIndex) = index;
}
this->attachmentLayout = attachmentLayout;
}
VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO};
@@ -704,7 +783,11 @@ std::shared_ptr<Framebuffer> Pipeline::AcquireFramebuffer(std::span<const VkImag
return framebuffers.back().framebuffer;
}
}
auto framebuffer = std::make_shared<Framebuffer>(context, renderPass, targets, extent, layers);
std::vector<VkImageView> boundViews;
for (std::size_t i = 0; i < targets.size(); ++i) {
if (boundAttachments[i]) boundViews.push_back(targets[i]);
}
auto framebuffer = std::make_shared<Framebuffer>(context, renderPass, renderPass == VK_NULL_HANDLE ? targets : std::span<const VkImageView>(boundViews), extent, layers);
if (!resident) return framebuffer;
// Beyond the bound the least recently used unreferenced entry goes.
constexpr std::size_t bound = 8;
@@ -762,7 +845,7 @@ void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, V
const auto locations = SampleLocations(state);
const auto grid = sampleLocationGrid;
const VkSampleLocationsInfoEXT info{VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT, nullptr, static_cast<VkSampleCountFlagBits>(state.samples), grid, state.samples * grid.width * grid.height, locations.data()};
const VkAttachmentSampleLocationsEXT initial{static_cast<std::uint32_t>(state.colors.size()), info};
const VkAttachmentSampleLocationsEXT initial{boundColorAttachments, info};
const VkSubpassSampleLocationsEXT post{0, info};
VkRenderPassSampleLocationsBeginInfoEXT sampleLocations{VK_STRUCTURE_TYPE_RENDER_PASS_SAMPLE_LOCATIONS_BEGIN_INFO_EXT};
sampleLocations.attachmentInitialSampleLocationsCount = state.depth ? 1 : 0;
@@ -913,6 +996,7 @@ void pipelineKey(std::vector<std::byte>& key, const Context& context, const Stat
for (const auto& color : state.colors) {
append(key, color.format);
append(key, color.componentMapping);
append(key, color.partialWriteMask != 0u);
}
if (state.blends.size() != state.colors.size()) {
for (const auto& color : state.colors) append(key, color.exportIndex);
@@ -1166,6 +1250,7 @@ void ClearCachedPipelines(VkDevice device) {
it = store.entries.erase(it);
}
ClearPipelineLibraries(device);
ReleasePixelFunctionLds(device);
}
}
core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp
+4 -1
View File
@@ -44,7 +44,7 @@ static_assert(sizeof(ShaderRecompiler::TessellationConfiguration) == 40, "Tessel
constexpr std::uint32_t FileMagic = 0x4c4a5041u;
// Bumped when the recipe codec, CanonicalPipelineState or the store key change in a way the
// layout guards above do not catch (a field renumbered or reinterpreted at the same size).
constexpr std::uint32_t FileFormat = 8u;
constexpr std::uint32_t FileFormat = 9u;
constexpr std::uint32_t ModuleRecord = 1u;
constexpr std::uint32_t GraphicsRecord = 2u;
constexpr std::uint32_t ComputeRecord = 3u;
@@ -195,6 +195,7 @@ void encodeState(Writer& writer, const State& state) {
writer.U32(static_cast<std::uint32_t>(color.format));
writer.U32(color.exportIndex);
writer.Raw(color.componentMapping);
writer.Raw(color.partialWriteMask);
}
writer.Raws(std::span<const VkPipelineColorBlendAttachmentState>(state.blends));
for (const auto value : state.blendConstants) writer.Raw(value);
@@ -260,6 +261,7 @@ void decodeState(Reader& reader, State& state) {
color.format = static_cast<VkFormat>(reader.U32());
color.exportIndex = reader.U32();
color.componentMapping = reader.Raw<std::uint8_t>();
color.partialWriteMask = reader.Raw<std::uint16_t>();
state.colors.push_back(color);
}
reader.Raws(state.blends, 8u);
@@ -1038,6 +1040,7 @@ State CanonicalPipelineState(const State& state) {
canonicalColor.format = color.format;
canonicalColor.exportIndex = color.exportIndex;
canonicalColor.componentMapping = color.componentMapping;
canonicalColor.partialWriteMask = color.partialWriteMask;
canonical.colors.push_back(canonicalColor);
}
canonical.blends = state.blends;
core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp
+35 -1
View File
@@ -516,6 +516,7 @@ std::atomic<std::int64_t> pendingLabelSince{NoPendingLabel};
std::atomic<std::uint64_t> writeGeneration{0};
std::atomic<std::uint64_t> publishGeneration{0};
std::atomic<std::uint64_t> completionLabels{0};
std::atomic<std::uint64_t> completionStores{0};
std::atomic<std::uint64_t> writeBackCompletions{0};
std::atomic<std::uint64_t> completionStoresSkipped{0};
std::atomic<std::uint64_t> completionStoresRun{0};
@@ -1191,6 +1192,7 @@ Recorder::~Recorder() {
publishGeneration.fetch_add(1, std::memory_order_release);
pendingLabelSince.store(NoPendingLabel, std::memory_order_release);
completionLabels.store(0, std::memory_order_release);
completionStores.store(0, std::memory_order_release);
writeBackCompletions.store(0, std::memory_order_release);
workSinceSubmit.store(0, std::memory_order_release);
}
@@ -1266,6 +1268,10 @@ std::uint64_t Recorder::PendingCompletionLabels() {
return completionLabels.load(std::memory_order_acquire);
}
std::uint64_t Recorder::PendingCompletionStores() {
return completionStores.load(std::memory_order_acquire);
}
std::uint64_t Recorder::PendingWriteBackCompletions() {
return writeBackCompletions.load(std::memory_order_acquire);
}
@@ -2281,6 +2287,20 @@ void Recorder::EndPassSamples() {
if (open != nullptr && open->segmentActive) endSegment(*open);
}
void Recorder::RecordUncounted(const std::function<void(VkCommandBuffer)>& record) {
const auto commands = Commands();
if (open->segmentActive) endSegment(*open);
const bool counted = open->sampleActive;
if (counted) {
open->endedSamples.reserve(open->endedSamples.size() + 1);
endSamples(*open);
open->endedSamples.push_back(open->samples);
open->samples = VK_NULL_HANDLE;
}
record(commands);
if (counted) beginSamples(*open);
}
void Recorder::takeSegmentPool(Batch& batch) {
auto cache = std::static_pointer_cast<SamplePoolCache>(segmentPools);
if (cache == nullptr) {
@@ -2571,9 +2591,15 @@ void Recorder::beginSamples(Batch& batch) {
void Recorder::readSamples(Batch& batch) {
if (sampleCounterState > 0) return;
if (batch.samples == VK_NULL_HANDLE || !batch.submitted) return;
std::uint64_t total = 0;
for (const auto pool : batch.endedSamples) {
std::uint64_t samples = 0;
Check(context.Function<PFN_vkGetQueryPoolResults>("vkGetQueryPoolResults")(context.device, pool, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion");
total += samples;
}
std::uint64_t samples = 0;
Check(context.Function<PFN_vkGetQueryPoolResults>("vkGetQueryPoolResults")(context.device, batch.samples, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion");
samplesPassed.fetch_add(samples, std::memory_order_acq_rel);
samplesPassed.fetch_add(total + samples, std::memory_order_acq_rel);
}
void Recorder::readGpuTiming(Batch& batch) {
@@ -3450,6 +3476,8 @@ void Recorder::AfterCompletions(std::uint64_t address, std::span<const std::byte
} else {
++batch.completionLabelCount;
completionLabels.fetch_add(1, std::memory_order_acq_rel);
++batch.completionStoreCount;
completionStores.fetch_add(1, std::memory_order_acq_rel);
GuestMemory::BreakProofs();
}
// The table entry before the write note: the note bumps the generation a poller watches, and a
@@ -3497,6 +3525,8 @@ void Recorder::NoteWrittenBack(std::uint64_t address, std::size_t bytes) {
completionLabels.fetch_add(1, std::memory_order_acq_rel);
GuestMemory::BreakProofs();
completionLabelsCountedLate.fetch_add(1, std::memory_order_relaxed);
++batch.completionStoreCount;
completionStores.fetch_add(1, std::memory_order_acq_rel);
recorder->markBehindCompletion(batch, label.begin, label.end);
}
};
@@ -4001,6 +4031,7 @@ void Recorder::finish(std::unique_ptr<Batch> batch, bool wait, int source, bool
list.erase(std::remove(list.begin(), list.end(), batch), list.end());
// The batch's completion labels have landed, or never will (fence failure below).
if (batch->completionLabelCount != 0) completionLabels.fetch_sub(batch->completionLabelCount, std::memory_order_acq_rel);
if (batch->completionStoreCount != 0) completionStores.fetch_sub(batch->completionStoreCount, std::memory_order_acq_rel);
if (batch->writeBackCompletionCount != 0) writeBackCompletions.fetch_sub(batch->writeBackCompletionCount, std::memory_order_acq_rel);
// This also runs while the fence-failure throw unwinds: an allocation failure in the
// rebuild keeps the previous snapshot, which is a superset (a stale entry only causes a
@@ -4145,6 +4176,7 @@ void Recorder::finish(std::unique_ptr<Batch> batch, bool wait, int source, bool
void Recorder::discard(std::unique_ptr<Batch> batch) noexcept {
removeLabels(*batch);
if (batch->completionLabelCount != 0) completionLabels.fetch_sub(batch->completionLabelCount, std::memory_order_acq_rel);
if (batch->completionStoreCount != 0) completionStores.fetch_sub(batch->completionStoreCount, std::memory_order_acq_rel);
if (batch->writeBackCompletionCount != 0) writeBackCompletions.fetch_sub(batch->writeBackCompletionCount, std::memory_order_acq_rel);
if (activeRecorder == this) pendingLabelSince.store(NoPendingLabel, std::memory_order_release);
release(*batch);
@@ -4183,6 +4215,8 @@ void Recorder::release(Batch& batch) noexcept {
if (batch.samples != VK_NULL_HANDLE && batch.samplePool == nullptr) context.Function<PFN_vkDestroyQueryPool>("vkDestroyQueryPool")(context.device, batch.samples, nullptr);
batch.samples = VK_NULL_HANDLE;
batch.samplePool.reset();
for (const auto pool : batch.endedSamples) context.Function<PFN_vkDestroyQueryPool>("vkDestroyQueryPool")(context.device, pool, nullptr);
batch.endedSamples.clear();
batch.segments = VK_NULL_HANDLE;
batch.segmentPool.reset();
// A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or
core/libs/prx/libSceAgcDriver/Graphics/src/SampleOrder.cpp
+2 -1
View File
@@ -185,6 +185,7 @@ public:
auto copy = copyFor(target.format, target.samples, target.extent, stencil);
kept.push_back(copy);
const VkImageAspectFlags aspects = VK_IMAGE_ASPECT_DEPTH_BIT | (stencil ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
const VkImageAspectFlags layoutAspects = VK_IMAGE_ASPECT_DEPTH_BIT | (HasStencil(target.format) ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
if (!copy->general) {
VkImageMemoryBarrier toGeneral{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toGeneral.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
@@ -193,7 +194,7 @@ public:
toGeneral.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toGeneral.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toGeneral.image = copy->image;
toGeneral.subresourceRange = {aspects, 0, 1, 0, 1};
toGeneral.subresourceRange = {layoutAspects, 0, 1, 0, 1};
context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral);
copy->general = true;
}
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp
+1 -1
View File
@@ -138,7 +138,7 @@ void ShaderResources::prepareAddressBindings(std::span<const CompiledShader> sha
if (tables != 0 && storesThroughAddressesCached(shader.program->spirv)) bdaReadOnly = false;
usesFaultBuffer = usesFaultBuffer || faults != 0;
const bool tableFaultsOnly = std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::ImageTableMap; });
faultChecks = faultChecks || tables != 0 || (faults != 0 && (!tableFaultsOnly || shader.program->imageTableFaults != 0 || shader.program->poisonedSrtReads != 0 || loopGuarded));
faultChecks = faultChecks || tables != 0 || (faults != 0 && (!tableFaultsOnly || shader.program->imageTableFaults != 0 || shader.program->poisonedSrtReads != 0 || shader.program->windowLaneGuards != 0 || loopGuarded));
}
if (usesBda) {
Require(context.bufferDeviceAddress, "buffer device address is not enabled");
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp
+215 -79
View File
@@ -9,6 +9,7 @@
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
@@ -16,12 +17,12 @@
#include <atomic>
#include <memory>
#include <mutex>
#include <thread>
#include <array>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <exception>
#include <functional>
#include <stdexcept>
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
@@ -133,6 +134,8 @@ struct TextureCache {
std::list<CachedTexture> entries;
std::unordered_map<TextureKey, std::list<CachedTexture>::iterator, TextureKeyHash> index;
std::unordered_multimap<const StorageTexture*, std::list<CachedTexture>::iterator> views;
std::vector<std::pair<std::vector<std::byte>, std::uint64_t>> spare;
std::uint64_t spareBytes = 0;
std::uint64_t bytes = 0;
std::size_t passed = 0;
std::chrono::steady_clock::time_point passedSince{};
@@ -162,6 +165,8 @@ TextureCache& Textures() {
thread_local unsigned sampledCacheHeld = 0;
thread_local unsigned storageCacheHeld = 0;
thread_local unsigned sampledCacheReclaimable = 0;
thread_local unsigned storageCacheReclaimable = 0;
struct CacheHold {
unsigned& depth;
@@ -171,15 +176,6 @@ struct CacheHold {
CacheHold& operator=(const CacheHold&) = delete;
};
bool AcquireCache(AgcDriver::Mutex& mutex) {
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(20);
do {
if (mutex.try_lock()) return true;
std::this_thread::yield();
} while (std::chrono::steady_clock::now() < deadline);
return false;
}
struct UncachedImages {
AgcDriver::Mutex mutex;
std::vector<std::weak_ptr<StorageTexture>> images;
@@ -267,6 +263,39 @@ void touchTexture(TextureCache& cache, std::list<CachedTexture>::iterator it) {
it->present = Recorder::Presents();
}
constexpr std::uint64_t SpareSnapshotBytes = 256ull << 20;
void parkSnapshot(TextureCache& cache, std::vector<std::byte>& bytes) {
const auto size = bytes.size();
if (size == 0) return;
const auto present = Recorder::Presents();
while (!cache.spare.empty() && (present - cache.spare.front().second > SpareSnapshotPresents || cache.spareBytes + size > SpareSnapshotBytes)) {
cache.spareBytes -= cache.spare.front().first.size();
cache.spare.erase(cache.spare.begin());
}
if (cache.spareBytes + size <= SpareSnapshotBytes) {
cache.spareBytes += size;
cache.spare.emplace_back(std::move(bytes), present);
}
bytes = std::vector<std::byte>{};
}
std::vector<std::byte> spareSnapshot(TextureCache& cache, std::size_t bytes) {
const auto present = Recorder::Presents();
while (!cache.spare.empty() && present - cache.spare.front().second > SpareSnapshotPresents) {
cache.spareBytes -= cache.spare.front().first.size();
cache.spare.erase(cache.spare.begin());
}
for (auto spare = cache.spare.begin(); spare != cache.spare.end(); ++spare) {
if (spare->first.size() != bytes) continue;
cache.spareBytes -= bytes;
std::vector<std::byte> taken = std::move(spare->first);
cache.spare.erase(spare);
return taken;
}
return {};
}
bool admitTexture(TextureCache& cache, const TextureKey& key, std::uint64_t incoming, std::uint64_t budget) {
if (cache.bytes + incoming <= budget) return true;
if (const auto now = std::chrono::steady_clock::now(); now - cache.passedSince >= std::chrono::seconds(1)) {
@@ -396,6 +425,8 @@ TextureCounters& TextureCounts() {
return counters;
}
thread_local std::uint64_t sampledLookupKeyScans = 0;
void reportTextureCounters() {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (!profile) return;
@@ -572,7 +603,7 @@ std::uint64_t forcedCacheBudget() {
return forced;
}
std::uint64_t storageBudget(const Context& context) {
std::uint64_t storageBudget(const Context& context, std::uint64_t required = 0) {
const auto forced = forcedCacheBudget();
if (forced != 0) return forced;
if (context.memoryProperties2 == nullptr || context.physical == VK_NULL_HANDLE) return TextureCacheBudget(context.memory);
@@ -581,17 +612,26 @@ std::uint64_t storageBudget(const Context& context) {
VkPhysicalDevice device{VK_NULL_HANDLE};
std::chrono::steady_clock::time_point read{};
std::uint64_t budget{0};
std::uint64_t ceiling{0};
} cached;
const auto now = std::chrono::steady_clock::now();
std::lock_guard lock(cached.mutex);
if (cached.device == context.physical && now - cached.read < std::chrono::seconds(1)) return cached.budget;
VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2};
properties.pNext = &reported;
context.memoryProperties2(context.physical, &properties);
cached.device = context.physical;
cached.read = now;
cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
const auto read = [&] {
VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2};
properties.pNext = &reported;
context.memoryProperties2(context.physical, &properties);
cached.device = context.physical;
cached.read = now;
cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes());
cached.ceiling = cached.budget;
if (const auto heap = largestDeviceLocalHeap(context.memory); heap && reported.heapBudget[*heap] != 0) {
cached.ceiling = std::max<std::uint64_t>(512ull << 20u, std::min<std::uint64_t>(TextureCacheBudget(context.memory), reported.heapBudget[*heap] / 4u));
}
};
if (cached.device != context.physical || now - cached.read >= std::chrono::seconds(1)) read();
const auto needed = std::min(required, cached.ceiling);
if (needed > cached.budget && context.bufferPool != nullptr && context.bufferPool->ReclaimIdleDevice(needed - cached.budget, now) != 0) read();
return cached.budget;
}
@@ -730,11 +770,14 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr;
const bool profile = LookupOutcomes::Profiled();
const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
std::optional<DccKeys> scanned;
const auto scanKeys = [&] {
if (scanned.has_value()) return *scanned;
const auto scanStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
const auto keys = TextureClearKeys(resource, guestBytes);
scanned = TextureClearKeys(resource, guestBytes);
if (resource.dccAddress != 0) ++sampledLookupKeyScans;
if (profile && resource.dccAddress != 0) LookupOutcomes::Add(LookupOutcomes::DccScan, scanStart);
return keys;
return *scanned;
};
auto& counters = TextureCounts();
const auto address = resource.baseAddress;
@@ -790,6 +833,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
auto candidate = sampledStorageSource(context, resource, guestBytes);
if (candidate != nullptr && candidate->Descriptor().dccAddress == resource.dccAddress) source = std::move(candidate);
}
scanned.reset();
// The lookup's Refresh may have flushed another image's results over the memory (which
// marks the keys uncompressed): a surface that stays a snapshot re-reads them behind that
// flush, as it does behind its own. APS5_NO_KEYS_RESCAN=1 keeps the keys scanned above.
@@ -805,15 +849,19 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
if (source != nullptr && !GuestMemory::UnchangedSince(address, bytes, source->GenerationFor(address, bytes))) {
// The CPU wrote the memory while results were pending: Refresh merges them the usual way.
refreshForView(*source, resource);
scanned.reset();
}
// Results stored to guest memory just now are stamped newer than `generation`; a snapshot read
// after them is current at the generation of a second (memoized) collect. The store marks the
// surface's DCC keys uncompressed, so the keys are read after it.
const auto flushStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
const auto observedKeys = keys.has_value() ? *keys : scanKeys();
const bool stagedClear = source != nullptr && resource.dccAddress != 0 && (IsDccClear(observedKeys) || observedKeys == DccKeys::ClearSingle);
const bool stagedClear = source != nullptr && resource.dccAddress != 0 && [&] {
const auto observedKeys = keys.has_value() ? *keys : scanKeys();
return IsDccClear(observedKeys) || observedKeys == DccKeys::ClearSingle;
}();
bool stagedFlushed = false;
if ((source == nullptr || stagedClear) && !clearThroughKeys && StorageTexture::FlushPending(address, bytes, nullptr, "sampled texture")) {
const bool flushAsked = (source == nullptr || stagedClear) && !clearThroughKeys;
if (flushAsked && StorageTexture::FlushPending(address, bytes, nullptr, "sampled texture")) {
if (profile) LookupOutcomes::Add(LookupOutcomes::PendingFlush, flushStart);
generation = GuestMemory::CollectWrites(address, bytes);
if (stagedClear) {
@@ -824,6 +872,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
}
if (stagedFlushed) source.reset();
}
if (flushAsked) scanned.reset();
if (!keys.has_value()) keys = scanKeys();
if (disabled) {
if (source != nullptr && depthCompare) {
@@ -866,6 +915,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
std::lock_guard lock(cache.mutex);
const CacheHold held(sampledCacheHeld);
dropUncachedViews(cache);
std::vector<std::byte> recycled;
if (auto it = findTexture(cache, key); it != cache.entries.end()) {
if (it->source != nullptr) {
// The view follows the storage image, whatever the GPU wrote to it since; guest memory
@@ -912,29 +962,37 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
return it->texture;
}
}
if (it->bytes.size() == bytes) recycled = std::move(it->bytes);
else parkSnapshot(cache, it->bytes);
eraseTexture(cache, it);
++cache.counts.stale;
}
CachedTexture entry{key, address, std::vector<std::byte>(source != nullptr ? 0u : bytes), nullptr, *keys, snapshotGeneration};
if (source != nullptr) parkSnapshot(cache, recycled);
else if (recycled.empty()) recycled = spareSnapshot(cache, bytes);
CachedTexture entry{key, address, source != nullptr ? std::vector<std::byte>{} : (recycled.empty() ? std::vector<std::byte>(bytes) : std::move(recycled)), nullptr, *keys, snapshotGeneration};
entry.committed = std::move(committed);
entry.surfaceBytes = bytes;
const auto createTexture = [&](auto&&... args) {
const CacheHold reclaimable(sampledCacheReclaimable);
return std::make_shared<Texture>(context, std::forward<decltype(args)>(args)...);
};
if (source != nullptr) {
entry.source = source;
entry.sourceVersion = source->Version();
if (depthCompare) {
entry.texture = std::make_shared<Texture>(context, source, resource, components, Texture::DepthCopyTag{});
entry.texture = createTexture(source, resource, components, Texture::DepthCopyTag{});
entry.texture->RefreshDepthCopy();
} else if (blockCopy) {
entry.texture = std::make_shared<Texture>(context, source, resource, components, Texture::BlockCopyTag{});
entry.texture = createTexture(source, resource, components, Texture::BlockCopyTag{});
entry.texture->RefreshDepthCopy();
} else if (IsRepackedTextureFormat(resource.format)) {
entry.texture = std::make_shared<Texture>(context, source, resource, components, Texture::RepackCopyTag{});
entry.texture = createTexture(source, resource, components, Texture::RepackCopyTag{});
entry.texture->RefreshDepthCopy();
} else if (srgbExpand) {
entry.texture = std::make_shared<Texture>(context, source, resource, components, Texture::SrgbExpandTag{});
entry.texture = createTexture(source, resource, components, Texture::SrgbExpandTag{});
entry.texture->RefreshDepthCopy();
} else {
entry.texture = std::make_shared<Texture>(context, source, resource, components);
entry.texture = createTexture(source, resource, components);
}
entry.accounted = source->Cached() ? 0u : heldBytes(*source);
counters.fromStorage.fetch_add(1, std::memory_order_relaxed);
@@ -950,10 +1008,10 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
for (std::size_t i = 0; i < entry.bytes.size(); i += 64) nonzero += entry.bytes[i] != std::byte{0};
std::fprintf(stderr, "[texture] 0x%llx %ux%u format %u tile %d: %zu of %zu sampled bytes nonzero\n", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), nonzero, entry.bytes.size() / 64);
}
entry.texture = std::make_shared<Texture>(context, *context.detiler, resource, components, entry.bytes, depthCompare);
entry.texture = createTexture(*context.detiler, resource, components, entry.bytes, depthCompare);
entry.accounted = entry.texture->AllocationBytes();
constexpr std::size_t SnapshotCompareBytes = 256u << 10u;
if (entry.bytes.size() > SnapshotCompareBytes && (entry.generation != 0 || entry.untracked[0] != 0)) std::vector<std::byte>().swap(entry.bytes);
if (entry.bytes.size() > SnapshotCompareBytes && (entry.generation != 0 || entry.untracked[0] != 0)) parkSnapshot(cache, entry.bytes);
counters.snapshots.fetch_add(1, std::memory_order_relaxed);
}
const auto budget = sampledBudget(context, cache);
@@ -1178,11 +1236,16 @@ std::shared_ptr<StorageTexture> cachedStorageTexture(const Context& context, std
} else {
if (guestBytes == 0) guestBytes = DescribeSurface(resource).guestBytes;
if (StorageTexture::FlushPending(resource.baseAddress, static_cast<std::size_t>(guestBytes), nullptr, "storage image creation", PublishScope::None) && profile) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start);
CachedStorageTexture entry{key, mip, StorageTexture::Create(context, *context.detiler, resource, mip)};
std::shared_ptr<StorageTexture> created;
{
const CacheHold reclaimable(storageCacheReclaimable);
created = StorageTexture::Create(context, *context.detiler, resource, mip);
}
CachedStorageTexture entry{key, mip, std::move(created)};
static const bool keepNew = std::getenv("APS5_NO_KEEP_NEW_STORAGE") == nullptr;
if (auto* recorder = Recorder::Active(); keepNew && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread() && recorder->Recording()) recorder->Keep(entry.texture);
entry.accounted = heldBytes(*entry.texture);
const auto budget = storageBudget(context);
const auto budget = storageBudget(context, cache.bytes + entry.accounted);
while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) evicted.push_back(evictStorage(cache, std::prev(cache.entries.end())));
cache.bytes += entry.accounted;
texture = entry.texture;
@@ -1264,6 +1327,8 @@ TextureCacheUse TextureCacheUsage() {
dropUncachedViews(sampled);
use.sampledEntries = sampled.entries.size();
use.sampledBytes = sampled.bytes;
use.spareEntries = sampled.spare.size();
use.spareBytes = sampled.spareBytes;
}
use.storageBytes = storageCacheBytes();
return use;
@@ -1293,6 +1358,7 @@ void FlushCachedTextures(VkDevice device) {
GuestMemory::AssertGpuLockHeld("FlushCachedTextures");
auto& cache = StorageTextures();
std::lock_guard lock(cache.mutex);
const CacheHold held(storageCacheHeld);
for (const auto& entry : cache.entries) {
if (entry.key.device == device) entry.texture->Flush();
}
@@ -1303,15 +1369,19 @@ void ClearCachedTextures(VkDevice device) {
auto& sampled = Textures();
{
std::lock_guard lock(sampled.mutex);
const CacheHold held(sampledCacheHeld);
for (auto it = sampled.entries.begin(); it != sampled.entries.end();) {
if (it->key.device == device) eraseTexture(sampled, it++);
else ++it;
}
sampled.spare.clear();
sampled.spareBytes = 0;
}
auto& storage = StorageTextures();
std::vector<std::shared_ptr<StorageTexture>> cleared;
{
std::lock_guard lock(storage.mutex);
const CacheHold held(storageCacheHeld);
for (auto it = storage.entries.begin(); it != storage.entries.end();) {
if (it->key.device != device) {
++it;
@@ -1338,7 +1408,7 @@ void ClearCachedTextures(VkDevice device) {
}
}
std::uint64_t ReleaseTextureCaches(std::uint64_t bytes) {
std::uint64_t ReleaseTextureCaches(std::uint64_t bytes, bool newImage) {
thread_local bool releasing = false;
if (releasing) return 0;
releasing = true;
@@ -1354,63 +1424,67 @@ std::uint64_t ReleaseTextureCaches(std::uint64_t bytes) {
bool storageSkipped = false;
{
auto& cache = Textures();
if (sampledCacheHeld != 0 || !AcquireCache(cache.mutex)) {
std::unique_lock lock(cache.mutex, std::defer_lock);
const bool inherited = sampledCacheHeld != 0;
if ((inherited && (!newImage || sampledCacheReclaimable == 0)) || (!inherited && !lock.try_lock())) {
sampledSkipped = true;
} else {
std::lock_guard lock(cache.mutex, std::adopt_lock);
dropUncachedViews(cache);
while (freed < bytes && !cache.entries.empty()) {
const auto back = std::prev(cache.entries.end());
const auto accounted = back->accounted;
const bool last = back->texture.use_count() == 1;
eraseTexture(cache, back);
for (auto it = cache.entries.end(); it != cache.entries.begin() && freed < bytes;) {
--it;
if (it->texture.use_count() != 1 || (it->source != nullptr && it->source->Dirty())) continue;
const auto allocated = it->texture->AllocationBytes();
const auto sourceBytes = it->source != nullptr ? it->source->AllocationBytes() : 0u;
const std::weak_ptr<StorageTexture> source = it->source;
const auto next = std::next(it);
eraseTexture(cache, it);
++cache.counts.evicted;
++sampledEntries;
if (last) freed += accounted;
freed += allocated;
if (source.expired()) freed += sourceBytes;
it = next;
}
}
}
if (freed < bytes) {
auto& cache = StorageTextures();
std::unique_lock<GuestMemory::GpuMutexType> gpu(GuestMemory::GpuMutex(), std::defer_lock);
if (storageCacheHeld != 0 || !gpu.try_lock()) {
const bool inherited = storageCacheHeld != 0;
if ((inherited && (!newImage || storageCacheReclaimable == 0)) || !gpu.try_lock()) {
storageSkipped = true;
} else {
try {
while (freed < bytes) {
std::vector<std::pair<std::shared_ptr<StorageTexture>, std::uint64_t>> evicted;
std::shared_ptr<StorageTexture> image;
{
if (!AcquireCache(cache.mutex)) {
std::unique_lock lock(cache.mutex, std::defer_lock);
if (!inherited && !lock.try_lock()) {
storageSkipped = true;
break;
}
std::lock_guard lock(cache.mutex, std::adopt_lock);
std::uint64_t batch = 0;
storageKept = 0;
for (auto it = cache.entries.end(); it != cache.entries.begin() && batch + freed < bytes;) {
for (auto it = cache.entries.end(); it != cache.entries.begin();) {
--it;
if (it->texture->Dirty()) {
++storageKept;
continue;
}
const auto accounted = it->accounted;
const auto next = std::next(it);
evicted.emplace_back(evictStorage(cache, it), accounted);
batch += accounted;
it = next;
image = evictStorage(cache, it);
break;
}
if (evicted.empty()) break;
if (image == nullptr) break;
}
for (auto& [image, accounted] : evicted) {
std::vector<std::shared_ptr<ShaderResources>> resources;
SharedResourceCache().RemoveImage(image.get(), &resources);
if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) {
for (auto& resource : resources) recorder->Keep(std::move(resource));
}
resources.clear();
++storageEntries;
if (image.use_count() == 1) freed += accounted;
image.reset();
std::vector<std::shared_ptr<ShaderResources>> resources;
const bool removed = SharedResourceCache().TryRemoveImage(image.get(), &resources);
if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) {
for (auto& resource : resources) recorder->Keep(std::move(resource));
}
resources.clear();
++storageEntries;
if (image.use_count() == 1) freed += image->AllocationBytes();
image.reset();
if (!removed) {
storageSkipped = true;
break;
}
}
} catch (const std::exception& error) {
@@ -2371,7 +2445,7 @@ void ShaderResources::buildComplete() {
for (auto element = first; element < last; ++element) {
const auto index = binding.imageAllocations[element];
if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED});
else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
}
if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount);
@@ -2911,21 +2985,57 @@ bool TemplateAuditDue(std::chrono::steady_clock::time_point last) {
}
void LastRecordIndices(std::span<const void* const> logged, std::span<const void* const> objects, std::vector<std::size_t>& indices) {
indices.assign(objects.size(), logged.size());
thread_local std::vector<std::pair<const void*, std::size_t>>* wantedSlot = nullptr;
auto& wanted = ShaderRecompiler::ThreadOwned(wantedSlot);
wanted.clear();
for (std::size_t i = 0; i < objects.size(); ++i) wanted.emplace_back(objects[i], i);
const auto byObject = [](const std::pair<const void*, std::size_t>& left, const std::pair<const void*, std::size_t>& right) { return std::less<const void*>{}(left.first, right.first); };
std::sort(wanted.begin(), wanted.end(), byObject);
std::size_t open = wanted.size();
for (auto at = logged.size(); at-- > 0 && open != 0;) {
const auto [first, last] = std::equal_range(wanted.begin(), wanted.end(), std::pair<const void*, std::size_t>{logged[at], 0}, byObject);
if (first == last || indices[first->second] != logged.size()) continue;
for (auto it = first; it != last; ++it) indices[it->second] = at;
open -= static_cast<std::size_t>(last - first);
}
if (wanted.capacity() > LookupLogBound + LookupLogBound / 2) {
wanted.clear();
wanted.shrink_to_fit();
}
}
// Turns the lookups' records into this object's per-texture validation records (see
// ValidatedSurface); the last record of an object is the freshest.
void ShaderResources::captureValidation() {
const auto record = [](const void* object) -> const LookupRecord* {
const auto& log = lookupLog();
for (auto it = log.rbegin(); it != log.rend(); ++it) {
if (it->object == object) return &*it;
}
return nullptr;
};
const auto& log = lookupLog();
thread_local std::vector<const void*>* loggedSlot = nullptr;
auto& logged = ShaderRecompiler::ThreadOwned(loggedSlot);
thread_local std::vector<const void*>* objectsSlot = nullptr;
auto& objects = ShaderRecompiler::ThreadOwned(objectsSlot);
thread_local std::vector<std::size_t>* indicesSlot = nullptr;
auto& indices = ShaderRecompiler::ThreadOwned(indicesSlot);
logged.clear();
for (const auto& entry : log) logged.push_back(entry.object);
objects.clear();
for (const auto& texture : textures) objects.push_back(texture.get());
LastRecordIndices(logged, objects, indices);
validatedTextures.assign(textures.size(), {});
for (std::size_t i = 0; i < textures.size(); ++i) {
if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, found->depth, true};
if (indices[i] == log.size()) continue;
const auto& found = log[indices[i]];
validatedTextures[i] = {found.resource, found.bytes, found.keys, found.generation, 0, found.source, found.depth, true};
}
lookupLog().clear();
const auto trim = [](auto& scratch) {
if (scratch.capacity() <= LookupLogBound + LookupLogBound / 2) return;
scratch.clear();
scratch.shrink_to_fit();
};
trim(logged);
trim(objects);
trim(indices);
}
std::size_t ShaderResources::LookupLogCapacity() {
@@ -3258,6 +3368,11 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span
return OwnRefreshFallback::Count;
}
bool AnyPendingCpuWrites(const Context& context) {
if (Recorder::InCompletion() || Recorder::Active() == nullptr) return false;
return (context.copiedWriters != nullptr && !context.copiedWriters->empty()) || !DrawCopiedWriters()->empty() || Recorder::PendingCompletionLabels() != 0;
}
bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std::size_t bytes) {
if (bytes == 0 || Recorder::InCompletion()) return false;
auto* recorder = Recorder::Active();
@@ -3906,6 +4021,17 @@ void ResourceCache::Remove(const Key& key, const ShaderResources* object) {
void ResourceCache::RemoveImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted) {
std::lock_guard lock(mutex);
removeImage(image, evicted);
}
bool ResourceCache::TryRemoveImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted) {
std::unique_lock lock(mutex, std::try_to_lock);
if (!lock.owns_lock()) return false;
removeImage(image, evicted);
return true;
}
void ResourceCache::removeImage(const StorageTexture* image, std::vector<std::shared_ptr<ShaderResources>>* evicted) {
for (auto it = entries.begin(); it != entries.end();) {
if (!it->second->ReadsImage(image)) {
++it;
@@ -4711,18 +4837,23 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind
const bool depthCompare = !binding.imageDepthCompare.empty() && binding.imageDepthCompare.at(element);
const bool firstLayer = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && (resource.dimension == TextureDimension::k2DArray || (depthCompare && resource.dimension == TextureDimension::k1DArray));
const bool depth1D = depthCompare && ((binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k1D) || (binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2DArray && resource.dimension == TextureDimension::k1DArray));
if (!firstLayer && !depth1D && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast<int>(*binding.imageShape)) + ", dimension " + std::to_string(static_cast<int>(resource.dimension)) + ")");
const bool firstSample = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DMsaa;
if (!firstLayer && !depth1D && !firstSample && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast<int>(*binding.imageShape)) + ", dimension " + std::to_string(static_cast<int>(resource.dimension)) + ")");
const VkComponentMapping components = ViewComponents(resource);
const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes;
CompletePendingCpuWrites(context, resource.baseAddress, static_cast<std::size_t>(guestBytes));
if (resource.dccAddress != 0) CompletePendingCpuWrites(context, resource.dccAddress, DccKeyBytes(guestBytes));
std::shared_ptr<Texture> texture;
const VkImageViewType multisampleViewType = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2DMultisampleArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
if (previousSampled.texture != nullptr && previousSampled.depthCompare == depthCompare && previousSampled.multisampleViewType == multisampleViewType && words.size() == 8 && std::equal(words.begin(), words.end(), previousSampled.words.begin())) texture = previousSampled.texture;
if (texture == nullptr && record != nullptr && record->texture != nullptr) {
if (previousSampled.texture != nullptr && previousSampled.depthCompare == depthCompare && previousSampled.multisampleViewType == multisampleViewType && previousSampled.firstSample == firstSample && words.size() == 8 && std::equal(words.begin(), words.end(), previousSampled.words.begin())) texture = previousSampled.texture;
if (texture == nullptr && !firstSample && record != nullptr && record->texture != nullptr) {
texture = fastTexture(*record);
(texture != nullptr ? counters.fastHits : counters.fastMisses).fetch_add(1, std::memory_order_relaxed);
}
if (texture == nullptr && firstSample) {
Require(!depthCompare && !DepthSurfaceAt(resource.baseAddress), "reading sample 0 of a multisampled depth surface through a 2D image is not implemented");
texture = MultisampleTargetFirstSample(context, resource, components);
}
if (texture == nullptr) texture = cachedTexture(context, words, resource, components, guestBytes, depthCompare, multisampleViewType);
if (resource.dimension == TextureDimension::k2DMsaa) multisampleSources.push_back(resource.baseAddress);
bool baseLevel = false;
@@ -4750,6 +4881,7 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind
std::copy(words.begin(), words.end(), previousSampled.words.begin());
previousSampled.depthCompare = depthCompare;
previousSampled.multisampleViewType = multisampleViewType;
previousSampled.firstSample = firstSample;
previousSampled.texture = texture;
}
textures.push_back(std::move(texture));
@@ -5094,7 +5226,7 @@ void ShaderResources::resolveImageTables() {
storageKeys.push_back(resource.dccAddress);
storageFirstLayer.push_back(firstLayer);
storageWritten.push_back(true);
storageAtomic.push_back(false);
storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]);
storageAtomic64.push_back(false);
describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast<int>(resource.tileMode), resource.dccAddress, true});
bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u;
@@ -5153,6 +5285,10 @@ void ShaderResources::resolveImageTables() {
reportTextureCounters();
}
std::uint64_t SampledLookupKeyScans() {
return sampledLookupKeyScans;
}
ImageTableReuseCounts ImageTableReuses() {
const auto& counters = TextureCounts();
return {counters.tablesReused.load(std::memory_order_relaxed) + counters.tablesReplayed.load(std::memory_order_relaxed), counters.tablesResolved.load(std::memory_order_relaxed), counters.tablesReplayed.load(std::memory_order_relaxed), counters.tableWritesKept.load(std::memory_order_relaxed), counters.tableWritesMade.load(std::memory_order_relaxed)};
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp
+5
View File
@@ -1,4 +1,5 @@
#include "BdaAbi.hpp"
#include "SpirvBackend/SpirvAnalysis.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp"
#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp"
@@ -594,6 +595,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
} else {
Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource");
const auto key = std::make_pair(*decoration.set, *decoration.binding);
if (*decoration.set == ShaderRecompiler::PixelFunctionLdsDescriptorSet) {
Require(fragment && *decoration.binding == 0u, "pixel function LDS buffer binding is wrong");
continue;
}
const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; });
const bool array = binding != shader.bindings.end() && binding->role == ShaderRecompiler::DescriptorRole::GuestBuffers;
const auto [view, first] = bufferViews.emplace(key, decoration.aliased);
core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp
+98 -6
View File
@@ -605,6 +605,25 @@ bool sampleCoverageSupported(const QueueState& queue, std::uint32_t samples, boo
return (inputs & PixelSampleCoverageInput) == 0u || (!perSampleShading && (inputs & PixelSampleRateInputs) == 0u);
}
bool pixelRateInputsKept(const QueueState& queue, const State& state) {
using ShaderRecompiler::PixelInput;
using ShaderRecompiler::PixelInputBit;
constexpr std::uint32_t numbered = PixelInputBit(PixelInput::Ancillary) | PixelInputBit(PixelInput::SampleCoverage);
constexpr std::uint32_t located = PixelInputBit(PixelInput::PerspectiveSample) | PixelInputBit(PixelInput::PerspectiveCentroid) | PixelInputBit(PixelInput::PerspectivePullModel) | PixelInputBit(PixelInput::LinearSample) | PixelInputBit(PixelInput::LinearCentroid) | PixelInputBit(PixelInput::PositionZ) | PixelInputBit(PixelInput::PositionW);
const bool centred = state.customSampleLocations && std::ranges::all_of(state.sampleLocations, [](std::uint8_t location) { return location == 0u; });
const auto moving = numbered | (centred ? 0u : located);
if (PixelProgramSkipped(queue)) return true;
const auto ena = find(queue.context, 0x1b3);
const auto addr = find(queue.context, 0x1b4);
if (ena == queue.context.end() || addr == queue.context.end()) return true;
const auto inputs = ena->second & addr->second;
if ((inputs & moving) != 0u) return false;
if (centred || (inputs & (PixelInputBit(PixelInput::PositionX) | PixelInputBit(PixelInput::PositionY))) == 0u) return true;
const auto position = find(queue.context, 0x1b8);
const auto control = position == queue.context.end() ? 0u : position->second;
return (control & 0x100000u) != 0u || ((control >> 16u) & 3u) == 0u;
}
bool coveredCentroidSupported(const QueueState& queue) {
const auto config = find(queue.context, 0x2f8);
if (config == queue.context.end() || (config->second & CoveredCentroidIsCenter) == 0 || PixelProgramSkipped(queue)) return true;
@@ -907,6 +926,7 @@ struct DecodedColorFormat {
};
const char* PackedColorTargetName(ShaderRecompiler::ColorExportPacking packing) {
if (packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4) return "4_4_4_4 unorm";
return packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "11_11_10 float" : "10_11_11 unorm";
}
@@ -920,7 +940,7 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
};
const bool alternate = swap == 1;
const auto single = [&](VkFormat vkFormat, std::uint32_t bytes) { return DecodedColorFormat{vkFormat, bytes, static_cast<std::uint8_t>((0xe4u & ~3u) | swap)}; };
if (swap > 1 && format != 1 && format != 2 && format != 4 && format != 7 && format != 10 && format != 12 && format != 16) return fail();
if (swap > 1 && format != 1 && format != 2 && format != 4 && format != 7 && format != 10 && format != 12 && format != 16 && format != 19) return fail();
switch (format) {
case 1:
if (number == unorm) return single(VK_FORMAT_R8_UNORM, 1);
@@ -998,6 +1018,9 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
case 17:
if (swap != 0 || number != unorm) return fail();
return {VK_FORMAT_A1R5G5B5_UNORM_PACK16, 2, 0xc6u};
case 19:
if (swap != 2 || number != unorm) return fail();
return {VK_FORMAT_R16_UINT, 2, 0x1bu, ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4};
default:
return fail();
}
@@ -1098,7 +1121,8 @@ std::array<std::uint8_t, 32> PixelSampleLocations(const State& state, const Shad
std::array<std::uint8_t, 32> locations{};
const bool sampleBarycentrics = (pixel.inputAddr & (PixelInputBit(PixelInput::PerspectiveSample) | PixelInputBit(PixelInput::LinearSample))) != 0u;
const bool samplePosition = !state.perSampleShading && (pixel.posX || pixel.posY) && pixel.positionLocation == ShaderRecompiler::PixelPositionLocation::Sample;
if (!sampleBarycentrics && !samplePosition) return locations;
const bool mergedCentres = pixel.targetMerge.samples > 1u && (pixel.inputAddr & (PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::LinearCenter))) != 0u;
if (!sampleBarycentrics && !samplePosition && !mergedCentres) return locations;
const auto samples = state.perSampleShading ? state.samples : 1u;
for (std::uint32_t pixelIndex = 0; pixelIndex < 4u; ++pixelIndex) {
for (std::uint32_t sample = 0; sample < samples; ++sample) {
@@ -1372,7 +1396,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
if (depthKeepsSampleZero && (read(cx, 0x292) & 1u) == 0) {
const bool filled = result.topology != VK_PRIMITIVE_TOPOLOGY_POINT_LIST && result.topology != VK_PRIMITIVE_TOPOLOGY_LINE_LIST && result.topology != VK_PRIMITIVE_TOPOLOGY_LINE_STRIP;
const auto sampleInputs = read(cx, 0x1b3) & read(cx, 0x1b4) & (ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage));
Require(filled && !result.alphaToCoverage && (PixelProgramSkipped(queue) || sampleInputs == 0) && (read(cx, 0x203) & 0x100u) == 0 && (read(cx, 0x201) & 0x0f0c0000u) == 0, "a one-sample depth target under a multisampled rasterizer without MSAA_ENABLE is unsupported for points or lines and with alpha-to-coverage, a sample index or coverage input, a sample-mask export, or DB_EQAA Z interpolation or overrasterization");
Require(filled && !result.alphaToCoverage && (PixelProgramSkipped(queue) || sampleInputs == 0) && (read(cx, 0x203) & 0x100u) == 0 && (read(cx, 0x201) & 0x0f080000u) == 0, "a one-sample depth target under a multisampled rasterizer without MSAA_ENABLE is unsupported for points or lines and with alpha-to-coverage, a sample index or coverage input, a sample-mask export, or DB_EQAA source Z interpolation or overrasterization");
result.samples = 1;
result.customSampleLocations = false;
result.sampleLocations.fill(0);
@@ -1511,6 +1535,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
}
result.quadSampleMask = decodeQuadSampleMask(cx, result.samples);
result.pixelEarlyZ = !PixelProgramSkipped(queue) && PixelEarlyZ(read(cx, 0x203));
result.pixelKill = !PixelProgramSkipped(queue) && (read(cx, 0x203) & 0x40u) != 0;
if ((read(cx, 0x207) & LayerExports & ~result.layerExports & ~(1u << 21u)) != 0) {
static bool reported = false;
if (!reported) {
@@ -1556,7 +1581,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
result.blends.assign(exportCount, VkPipelineColorBlendAttachmentState{});
std::array<float, 3> requiredBlendConstants{};
std::uint32_t requiredBlendMask = 0;
for (const auto& color : result.colors) {
for (auto& color : result.colors) {
const auto slot = color.slot;
const auto blend = read(cx, 0x1e0 + slot);
APS5_LOG_OUT_DEBUG("Blend %u control=0x%x", slot, blend);
@@ -1624,7 +1649,25 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into " + packed + " target is unsupported");
Require(!state.blendEnable, "blending into " + packed + " color target is unsupported");
Require((exportedMask & 7u) == 7u, "partial writes of " + packed + " color target are unsupported");
const bool nibbles = color.packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4;
const std::uint32_t packedMask = nibbles ? 0xfu : 7u;
Require(nibbles || (exportedMask & packedMask) == packedMask, "partial writes of " + packed + " color target are unsupported");
if ((exportedMask & packedMask) != packedMask) {
Require(color.mipCount == 1u && color.depth == 1u && color.layers == 1u && color.arrayLastSlice == 0u, "partial writes of a mipmapped, volume or array " + std::string(PackedColorTargetName(color.packing)) + " color target are not implemented");
Require(std::ranges::none_of(result.colors, [](const ColorTarget& other) { return other.partialWriteMask != 0; }), "partial writes of more than one packed color target are not implemented");
color.partialWriteMask = PackedWriteMask(mapping, exportedMask);
state.colorWriteMask = 0;
if (color.samples > 1u) {
Require(sampleCoverageSupported(queue, result.samples, true), "SAMPLE_COVERAGE in a multisampled pixel shader that runs per sample, or reads sample-rate or ancillary inputs, is unsupported (the input holds the whole pixel's coverage, gl_SampleMaskIn only the invocation's samples)");
if (!result.perSampleShading && !pixelRateInputsKept(queue, result)) {
const auto value = [&](std::uint32_t offset) { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; };
char detail[160];
std::snprintf(detail, sizeof(detail), " (SPI_PS_INPUT_ENA 0x%x, SPI_PS_INPUT_ADDR 0x%x, SPI_BARYC_CNTL 0x%x, sample 1 location 0x%02x)", value(0x1b3), value(0x1b4), value(0x1b8), result.sampleLocations[1]);
Require(false, std::string("partial writes of a multisampled 4_4_4_4 unorm color target run the pixel shader once per sample, so a shader the guest runs once per pixel cannot read the ancillary word or, unless every sample sits at the pixel centre, centroid, sample-located or pull-model barycentrics, Z, 1/W or a position at the centroid or a sample: they would follow the invocation's sample") + detail);
}
result.perSampleShading = true;
}
}
}
if (state.blendEnable && !reversed && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) {
const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; };
@@ -1641,7 +1684,11 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
for (std::uint32_t i = 0; i < 3; ++i) {
if ((requiredBlendMask & (1u << i)) != 0) result.blendConstants[i] = requiredBlendConstants[i];
}
if (!result.colors.empty()) result.blend = result.blends[result.colors.front().exportIndex];
Require((!result.mergedCoverage && !result.truncatedCoverage) || std::ranges::none_of(result.colors, [](const ColorTarget& color) { return color.partialWriteMask != 0; }), "partial writes of a 4_4_4_4 unorm color target under merged or truncated coverage (a one-sample target under a multisampled rasterizer) are not implemented");
if (!result.colors.empty()) {
result.color = result.colors.front();
result.blend = result.blends[result.colors.front().exportIndex];
}
Require(result.reversedBlend == 0 || (exportSlots.size() == 1 && exportCount == 1), "blending into a color target with a reversed component order through a second source color needs that target to be the shader's only color export");
if (result.dualSourceBlend) {
Require(exportSlots.size() == 2 && exportSlots[0] == 0 && exportSlots[1] == 1 && exportCount == 1, "dual-source blending needs pixel exports to MRT slots 0 and 1 and color writes to slot 0 alone");
@@ -1668,6 +1715,50 @@ std::array<ShaderRecompiler::ColorExportPacking, 8> ExportPackings(const State&
return packings;
}
std::uint16_t PackedWriteMask(std::uint8_t componentMapping, std::uint32_t exportedMask) {
std::uint16_t mask = 0;
for (std::uint32_t component = 0; component < 4; ++component) {
if (((exportedMask >> ((componentMapping >> (2u * component)) & 3u)) & 1u) != 0) mask = static_cast<std::uint16_t>(mask | (0xfu << (4u * component)));
}
return mask;
}
GuestTextureResource TargetMergeSurface(const ColorTarget& color) {
auto surface = SurfaceForTarget(color);
if (color.samples > 1u) {
Require(surface.baseAddress == color.surfaceAddress, "a merged multisampled color target does not start at its surface");
surface.dimension = TextureDimension::k2DMsaa;
surface.samples = color.samples;
}
return surface;
}
bool DepthStencilWrites(const State& state) {
if (!state.depth) return false;
const auto writes = [](const VkStencilOpState& face) { return (face.writeMask & 0xffu) != 0 && (face.failOp != VK_STENCIL_OP_KEEP || face.passOp != VK_STENCIL_OP_KEEP || face.depthFailOp != VK_STENCIL_OP_KEEP); };
return state.depthWrite || (state.stencilTest && (writes(state.stencilFront) || writes(state.stencilBack)));
}
bool DepthStencilTests(const State& state) {
return state.depth.has_value() && (state.depthTest || state.stencilTest || state.depthBoundsTest);
}
ShaderRecompiler::TargetMerge ExportTargetMerge(const State& state) {
ShaderRecompiler::TargetMerge merge;
for (const auto& color : state.colors) {
if (color.partialWriteMask == 0) continue;
Require(merge.mask == 0 && color.exportIndex < 8u && color.samples <= 8u, "a draw merges into one packed color target at most");
Require(color.samples == 1u || state.perSampleShading, "a merged multisampled color target needs per-sample shading");
merge.target = static_cast<std::uint8_t>(color.exportIndex);
merge.mask = color.partialWriteMask;
merge.samples = static_cast<std::uint8_t>(color.samples);
merge.descriptor = EncodeTextureResource(TargetMergeSurface(color));
merge.depthStencilWrites = DepthStencilWrites(state);
merge.depthStencilTests = DepthStencilTests(state);
}
return merge;
}
bool PartialSampleMask(const State& state) {
const auto full = (1u << state.samples) - 1u;
return std::ranges::any_of(state.quadSampleMask, [&](std::uint16_t mask) { return (mask & full) != full; });
@@ -1885,6 +1976,7 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot, bool allo
color.packing = decoded.packing;
if (color.packing != ShaderRecompiler::ColorExportPacking::None && (info & 0x10000000u) != 0) throw std::runtime_error("AGC graphics: DCC-compressed " + std::string(PackedColorTargetName(color.packing)) + " color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4 || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 4_4_4_4 unorm color targets are unsupported");
Require(color.packing != ShaderRecompiler::ColorExportPacking::Float11_11_10 || (info & 0x00040000u) != 0, "rounding (ROUND_MODE 0) 11_11_10 float color targets are unsupported");
for (std::uint32_t word = 0; word < 2; ++word) {
const auto clear = find(cx, 0x323 + word + stride);
core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp
+51 -10
View File
@@ -48,8 +48,8 @@ OwnedImage::~OwnedImage() {
ReleaseImageMemory(context, memory, memoryBytes, memoryType);
}
void OwnedImage::Allocate(const Context& device, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation) {
memory = AllocateImageMemory(device, allocation, mask, operation, &memoryType);
void OwnedImage::Allocate(const Context& device, const VkMemoryAllocateInfo& allocation, std::uint32_t mask, const char* operation, bool newImage) {
memory = AllocateImageMemory(device, allocation, mask, operation, &memoryType, newImage);
memoryBytes = allocation.allocationSize;
context.bufferPool = device.bufferPool;
}
@@ -282,7 +282,7 @@ ExpandImage CreateExpandImage(const Context& context, const GuestTextureResource
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture");
result.owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory sRGB copy");
result.owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory sRGB copy", true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, result.image, result.owned->memory, 0), "vkBindImageMemory sRGB copy");
result.bytes = requirements.size;
return result;
@@ -366,7 +366,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture");
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture");
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture", true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory");
const VkImage uploadImage = image;
@@ -642,7 +642,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory repacked copy");
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory repacked copy", true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory repacked copy");
const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
@@ -689,6 +689,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
imageInfo.arrayLayers = copyLayers;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.flags = SrgbTwinFormat(context, blockFormat) != VK_FORMAT_UNDEFINED ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0u;
imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
@@ -700,7 +701,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory block copy");
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory block copy", true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory block copy");
const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
@@ -718,6 +719,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
viewRange = {viewInfo.viewType, viewInfo.subresourceRange.levelCount, viewInfo.subresourceRange.layerCount};
createFirstLayerView(descriptor, viewInfo);
prepareBaseLevelView(descriptor, viewInfo);
prepareSrgbView(descriptor, viewInfo);
} catch (...) {
release();
throw;
@@ -758,7 +760,7 @@ Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>&
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth copy");
owned->Allocate(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth copy", true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy");
const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
@@ -1331,7 +1333,10 @@ VkFormat SrgbTwinFormat(const Context& context, VkFormat format) {
switch (format) {
case VK_FORMAT_R8_UNORM: return decoded(ShaderRecompiler::IrBufferFormat::Format8Srgb) ? VK_FORMAT_UNDEFINED : VK_FORMAT_R8_SRGB;
case VK_FORMAT_R8G8_UNORM: return decoded(ShaderRecompiler::IrBufferFormat::Format8_8Srgb) ? VK_FORMAT_UNDEFINED : VK_FORMAT_R8G8_SRGB;
case VK_FORMAT_R8G8B8A8_UNORM: return VK_FORMAT_R8G8B8A8_SRGB;
case VK_FORMAT_R8_SNORM: return decoded(ShaderRecompiler::IrBufferFormat::Format8Srgb) ? VK_FORMAT_UNDEFINED : VK_FORMAT_R8_SRGB;
case VK_FORMAT_R8G8_SNORM: return decoded(ShaderRecompiler::IrBufferFormat::Format8_8Srgb) ? VK_FORMAT_UNDEFINED : VK_FORMAT_R8G8_SRGB;
case VK_FORMAT_R8G8B8A8_UNORM:
case VK_FORMAT_R8G8B8A8_SNORM: return VK_FORMAT_R8G8B8A8_SRGB;
case VK_FORMAT_B8G8R8A8_UNORM: return VK_FORMAT_B8G8R8A8_SRGB;
case VK_FORMAT_A8B8G8R8_UNORM_PACK32: return VK_FORMAT_A8B8G8R8_SRGB_PACK32;
case VK_FORMAT_BC1_RGB_UNORM_BLOCK: return VK_FORMAT_BC1_RGB_SRGB_BLOCK;
@@ -1450,7 +1455,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler,
allocation.allocationSize = requirements.size;
memoryBytes = requirements.size;
allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "storage texture");
memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory storage texture", &memoryType);
memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory storage texture", &memoryType, true);
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage");
} catch (...) {
release();
@@ -3583,7 +3588,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
toRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
toRead.newLayout = VK_IMAGE_LAYOUT_GENERAL;
const BarrierMask ready{VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, importReady.srcAccessMask | toRead.srcAccessMask, importReady.dstAccessMask | toRead.dstAccessMask, toRead.oldLayout != toRead.newLayout, toRead.srcQueueFamilyIndex != toRead.dstQueueFamilyIndex};
const bool ordered = recorder != nullptr && seeds.empty() && padding.copies.empty() && Recorder::ElideCovered(Recorder::CommandClass::StorageWriteBack, ready, covered);
const bool ordered = recorder != nullptr && seeds.empty() && padding.copies.empty() && holeCopies.empty() && Recorder::ElideCovered(Recorder::CommandClass::StorageWriteBack, ready, covered);
if (!ordered) context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toRead);
for (std::size_t i = 0; i < windows.size(); ++i) {
const auto& window = windows[i];
@@ -4312,6 +4317,42 @@ bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const Stor
return false;
}
std::uint64_t StorageTexture::StampKeepingNewerResults(std::uint64_t address, std::size_t bytes, std::uint64_t recorded, const std::function<std::uint64_t()>& stamp) {
std::vector<std::pair<std::shared_ptr<StorageTexture>, std::uint32_t>> kept;
if (recorded != 0 && bytes != 0) {
auto& pending = Pending();
std::lock_guard lock(pending.mutex);
for (auto* texture : pending.textures) {
if (!texture->blockUnits || !texture->overlaps(address, bytes)) continue;
auto alive = texture->weak_from_this().lock();
if (alive == nullptr) continue;
for (std::uint32_t unit = 0; unit < texture->trackedLayers; ++unit) {
const auto begin = texture->layerBegin(unit);
const auto unitBytes = texture->layerBytes(unit);
if (!texture->layerPending[unit] || begin >= address + bytes || address >= begin + unitBytes) continue;
if (texture->layerGeneration[unit] >= recorded) kept.emplace_back(alive, unit);
}
}
}
std::erase_if(kept, [](const auto& entry) {
const auto& [texture, unit] = entry;
const auto begin = texture->layerBegin(unit);
const auto unitBytes = static_cast<std::size_t>(texture->layerBytes(unit));
GuestMemory::CollectWritesUncached(begin, unitBytes);
return !GuestMemory::UnchangedSince(begin, unitBytes, texture->layerGeneration[unit]);
});
const auto generation = stamp();
if (generation == 0 || kept.empty()) return generation;
auto& pending = Pending();
std::lock_guard lock(pending.mutex);
for (const auto& [texture, unit] : kept) {
if (!texture->layerPending[unit]) continue;
texture->layerGeneration[unit] = generation;
texture->refreshGeneration();
}
return generation;
}
bool StorageTexture::AnyPendingOverlaps(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
if (ranges.empty()) return false;
auto& pending = Pending();
core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp
+12 -8
View File
@@ -9,6 +9,9 @@
// Resource registration feeds GPU debugging tools, none of which are attached; the game treats this code as benign.
static constexpr int SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE = static_cast<int>(0x8A6C9018);
static constexpr uint32_t RESOURCE_REGISTRATION_MAX_NAME_LENGTH = 0xfc;
static constexpr int SCE_AGC_ERROR_WORKLOAD_STREAM_INVALID_ID = static_cast<int>(0x8A6C0033);
static constexpr int SCE_AGC_ERROR_WORKLOAD_STREAM_NULL = static_cast<int>(0x8A6C0035);
static constexpr int SCE_AGC_ERROR_WORKLOAD_STREAM_NOT_REGISTERED = static_cast<int>(0x8A6C003A);
static constexpr uint32_t WORKLOAD_STREAM_MIN_ID = 1;
static constexpr uint32_t WORKLOAD_STREAM_MAX_ID = 31;
@@ -49,10 +52,11 @@ int APS5_VABI sceAgcDriverRegisterMultipleResources(uint32_t* resourceHandle, ui
}
int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) {
if (stream != nullptr && stream_id >= WORKLOAD_STREAM_MIN_ID && stream_id <= WORKLOAD_STREAM_MAX_ID) {
std::lock_guard lock(g_workloadStreamLock);
g_workloadStreams |= 1u << stream_id;
}
if (stream_id < WORKLOAD_STREAM_MIN_ID || stream_id > WORKLOAD_STREAM_MAX_ID) return SCE_AGC_ERROR_WORKLOAD_STREAM_INVALID_ID;
std::lock_guard lock(g_workloadStreamLock);
if ((g_workloadStreams & (1u << stream_id)) != 0) return SCE_AGC_ERROR_WORKLOAD_STREAM_INVALID_ID;
if (stream == nullptr) return SCE_AGC_ERROR_WORKLOAD_STREAM_NULL;
g_workloadStreams |= 1u << stream_id;
return 0;
}
@@ -83,10 +87,10 @@ int APS5_VABI sceAgcDriverUnregisterAllResourcesForOwner(uint32_t owner_handle)
}
int APS5_VABI sceAgcDriverUnregisterWorkloadStream(uint32_t stream_id) {
if (stream_id >= WORKLOAD_STREAM_MIN_ID && stream_id <= WORKLOAD_STREAM_MAX_ID) {
std::lock_guard lock(g_workloadStreamLock);
g_workloadStreams &= ~(1u << stream_id);
}
if (stream_id < WORKLOAD_STREAM_MIN_ID || stream_id > WORKLOAD_STREAM_MAX_ID) return SCE_AGC_ERROR_WORKLOAD_STREAM_INVALID_ID;
std::lock_guard lock(g_workloadStreamLock);
if ((g_workloadStreams & (1u << stream_id)) == 0) return SCE_AGC_ERROR_WORKLOAD_STREAM_NOT_REGISTERED;
g_workloadStreams &= ~(1u << stream_id);
return 0;
}
core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp
+31 -2
View File
@@ -1023,6 +1023,34 @@ void importRefusedWholeRangeTests(const Context& context) {
Require(kept != nullptr && kept->buffer == buffer, "the range's window was imported again after a refused whole-range import");
}
void importPiecesOverAllocationLimitTests(const Context& context) {
WindowedRange range(context, 2 * WindowedRange::piece, 5);
const auto first = range.First();
const auto limit = 3 * range.window;
SetHostImportMaxBytes(limit);
{
GuestBufferMemory leased(range.importing);
leased.AcquireRegistered();
leased.Upload(true);
std::uint64_t covered = first;
for (const auto& entry : leased.AddressRanges()) {
if (entry.end <= first || entry.begin >= first + range.bytes) continue;
Require(entry.begin == covered, "the pieces of a range over the allocation limit leave a gap or overlap in the BDA table");
Require(entry.end - entry.begin <= limit, "a BDA table entry of a range over the allocation limit is larger than the limit");
const auto* piece = ExistingHostImportFor(range.importing, entry.begin, static_cast<std::size_t>(entry.end - entry.begin));
Require(piece != nullptr && piece->bytes <= limit && entry.deviceAddress == piece->address + (entry.begin - piece->base), "a BDA table entry of a range over the allocation limit is not served in place by an import within the limit");
covered = entry.end;
}
Require(covered == first + range.bytes, "the pieces of a range over the allocation limit do not cover it");
for (std::uint64_t at = first; at < first + range.bytes; at += range.window) {
const auto* piece = ExistingHostImportFor(range.importing, at, 16);
Require(piece != nullptr && piece->bytes <= limit, "a range over the allocation limit has an import larger than the limit");
}
leased.WriteBack();
}
SetHostImportMaxBytes(0);
}
// Two uses across neighbouring cell edges of one range, each in a build of its own, share one import: the second grows
// the first's window to their union, and later rounds of both keep it. Before, each replaced the other's window at
// every use.
@@ -1223,7 +1251,7 @@ public:
static constexpr std::size_t Window = std::size_t{8} << 20u;
explicit PieceWindow(const Context& context, bool views = false) : importing(context) {
#ifndef _WIN32
#ifdef __linux__
if (views) {
reserved = mmap(nullptr, 2 * Window, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
Require(reserved != MAP_FAILED, "cannot reserve the block of 16 KiB views");
@@ -3645,12 +3673,13 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
importMappedFailureGenerationTests(context);
importStaleProbeTests(context, access);
importRefusedWholeRangeTests(context);
importPiecesOverAllocationLimitTests(context);
importNeighbouringEdgesTests(context);
importAtMapOverBudgetTests(context);
importRequesterTests(context, access);
hostImportReportTests();
importBlockMapTests(context, false);
#ifndef _WIN32
#ifdef __linux__
importBlockMapTests(context, true);
#endif
importBlockBuildTests(context);
core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp
+156 -4
View File
@@ -11,6 +11,7 @@
#include <deque>
#include <initializer_list>
#include <map>
#include <limits>
#include <memory>
#include <new>
#include <set>
@@ -675,6 +676,7 @@ struct ReclaimMock {
VkDeviceSize externalBytes = 0;
std::uint64_t requests = 0;
VkDeviceSize asked = 0;
bool newImage = false;
};
ReclaimMock reclaimMock;
@@ -689,7 +691,8 @@ VkDeviceMemory ExternalAllocation(VkDeviceSize bytes) {
return memory;
}
VkDeviceSize mockCacheReclaim(VkDeviceSize bytes) {
VkDeviceSize mockCacheReclaim(VkDeviceSize bytes, bool newImage) {
Expect(newImage == reclaimMock.newImage, "an allocation used the wrong image-creation cache access");
++reclaimMock.requests;
reclaimMock.asked = bytes;
if (reclaimMock.external == VK_NULL_HANDLE) return 0;
@@ -701,6 +704,10 @@ VkDeviceSize mockCacheReclaim(VkDeviceSize bytes) {
void CacheReclaimRecovery() {
Expect(BufferPool::ReclaimTarget(4 * MiB) == 260 * MiB, "a 4 MiB request does not ask the caches for 256 MiB on top of itself");
Expect(BufferPool::ReclaimTarget(512 * MiB) == 1024 * MiB, "a 512 MiB request does not ask the caches for twice its size");
const auto limit = std::numeric_limits<VkDeviceSize>::max();
Expect(BufferPool::ReclaimTarget(limit / 2u) == limit - 1u, "the largest doubled request without overflow lost its reserve");
Expect(BufferPool::ReclaimTarget(limit / 2u + 1u) == limit, "a reclaim target wrapped when twice the request overflowed");
Expect(BufferPool::ReclaimTarget(limit) == limit, "the largest reclaim request wrapped to a smaller target");
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
mock = MockDevice{};
reclaimMock = ReclaimMock{};
@@ -775,18 +782,23 @@ void ImageCacheReclaim() {
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = 8 * MiB;
allocation.memoryTypeIndex = 0;
const auto refuse = [&](std::initializer_list<VkResult> results) {
const auto refuse = [&](std::initializer_list<VkResult> results, bool newImage = false) {
ExternalAllocation(4 * MiB);
reclaimMock.newImage = newImage;
mock.results = results;
const auto requests = reclaimMock.requests;
const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory texture");
const auto before = mock.attempts;
const auto memory = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory texture", nullptr, newImage);
Expect(reclaimMock.requests == requests + 1 && reclaimMock.external == VK_NULL_HANDLE, "a texture refused device memory did not release the texture caches before retrying");
Expect(mock.attempts == before + results.size() + 1, "image recovery or host fallback exceeded its bounded attempts");
const auto type = mock.memoryTypes.at(memory);
mockFreeMemory(VK_NULL_HANDLE, memory, nullptr);
return type;
};
Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 0, "a texture was placed in host memory although releasing the texture caches made room in device memory");
Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}) == 1, "a texture still refused device memory after releasing the texture caches was not placed in host memory");
Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY}, true) == 0, "a new image was placed in host memory before retrying after cache release");
Expect(refuse({VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_DEVICE_MEMORY}, true) == 1, "a new image refused device memory after cache release lost its host fallback");
Expect(mock.liveBytes == 0, "a texture allocation leaked memory");
}
@@ -832,6 +844,103 @@ void AllocationFailureReportsHeapBudgets() {
Expect(diagnosticBudgetQueries == 0, "missing-extension fallback attempted a memory budget query");
}
Context reclaimContext() {
auto context = mockContext();
context.memory.memoryHeapCount = 2;
context.memory.memoryHeaps[0] = {4096 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT};
context.memory.memoryHeaps[1] = {8192 * MiB, 0};
context.memory.memoryTypes[1].heapIndex = 1;
return context;
}
void IdleDeviceReclaim() {
mock = MockDevice{};
{
auto context = reclaimContext();
VkBuffer cold;
VkBuffer nextCold;
VkBuffer hot;
VkBuffer host;
VkBuffer small;
{ DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); cold = buffer.Handle(); }
{ DeviceBuffer buffer(context, 3 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); nextCold = buffer.Handle(); }
{ DeviceBuffer buffer(context, 4 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); hot = buffer.Handle(); }
{ Buffer buffer(context, 8 * MiB, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); host = buffer.Handle(); }
{ Buffer buffer(context, 1024, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); small = buffer.Handle(); }
DeviceBuffer active(context, 16 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
auto& pool = *context.bufferPool;
const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)};
Expect(pool.ReclaimIdleDevice(MiB, start) == 0, "the first pressure observation reclaimed a buffer without an idle interval");
Expect(pool.ReclaimIdleDevice(MiB, start + std::chrono::milliseconds(999)) == 0, "a partial idle interval reclaimed a buffer");
{
DeviceBuffer reused(context, 4 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
Expect(reused.Handle() == hot, "the warm device buffer was not reused");
}
const auto next = start + std::chrono::seconds(1);
SetAllocationFailureCountdown(0);
bool threw = false;
try {
pool.ReclaimIdleDevice(MiB, next);
} catch (const std::bad_alloc&) {
threw = true;
}
SetAllocationFailureCountdown(-1);
Expect(threw && mock.liveBuffers.contains(cold) && mock.liveBuffers.contains(nextCold), "a failed reclaim collection lost ownership of an allocation");
Expect(pool.ReclaimIdleDevice(MiB, next) == 2 * MiB, "reclaim did not release exactly the oldest complete idle allocation");
Expect(!mock.liveBuffers.contains(cold) && mock.liveBuffers.contains(nextCold), "bounded reclaim released the wrong allocation or exceeded its request");
Expect(pool.ReclaimIdleDevice(0, next) == 0, "zero pressure released a buffer");
Expect(pool.ReclaimIdleDevice(32 * MiB, next) == 3 * MiB, "reclaim counted a hot, live, host or slab allocation as idle device memory");
for (const auto buffer : {hot, host, small, active.Handle()}) Expect(mock.liveBuffers.contains(buffer), "reclaim destroyed a protected buffer");
SetAllocationFailureCountdown(0);
const auto empty = pool.ReclaimIdleDevice(32 * MiB, next);
SetAllocationFailureCountdown(-1);
Expect(empty == 0, "an unchanged empty eligible set did not stay empty");
Expect(pool.ReclaimIdleDevice(32 * MiB, start) == 0, "a backwards time observation reclaimed a buffer");
Expect(pool.ReclaimIdleDevice(32 * MiB, next + std::chrono::seconds(1)) == 4 * MiB, "a device buffer did not become reclaimable after a complete idle interval");
Expect(mock.liveBuffers.contains(active.Handle()), "reclaim touched an allocation still in use");
Expect(pool.ReclaimIdleDevice(32 * MiB, next + std::chrono::seconds(2)) == 0, "an empty device tier reported reclaimed memory");
}
Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "idle device reclaim leaked an allocation");
}
void IdleReclaimPreservesWorkingSet() {
mock = MockDevice{};
{
auto context = reclaimContext();
const auto useWorkingSet = [&] {
std::vector<std::unique_ptr<DeviceBuffer>> buffers;
for (unsigned i = 0; i < 4; ++i) buffers.push_back(std::make_unique<DeviceBuffer>(context, 96 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT));
};
useWorkingSet();
const auto allocations = mock.allocations;
const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)};
auto& pool = *context.bufferPool;
Expect(pool.ReclaimIdleDevice(1024 * MiB, start) == 0, "initial pressure discarded the working set");
for (unsigned second = 1; second <= 4; ++second) {
useWorkingSet();
Expect(pool.ReclaimIdleDevice(1024 * MiB, start + std::chrono::seconds(second)) == 0, "pressure discarded buffers reused during the observation interval");
Expect(mock.allocations == allocations && mock.liveBytes == 384 * MiB, "a reusable working set larger than 128 MiB was recreated or shrunk");
}
Expect(pool.ReclaimIdleDevice(1024 * MiB, start + std::chrono::seconds(5)) == 384 * MiB, "the former working set remained retained after becoming idle");
Expect(mock.liveBytes == 0, "reclaim did not return the idle working set to the device");
}
Expect(mock.liveBuffers.empty(), "working-set reclaim leaked a buffer");
for (const auto heaps : {0u, 2u}) {
mock = MockDevice{};
{
auto context = reclaimContext();
context.memory.memoryHeapCount = heaps;
context.memory.memoryHeaps[1].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT;
{ DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); }
auto& pool = *context.bufferPool;
const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)};
Expect(pool.ReclaimIdleDevice(4 * MiB, start) == 0 && pool.ReclaimIdleDevice(4 * MiB, start + std::chrono::seconds(1)) == 0, "reclaim assumed a device heap without unambiguous ownership");
Expect(mock.liveBytes == 2 * MiB, "reclaim released memory from an unknown or different local heap");
}
Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "unsupported-heap reclaim leaked an allocation");
}
}
void SuccessfulAllocationKeepsPool() {
mock = MockDevice{};
{
@@ -900,7 +1009,7 @@ void DeviceBindingFailures() {
VkDeviceMemory reclaimImage = VK_NULL_HANDLE;
const Context* reclaimImageContext = nullptr;
VkDeviceSize mockImageReclaim(VkDeviceSize) {
VkDeviceSize mockImageReclaim(VkDeviceSize, bool) {
if (reclaimImage == VK_NULL_HANDLE) return 0;
ReleaseImageMemory(*reclaimImageContext, reclaimImage, 10 * MiB, 0);
reclaimImage = VK_NULL_HANDLE;
@@ -1001,10 +1110,50 @@ void ImageMemoryReuse() {
Expect(mock.frees == 35 && context.bufferPool->RetainedImages() == 1, "the pool retains image memory again after a trim");
}
void IdleReclaimPreservesImages() {
mock = MockDevice{};
{
auto context = reclaimContext();
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = 4 * MiB;
allocation.memoryTypeIndex = 0;
std::uint32_t type = 99;
const auto image = AllocateImageMemory(context, allocation, 1u, "test idle image", &type);
ReleaseImageMemory(context, image, 4 * MiB, type);
VkBuffer buffer;
{ DeviceBuffer idle(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); buffer = idle.Handle(); }
auto& pool = *context.bufferPool;
const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)};
Expect(pool.RetainedImages() == 1 && mock.liveBytes == 6 * MiB, "the mixed pool fixture was not retained");
Expect(pool.ReclaimIdleDevice(MiB, start) == 0, "initial pressure reclaimed mixed pool memory");
const auto next = start + std::chrono::seconds(1);
const auto released = pool.ReclaimIdleDevice(MiB, next);
Expect(released == 2 * MiB, "idle buffer reclaim selected image memory instead of the oldest buffer");
if (released != 2 * MiB) return;
Expect(!mock.liveBuffers.contains(buffer) && mock.liveBytes == 4 * MiB && pool.RetainedImages() == 1, "buffer reclaim changed image ownership or accounting");
Expect(pool.ReclaimIdleDevice(32 * MiB, next) == 0, "an image-only pool was reclaimed as idle buffers");
SetAllocationFailureCountdown(0);
const auto empty = pool.ReclaimIdleDevice(32 * MiB, next);
SetAllocationFailureCountdown(-1);
Expect(empty == 0, "a repeated image-only scan allocated or reclaimed memory");
const auto allocations = mock.allocations;
const auto reused = AllocateImageMemory(context, allocation, 1u, "test idle image", &type);
Expect(reused == image && mock.allocations == allocations && pool.RetainedImages() == 0, "the retained image was not reusable after buffer reclaim");
ReleaseImageMemory(context, reused, 4 * MiB, type);
Expect(pool.RetainedImages() == 1 && mock.liveBytes == 4 * MiB, "the reused image was not retained exactly once");
}
Expect(mock.liveBytes == 0 && mock.liveBuffers.empty() && mock.frees == 2, "mixed pool cleanup leaked or freed an allocation twice");
}
int main(int argc, char** argv) {
try {
if (argc == 2 && std::string_view(argv[1]) == "--small-device-retention") {
SmallDeviceRetention();
if (std::getenv("APS5_STAGING_POOL_MIB") == nullptr) {
IdleDeviceReclaim();
IdleReclaimPreservesWorkingSet();
}
return failures == 0 ? 0 : 1;
}
if (argc == 2) {
@@ -1040,6 +1189,9 @@ int main(int argc, char** argv) {
AllocationFailure(VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_DEVICE_LOST, true);
AllocationFailureReportsHeapBudgets();
SuccessfulAllocationKeepsPool();
IdleDeviceReclaim();
IdleReclaimPreservesWorkingSet();
IdleReclaimPreservesImages();
PartialSlabDoesNotRetry();
CacheReclaimRecovery();
ImageCacheReclaim();
core/libs/prx/libSceAgcDriver/tests/CollectMemoGeneration.cpp
+127
View File
@@ -0,0 +1,127 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include <algorithm>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <iostream>
#include <mutex>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <vector>
namespace {
constexpr std::uint64_t Block = 65536;
constexpr std::uint64_t Page = 4096;
constexpr std::uint64_t Base = std::uint64_t{1} << 32;
constexpr std::chrono::seconds Patience{10};
struct FakeWatch {
std::mutex mutex;
std::condition_variable changed;
std::vector<std::pair<std::uintptr_t, std::uintptr_t>> written;
bool holdNextWalk = false;
bool holding = false;
bool released = false;
bool timedOut = false;
};
FakeWatch& Fake() {
static FakeWatch fake;
return fake;
}
void Require(bool condition, const std::string& message) {
if (!condition) throw std::runtime_error(message);
}
}
namespace GuestWriteWatch {
extern "C" {
bool GuestWriteWatchAvailable_nid_postfix() {
return true;
}
bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes) {
return address >= Base && bytes <= 2 * Block && address - Base <= 2 * Block - bytes;
}
bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) {
auto& fake = Fake();
const auto end = address + bytes;
std::vector<std::pair<std::uintptr_t, std::uintptr_t>> runs;
{
std::unique_lock lock(fake.mutex);
if (fake.holdNextWalk) {
fake.holdNextWalk = false;
fake.holding = true;
fake.changed.notify_all();
fake.timedOut = !fake.changed.wait_for(lock, Patience, [&] { return fake.released; });
}
for (auto it = fake.written.begin(); it != fake.written.end();) {
if (it->first < end && address < it->second) {
runs.emplace_back(std::max(it->first, address), std::min(it->second, end));
it = fake.written.erase(it);
} else ++it;
}
}
for (const auto& [from, to] : runs) written(context, from, to);
return true;
}
}
}
namespace {
using namespace AgcDriver::GuestMemory;
void CheckMemoHitDuringWalk() {
auto& fake = Fake();
BumpCollectEpoch();
const auto collected = CollectWrites(Base, Block);
Require(collected != 0 && UnchangedSince(Base, Block, collected), "the block was not collected");
{
const std::lock_guard lock(fake.mutex);
fake.written.emplace_back(Base + Page, Base + 2 * Page);
fake.holdNextWalk = true;
}
std::thread walker([] { CollectWritesUncached(Base, 2 * Block); });
bool holding = false;
{
std::unique_lock lock(fake.mutex);
holding = fake.changed.wait_for(lock, Patience, [&] { return fake.holding; });
}
const auto memoized = holding ? CollectWrites(Base, Block) : 0;
{
const std::lock_guard lock(fake.mutex);
fake.released = true;
}
fake.changed.notify_all();
walker.join();
Require(holding, "the second walk did not reach the write watch");
Require(!fake.timedOut, "the memoized collect waited for the walk in progress");
Require(!UnchangedSince(Base + Page, Page, memoized), "a memoized collect returned the generation a walk in progress then stamped a write with");
BumpCollectEpoch();
Require(CollectWrites(Base, Block) > memoized && !UnchangedSince(Base, Block, memoized), "a later epoch trusts the write stamped by the walk in progress");
}
}
int main() {
try {
CheckMemoHitDuringWalk();
} catch (const std::exception& error) {
std::cerr << "collect memo generation test failed: " << error.what() << "\n";
return 1;
}
std::cout << "collect memo generation test passed\n";
return 0;
}
core/libs/prx/libSceAgcDriver/tests/ConstantPhi.cpp
+78
View File
@@ -79,6 +79,83 @@ void CheckDiamond(unsigned mode) {
}
}
void CheckUnobservedPhiCycles() {
IrProgram empty;
DeadCodeEliminator{}.EliminateUnobservedPhis(empty);
IrProgram program;
for (std::uint32_t id = 0; id < 4u; ++id) {
auto& block = program.CreateBlock();
program.BlockOrder().push_back(&block);
BlockInfo info;
info.id = id;
info.terminator.kind = TerminatorKind::Return;
program.Metadata().blockInfo.push_back(info);
}
auto& entry = *program.BlockOrder()[0];
auto& header = *program.BlockOrder()[1];
auto& latch = *program.BlockOrder()[2];
auto& exit = *program.BlockOrder()[3];
program.SetEntryBlock(entry);
entry.AddBranch(&header);
header.AddBranch(&latch);
header.AddBranch(&exit);
latch.AddBranch(&header);
program.Metadata().blockInfo[0].terminator.kind = TerminatorKind::Branch;
program.Metadata().blockInfo[0].terminator.trueBlock = 1u;
auto& headerInfo = program.Metadata().blockInfo[1];
headerInfo.terminator.kind = TerminatorKind::ConditionalBranch;
headerInfo.terminator.trueBlock = 2u;
headerInfo.terminator.falseBlock = 3u;
program.Metadata().blockInfo[2].terminator.kind = TerminatorKind::Branch;
program.Metadata().blockInfo[2].terminator.trueBlock = 1u;
IrBuilder builder(program);
builder.SetInsertionPoint(entry);
auto& zero = builder.Constant(0u);
auto& producer = builder.Emit(IrOpcode::LaneId, IrType::U32, {});
const auto phi = [&](IrBlock& block, IrType type = IrType::U32) -> IrValue& {
auto& value = program.CreateValue(IrOpcode::Phi, type);
block.AppendInstruction(&value);
return value;
};
auto& first = phi(header);
auto& second = phi(latch);
auto& tail = phi(header);
auto& self = phi(header);
auto& liveFirst = phi(header);
auto& liveSecond = phi(latch);
auto& condition = phi(header, IrType::Bool);
first.AddPhiOperand(&entry, &producer);
first.AddPhiOperand(&latch, &second);
second.AddPhiOperand(&header, &first);
tail.AddPhiOperand(&entry, &zero);
tail.AddPhiOperand(&latch, &second);
self.AddPhiOperand(&entry, &zero);
self.AddPhiOperand(&latch, &self);
liveFirst.AddPhiOperand(&entry, &zero);
liveFirst.AddPhiOperand(&latch, &liveSecond);
liveSecond.AddPhiOperand(&header, &liveFirst);
condition.AddPhiOperand(&entry, &builder.ConstantBool(true));
condition.AddPhiOperand(&latch, &condition);
headerInfo.condition = &condition;
builder.SetInsertionPoint(header);
static_cast<void>(builder.Emit(IrOpcode::Reference, IrType::Void, {&condition}));
builder.SetInsertionPoint(latch);
auto& store = builder.Emit(IrOpcode::StoreVectorWindow, IrType::Void, {&zero, &liveSecond});
ValidateProgram(program, true);
DeadCodeEliminator{}.EliminateUnobservedPhis(program);
ValidateProgram(program, true);
Require(first.Parent() == nullptr && second.Parent() == nullptr && tail.Parent() == nullptr && self.Parent() == nullptr, "phi cycles: an unobserved cycle or its tail survived");
Require(!first.HasUses() && !second.HasUses() && !tail.HasUses() && !self.HasUses(), "phi cycles: a removed cycle retained uses");
Require(liveFirst.Parent() == &header && liveSecond.Parent() == &latch && liveFirst.Argument(1) == &liveSecond && liveSecond.Argument(0) == &liveFirst, "phi cycles: an observed cycle was changed");
Require(store.Parent() == &latch && store.Argument(1) == &liveSecond, "phi cycles: an observable store was changed");
Require(condition.Parent() == &header && headerInfo.condition == &condition && condition.Argument(1) == &condition, "phi cycles: a branch condition was changed");
DeadCodeEliminator{}.Eliminate(program);
Require(producer.Parent() == nullptr && !producer.HasUses(), "phi cycles: ordinary DCE retained a released producer");
DeadCodeEliminator{}.EliminateUnobservedPhis(program);
ValidateProgram(program, true);
Require(entry.Instructions().empty() && header.Instructions().size() == 3u && latch.Instructions().size() == 2u, "phi cycles: repeated elimination changed the live graph");
}
void CheckLongAncillaryForwarding() {
IrProgram program;
auto& block = program.CreateBlock();
@@ -127,6 +204,7 @@ int main() {
try {
for (unsigned mode = 0; mode < 5u; ++mode) CheckDiamond(mode);
CheckLongAncillaryForwarding();
CheckUnobservedPhiCycles();
std::puts("constant phi dominance tests passed");
return 0;
} catch (const std::exception& error) {
core/libs/prx/libSceAgcDriver/tests/F32DenormPreserve.cpp
+16 -3
View File
@@ -4,6 +4,7 @@
#include <cstdint>
#include <cstdio>
#include <exception>
#include <optional>
#include <span>
#include <stdexcept>
#include <string>
@@ -19,17 +20,19 @@ void require(bool condition, const std::string& message) {
}
constexpr std::array<std::uint32_t, 4> CvtF16F32Store{0x7e021500u, 0xe0700000u, 0x80000100u, 0xbf810000u};
constexpr std::array<std::uint32_t, 4> MovStore{0x7e020300u, 0xe0700000u, 0x80000100u, 0xbf810000u};
std::vector<std::uint32_t> recompile(bool preserve, bool flush, bool flushCapability = true) {
std::vector<std::uint32_t> recompile(bool preserve, bool flush, bool flushCapability = true, std::optional<ShaderFloatMode> floatMode = std::nullopt, std::span<const std::uint32_t> code = CvtF16F32Store) {
const std::array<std::uint32_t, 4> capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityDenormPreserve, spv::CapabilityDenormFlushToZero};
const std::array<std::uint32_t, 4> userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu};
const std::array<std::string_view, 2> extensions{"SPV_KHR_storage_buffer_storage_class", "SPV_KHR_float_controls"};
RecompileRequest request{};
request.shader = {ShaderStage::Compute, 0x20000u, CvtF16F32Store, 0, {}};
request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
request.context.waveSize = 64;
request.context.userDataBaseRegister = 0;
request.context.userData = userData;
request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
request.context.floatMode = floatMode;
request.target.vulkanVersion = 0x00403000u;
request.target.spirvVersion = 0x00010600u;
request.target.subgroupSize = 64;
@@ -79,6 +82,16 @@ void verify() {
require(flushed.flush32 && !flushed.preserve32, "the hardware 32-bit flush did not take precedence over the preserve flag");
const auto nativeFlush = scan(recompile(true, true, false));
require(!nativeFlush.preserve32 && !nativeFlush.flush32, "a hardware flush without the DenormFlushToZero capability declared a 32-bit denormal execution mode");
const auto flushingMode = scan(recompile(true, true, true, ShaderFloatMode{0xc0u, true, false, false}, MovStore));
require(flushingMode.flush32 && !flushingMode.preserve32, "FLOAT_MODE 0xc0 flushes f32 denormals but the module did not take the hardware flush");
for (const std::uint32_t mode : {0xd0u, 0xe0u, 0xf0u}) {
char name[64];
std::snprintf(name, sizeof(name), "FLOAT_MODE 0x%02x keeps f32 denormals", mode);
const auto kept = scan(recompile(true, true, true, ShaderFloatMode{mode, true, false, false}, MovStore));
require(kept.preserve32 && kept.preserveCapability && !kept.flush32, std::string(name) + ", but the module did not declare DenormPreserve 32 instead of the hardware flush");
const auto unpreserved = scan(recompile(false, true, true, ShaderFloatMode{mode, true, false, false}, MovStore));
require(!unpreserved.flush32 && !unpreserved.preserve32, std::string(name) + ", but the module took the hardware flush on a device that cannot preserve them");
}
}
}
@@ -90,6 +103,6 @@ int main() {
std::fprintf(stderr, "F32DenormPreserve: %s\n", error.what());
return 1;
}
std::puts("F32DenormPreserve: the 32-bit DenormPreserve execution mode follows the target flag and yields to the hardware flush");
std::puts("F32DenormPreserve: the 32-bit DenormPreserve execution mode follows the target flag and yields to the hardware flush in shaders that flush f32 denormals");
return 0;
}
core/libs/prx/libSceAgcDriver/tests/FlatScratch.cpp
+118
View File
@@ -0,0 +1,118 @@
#include "RdnaDecoder/RdnaMemoryOpDecoder.hpp"
#include "Translation/TranslationContext.hpp"
#include <array>
#include <cstdio>
#include <stdexcept>
using namespace ShaderRecompiler;
static unsigned tests = 0;
#define Require(value) \
do { \
if (!(value)) { \
std::fprintf(stderr, "flat scratch assertion line %u: %s\n", __LINE__, \
#value); \
throw std::runtime_error("flat scratch regression"); \
} \
} while (false)
static RdnaInstruction Decode(unsigned seg, unsigned saddr, unsigned addr,
unsigned offset = 0, unsigned opcode = 12) {
const std::array<unsigned, 2> code{(0x37u << 26) | (opcode << 18) |
(seg << 14) | offset,
(2u << 24) | (saddr << 16) | addr};
return DecodeRdnaFlat(0x154cu, code, 0);
}
static void Reject(unsigned seg, unsigned saddr, unsigned addr) {
try {
(void)Decode(seg, saddr, addr);
} catch (const std::runtime_error &) {
++tests;
return;
}
throw std::runtime_error("invalid FLAT accepted");
}
static void Scratch(unsigned saddr, unsigned addr, unsigned expectedVector,
unsigned expectedScalar) {
auto op = Decode(1, saddr, addr, 0xffcu);
Require(op.memorySegment == 1 && op.memoryOffset == -4 &&
op.sourceCount == 2);
Require(op.source0.kind == (expectedVector
? RdnaOperandKind::VectorRegister
: RdnaOperandKind::IntegerInlineConstant));
Require(op.source1.kind == (expectedScalar
? RdnaOperandKind::ScalarRegister
: RdnaOperandKind::IntegerInlineConstant));
if (expectedVector)
Require(op.source0.reg == addr);
else
Require(op.source0.value == 0);
if (expectedScalar)
Require(op.source1.reg == saddr);
else
Require(op.source1.value == 0);
IrProgram program;
auto &block = program.CreateBlock();
program.SetEntryBlock(block);
TranslationContext context(program, block, 256);
context.TranslateInstruction(op);
unsigned vectors = 0, addressReads = 0, destinationReads = 0, scalars = 0,
loads = 0, scratch = 0;
for (auto *inst : block.Instructions()) {
if (inst->Opcode() == IrOpcode::GetVectorRegister) {
++vectors;
auto reg = inst->Arguments().at(0)->Register().index;
Require(reg < 256);
if (reg == addr)
++addressReads;
if (reg == 2)
++destinationReads;
}
if (inst->Opcode() == IrOpcode::GetScalarRegister)
++scalars;
if (inst->Opcode() == IrOpcode::LoadAddressU32)
++loads;
if (inst->Opcode() == IrOpcode::GetScratchResource)
++scratch;
}
std::printf(
"SV/SS/ST saddr=%u addr=%u vector=%u scalar=%u loads=%u scratch=%u\n",
saddr, addr, vectors, scalars, loads, scratch);
std::fflush(stdout);
Require(vectors == expectedVector + 1 && addressReads == expectedVector &&
destinationReads == 1 && scalars == expectedScalar && loads == 1 &&
scratch == 1);
++tests;
}
int main() {
const std::array<unsigned, 2> code{0xdc304000u, 0x027d00ffu};
auto actual = DecodeRdnaFlat(0x154cu, code, 0);
Require(actual.destination.reg == 2 && actual.source0.reg == 255 &&
actual.source1.value == 0);
++tests;
Scratch(125, 255, 1, 0);
Scratch(125, 254, 1, 0);
Scratch(0, 255, 0, 1);
Scratch(105, 255, 0, 1);
Scratch(127, 255, 0, 0);
Reject(0, 125, 255);
Reject(2, 125, 255);
Reject(2, 127, 255);
Reject(1, 108, 0);
Reject(3, 0, 0);
auto flat = Decode(0, 125, 254);
Require(flat.source1.kind == RdnaOperandKind::VectorRegister &&
flat.source1.reg == 255);
++tests;
auto global = Decode(2, 0, 255);
Require(global.source0.reg == 255 &&
global.source1.kind == RdnaOperandKind::ScalarRegister);
++tests;
try {
const std::array<unsigned, 1> truncated{0xdc304000u};
(void)DecodeRdnaFlat(0, truncated, 0);
throw std::logic_error("truncated accepted");
} catch (const std::out_of_range &) {
++tests;
}
std::printf("PASS %u flat scratch decode/IR and invalid boundary cases\n",
tests);
}
core/libs/prx/libSceAgcDriver/tests/Graphics.cpp
+487 -4
View File
@@ -906,7 +906,7 @@ void msaaEnableClearOneSampleTargetTests() {
queue.context[0x1b3] = queue.context[0x1b4] = 2u | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::Ancillary) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::SampleCoverage);
Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "the depth-only draw did not skip its pixel program");
requireOneSampleDepth(queue, 0, "a one-sample depth target whose skipped pixel program has PS_ITER_SAMPLE and sample index and coverage inputs");
constexpr std::string_view depthRefused = "a one-sample depth target under a multisampled rasterizer without MSAA_ENABLE is unsupported for points or lines and with alpha-to-coverage, a sample index or coverage input, a sample-mask export, or DB_EQAA Z interpolation or overrasterization";
constexpr std::string_view depthRefused = "a one-sample depth target under a multisampled rasterizer without MSAA_ENABLE is unsupported for points or lines and with alpha-to-coverage, a sample index or coverage input, a sample-mask export, or DB_EQAA source Z interpolation or overrasterization";
for (const bool color : {false, true}) {
for (const auto input : {ShaderRecompiler::PixelInput::Ancillary, ShaderRecompiler::PixelInput::SampleCoverage}) {
queue = depthTarget(color);
@@ -934,7 +934,12 @@ void msaaEnableClearOneSampleTargetTests() {
queue.userConfig[0x242] = primitive;
expectFailure([&] { DecodeState(queue); }, depthRefused);
}
for (const auto eqaa : {0x00050002u, 0x00090002u, 0x01010002u, 0x08010002u}) {
for (const auto eqaa : {0x00040000u, 0x00050000u}) {
queue = depthTarget(color);
queue.context[0x201] = eqaa;
requireOneSampleDepth(queue, color ? 1u : 0u, "a one-sample depth target under a 4x PA_SC_AA_CONFIG without MSAA_ENABLE with DB_EQAA INTERPOLATE_COMP_Z");
}
for (const auto eqaa : {0x00090002u, 0x000d0000u, 0x01010002u, 0x08010002u}) {
queue = depthTarget(color);
queue.context[0x201] = eqaa;
expectFailure([&] { DecodeState(queue); }, depthRefused);
@@ -4231,6 +4236,24 @@ void multisampleDccStateTests() {
changed.context[0x3b0] |= 1u << 28u;
expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "multisampled color mips");
}
for (const auto samples : {2u, 4u, 8u}) {
AgcDriver::Graphics::ColorTarget target{};
target.extent = {64, 64};
target.format = VK_FORMAT_R8G8B8A8_UNORM;
target.tileMode = AgcDriver::Graphics::ColorTileMode::RenderTarget;
target.samples = samples;
target.dccAddress = 0x200000u;
target.dccPipeAligned = true;
target.cmaskAddress = 0x300000u;
target.cmaskBytes = AgcDriver::Graphics::CmaskBytes(64, 64);
const std::vector<std::byte> dcc(AgcDriver::Graphics::MultisampledDccBytes(target), std::byte{0x00});
std::vector<std::byte> cmask(target.cmaskBytes);
for (std::size_t index = 0; index < cmask.size(); ++index) cmask[index] = std::byte{static_cast<std::uint8_t>(index * 37u)};
const auto clear = AgcDriver::Graphics::MultisampledDccClear(target, dcc, cmask);
Require(clear.float32[0] == 0.0f && clear.float32[1] == 0.0f && clear.float32[2] == 0.0f && clear.float32[3] == 0.0f, "a uniform 0000 DCC clear of a multisampled target without CMASK fast clears did not import as 0000");
target.cmaskFastClear = true;
expectFailure([&] { static_cast<void>(AgcDriver::Graphics::MultisampledDccClear(target, dcc, cmask)); }, "multisampled DCC CMASK compression or register clear");
}
}
void multisampledDepthTests() {
@@ -8691,6 +8714,24 @@ std::string interlockShape(std::span<const std::uint32_t> words) {
return shape;
}
bool coherentImageRead(std::span<const std::uint32_t> words) {
std::map<std::uint32_t, std::uint32_t> sources;
std::set<std::uint32_t> coherent;
std::uint32_t image = 0;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
const auto op = static_cast<spv::Op>(words[at] & 0xffffu);
if (op == spv::OpDecorate && words[at + 2] == spv::DecorationCoherent) coherent.insert(words[at + 1]);
if (op == spv::OpLoad || op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain) sources[words[at + 2]] = words[at + 3];
if (op == spv::OpImageRead) image = words[at + 3];
}
for (std::size_t step = 0; image != 0 && step < 8u; ++step) {
if (coherent.contains(image)) return true;
const auto source = sources.find(image);
image = source == sources.end() ? 0u : source->second;
}
return false;
}
std::string interlockExits(std::span<const std::uint32_t> words, std::size_t& kills, std::size_t& terminates) {
struct Block {
std::vector<spv::Op> ops;
@@ -9354,7 +9395,7 @@ void ProvokingVertexTests() {
ValidateProvokingVertex(unsupported, first);
auto last = first;
last.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT;
expectFailure([&] { ValidateProvokingVertex(unsupported, last); }, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
ValidateProvokingVertex(unsupported, last);
auto supported = unsupported;
supported.provokingVertexLast = true;
ValidateProvokingVertex(supported, last);
@@ -9477,6 +9518,105 @@ void ProvokingVertexTests() {
expectFailure([&] { ValidateProvokingVertex(supported, last); }, "invalid provoking vertex mode");
}
void UnobservedProvokingVertexTests() {
using namespace AgcDriver::Graphics;
auto last = DecodeState(makeState());
last.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT;
const Context unsupported{};
const auto module = [](std::initializer_list<std::uint32_t> instructions) {
std::vector<std::uint32_t> words{spv::MagicNumber, 0x10300u, 0u, 4u, 0u};
words.insert(words.end(), instructions);
return words;
};
ShaderRecompiler::RecompileResult fragment{};
const std::array<CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}};
const ShaderRecompiler::FragmentParameter smooth{0, 0, false, false};
const auto refused = [&](const State& state) {
expectFailure([&] { ValidateProvokingVertex(unsupported, state, shaders); }, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
};
for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN}) {
auto state = last;
state.topology = topology;
ValidateProvokingVertex(unsupported, state);
fragment.fragmentParameters.clear();
fragment.spirv = module({});
ValidateProvokingVertex(unsupported, state, shaders);
fragment.fragmentParameters = {smooth};
fragment.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationLocation, 0u});
ValidateProvokingVertex(unsupported, state, shaders);
for (std::uint32_t read = 0; read < 4; ++read) {
const ShaderRecompiler::FragmentParameter observed{1, 1, read == 0, read == 1, read == 2, read == 3, false};
fragment.fragmentParameters = {smooth, observed};
refused(state);
}
fragment.fragmentParameters = {smooth};
for (const std::uint32_t decoration : {static_cast<std::uint32_t>(spv::DecorationFlat), static_cast<std::uint32_t>(spv::DecorationPerVertexKHR)}) {
fragment.spirv = module({(3u << 16u) | spv::OpDecorate, 1u, decoration});
refused(state);
fragment.spirv = module({(4u << 16u) | spv::OpMemberDecorate, 1u, 0u, decoration});
refused(state);
}
fragment.spirv = module({(5u << 16u) | spv::OpDecorate, 1u, spv::DecorationLocation});
refused(state);
fragment.spirv = std::vector<std::uint32_t>{};
refused(state);
}
fragment.fragmentParameters = {smooth};
for (const std::uint32_t generated : {static_cast<std::uint32_t>(spv::BuiltInSampleId), static_cast<std::uint32_t>(spv::BuiltInFrontFacing), static_cast<std::uint32_t>(spv::BuiltInSubgroupLocalInvocationId)}) {
fragment.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, generated, (3u << 16u) | spv::OpDecorate, 1u, spv::DecorationFlat});
ValidateProvokingVertex(unsupported, last, shaders);
fragment.spirv = module({(3u << 16u) | spv::OpDecorate, 1u, spv::DecorationFlat, (4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, generated});
ValidateProvokingVertex(unsupported, last, shaders);
fragment.spirv = module({(5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, generated, (4u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationFlat});
ValidateProvokingVertex(unsupported, last, shaders);
fragment.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, generated, (3u << 16u) | spv::OpDecorate, 2u, spv::DecorationFlat});
refused(last);
fragment.spirv = module({(5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, generated, (4u << 16u) | spv::OpMemberDecorate, 1u, 1u, spv::DecorationFlat});
refused(last);
}
for (const std::uint32_t passed : {static_cast<std::uint32_t>(spv::BuiltInLayer), static_cast<std::uint32_t>(spv::BuiltInViewportIndex), static_cast<std::uint32_t>(spv::BuiltInPrimitiveId)}) {
fragment.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, passed, (3u << 16u) | spv::OpDecorate, 1u, spv::DecorationFlat});
refused(last);
fragment.spirv = module({(5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, passed, (4u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationFlat});
refused(last);
}
ShaderRecompiler::RecompileResult vertex{};
vertex.fragmentParameters = {{0, 0, true, true, true, true, true}};
vertex.spirv = module({(3u << 16u) | spv::OpDecorate, 1u, spv::DecorationFlat});
fragment.fragmentParameters = {smooth};
fragment.spirv = module({});
const std::array<CompiledShader, 2> stages{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}};
ValidateProvokingVertex(unsupported, last, stages);
Context supported{};
supported.provokingVertexLast = true;
for (const auto stage : {ShaderRecompiler::ShaderStage::Vertex, ShaderRecompiler::ShaderStage::TessellationEvaluation, ShaderRecompiler::ShaderStage::Geometry, ShaderRecompiler::ShaderStage::Mesh}) {
ShaderRecompiler::RecompileResult writer{};
const std::array<CompiledShader, 2> written{{{stage, &writer, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}};
const auto refusedWrite = [&] {
expectFailure([&] { ValidateProvokingVertex(unsupported, last, written); }, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
ValidateProvokingVertex(supported, last, written);
};
for (const std::uint32_t selected : {static_cast<std::uint32_t>(spv::BuiltInLayer), static_cast<std::uint32_t>(spv::BuiltInViewportIndex)}) {
writer.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, selected});
refusedWrite();
writer.spirv = module({(5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, selected});
refusedWrite();
writer.spirv = module({(5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, selected, (4u << 16u) | spv::OpMemberDecorate, 1u, 1u, spv::DecorationPerPrimitiveEXT});
refusedWrite();
writer.spirv = module({(4u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationPerPrimitiveEXT, (5u << 16u) | spv::OpMemberDecorate, 1u, 0u, spv::DecorationBuiltIn, selected});
ValidateProvokingVertex(unsupported, last, written);
writer.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, selected, (3u << 16u) | spv::OpDecorate, 1u, spv::DecorationPerPrimitiveEXT});
ValidateProvokingVertex(unsupported, last, written);
}
writer.spirv = module({(4u << 16u) | spv::OpDecorate, 1u, spv::DecorationBuiltIn, spv::BuiltInPosition, (5u << 16u) | spv::OpMemberDecorate, 2u, 0u, spv::DecorationBuiltIn, spv::BuiltInPointSize});
ValidateProvokingVertex(unsupported, last, written);
writer.spirv = std::vector<std::uint32_t>{};
refusedWrite();
}
const std::array<CompiledShader, 1> missing{{{ShaderRecompiler::ShaderStage::Fragment, nullptr, 0}}};
expectFailure([&] { ValidateProvokingVertex(unsupported, last, missing); }, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled");
}
void ProvokingVertexRenderPassTests() {
using namespace AgcDriver::Graphics;
const auto first = DecodeState(makeState());
@@ -9743,6 +9883,332 @@ void unorm10_11_11TargetTests() {
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(uintExport)); }, "fragment output 0 has no color export packing path");
}
void unorm4_4_4_4TargetTests() {
using ShaderRecompiler::ColorExportPacking;
auto queue = makeState();
queue.context[0x31c] = (queue.context.at(0x31c) & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (19u << 2u) | (2u << 11u);
queue.context[0x1b3] = 2;
queue.context[0x1b4] = 2;
const auto state = AgcDriver::Graphics::DecodeState(queue);
Require(state.color.format == VK_FORMAT_R16_UINT && state.color.elementBytes == 2 && state.color.componentMapping == 0x1bu && state.color.packing == ColorExportPacking::Unorm4_4_4_4, "a 4_4_4_4 unorm target did not decode to an R16_UINT attachment with reversed exports");
const auto packings = AgcDriver::Graphics::ExportPackings(state);
Require(packings[0] == ColorExportPacking::Unorm4_4_4_4 && std::all_of(packings.begin() + 1, packings.end(), [](ColorExportPacking packing) { return packing == ColorExportPacking::None; }), "export 0 did not carry the 4_4_4_4 unorm packing");
Require(state.color.partialWriteMask == 0u && state.blends[0].colorWriteMask == 0xfu && AgcDriver::Graphics::ExportTargetMerge(state).mask == 0u, "a full write of a 4_4_4_4 unorm target merged into the target");
auto changed = queue;
for (const auto& [targetMask, wordMask] : {std::pair{1u, 0xf000u}, std::pair{7u, 0xfff0u}, std::pair{8u, 0x000fu}, std::pair{0xau, 0x0f0fu}}) {
changed.context[0x8e] = targetMask;
Require(AgcDriver::Graphics::DrawRejection(changed, false).empty(), "the draw precheck refused CB_TARGET_MASK " + std::to_string(targetMask) + " of a 4_4_4_4 unorm target");
const auto partial = AgcDriver::Graphics::DecodeState(changed);
Require(partial.color.partialWriteMask == wordMask && partial.colors.front().partialWriteMask == wordMask && partial.blends[0].colorWriteMask == 0u && partial.blend.colorWriteMask == 0u, "CB_TARGET_MASK " + std::to_string(targetMask) + " of a 4_4_4_4 unorm target did not merge its nibbles behind a closed write mask");
const auto merge = AgcDriver::Graphics::ExportTargetMerge(partial);
const auto expected = AgcDriver::Graphics::SurfaceForTarget(partial.color);
const auto surface = AgcDriver::Graphics::DecodeTextureResource(merge.descriptor);
Require(merge.target == 0u && merge.mask == wordMask && merge.samples == 1u && !merge.depthStencilWrites && surface.baseAddress == expected.baseAddress && surface.pipeBankXor == expected.pipeBankXor && surface.width == expected.width && surface.height == expected.height && surface.format == expected.format && surface.tileMode == expected.tileMode && surface.dimension == AgcDriver::Graphics::TextureDimension::k2D && surface.mipCount == 1u && surface.samples == 1u, "the merge descriptor of CB_TARGET_MASK " + std::to_string(targetMask) + " does not decode to the target's surface");
}
{
auto depthQueue = queue;
depthQueue.context[0x8e] = 1;
depthQueue.context[0x000] = 0;
depthQueue.context[0x002] = 0;
depthQueue.context[0x007] = (1u << 16u) | 3u;
depthQueue.context[0x00a] = 7;
depthQueue.context[0x00b] = 0;
depthQueue.context[0x010] = 0x22900983;
depthQueue.context[0x011] = 0x20000181;
for (const auto offset : {0x012u, 0x014u}) depthQueue.context[offset] = 0x100;
for (const auto offset : {0x013u, 0x015u}) depthQueue.context[offset] = 0x200;
depthQueue.context[0x005] = 0x300;
depthQueue.context[0x01e] = 0x1;
depthQueue.context[0x008] = std::bit_cast<std::uint32_t>(0.25f);
depthQueue.context[0x009] = std::bit_cast<std::uint32_t>(0.75f);
struct DepthStencilCase {
std::uint32_t control;
std::uint32_t ops;
std::uint32_t masks;
bool writes;
};
for (const auto& [control, ops, masks, writes] : {DepthStencilCase{0x00700710u, 0x00050050u, 0x01ffff01u, false}, DepthStencilCase{0x00700712u, 0x00050050u, 0x01ffff01u, false}, DepthStencilCase{0x00700714u, 0x00050050u, 0x01ffff01u, false}, DepthStencilCase{0x00700716u, 0x00050050u, 0x01ffff01u, true}, DepthStencilCase{0x00700711u, 0x00050050u, 0x01ffff01u, true}, DepthStencilCase{0x00700711u, 0u, 0x01ffff01u, false}, DepthStencilCase{0x00700711u, 0x00050050u, 0x0100ff01u, false}, DepthStencilCase{0x00700713u, 0u, 0x01ffff01u, false}, DepthStencilCase{0x00700718u, 0x00050050u, 0x01ffff01u, false}}) {
depthQueue.context[0x200] = control;
depthQueue.context[0x10b] = ops;
depthQueue.context[0x10c] = depthQueue.context[0x10d] = masks;
const auto what = "DB_DEPTH_CONTROL " + std::to_string(control) + " with stencil operations " + std::to_string(ops) + " and masks " + std::to_string(masks);
Require(AgcDriver::Graphics::DrawRejection(depthQueue, false).empty(), "the draw precheck refused a partial 4_4_4_4 write under " + what);
const auto partial = AgcDriver::Graphics::DecodeState(depthQueue);
const auto merge = AgcDriver::Graphics::ExportTargetMerge(partial);
Require(partial.depth.has_value() == ((control & 0xbu) != 0u) && partial.depthBoundsTest == ((control & 8u) != 0u) && merge.mask == 0xf000u && AgcDriver::Graphics::DepthStencilWrites(partial) == writes && merge.depthStencilWrites == writes && merge.depthStencilTests == ((control & 0xbu) != 0u), "a partial 4_4_4_4 write under " + what + (writes ? " did not report" : " reported") + " depth or stencil writes");
}
auto truncated = depthQueue;
truncated.context[0x200] = 0x00700712u;
truncated.context[0x010] = 0x22900987u;
truncated.context[0x2f8] = 0x00100001u;
truncated.context[0x292] = 3;
for (std::uint32_t pixel = 0; pixel < 4; ++pixel) truncated.context[0x2fe + pixel * 4u] = 0xcc44;
truncated.context[0x31d] = truncated.context.at(0x31d) & ~0x3f000u;
truncated.context[0x8e] = 0xf;
const auto full = AgcDriver::Graphics::DecodeState(truncated);
Require(full.samples == 2 && full.truncatedCoverage && !full.mergedCoverage && full.color.samples == 1 && full.color.partialWriteMask == 0u, "a full write of a one-sample 4_4_4_4 unorm target beside a 2x depth target did not decode as truncated coverage");
truncated.context[0x8e] = 1;
expectFailure([&] { AgcDriver::Graphics::DecodeState(truncated); }, "under merged or truncated coverage");
}
{
auto smoothed = queue;
smoothed.context[0x2f8] = 0x0020c002u;
smoothed.context[0x292] = 3;
smoothed.context[0x3b8] = 0x0dc00000u;
for (std::uint32_t pixel = 0; pixel < 4; ++pixel) smoothed.context[0x2fe + pixel * 4u] = 0xcc44;
smoothed.context[0x31d] = smoothed.context.at(0x31d) & ~0x3f000u;
smoothed.context[0x201] = 0x02130000u;
const auto merged = AgcDriver::Graphics::DecodeState(smoothed);
Require(merged.samples == 4 && merged.mergedCoverage && merged.color.samples == 1 && merged.color.partialWriteMask == 0u, "a full write of a one-sample 4_4_4_4 unorm target under OVERRASTERIZATION_AMOUNT 2 did not decode as merged coverage");
smoothed.context[0x8e] = 1;
expectFailure([&] { AgcDriver::Graphics::DecodeState(smoothed); }, "under merged or truncated coverage");
}
{
changed.context[0x8e] = 1;
const auto partial = AgcDriver::Graphics::DecodeState(changed);
auto killQueue = changed;
killQueue.context[0x203] = 0x40u;
const auto killState = AgcDriver::Graphics::DecodeState(killQueue);
killQueue.context[0x203] = 0x10u;
const auto earlyState = AgcDriver::Graphics::DecodeState(killQueue);
Require(killState.pixelKill && !killState.pixelEarlyZ && !earlyState.pixelKill && earlyState.pixelEarlyZ, "DB_SHADER_CONTROL.KILL_ENABLE of a 4_4_4_4 merge did not decode as a kill under late tests");
constexpr std::array<std::uint32_t, 3> mergeCode{0xf800180fu, 0x07060504u, 0xbf810000u};
auto merging = packedExportRequest(changed, partial, mergeCode);
merging.context.pixel->targetMerge = AgcDriver::Graphics::ExportTargetMerge(partial);
const std::array<std::uint32_t, 3> capabilities{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityStorageImageReadWithoutFormat};
merging.target.supportedCapabilities = capabilities;
merging.target.registrationWordLimit = 1u << 20u;
const ShaderRecompiler::RequestSerializer mergeSerializer;
Require(mergeSerializer.Deserialize(mergeSerializer.Serialize(merging)).request.context.pixel->targetMerge == merging.context.pixel->targetMerge, "the 4_4_4_4 target merge was lost in serialization");
auto unmerged = merging;
unmerged.context.pixel->targetMerge = {};
auto moved = merging;
moved.context.pixel->targetMerge.descriptor[0] += 0x100u;
std::vector<std::uint64_t> mergedKey, unmergedKey, movedKey;
ShaderRecompiler::RecompileCacheKey::BuildInterface(merging, mergedKey);
ShaderRecompiler::RecompileCacheKey::BuildInterface(unmerged, unmergedKey);
ShaderRecompiler::RecompileCacheKey::BuildInterface(moved, movedKey);
Require(mergedKey != unmergedKey && mergedKey == movedKey, "the static shader key does not separate merging from plain writes, or depends on the merged target's address");
for (const bool registration : {false, true}) {
merging.target.registrationModules = registration;
const auto merged = ShaderRecompiler::Recompile(merging);
const auto words = merged.spirv.Copy();
std::size_t reads = 0, writes = 0;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
reads += static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpImageRead ? 1u : 0u;
writes += static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpImageWrite ? 1u : 0u;
}
const auto what = std::string(registration ? "a registration" : "a specialized") + " 4_4_4_4 merge module";
Require(merged.registrationModule == registration && reads == 1u && writes == 1u && interlockShape(words) == "capability extension ordered begin end " && declaresEarlyFragmentTests(merging), what + " did not read and write the target once inside a pixel interlock after early tests: " + interlockShape(words));
Require(coherentImageRead(words), what + " does not read the target through a Coherent image variable");
const auto bound = std::ranges::any_of(merged.bindings, [&](const ShaderRecompiler::DescriptorBinding& binding) {
if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) return false;
for (std::size_t offset = 0; offset + 8u <= binding.guestDescriptor.size(); offset += 8u) {
if (std::equal(merging.context.pixel->targetMerge.descriptor.begin(), merging.context.pixel->targetMerge.descriptor.end(), binding.guestDescriptor.begin() + static_cast<std::ptrdiff_t>(offset))) return true;
}
return false;
});
Require(bound, what + " does not bind the merged target as a storage image");
ShaderRecompiler::RecompileResult vertex;
vertex.spirv = makeModule({});
const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &merged, 0}}};
Require(AgcDriver::Graphics::ValidateShaders(shaders, partial, subgroup, false) == std::set<std::uint32_t>{0u}, what + " lost its export to attachment 0");
}
auto lacking = merging;
const std::array<std::uint32_t, 2> noInterlock{spv::CapabilityShader, spv::CapabilityStorageImageReadWithoutFormat};
lacking.target.supportedCapabilities = noInterlock;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(lacking)); }, "fragmentShaderPixelInterlock");
auto unformatted = merging;
const std::array<std::uint32_t, 2> noFormatlessReads{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT};
unformatted.target.supportedCapabilities = noFormatlessReads;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(unformatted)); }, "storage image reads without a format");
auto killing = merging;
killing.context.pixel->pixelKillEnable = true;
killing.context.pixel->earlyZ = false;
for (const bool registration : {false, true}) {
killing.target.registrationModules = registration;
const auto words = ShaderRecompiler::Recompile(killing).spirv.Copy();
std::size_t reads = 0, writes = 0;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
reads += static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpImageRead ? 1u : 0u;
writes += static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpImageWrite ? 1u : 0u;
}
Require(reads == 1u && writes == 1u && interlockShape(words) == "capability extension ordered begin end " && declaresEarlyFragmentTests(killing), "a killing 4_4_4_4 merge module under late tests of a draw without depth or stencil writes did not merge inside a pixel interlock after early tests");
}
killing.context.pixel->targetMerge.depthStencilWrites = true;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(killing)); }, "kills pixels under late tests of a draw writing depth or stencil");
auto earlyKilling = killing;
earlyKilling.context.pixel->earlyZ = true;
static_cast<void>(ShaderRecompiler::Recompile(earlyKilling));
auto exporting = merging;
exporting.context.pixel->depthExportEnable = true;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(exporting)); }, "exports depth");
alignas(16) static std::array<std::uint32_t, 4> counted{};
static const std::array<std::uint32_t, 4> counter{static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(counted.data())), static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(counted.data()) >> 32u) & 0xffffu, 16u, 0x31016facu};
constexpr std::array<std::uint32_t, 6> countingCode{0x7e000281u, 0xe0c80000u, 0x80000000u, 0xf800180fu, 0x07060504u, 0xbf810000u};
auto counting = packedExportRequest(changed, partial, countingCode);
counting.context.userData = counter;
counting.context.pixel->targetMerge = merging.context.pixel->targetMerge;
counting.target = merging.target;
counting.target.registrationModules = false;
counting.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version;
static const std::array<std::uint32_t, 4> atomicCapabilities{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityStorageImageReadWithoutFormat, spv::CapabilityInt64};
counting.target.supportedCapabilities = atomicCapabilities;
counting.context.pixel->earlyZ = false;
static_cast<void>(ShaderRecompiler::Recompile(counting));
counting.context.pixel->targetMerge.depthStencilTests = true;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(counting)); }, "writes guest memory (stores, atomics or GDS) under late tests");
counting.context.pixel->earlyZ = true;
const auto earlyWords = ShaderRecompiler::Recompile(counting).spirv.Copy();
std::size_t atomics = 0;
for (std::size_t at = 5; at < earlyWords.size() && (earlyWords[at] >> 16u) != 0; at += earlyWords[at] >> 16u) atomics += static_cast<spv::Op>(earlyWords[at] & 0xffffu) == spv::OpAtomicIAdd ? 1u : 0u;
Require(atomics == 1u && interlockShape(earlyWords) == "capability extension ordered begin end " && declaresEarlyFragmentTests(counting), "a 4_4_4_4 merge whose shader adds to a counter under early tests did not keep its one atomic add beside the merge");
constexpr std::array<std::uint32_t, 9> gdsReadCode{0xbefc03ffu, 0x00000100u, 0x7e000280u, 0xd8da0000u, 0x04000000u, 0xbf8c0000u, 0xf800180fu, 0x07060504u, 0xbf810000u};
auto gdsReading = packedExportRequest(changed, partial, gdsReadCode);
gdsReading.context.pixel->targetMerge = counting.context.pixel->targetMerge;
gdsReading.context.pixel->earlyZ = false;
gdsReading.target = counting.target;
static_cast<void>(ShaderRecompiler::Recompile(gdsReading));
constexpr std::array<std::uint32_t, 9> gdsAddCode{0xbefc03ffu, 0x00000100u, 0x7e000280u, 0xd8020000u, 0x00000000u, 0xbf8c0000u, 0xf800180fu, 0x07060504u, 0xbf810000u};
auto gdsAdding = gdsReading;
gdsAdding.shader.code = gdsAddCode;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(gdsAdding)); }, "writes guest memory (stores, atomics or GDS) under late tests");
auto lateMerge = counting;
lateMerge.context.pixel->earlyZ = false;
lateMerge.context.pixel->targetMerge = {};
static_cast<void>(ShaderRecompiler::Recompile(lateMerge));
}
changed = queue;
changed.context[0x8e] = 1u;
changed.context[0x31d] = (changed.context.at(0x31d) & ~0x3f000u) | (1u << 12u) | (1u << 15u);
{
const auto multisampled = AgcDriver::Graphics::DecodeState(changed);
Require(multisampled.samples == 2u && multisampled.color.samples == 2u && multisampled.color.partialWriteMask == 0xf000u && multisampled.blends[0].colorWriteMask == 0u && multisampled.perSampleShading, "a partial write of a 2-sample 4_4_4_4 unorm target did not merge per sample behind a closed write mask");
auto coverage = changed;
coverage.context[0x1b3] = 0x4002u;
coverage.context[0x1b4] = 0x4002u;
expectFailure([&] { AgcDriver::Graphics::DecodeState(coverage); }, "SAMPLE_COVERAGE in a multisampled pixel shader that runs per sample");
coverage.context[0x8e] = 0xfu;
Require(!AgcDriver::Graphics::DecodeState(coverage).perSampleShading, "a full write of a 2-sample 4_4_4_4 unorm target with a SAMPLE_COVERAGE input was refused or ran per sample");
struct InputCase {
std::uint32_t inputs;
std::uint32_t position;
const char* what;
};
auto rasterized = changed;
rasterized.context[0x2f8] = 0x00100001u;
rasterized.context[0x292] = 3;
rasterized.context[0x3b8] = 0x0dc00000u;
for (std::uint32_t pixel = 0; pixel < 4; ++pixel) rasterized.context[0x2fe + pixel * 4u] = 0xcc44;
const auto inputState = [](AgcDriver::QueueState queue, std::uint32_t inputs, std::uint32_t position) {
queue.context[0x1b3] = inputs;
queue.context[0x1b4] = inputs;
queue.context[0x1b8] = position;
return queue;
};
for (const auto& [inputs, position, what] : {InputCase{0x0004u, 0u, "PERSP_CENTROID"}, InputCase{0x0040u, 0u, "LINEAR_CENTROID"}, InputCase{0x0001u, 0u, "PERSP_SAMPLE"}, InputCase{0x0008u, 0u, "PERSP_PULL_MODEL"}, InputCase{0x2002u, 0u, "ANCILLARY"}, InputCase{0x0402u, 0u, "POS_Z_FLOAT"}, InputCase{0x0802u, 0u, "POS_W_FLOAT"}, InputCase{0x0302u, 0x20000u, "POS_X_FLOAT and POS_Y_FLOAT at a sample"}, InputCase{0x0102u, 0x10000u, "POS_X_FLOAT at the centroid"}}) {
auto moved = inputState(rasterized, inputs, position);
expectFailure([&] { AgcDriver::Graphics::DecodeState(moved); }, "cannot read the ancillary word or, unless every sample sits at the pixel centre");
moved.context[0x293] = moved.context.at(0x293) | 0x10000u;
Require(AgcDriver::Graphics::DecodeState(moved).perSampleShading, std::string("a partial write of a 2-sample 4_4_4_4 unorm target whose shader the guest runs per sample was refused for reading ") + what);
if (inputs == 0x2002u) {
expectFailure([&] { AgcDriver::Graphics::DecodeState(inputState(changed, inputs, position)); }, "cannot read the ancillary word");
continue;
}
Require(AgcDriver::Graphics::DecodeState(inputState(changed, inputs, position)).perSampleShading, std::string("a partial write of a 2-sample 4_4_4_4 unorm target raised from a one-sample rasterizer, every sample at the pixel centre, was refused for reading ") + what);
}
for (const auto& [inputs, position, what] : {InputCase{0x0302u, 0u, "POS_X_FLOAT and POS_Y_FLOAT at the centre"}, InputCase{0x0302u, 0x100000u, "POS_X_FLOAT and POS_Y_FLOAT at the upper-left corner"}, InputCase{0x9022u, 0u, "PERSP_CENTER, LINEAR_CENTER, FRONT_FACE and POS_FIXED_PT"}}) {
Require(AgcDriver::Graphics::DecodeState(inputState(rasterized, inputs, position)).perSampleShading, std::string("a partial write of a 2-sample 4_4_4_4 unorm target whose pixel-rate shader reads ") + what + " was refused");
}
const auto merge = AgcDriver::Graphics::ExportTargetMerge(multisampled);
const auto surface = AgcDriver::Graphics::DecodeTextureResource(merge.descriptor);
Require(merge.samples == 2u && merge.mask == 0xf000u && surface.dimension == AgcDriver::Graphics::TextureDimension::k2DMsaa && surface.samples == 2u && surface.baseAddress == multisampled.color.surfaceAddress && surface.width == multisampled.color.extent.width && surface.height == multisampled.color.extent.height && surface.format == AgcDriver::Graphics::TargetMergeSurface(multisampled.color).format, "the merge descriptor of a 2-sample 4_4_4_4 unorm target does not decode to its multisampled surface");
auto unshaded = multisampled;
unshaded.perSampleShading = false;
expectFailure([&] { static_cast<void>(AgcDriver::Graphics::ExportTargetMerge(unshaded)); }, "needs per-sample shading");
constexpr std::array<std::uint32_t, 3> mergeCode{0xf800180fu, 0x07060504u, 0xbf810000u};
auto merging = packedExportRequest(changed, multisampled, mergeCode);
merging.context.pixel->targetMerge = merge;
const std::array<std::uint32_t, 5> capabilities{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityStorageImageReadWithoutFormat, spv::CapabilityStorageImageMultisample, spv::CapabilityInterpolationFunction};
merging.target.supportedCapabilities = capabilities;
merging.target.fragmentShaderBarycentricEnabled = true;
const auto words = ShaderRecompiler::Recompile(merging).spirv.Copy();
std::size_t sampled = 0;
bool sampleRate = false;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
const auto op = static_cast<spv::Op>(words[at] & 0xffffu);
if (op == spv::OpCapability && words[at + 1] == spv::CapabilitySampleRateShading) sampleRate = true;
if (op == spv::OpImageRead && (words[at] >> 16u) == 7u && words[at + 5] == spv::ImageOperandsSampleMask) ++sampled;
if (op == spv::OpImageWrite && (words[at] >> 16u) == 6u && words[at + 4] == spv::ImageOperandsSampleMask) ++sampled;
}
Require(sampleRate && sampled == 2u && interlockShape(words) == "capability extension ordered begin end ", "a 2-sample 4_4_4_4 merge module does not read and write the shaded sample inside a pixel interlock");
auto single = merging;
const std::array<std::uint32_t, 4> noMultisampleStorage{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityStorageImageReadWithoutFormat, spv::CapabilityInterpolationFunction};
single.target.supportedCapabilities = noMultisampleStorage;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(single)); }, "storage image multisampling");
auto centred = merging;
centred.target.fragmentShaderBarycentricEnabled = false;
centred.context.pixel->sampleLocations = AgcDriver::Graphics::PixelSampleLocations(multisampled, *centred.context.pixel);
Require(std::ranges::all_of(centred.context.pixel->sampleLocations, [](std::uint8_t location) { return location == 0u; }), "a merge raised from a one-sample rasterizer did not keep every sample at the pixel centre");
static_cast<void>(ShaderRecompiler::Recompile(centred));
const auto rasterizedState = AgcDriver::Graphics::DecodeState(rasterized);
auto offCentre = packedExportRequest(rasterized, rasterizedState, mergeCode);
offCentre.context.pixel->targetMerge = AgcDriver::Graphics::ExportTargetMerge(rasterizedState);
offCentre.context.pixel->iteratesSamples = rasterizedState.perSampleShading;
offCentre.context.pixel->sampleLocations = AgcDriver::Graphics::PixelSampleLocations(rasterizedState, *offCentre.context.pixel);
Require(std::ranges::any_of(offCentre.context.pixel->sampleLocations, [](std::uint8_t location) { return location != 0u; }), "a 2-sample rasterizer's merge did not pass its off-centre sample locations");
offCentre.target.supportedCapabilities = capabilities;
offCentre.target.fragmentShaderBarycentricEnabled = false;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(offCentre)); }, "its centre-located barycentrics would follow the sample");
offCentre.target.fragmentShaderBarycentricEnabled = true;
static_cast<void>(ShaderRecompiler::Recompile(offCentre));
alignas(16) static std::array<std::uint32_t, 4> counted{};
static const std::array<std::uint32_t, 4> counter{static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(counted.data())), static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(counted.data()) >> 32u) & 0xffffu, 16u, 0x31016facu};
constexpr std::array<std::uint32_t, 6> countingCode{0x7e000281u, 0xe0c80000u, 0x80000000u, 0xf800180fu, 0x07060504u, 0xbf810000u};
auto counting = packedExportRequest(changed, multisampled, countingCode);
counting.context.userData = counter;
counting.context.pixel->targetMerge = merge;
counting.context.pixel->iteratesSamples = true;
counting.context.pixel->earlyZ = true;
counting.target = merging.target;
counting.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version;
static const std::array<std::uint32_t, 6> atomicCapabilities{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityStorageImageReadWithoutFormat, spv::CapabilityStorageImageMultisample, spv::CapabilityInterpolationFunction, spv::CapabilityInt64};
counting.target.supportedCapabilities = atomicCapabilities;
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(counting)); }, "each covered sample would repeat its writes");
auto unmergedCounting = counting;
unmergedCounting.context.pixel->targetMerge = {};
static_cast<void>(ShaderRecompiler::Recompile(unmergedCounting));
}
for (const auto& [number, swap] : {std::pair{1u, 2u}, std::pair{4u, 2u}, std::pair{0u, 0u}, std::pair{0u, 1u}, std::pair{0u, 3u}}) {
changed = queue;
changed.context[0x31c] = (changed.context[0x31c] & ~((7u << 8u) | (3u << 11u))) | (number << 8u) | (swap << 11u);
expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "unsupported color format 19 number type " + std::to_string(number) + " component swap " + std::to_string(swap));
}
changed = queue;
changed.context[0x31c] |= 0x10000000u;
changed.context[0x325] = 0x1234;
expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "DCC-compressed 4_4_4_4 unorm color targets");
changed = queue;
changed.context[0x31c] |= 0x40000u;
expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "truncating (ROUND_MODE) 4_4_4_4 unorm color targets");
constexpr std::array<std::uint32_t, 3> code{0xf800180fu, 0x07060504u, 0xbf810000u};
const auto request = packedExportRequest(queue, state, code);
auto plain = request;
plain.context.pixel->targetExportPacking.fill(ColorExportPacking::None);
const ShaderRecompiler::RequestSerializer serializer;
Require(serializer.Deserialize(serializer.Serialize(request)).request.context.pixel->targetExportPacking == request.context.pixel->targetExportPacking, "the 4_4_4_4 unorm packing was lost in serialization");
const auto packed = ShaderRecompiler::Recompile(request);
const auto unpacked = ShaderRecompiler::Recompile(plain);
Require(packed.PipelineVariantId() != unpacked.PipelineVariantId() && !(packed.spirv == unpacked.spirv), "the 4_4_4_4 export packing did not specialize the module");
ShaderRecompiler::RecompileResult vertex;
vertex.spirv = makeModule({});
const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES};
const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &packed, 0}}};
Require(AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false) == std::set<std::uint32_t>{0u}, "the 4_4_4_4 packed export did not reach attachment 0");
}
void float11_11_10TargetTests() {
using ShaderRecompiler::ColorExportPacking;
auto queue = makeState();
@@ -9809,7 +10275,7 @@ void float11_11_10TargetTests() {
const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &packed, 0}}};
Require(AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false) == std::set<std::uint32_t>{0u}, "the packed float export did not reach attachment 0");
auto invalid = request;
invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(3);
invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(4);
expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(invalid)); }, "invalid prepared fragment export packing");
}
@@ -11038,6 +11504,15 @@ void pipelineJournalTests(bool longPath = false, bool extended = false) {
BuildPipelineStoreKey(mergedKey, context, mergedState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL);
BuildPipelineStoreKey(mergedReplayed, context, mergedDecoded.state, mergedDecoded.vertexInput, layouts.View(), shaders, mergedDecoded.attachmentLayout);
Require(mergedKey != fourSamples && mergedKey == mergedReplayed, "the decoded merged coverage state keys another pipeline");
auto partialState = state;
partialState.colors[0].partialWriteMask = 0xf000u;
GraphicsPipelineRecipe partialDecoded;
Require(DecodeGraphicsPipelineRecipe(EncodeGraphicsPipelineRecipe(MakeGraphicsPipelineRecipe(partialState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL)), partialDecoded), "the partial write recipe does not decode");
Require(partialDecoded.state.colors.size() == partialState.colors.size() && partialDecoded.state.colors[0].partialWriteMask == 0xf000u, "the recipe dropped the merged target's partial write mask");
std::vector<std::byte> partialKey, partialReplayed;
BuildPipelineStoreKey(partialKey, context, partialState, input, layouts.View(), shaders, VK_IMAGE_LAYOUT_GENERAL);
BuildPipelineStoreKey(partialReplayed, context, partialDecoded.state, partialDecoded.vertexInput, layouts.View(), shaders, partialDecoded.attachmentLayout);
Require(partialKey != original && partialKey == partialReplayed, "the decoded partial write state keys another pipeline");
auto movedState = state;
movedState.viewport.x = 5;
movedState.colors[0].address = 0x9000;
@@ -11397,6 +11872,12 @@ int main(int argc, char** argv) {
std::cerr << "descriptor pool reset failure did not terminate\n";
return 1;
}
if (argc == 2 && std::string_view(argv[1]) == "--unobserved-provoking-vertex") {
UnobservedProvokingVertexTests();
ProvokingVertexTests();
std::cout << "Unobserved provoking vertex checks passed\n";
return 0;
}
if (argc == 2 && std::string_view(argv[1]) == "--covered-centroid-state") {
multisampleTests();
std::cout << "Covered-centroid and multisample state checks passed\n";
@@ -11465,6 +11946,7 @@ int main(int argc, char** argv) {
ZExportTests();
unorm10_11_11TargetTests();
float11_11_10TargetTests();
unorm4_4_4_4TargetTests();
DepthBoundsBiasTests();
conservativeZExportTests();
earlyFragmentTestsTests();
@@ -11599,6 +12081,7 @@ int main(int argc, char** argv) {
RunGuestTextureResourceTests();
RunDepthSurfaceReuseTests();
RunGuestSamplerResourceTests();
RunLastRecordIndicesTests();
mock = MockVulkan{};
auto textureDetilerContext = mockContext();
textureDetilerContext.limits.minStorageBufferOffsetAlignment = 16;
core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp
+1
View File
@@ -36,6 +36,7 @@ void RunTextureTilingTests();
void RunGuestTextureResourceTests();
void RunDepthSurfaceReuseTests();
void RunGuestSamplerResourceTests();
void RunLastRecordIndicesTests();
void RunTextureDetilerTests(const AgcDriver::Graphics::Context& context, const TextureDetilerTestAccess& access);
#endif
core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp
+23 -2
View File
@@ -184,8 +184,29 @@ void RunGuestTextureResourceTests() {
fmaskArray.baseArray = 1u;
const auto fmaskArrayResource = DecodeTextureResource(pack(fmaskArray));
Require(fmaskArrayResource.format == 20u && fmaskArrayResource.baseArray == 1u && fmaskArrayResource.depthOrLastArray == 2u, "FMASK normalization changed the array view");
for (std::uint32_t format = 156u; format <= 168u; ++format) {
if (format == 166u) continue;
constexpr std::array<std::pair<std::uint32_t, std::uint32_t>, 8> narrowFormats{{{156u, 5u}, {157u, 5u}, {158u, 5u}, {159u, 5u}, {160u, 5u}, {161u, 5u}, {163u, 11u}, {165u, 20u}}};
for (const auto& [format, surface] : narrowFormats) {
auto narrow = fmask;
narrow.format = format;
const auto narrowResource = DecodeTextureResource(pack(narrow));
Require(narrowResource.format == surface && narrowResource.tileMode == TextureTileMode::kZ64KBX, "FMASK " + std::to_string(format) + " did not use an integer Z surface of its storage width");
narrow.dstSelX = 4u;
narrow.dstSelY = 5u;
narrow.dstSelZ = 6u;
narrow.dstSelW = 7u;
Require(DecodeTextureResource(pack(narrow)).format == surface, "an expanded FMASK " + std::to_string(format) + " view with identity selectors was refused");
auto compressedNarrow = narrow;
compressedNarrow.metaCompress = compressedNarrow.metaPipeAligned = true;
compressedNarrow.metaAddr = 0x123456789aull;
rejectFields(compressedNarrow, "FMASK CMASK compression is implemented only for 8-sample, 8-fragment FMASK");
}
auto identityWord = fmask;
identityWord.dstSelX = 4u;
identityWord.dstSelY = 5u;
identityWord.dstSelZ = 6u;
identityWord.dstSelW = 7u;
rejectFields(identityWord, "FMASK destination channel selector is not implemented");
for (const auto format : {162u, 164u, 167u, 168u}) {
auto unsupported = fmask;
unsupported.format = format;
rejectFields(unsupported, "FMASK format is not implemented");
core/libs/prx/libSceAgcDriver/tests/LastRecordIndices.cpp
+64
View File
@@ -0,0 +1,64 @@
#include "GraphicsTests.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
#include <array>
#include <cstddef>
#include <random>
#include <string>
#include <vector>
namespace {
using namespace AgcDriver::Graphics;
std::vector<std::size_t> BackwardScan(const std::vector<const void*>& logged, const std::vector<const void*>& objects) {
std::vector<std::size_t> indices(objects.size(), logged.size());
for (std::size_t i = 0; i < objects.size(); ++i) {
for (auto at = logged.size(); at-- > 0;) {
if (logged[at] == objects[i]) {
indices[i] = at;
break;
}
}
}
return indices;
}
void CheckIndices(const std::vector<const void*>& logged, const std::vector<const void*>& objects, const std::string& what) {
std::vector<std::size_t> indices;
LastRecordIndices(logged, objects, indices);
Require(indices == BackwardScan(logged, objects), "last record indices, " + what + ": an object did not get the index of its last record");
}
}
void RunLastRecordIndicesTests() {
std::array<int, 4096> pool{};
const auto object = [&pool](std::size_t index) -> const void* { return &pool[index]; };
CheckIndices({object(0), object(1), object(0), object(2)}, {object(0), object(1), object(2), object(3), object(0), nullptr}, "repeated records and objects");
CheckIndices({}, {object(0), nullptr}, "an empty log");
CheckIndices({object(0), nullptr, object(1)}, {}, "no objects");
CheckIndices({nullptr, object(0)}, {nullptr, object(0)}, "a null object");
std::vector<const void*> table;
for (std::size_t texture = 0; texture < 1500; ++texture) table.push_back(object(texture));
CheckIndices(table, table, "a table of 1500 textures over its own records in table order");
std::vector<const void*> repeated = table;
repeated.insert(repeated.end(), table.begin(), table.begin() + 548);
CheckIndices(repeated, table, "a table of 1500 textures whose first 548 were looked up twice");
std::vector<const void*> missing(table.begin() + 476, table.end());
CheckIndices(missing, table, "a table of 1500 textures over a log without its first 476 records");
std::mt19937 generator(0x5eed);
for (const std::size_t records : {std::size_t{1}, std::size_t{7}, std::size_t{64}, std::size_t{1023}, std::size_t{1024}, std::size_t{1500}, std::size_t{2048}}) {
for (const std::size_t textures : {std::size_t{1}, std::size_t{5}, std::size_t{300}, std::size_t{1500}}) {
for (const std::size_t distinct : {std::size_t{8}, pool.size()}) {
std::vector<const void*> logged(records);
for (auto& entry : logged) entry = object(generator() % distinct);
std::vector<const void*> objects(textures);
for (auto& entry : objects) entry = object(generator() % distinct);
CheckIndices(logged, objects, std::to_string(records) + " records, " + std::to_string(textures) + " objects of " + std::to_string(distinct));
}
}
}
}
core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp
+32
View File
@@ -1066,6 +1066,38 @@ int main() {
return StencilReplaySafe(b.program);
});
passed &= run("read-only BDA reads can be replayed and BDA writes cannot", [] {
Builder b;
b.program.Info().usesDma = true;
if (!StencilReplaySafe(b.program)) return false;
b.program.Info().bdaWrites = true;
return !StencilReplaySafe(b.program);
});
const auto execCopy = [](Builder& b, IrValue& low, IrValue& high) -> IrValue& {
auto& active = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
auto& lanes = b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(1u)})});
auto& wqm = b.emit(IrOpcode::WqmU64, IrType::U64, {&lanes});
return b.threadBit(b.wordAnd(low, b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(0u)})), b.wordAnd(high, b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(1u)})));
};
passed &= run("an exec copy masked by the whole-quad active lanes is a stable export guard", [&] {
Builder b;
b.program.Resources().stage = IrShaderStage::Pixel;
const auto [low, high] = b.ballotWords(pixelPredicate(b));
exportUnder(b, execCopy(b, *low, *high), true, true);
return StencilReplaySafe(b.program);
});
passed &= run("an exec copy does not hide helper participation in a whole-quad expansion", [&] {
Builder b;
b.program.Resources().stage = IrShaderStage::Pixel;
const auto [low, high] = b.ballotWords(helperPredicate(b, true));
auto& wqm = b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {low, high})});
stencilByMask(b, execCopy(b, b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqm, &b.constant(1u)})));
return !StencilReplaySafe(b.program);
});
passed &= run("only guarded read-only SRT diagnostics can be replayed", [] {
Builder b;
b.program.Info().usesFaultBuffer = true;
core/libs/prx/libSceAgcDriver/tests/PageStates.cpp
+145
View File
@@ -0,0 +1,145 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "prx/libc/include/GuestArena.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <cstdio>
#include <cstring>
#include <stdexcept>
#include <string_view>
#include <thread>
#include <vector>
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <psapi.h>
namespace {
using namespace AgcDriver::GuestMemory;
constexpr std::size_t Unit = 65536;
constexpr std::size_t ChunkGuestBytes = 256ull << 20u;
void Require(bool condition, const char* message) {
if (!condition) throw std::runtime_error(message);
}
class Range {
public:
explicit Range(std::size_t bytes) : bytes(bytes) {
data = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, ChunkGuestBytes));
Require(data != nullptr, "cannot reserve page-state fixture");
}
~Range() {
{
GuestAllocations::Mutation mutation;
for (const auto offset : committed) mutation.Remove(data + offset);
}
GuestArena::GuestArenaReset_nid_postfix(data, bytes);
GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
}
std::byte* Commit(std::size_t offset) {
auto* at = data + offset;
GuestArena::GuestArenaCommit_nid_postfix(at, Unit, PAGE_READWRITE, Unit);
std::memset(at, 0x5b, Unit);
GuestAllocations::Mutation mutation;
mutation.Add(at, Unit, true, true, true);
committed.push_back(offset);
return at;
}
std::byte* Data() const { return data; }
private:
std::byte* data = nullptr;
std::size_t bytes;
std::vector<std::size_t> committed;
};
std::size_t PrivateBytes() {
PROCESS_MEMORY_COUNTERS_EX counters{};
Require(K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters)), "cannot query page-state fixture private commit");
return counters.PrivateUsage;
}
void CheckMetadata() {
Range range(3 * ChunkGuestBytes);
auto* data = range.Commit(0);
const auto before = PrivateBytes();
Require(Accessible(data, Unit, true), "committed fixture is inaccessible");
const auto after = PrivateBytes();
const auto added = after > before ? after - before : 0;
std::printf("page-permission initialization: %zu bytes of private commit\n", added);
Require(added < (16ull << 20u), "page permissions committed a dense arena table");
const auto queried = PrivateBytes();
Require(!Accessible(range.Data() + 2 * ChunkGuestBytes, Unit), "reserved fixture tail is accessible");
Require(PrivateBytes() < queried + (4ull << 20u), "querying reserved pages allocated untouched metadata chunks");
}
void CheckBoundaries() {
constexpr auto span = 3 * ChunkGuestBytes;
Range range(span);
const std::array<std::size_t, 4> offsets{0, ChunkGuestBytes - Unit, ChunkGuestBytes, 2 * ChunkGuestBytes + Unit};
for (const auto offset : offsets) range.Commit(offset);
auto* block = range.Data();
std::atomic<bool> begin{false};
std::atomic<unsigned> errors{0};
std::vector<std::thread> readers;
for (unsigned reader = 0; reader < 8; ++reader) {
readers.emplace_back([&] {
while (!begin.load(std::memory_order_acquire)) std::this_thread::yield();
for (unsigned round = 0; round < 500; ++round) {
if (!Accessible(block + offsets.back(), Unit, true)) errors.fetch_add(1, std::memory_order_relaxed);
}
});
}
begin.store(true, std::memory_order_release);
for (auto& reader : readers) reader.join();
Require(errors.load() == 0, "concurrent page-state publication changed access");
for (const auto offset : offsets) {
std::array<std::byte, 16> copy{};
Read(reinterpret_cast<std::uint64_t>(block + offset), copy);
Require(std::all_of(copy.begin(), copy.end(), [](std::byte value) { return value == std::byte{0x5b}; }), "page cache changed guest bytes");
}
Require(Accessible(block + ChunkGuestBytes - Unit, 2 * Unit, true), "adjacent chunks lost their boundary pages");
const auto described = DescribeCommitted(reinterpret_cast<std::uint64_t>(block), span, true);
std::uint64_t total = 0;
for (const auto& [first, last] : described.ranges) total += last - first;
Require(!described.whole && total == offsets.size() * Unit, "reserved holes were reported committed");
auto* changed = block + offsets[1];
const auto protect = [&](bool writable) {
GuestAllocations::Mutation mutation;
mutation.Protect(changed, Unit, true, writable, true, [&] {
const DWORD protection = writable ? PAGE_READWRITE : PAGE_READONLY;
DWORD previous = 0;
Require(VirtualProtect(changed, Unit, protection, &previous) != 0, "cannot protect page-state fixture");
GuestArena::GuestArenaSetProtection_nid_postfix(reinterpret_cast<std::uintptr_t>(changed), Unit, protection);
});
};
protect(false);
Require(Accessible(changed, Unit) && !Accessible(changed, Unit, true), "protection invalidation retained writable state");
protect(true);
Require(Accessible(changed, Unit, true), "restored writable state was not discovered");
GuestArena::GuestArenaReset_nid_postfix(block, Unit);
GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast<std::uintptr_t>(block), Unit);
Require(!Accessible(block, Unit), "decommitted pages remained readable");
GuestArena::GuestArenaCommit_nid_postfix(block, Unit, PAGE_READWRITE, Unit);
Require(Accessible(block, Unit, true), "access observer did not invalidate inaccessible state after commit");
std::puts("page-state boundaries, publication, holes and invalidation passed");
}
}
int main(int argc, char** argv) {
try {
Require(GuestArena::GuestArenaAvailable_nid_postfix(), "page-state fixture needs the guest arena");
if (argc == 2 && std::string_view(argv[1]) == "--metadata") CheckMetadata();
else CheckBoundaries();
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "%s\n", error.what());
return 1;
}
}
core/libs/prx/libSceAgcDriver/tests/Pm4Replay.cpp
+5 -1
View File
@@ -693,8 +693,12 @@ void testCorrupt(const std::filesystem::path& directory) {
int main() {
std::filesystem::path scratch;
try {
#ifdef _WIN32
const wchar_t* captured = _wgetenv(L"APS5_DUMP_SUBMITS");
#else
const char* captured = std::getenv("APS5_DUMP_SUBMITS");
check(captured != nullptr && *captured != '\0', "set APS5_DUMP_SUBMITS to the capture directory of this test");
#endif
check(captured != nullptr && *captured != 0, "set APS5_DUMP_SUBMITS to the capture directory of this test");
const std::filesystem::path directory = captured;
std::filesystem::remove_all(directory);
scratch = std::filesystem::temp_directory_path() / ("anyps5-pm4-replay-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
core/libs/prx/libSceAgcDriver/tests/Recorder.cpp
+401 -43
View File
@@ -1,3 +1,4 @@
#include "prx/libSceAgc/Command/include/Packet.hpp"
#include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp"
#include "Triangle_frag_spv.h"
#include "Triangle_vert_spv.h"
@@ -30,6 +31,7 @@
#include "ResidentPresent.hpp"
#include "ImageTableAbi.hpp"
#include "HostImportLifetime.hpp"
#include "ShaderCacheDirectory.hpp"
#include "SampleLod_spv.h"
#include "BufferUpdate_spv.h"
#include "SampleArray_spv.h"
@@ -1222,8 +1224,10 @@ void completionCountTests(const Device& device, Recorder& recorder) {
recorder.AfterCompletions(base, value, 3, 0, true);
recorder.AfterCompletions(base + 8, value, 4, 0, true);
Require(Recorder::PendingCompletionLabels() == registered, "a GPU-stored completion label counts before any write-back");
Require(Recorder::PendingCompletionStores() == registered, "a GPU-stored completion label counts as a completion store before any write-back");
recorder.AfterCompletions(base + 0x40, value, 5, 0, false);
Require(Recorder::PendingCompletionLabels() == registered + 1, "a label the GPU has no view of is not pending");
Require(Recorder::PendingCompletionStores() == registered + 1, "a label the GPU has no view of is not a pending completion store");
// A late lookup (stamp <= afterStamp) of a label whose value reaches memory only through the
// batch's completion action is refused as BehindCompletion (a poller reaps for it); a GPU-stored
// one keeps the ordinary reason (unclosed group, or trust off) until a write-back overlaps it.
@@ -1239,16 +1243,20 @@ void completionCountTests(const Device& device, Recorder& recorder) {
Recorder::NoteWrittenBack(base + 2, 4);
const std::uint64_t counted = countAll ? 3 : 2;
Require(Recorder::PendingCompletionLabels() == counted, "a write-back over a GPU-stored label did not count it exactly once");
Require(Recorder::PendingCompletionStores() == counted, "a write-back over a GPU-stored label did not make it a completion store exactly once");
int delivered = 0;
Require(recorder.AfterRecordedWork([&] { ++delivered; }), "an action after recorded work was refused");
Require(Recorder::PendingCompletionLabels() == counted + 1 && Recorder::PendingCompletionStores() == counted, "an action after recorded work counted as a completion store");
Require(!Recorder::LookupLabel(base, 4, 3, &refusal).has_value() && refusal == Refusal::BehindCompletion, "a written-back GPU-stored label is not refused as behind a completion");
Require(!Recorder::LookupLabel(base + 8, 4, 4, &refusal).has_value() && refusal == (countAll ? Refusal::BehindCompletion : ordinary), "an untouched GPU-stored label became behind a completion");
recorder.Submit();
device.WaitQueue();
Require(Recorder::PendingCompletionLabels() == counted && Recorder::PendingWriteBackCompletions() == 1, "counts dropped before the batch finished");
Require(Recorder::PendingCompletionLabels() == counted + 1 && Recorder::PendingCompletionStores() == counted && Recorder::PendingWriteBackCompletions() == 1, "counts dropped before the batch finished");
memory[0] = 7;
memory[2] = 7;
memory[16] = 7;
recorder.Sync();
Require(ran == 1 && Recorder::PendingCompletionLabels() == 0 && Recorder::PendingWriteBackCompletions() == 0 && recorder.Idle(), "counts did not return to 0 at finish");
Require(ran == 1 && delivered == 1 && Recorder::PendingCompletionLabels() == 0 && Recorder::PendingCompletionStores() == 0 && Recorder::PendingWriteBackCompletions() == 0 && recorder.Idle(), "counts did not return to 0 at finish");
const bool storeAlways = std::getenv("APS5_LABEL_STORE_ALWAYS") != nullptr;
Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)");
}
@@ -1839,6 +1847,16 @@ void storeRunTests(const Device& device, Recorder& recorder) {
Require(words[0] == 1 && words[1] == 2 && words[2] == 5 && words[4] == 4 && words[3] == 0, "the stores did not land (or the later store of a dword lost)");
const auto after = Recorder::StoreCounts();
Require(after.stores == before.stores + 1 && after.replaced == before.replaced + 0 && after.joined == before.joined + 0, "store counters are off");
store(8, 1);
recorder.NotePendingWrite(address + 32, 4);
auto* commandWords = static_cast<std::uint32_t*>(block);
CommandBuffer disjoint{commandWords + 12, commandWords + 16, commandWords + 12, nullptr, nullptr, nullptr, 0};
Agc::Command::Emit(&disjoint, 0x7a, {0x20000243, 0x400}, __func__);
Require(recorder.Recording(), "a disjoint command allocation drained the GPU store");
CommandBuffer reused{commandWords + 6, commandWords + 10, commandWords + 6, nullptr, nullptr, nullptr, 0};
Agc::Command::Emit(&reused, 0x7a, {0x20000243, 0x400}, __func__);
recorder.Sync();
Require(words[8] == 0x400, "a pending GPU label overwrote a reused command word");
// The covered-access mask: reported once by the next Commands() call, then cleared.
VkAccessFlags covered = 0xffffffffu;
recorder.Commands(&covered);
@@ -2028,7 +2046,10 @@ bool settleFirstUseRaced(const Context& context, int copies) {
const auto* import = HostImportFor(context, address, unit);
Require(import != nullptr && import->unwatched && !Watched(address, unit), "(f) a 64 KiB import was not compared at first");
const auto commands = recorder.Commands();
for (int copy = 0; copy < copies; ++copy) CopyBuffer(context, commands, import->buffer, 0, scratch.Handle(), 0, unit);
for (int copy = 0; copy < copies; ++copy) {
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
CopyBuffer(context, commands, import->buffer, 0, scratch.Handle(), 0, unit);
}
recorder.Submit();
const auto submittedAt = std::chrono::steady_clock::now();
const auto deadline = submittedAt + std::chrono::seconds(5);
@@ -2067,6 +2088,177 @@ bool settleFirstUseRaced(const Context& context, int copies) {
return racing;
}
bool settleWriteOrderTests(const Device& device) {
using namespace AgcDriver::GuestMemory;
const auto& context = device.GetContext();
constexpr std::size_t unit = 65536;
if (context.hostImportAlignment == 0 || context.hostImportAlignment > unit || context.settleQueue == VK_NULL_HANDLE || !WriteWatched()) {
std::cout << "host imports within 64 KiB, a settle queue or write watching unavailable: settle write order not tested\n";
return false;
}
std::unique_lock gpu(GpuMutex());
PrepareImportWatch(context);
if (!settledImportsWatchPrivate(context)) {
std::cout << "this device compares every Windows import: settle write order not tested\n";
return false;
}
void* block = AllocateWatched(unit, unit);
struct Release {
void* block;
~Release() { ReleaseWatched(block, unit); }
} release{block};
const auto address = reinterpret_cast<std::uint64_t>(block);
{
GuestAllocations::Mutation mutation;
mutation.Add(block, unit, true, true, true);
}
Recorder recorder(context);
recorder.Activate();
const auto* import = HostImportFor(context, address, unit);
Require(import != nullptr && import->unwatched, "the write-order import was not initially compared");
const auto fill = [&](std::uint32_t value) {
const auto commands = recorder.Commands();
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
context.Function<PFN_vkCmdFillBuffer>("vkCmdFillBuffer")(commands, import->buffer, HostImportOffset(*import, address), unit, value);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
recorder.NotePendingWrite(address, unit);
recorder.Submit();
};
const auto before = HostImportSettles().submitted;
fill(0x22222222u);
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
while (HostImportSettles().submitted == before && std::chrono::steady_clock::now() < deadline) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
Require(HostImportFor(context, address, unit) == import, "the write-order import changed while settling");
}
Require(HostImportSettles().submitted > before, "the write-order import did not submit a settle");
fill(0x33333333u);
recorder.Sync();
while (!Watched(address, unit) && std::chrono::steady_clock::now() < deadline) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
HostImportFor(context, address, unit);
}
Require(Watched(address, unit), "the write-order import did not return to the write watch");
const auto* bytes = static_cast<const std::uint8_t*>(block);
Require(std::all_of(bytes, bytes + unit, [](std::uint8_t value) { return value == 0x33; }), "the GPU store after the settle did not preserve its bytes");
{
GuestAllocations::Mutation mutation;
mutation.Remove(block);
}
HostImportFor(context, address, unit);
std::cout << "settle read between main-queue GPU writes passed\n";
return true;
}
struct SettleWaitFailure {
PFN_vkGetDeviceProcAddr resolve;
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
VkResult result = VK_SUCCESS;
std::uint64_t timeout = 0;
unsigned waits = 0;
unsigned destroys = 0;
unsigned frees = 0;
};
SettleWaitFailure* settleWaitFailure = nullptr;
VKAPI_ATTR VkResult VKAPI_CALL settleWaitForFences(VkDevice device, std::uint32_t count, const VkFence* fences, VkBool32 all, std::uint64_t timeout) {
auto& failure = *settleWaitFailure;
if (failure.result != VK_SUCCESS) {
failure.timeout = timeout;
++failure.waits;
return failure.result;
}
return reinterpret_cast<PFN_vkWaitForFences>(failure.resolve(device, "vkWaitForFences"))(device, count, fences, all, timeout);
}
VKAPI_ATTR void VKAPI_CALL settleDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) {
auto& failure = *settleWaitFailure;
if (buffer == failure.buffer) ++failure.destroys;
reinterpret_cast<PFN_vkDestroyBuffer>(failure.resolve(device, "vkDestroyBuffer"))(device, buffer, allocator);
}
VKAPI_ATTR void VKAPI_CALL settleFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) {
auto& failure = *settleWaitFailure;
if (memory == failure.memory) ++failure.frees;
reinterpret_cast<PFN_vkFreeMemory>(failure.resolve(device, "vkFreeMemory"))(device, memory, allocator);
}
PFN_vkVoidFunction VKAPI_CALL settleWaitDeviceProc(VkDevice device, const char* name) {
if (std::strcmp(name, "vkWaitForFences") == 0) return reinterpret_cast<PFN_vkVoidFunction>(settleWaitForFences);
if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast<PFN_vkVoidFunction>(settleDestroyBuffer);
if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast<PFN_vkVoidFunction>(settleFreeMemory);
return settleWaitFailure->resolve(device, name);
}
bool settleWaitFailureTests(const Device& device) {
using namespace AgcDriver::GuestMemory;
auto context = device.GetContext();
constexpr std::size_t unit = 65536;
if (context.hostImportAlignment == 0 || context.hostImportAlignment > unit || context.settleQueue == VK_NULL_HANDLE || !WriteWatched()) return false;
std::lock_guard gpu(GpuMutex());
PrepareImportWatch(context);
if (!settledImportsWatchPrivate(context)) return false;
ClearHostImports(context.device);
SettleWaitFailure failure{context.deviceProc};
settleWaitFailure = &failure;
context.deviceProc = settleWaitDeviceProc;
context.functions = nullptr;
void* block = AllocateWatched(unit, unit);
Require(block != nullptr, "no watched block for settle wait failure");
{
GuestAllocations::Mutation mutation;
mutation.Add(block, unit, true, true, true);
}
struct Release {
Context& context;
SettleWaitFailure& failure;
void* block;
~Release() {
failure.result = VK_SUCCESS;
ClearHostImports(context.device);
{
GuestAllocations::Mutation mutation;
mutation.Remove(block);
}
ReleaseWatched(block, unit);
settleWaitFailure = nullptr;
}
} release{context, failure, block};
const auto address = reinterpret_cast<std::uint64_t>(block);
const auto* import = HostImportFor(context, address, unit);
Require(import != nullptr && import->unwatched, "wait failure import was not initially compared");
failure.buffer = import->buffer;
failure.memory = import->memory;
const auto before = HostImportSettles().submitted;
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
while (HostImportSettles().submitted == before && std::chrono::steady_clock::now() < deadline) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
HostImportFor(context, address, unit);
}
Require(HostImportSettles().submitted > before, "wait failure did not submit a settle");
Check(context.Function<PFN_vkDeviceWaitIdle>("vkDeviceWaitIdle")(context.device), "vkDeviceWaitIdle before settle failure");
for (const auto result : {VK_TIMEOUT, VK_ERROR_DEVICE_LOST}) {
failure.result = result;
const auto waits = failure.waits;
std::string error;
try {
ClearHostImports(context.device);
} catch (const std::runtime_error& caught) {
error = caught.what();
}
failure.result = VK_SUCCESS;
Require(failure.waits == waits + 1 && failure.timeout > 0 && failure.timeout <= 60'000'000'000ull, "settle retirement did not use a bounded fence wait");
Require(error == "AGC graphics: vkWaitForFences (settle batch): Vulkan result " + std::to_string(result), "settle retirement did not propagate its Vulkan failure");
Require(failure.destroys == 0 && failure.frees == 0 && HostImportCovers(context, address, unit), "failed settle wait released its host import");
}
ClearHostImports(context.device);
Require(failure.destroys == 1 && failure.frees == 1, "successful settle retirement did not release the retained import exactly once");
std::cout << "bounded settle wait preserves imports on timeout and device loss; successful retry releases once\n";
return true;
}
bool settleFirstUseTests(const Device& device) {
using namespace AgcDriver::GuestMemory;
const auto& context = device.GetContext();
@@ -2081,15 +2273,9 @@ bool settleFirstUseTests(const Device& device) {
std::cout << "this device compares every Windows import: the first use of a settled import not tested\n";
return false;
}
for (int copies = 4096; copies <= 65536; copies *= 4) {
if (settleFirstUseRaced(context, copies)) {
std::cout << "settled import first use tests passed\n";
return true;
}
std::cout << "(f) the main-queue copies had finished before the import returned to the write watch, so its read did not race them\n";
}
std::cout << "(f) the main-queue copies finished first in every round: the first use of a settled import not tested\n";
return false;
Require(!settleFirstUseRaced(context, 4096), "(f) the import returned to the write watch while its earlier main-queue reads were still in flight");
std::cout << "settled import first use ordering tests passed\n";
return true;
}
bool settleEarlierImportTests(const Device& device) {
@@ -2128,7 +2314,10 @@ bool settleEarlierImportTests(const Device& device) {
const auto* earlier = HostImportFor(context, address, 16);
Require(earlier != nullptr && earlier->base == address && earlier->bytes == unit && earlier->unwatched, "(e) the first window of the range was not imported compared");
const auto commands = recorder.Commands();
for (int copy = 0; copy < copies; ++copy) CopyBuffer(context, commands, earlier->buffer, 0, scratch.Handle(), 0, unit);
for (int copy = 0; copy < copies; ++copy) {
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
CopyBuffer(context, commands, earlier->buffer, 0, scratch.Handle(), 0, unit);
}
const auto* whole = HostImportFor(context, address + unit - 16, 32);
Require(whole != nullptr && whole->base == address && whole->bytes == 2 * unit && whole->unwatched, "(e) an access across the windows did not replace the first window's import");
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
@@ -5106,6 +5295,11 @@ bool addressShadowTests(const Device& device, Recorder& recorder) {
}
const auto decided = PrepareImportWatch(base);
#ifndef _WIN32
const auto probe = ProbeImportWriteProtection(base);
if (probe.failure != nullptr || probe.writtenAtImport != 0 || probe.writtenAfterSubmit != 0) {
std::cout << "skipped, the device reports watched host memory written when it imports or reads it\n";
return false;
}
SetImportWatch(base, ImportWatch::Watch);
#endif
struct Restore {
@@ -5241,6 +5435,11 @@ bool completionStoreShadowTests(const Device& device, Recorder& recorder) {
}
const auto decided = PrepareImportWatch(base);
#ifndef _WIN32
const auto probe = ProbeImportWriteProtection(base);
if (probe.failure != nullptr || probe.writtenAtImport != 0 || probe.writtenAfterSubmit != 0) {
std::cout << "skipped, the device reports watched host memory written when it imports or reads it\n";
return false;
}
SetImportWatch(base, ImportWatch::Watch);
#endif
struct Restore {
@@ -7403,35 +7602,7 @@ public:
const auto commands = start.commands;
if (!continued) {
recorder.PrepareSampleSlot();
if (context.graphicsPipelineLibrary) {
if (!transitioned) {
transitioned = true;
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier.image = target.Image();
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
}
VkRenderingAttachmentInfoKHR attachment{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR};
attachment.imageView = target.View();
attachment.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
VkRenderingInfoKHR rendering{VK_STRUCTURE_TYPE_RENDERING_INFO_KHR};
rendering.renderArea = {{0, 0}, {8, 8}};
rendering.layerCount = 1;
rendering.colorAttachmentCount = 1;
rendering.pColorAttachments = &attachment;
context.Resolved(&DeviceFunctions::cmdBeginRendering, "vkCmdBeginRenderingKHR")(commands, &rendering);
} else {
VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
begin.renderPass = renderPass;
begin.framebuffer = framebuffer;
begin.renderArea = {{0, 0}, {8, 8}};
context.Function<PFN_vkCmdBeginRenderPass>("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE);
}
begin(commands);
}
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
recorder.NoteSampledDraw(commands);
@@ -7440,10 +7611,48 @@ public:
return continued;
}
void Record(VkCommandBuffer commands) const {
begin(commands);
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
context.Function<PFN_vkCmdDraw>("vkCmdDraw")(commands, 3, 1, 0, 0);
EndRenderPass(context, commands);
}
static constexpr std::uint64_t Samples = 64;
private:
static constexpr std::uint64_t PassKey = 0xc0de;
void begin(VkCommandBuffer commands) const {
if (context.graphicsPipelineLibrary) {
if (!transitioned) {
transitioned = true;
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier.image = target.Image();
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier);
}
VkRenderingAttachmentInfoKHR attachment{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR};
attachment.imageView = target.View();
attachment.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
VkRenderingInfoKHR rendering{VK_STRUCTURE_TYPE_RENDERING_INFO_KHR};
rendering.renderArea = {{0, 0}, {8, 8}};
rendering.layerCount = 1;
rendering.colorAttachmentCount = 1;
rendering.pColorAttachments = &attachment;
context.Resolved(&DeviceFunctions::cmdBeginRendering, "vkCmdBeginRenderingKHR")(commands, &rendering);
} else {
VkRenderPassBeginInfo info{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO};
info.renderPass = renderPass;
info.framebuffer = framebuffer;
info.renderArea = {{0, 0}, {8, 8}};
context.Function<PFN_vkCmdBeginRenderPass>("vkCmdBeginRenderPass")(commands, &info, VK_SUBPASS_CONTENTS_INLINE);
}
}
void release() noexcept {
const auto device = context.device;
if (pipeline != VK_NULL_HANDLE) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(device, pipeline, nullptr);
@@ -7791,6 +8000,24 @@ void sampleDumpTests(const Device& device, Recorder& recorder) {
inPassSampleDumpTests(recorder, context, cover, words, address, target);
}
void uncountedPassTests(const Device& device) {
auto context = device.GetContext();
context.bufferDeviceAddress = false;
Recorder recorder(context);
CoverPass cover(context);
recorder.CountSamples();
Require(!recorder.QueuesSampleDumps(), "a recorder without buffer device addresses counted samples in segments");
const auto base = recorder.SamplesTotal();
cover.Draw(recorder);
recorder.RecordUncounted([&](VkCommandBuffer commands) { cover.Record(commands); });
cover.Draw(recorder);
recorder.Submit();
recorder.Sync();
const auto counted = recorder.SamplesTotal() - base;
Require(counted != 0, "the draws around an uncounted pass were not counted");
if (context.occlusionQueryPrecise) Require(counted == 2 * CoverPass::Samples, "the batch-wide occlusion query counted " + std::to_string(counted) + " samples, not the " + std::to_string(2 * CoverPass::Samples) + " of the two draws around an uncounted pass");
}
void cmaskPassTests(const Device& device, Recorder& recorder) {
namespace GuestMemory = AgcDriver::GuestMemory;
auto context = device.GetContext();
@@ -9525,6 +9752,109 @@ void pipelineCacheTests(const Device& device, bool benchmark) {
std::cout << "Pipeline cache key and retained lifetime tests passed\n";
}
struct PipelineCacheFileProbe {
Context& context;
PFN_vkGetDeviceProcAddr resolver;
std::atomic<std::uint32_t> creations{0};
static inline PipelineCacheFileProbe* active = nullptr;
explicit PipelineCacheFileProbe(Context& context) : context(context), resolver(context.deviceProc) {
Require(active == nullptr, "pipeline cache file probe is already active");
active = this;
context.deviceProc = resolve;
}
~PipelineCacheFileProbe() {
context.deviceProc = resolver;
active = nullptr;
}
static VKAPI_ATTR VkResult VKAPI_CALL createCache(VkDevice device, const VkPipelineCacheCreateInfo* info, const VkAllocationCallbacks* callbacks, VkPipelineCache* created) {
++active->creations;
return reinterpret_cast<PFN_vkCreatePipelineCache>(active->resolver(device, "vkCreatePipelineCache"))(device, info, callbacks, created);
}
static VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL resolve(VkDevice device, const char* name) {
if (std::strcmp(name, "vkCreatePipelineCache") == 0) return reinterpret_cast<PFN_vkVoidFunction>(createCache);
return active->resolver(device, name);
}
};
void pipelineCacheHeaderTests(const Device& device) {
const auto root = std::filesystem::temp_directory_path() / ("aps5-pipeline-header-test-" + std::to_string(std::random_device{}()));
Require(root.native().size() < 225, "the pipeline cache header test requires a temporary root shorter than 225 characters");
#ifdef _WIN32
std::error_code cleanupError;
const auto cleanupRoot = ShaderRecompiler::NativeCachePath_nid_no_patch(root, cleanupError);
Require(!cleanupError, "the pipeline cache header test could not resolve its cleanup directory");
#else
const auto& cleanupRoot = root;
#endif
struct RemoveDirectory {
std::filesystem::path directory;
~RemoveDirectory() {
std::error_code error;
std::filesystem::remove_all(directory, error);
}
} removeDirectory{cleanupRoot};
auto directory = root;
char version[17];
std::snprintf(version, sizeof(version), "%016llx", static_cast<unsigned long long>(ShaderRecompiler::ShaderDiskCache::Generated::SourceVersion));
while ((directory / version).native().size() < 220) {
const auto length = (directory / version).native().size();
const auto remaining = 245 > length ? 245 - length : 1;
const auto chunk = std::min<std::size_t>(remaining, 16);
directory /= std::string(chunk, 'd');
}
const auto versionDirectory = directory / version;
Require(versionDirectory.native().size() >= 220 && versionDirectory.native().size() <= 249, "the pipeline cache directory is not between 220 and 249 characters");
#ifdef _WIN32
_putenv_s("ANYPS5_NO_SHADER_CACHE", "0");
_putenv_s("ANYPS5_SHADER_CACHE_DIR", directory.string().c_str());
#else
setenv("ANYPS5_NO_SHADER_CACHE", "0", 1);
setenv("ANYPS5_SHADER_CACHE_DIR", directory.string().c_str(), 1);
#endif
auto context = device.GetContext();
PipelineCacheFileProbe probe(context);
PipelineCache cache(context, device.Properties());
cache.SaveNow();
std::vector<std::filesystem::path> files;
for (const auto& entry : std::filesystem::directory_iterator(versionDirectory)) files.push_back(entry.path());
Require(files.size() == 1 && files.front().filename().string().starts_with("vk-pipelines-"), "the long pipeline cache directory does not hold exactly one cache file");
Require(files.front().native().size() > 260, "the pipeline cache header test did not exceed MAX_PATH");
std::vector<std::byte> stored;
Require(ShaderRecompiler::ReadWholeFile(files.front(), stored) && !stored.empty(), "the long pipeline cache file was not persisted");
const auto creations = probe.creations.load();
for (int repeat = 0; repeat < 3; ++repeat) cache.SaveNow();
Require(probe.creations.load() == creations, "an unchanged long-path pipeline cache was reloaded for an idle save");
std::vector<std::byte> after;
Require(ShaderRecompiler::ReadWholeFile(files.front(), after) && after == stored, "an idle save changed the long pipeline cache file");
#ifdef _WIN32
std::error_code lockError;
const auto lockNative = ShaderRecompiler::NativeCachePath_nid_no_patch(files.front(), lockError);
Require(!lockError, "could not resolve long cache path");
struct FileLockGuard {
HANDLE handle = INVALID_HANDLE_VALUE;
~FileLockGuard() {
if (handle != INVALID_HANDLE_VALUE) CloseHandle(handle);
}
} lockGuard{CreateFileW(lockNative.c_str(), GENERIC_READ, 0, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr)};
Require(lockGuard.handle != INVALID_HANDLE_VALUE, "could not lock pipeline cache file");
std::string capturedStderr;
{
StderrCapture capture;
PipelineCache lockedCache(context, device.Properties());
lockedCache.SaveNow();
capturedStderr = capture.Finish();
}
Require(capturedStderr.find("cannot read") != std::string::npos, "an unreadable long pipeline cache file did not report the cannot read diagnostic");
if (lockGuard.handle != INVALID_HANDLE_VALUE) {
CloseHandle(lockGuard.handle);
lockGuard.handle = INVALID_HANDLE_VALUE;
}
std::vector<std::byte> retained;
Require(ShaderRecompiler::ReadWholeFile(files.front(), retained) && retained == stored, "an unreadable long pipeline cache file was overwritten");
#endif
std::cout << "long-path pipeline cache header reused without reloading\n";
}
int pipelineCacheFileTests(const Device& device) {
const auto& context = device.GetContext();
const auto directory = std::filesystem::temp_directory_path() / ("aps5-pipeline-cache-file-test-" + std::to_string(std::random_device{}()));
@@ -10484,7 +10814,7 @@ int main(int argc, char** argv) {
}
std::unique_ptr<Device> created;
try {
created = std::make_unique<Device>(argc == 2 && (std::string_view(argv[1]) == "--import-watch-other-family-only" || std::string_view(argv[1]) == "--deferred-settle-other-family-only" || std::string_view(argv[1]) == "--host-import-lifetime-other-family-only"));
created = std::make_unique<Device>(argc == 2 && (std::string_view(argv[1]) == "--import-watch-other-family-only" || std::string_view(argv[1]) == "--deferred-settle-other-family-only" || std::string_view(argv[1]) == "--host-import-lifetime-other-family-only" || std::string_view(argv[1]) == "--settle-write-order-other-family-only"));
} catch (const std::exception& error) {
if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw;
std::cout << "skipped, no usable Vulkan device: " << error.what() << '\n';
@@ -10523,6 +10853,14 @@ int main(int argc, char** argv) {
}
return deferredSettleTests(device) ? 0 : 77;
}
if (argc == 2 && (std::string_view(argv[1]) == "--settle-write-order-only" || std::string_view(argv[1]) == "--settle-write-order-other-family-only")) {
if (std::string_view(argv[1]) == "--settle-write-order-other-family-only" && device.GetContext().settleFamily == device.GetContext().queueFamily) {
std::cout << "skipped, no other queue family to settle imports on\n";
return 77;
}
return settleWriteOrderTests(device) ? 0 : 77;
}
if (argc == 2 && std::string_view(argv[1]) == "--settle-wait-failure-only") return settleWaitFailureTests(device) ? 0 : 77;
if (argc == 2 && std::string_view(argv[1]) == "--settle-first-use-only") return settleFirstUseTests(device) ? 0 : 77;
if (argc == 2 && std::string_view(argv[1]) == "--settle-earlier-import-only") return settleEarlierImportTests(device) ? 0 : 77;
if (argc == 2 && std::string_view(argv[1]) == "--settle-retire-only") return settleRetireTests(device) ? 0 : 77;
@@ -10566,6 +10904,11 @@ int main(int argc, char** argv) {
return pipelineCacheFileTests(device);
}
if (argc == 2 && std::string_view(argv[1]) == "--pipeline-cache-header-only") {
pipelineCacheHeaderTests(device);
return 0;
}
if (argc == 2 && std::string_view(argv[1]) == "--descriptor-pool-only") {
descriptorPoolTests(device);
return 0;
@@ -10586,6 +10929,12 @@ int main(int argc, char** argv) {
std::cout << "host imports " << (PrepareImportWatch(device.GetContext()) == ImportWatch::Unwatch ? "are compared" : "stay watched") << '\n';
Recorder recorder(device.GetContext());
recorder.Activate();
if (argc == 2 && std::string_view(argv[1]) == "--command-reuse-only") {
if (device.GetContext().hostImportAlignment == 0) return 77;
const auto before = Recorder::StoreCounts().stores;
storeRunTests(device, recorder);
return Recorder::StoreCounts().stores != before ? 0 : 77;
}
if (argc == 2 && std::string_view(argv[1]) == "--copied-direct-only") {
copiedDirectOrderingTests(device, recorder);
return 0;
@@ -10787,6 +11136,15 @@ int main(int argc, char** argv) {
std::cout << "Occlusion counter dump tests passed\n";
return 0;
}
if (argc == 2 && std::string_view(argv[1]) == "--uncounted-passes-only") {
if (!device.Graphics()) {
std::cout << "skipped, the test queue has no graphics\n";
return 77;
}
uncountedPassTests(device);
std::cout << "Uncounted pass tests passed\n";
return 0;
}
if (argc == 2 && std::string_view(argv[1]) == "--shadowed-key-stores-only") {
if (!shadowedKeyStoreTests(device, recorder)) {
std::cout << "skipped, the device imports no watched host memory under a unit shadow\n";
core/libs/prx/libSceAgcDriver/tests/RedundantValueElimination.cpp
+143
View File
@@ -324,6 +324,149 @@ int main() {
return b.eliminate() == 1u && !dominated.HasUses() && first.UseCount() == 2u && AllUsed({&thenOnly, &sibling, &merged});
});
passed &= run("GetSrtResource and ReadConst with the same operands merge in a dominated block, while another slot or a sibling block stays", [] {
Builder b;
IrBlock& head = *b.block;
IrBlock& then = b.newBlock();
IrBlock& otherwise = b.newBlock();
IrBlock& merge = b.newBlock();
head.AddBranch(&then);
head.AddBranch(&otherwise);
then.AddBranch(&merge);
otherwise.AddBranch(&merge);
auto& srtHead = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {&b.constant(0u)});
auto& readHead = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtHead, &b.constant(2u)});
b.keep(readHead);
b.block = &then;
auto& srtDominated = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {&b.constant(0u)});
auto& readDominated = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtDominated, &b.constant(2u)});
auto& readOtherSlot = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtDominated, &b.constant(3u)});
auto& srtThenOnly = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {&b.constant(1u)});
auto& readThenOnly = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtThenOnly, &b.constant(4u)});
b.keep(readDominated);
b.keep(readOtherSlot);
b.keep(readThenOnly);
b.block = &otherwise;
auto& srtSibling = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {&b.constant(1u)});
auto& readSibling = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtSibling, &b.constant(4u)});
b.keep(readSibling);
b.block = &merge;
auto& srtMerge = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {&b.constant(0u)});
auto& readMerge = b.emit(IrOpcode::ReadConst, IrType::U32, {&srtMerge, &b.constant(2u)});
b.keep(readMerge);
return b.eliminate() == 4u && !srtDominated.HasUses() && !readDominated.HasUses() && !srtMerge.HasUses() && !readMerge.HasUses() && srtHead.UseCount() == 4u && readHead.UseCount() == 3u && AllUsed({&readOtherSlot, &srtThenOnly, &readThenOnly, &srtSibling, &readSibling});
});
passed &= run("two ReadConst instructions of the same slot separated by a store to the table address stay", [] {
Builder b;
auto& tableAddress = b.emit(IrOpcode::GetAddressResource, IrType::AddressResource, {&b.constant(0x1000u), &b.constant(0u)});
auto& tableRead = b.emit(IrOpcode::LoadAddressU32, IrType::U32, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.truth()});
b.program.Resources().srtReads.push_back({&tableRead, 0u});
auto& srt = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {});
auto& first = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
static_cast<void>(b.emit(IrOpcode::StoreAddressU32, IrType::Void, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.constant(7u), &b.truth()}));
auto& second = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(first);
b.keep(second);
return b.eliminate() == 0u && AllUsed({&first, &second});
});
passed &= run("a store on one branch keeps ReadConst in the merge block while the clean sibling branch reuses the head read", [] {
Builder b;
IrBlock& head = *b.block;
IrBlock& then = b.newBlock();
IrBlock& otherwise = b.newBlock();
IrBlock& merge = b.newBlock();
head.AddBranch(&then);
head.AddBranch(&otherwise);
then.AddBranch(&merge);
otherwise.AddBranch(&merge);
auto& tableAddress = b.emit(IrOpcode::GetAddressResource, IrType::AddressResource, {&b.constant(0x1000u), &b.constant(0u)});
auto& tableRead = b.emit(IrOpcode::LoadAddressU32, IrType::U32, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.truth()});
b.program.Resources().srtReads.push_back({&tableRead, 0u});
auto& srt = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {});
auto& readHead = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(readHead);
b.block = &then;
static_cast<void>(b.emit(IrOpcode::StoreAddressU32, IrType::Void, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.constant(7u), &b.truth()}));
auto& readThenAfterStore = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(readThenAfterStore);
b.block = &otherwise;
auto& readOtherwise = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(readOtherwise);
b.block = &merge;
auto& readMerge = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(readMerge);
return b.eliminate() == 1u && !readOtherwise.HasUses() && readHead.UseCount() == 2u && AllUsed({&readHead, &readThenAfterStore, &readMerge});
});
passed &= run("a store on a loop backedge prevents reusing a pre-loop ReadConst in the loop header", [] {
Builder b;
IrBlock& header = b.newBlock();
IrBlock& body = b.newBlock();
IrBlock& exit = b.newBlock();
b.block->AddBranch(&header);
header.AddBranch(&body);
header.AddBranch(&exit);
body.AddBranch(&header);
auto& tableAddress = b.emit(IrOpcode::GetAddressResource, IrType::AddressResource, {&b.constant(0x1000u), &b.constant(0u)});
auto& tableRead = b.emit(IrOpcode::LoadAddressU32, IrType::U32, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.truth()});
b.program.Resources().srtReads.push_back({&tableRead, 0u});
auto& srt = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {});
auto& beforeLoop = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(beforeLoop);
b.block = &header;
auto& inHeader = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inHeader);
b.block = &body;
auto& inBodyBeforeStore = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
static_cast<void>(b.emit(IrOpcode::StoreAddressU32, IrType::Void, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.constant(7u), &b.truth()}));
auto& inBodyAfterStore = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inBodyBeforeStore);
b.keep(inBodyAfterStore);
b.block = &exit;
auto& afterLoop = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(afterLoop);
return b.eliminate() == 2u && !inBodyBeforeStore.HasUses() && !afterLoop.HasUses() && beforeLoop.UseCount() == 1u && inHeader.UseCount() == 3u && AllUsed({&beforeLoop, &inHeader, &inBodyAfterStore});
});
passed &= run("a store in an inner loop propagates to the outer loop header across fixed-point iterations", [] {
Builder b;
IrBlock& outerHeader = b.newBlock();
IrBlock& innerHeader = b.newBlock();
IrBlock& innerBody = b.newBlock();
IrBlock& outerLatch = b.newBlock();
IrBlock& exit = b.newBlock();
b.block->AddBranch(&outerHeader);
outerHeader.AddBranch(&innerHeader);
outerHeader.AddBranch(&exit);
innerHeader.AddBranch(&innerBody);
innerHeader.AddBranch(&outerLatch);
innerBody.AddBranch(&innerHeader);
outerLatch.AddBranch(&outerHeader);
auto& tableAddress = b.emit(IrOpcode::GetAddressResource, IrType::AddressResource, {&b.constant(0x1000u), &b.constant(0u)});
auto& tableRead = b.emit(IrOpcode::LoadAddressU32, IrType::U32, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.truth()});
b.program.Resources().srtReads.push_back({&tableRead, 0u});
auto& srt = b.emit(IrOpcode::GetSrtResource, IrType::SrtResource, {});
auto& beforeOuter = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(beforeOuter);
b.block = &outerHeader;
auto& inOuterHeader = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inOuterHeader);
b.block = &innerHeader;
auto& inInnerHeader = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inInnerHeader);
b.block = &innerBody;
static_cast<void>(b.emit(IrOpcode::StoreAddressU32, IrType::Void, {&tableAddress, &b.constant(8u), &b.constant(0u), &b.constant(7u), &b.truth()}));
b.block = &outerLatch;
auto& inOuterLatch = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inOuterLatch);
b.block = &exit;
auto& inExit = b.emit(IrOpcode::ReadConst, IrType::U32, {&srt, &b.constant(0u)});
b.keep(inExit);
return b.eliminate() == 2u && !inOuterLatch.HasUses() && !inExit.HasUses() && beforeOuter.UseCount() == 1u && inOuterHeader.UseCount() == 2u && inInnerHeader.UseCount() == 2u && AllUsed({&beforeOuter, &inOuterHeader, &inInnerHeader});
});
passed &= run("a loop body reuses a value of its header, the loop exit does not reuse a value of the body", [] {
Builder b;
IrBlock& header = b.newBlock();
core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp
+3
View File
@@ -148,6 +148,7 @@ void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShade
require(left.parameterExports == right.parameterExports, prefix + "parameter exports differ");
require(left.stencilReferences == right.stencilReferences, prefix + "stencil reference set differs");
require(left.stencilReplaySafe == right.stencilReplaySafe, prefix + "stencil replay proof differs");
require(left.bdaWrites == right.bdaWrites, prefix + "BDA write flag differs");
require(left.stencilReferenceDword == right.stencilReferenceDword, prefix + "stencil replay push offset differs");
require(left.fragmentParameters.size() == right.fragmentParameters.size(), prefix + "fragment parameter count differs");
for (std::size_t i = 0; i < left.fragmentParameters.size(); ++i) {
@@ -259,6 +260,7 @@ RecompileResult sampleResult() {
result.fragmentParameters = {{0, 1, true, false, true}, {2, 3, false, true}};
result.stencilReferences = {0u, 17u, 255u};
result.stencilReplaySafe = true;
result.bdaWrites = true;
result.stencilReferenceDword = 23u;
result.poisonedSrtReads = 3;
result.shaderDataUserWords = {2u, 0xffffffffu, 9u};
@@ -748,6 +750,7 @@ void verifyKeySensitivity() {
changes("spanned waves", [](SampleRequest& sample) { sample.request.target.spannedWaves = true; });
changes("robust buffer access 2", [](SampleRequest& sample) { sample.request.target.robustBufferAccess2 = true; });
changes("wave LDS memory barriers", [](SampleRequest& sample) { sample.request.target.waveLdsMemoryBarriers = true; });
changes("device-scope LDS memory barriers", [](SampleRequest& sample) { sample.request.target.deviceScopeLdsMemoryBarriers = true; });
changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; });
changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; });
changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; });
core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp
+644 -36
View File
@@ -170,7 +170,6 @@ void verifyFrontendIdentity() {
require((preparation.AcquireFrontend(partial) == first) == reuse, "frontend pair: partial-thread state changed the frontend identity");
auto target = request;
target.target.subgroupSize = 64u;
target.context.waveSize = 32u;
target.context.compute->numThreads = {32u, 2u, 1u};
require((preparation.AcquireFrontend(target) == first) == reuse, "frontend pair: translation-only state changed the frontend identity");
const std::array<std::uint32_t, 2> changedUserData{1u, 2u};
@@ -199,6 +198,10 @@ void verifyFrontendIdentity() {
changed.context.userDataBaseRegister = 4u;
verifyMiss(changed, "frontend pair: changed user-data base reused the frontend");
changed = request;
changed.context.waveSize = 32u;
changed.context.compute->numThreads = {32u, 2u, 1u};
verifyMiss(changed, "frontend pair: a different wave size reused the frontend");
changed = request;
changed.context.userData = std::span<const std::uint32_t>(userData).first(1u);
verifyMiss(changed, "frontend pair: changed user-data count reused the frontend");
changed = request;
@@ -224,6 +227,25 @@ void verifyFrontendIdentity() {
require((preparation.AcquireFrontend(request) == first) == reuse, "frontend pair: a failed miss replaced the prior frontend");
}
void verifyFrontendWaveSizeCalls() {
using namespace ShaderRecompiler;
const std::vector<std::uint32_t> code{
0xbb0e0005u,
0xd4c2000eu, 0x00020300u,
0xd5010002u, 0x003a0300u,
0xbf820001u,
0xbe80200eu,
0xbf810000u};
const std::array<std::uint32_t, 2> userData{};
for (const std::uint32_t waveSize : {32u, 64u}) {
auto request = FrontendRequest(code, userData);
request.context.waveSize = waveSize;
request.context.compute->numThreads = {waveSize, 1u, 1u};
ShaderPreparationContext preparation;
require(preparation.AcquireFrontend(request) != nullptr, "frontend wave size: a mask write that reuses the link pair and its v_cndmask read were refused");
}
}
void verifyFrontendProgram(const ShaderRecompiler::IrProgram& reused, const ShaderRecompiler::IrProgram& fresh) {
using namespace ShaderRecompiler;
require(ProgramToString(reused) == ProgramToString(fresh), "frontend pair: reused frontend changed the IR");
@@ -258,6 +280,7 @@ void verifyFrontendProgram(const ShaderRecompiler::IrProgram& reused, const Shad
void verifyFrontendPair() {
using namespace ShaderRecompiler;
verifyFrontendIdentity();
verifyFrontendWaveSizeCalls();
const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr;
const std::vector<std::uint32_t> code{0xe0700000u, 0x80000000u, 0xbf810000u};
const std::array<std::uint32_t, 4> buffer{0x10000000u, 0u, 0x40u, 0x00027facu};
@@ -320,7 +343,7 @@ void verifyFrontendPair() {
const auto freshChanged = PrepareResourceProgram(translated);
verifyFrontendProgram(changed, freshChanged);
require(changed.WaveSize() != freshFull.WaveSize(), "frontend pair: changed translation inputs were ignored");
require((preparation.AcquireFrontend(request) == retained) == reuse, "frontend pair: translating a variant replaced the frontend");
require(preparation.AcquireFrontend(request) != retained, "frontend pair: a variant with another wave size reused the frontend");
}
for (const bool partialFirst : {false, true}) {
@@ -544,6 +567,297 @@ void verifyRegisterSources() {
require(!EquivalentValue(plan, &samplerRegister, &firstVector), "different register types were merged");
}
void verifyPlanConditionPruning() {
using namespace ShaderRecompiler;
std::vector<std::unique_ptr<IrValue>> values;
std::uint32_t ids = 0;
const auto make = [&](IrOpcode opcode, IrType type) -> IrValue& {
values.push_back(std::make_unique<IrValue>(opcode, type, ids++));
return *values.back();
};
const auto constant = [&](std::uint32_t value) -> IrValue& {
auto& immediate = make(IrOpcode::Void, IrType::U32);
immediate.SetImmediateU32(value);
return immediate;
};
const auto userData = [&](std::uint32_t index) -> IrValue& {
auto& reg = make(IrOpcode::Void, IrType::ScalarReg);
reg.SetRegister({RegisterBank::Scalar, index});
auto& read = make(IrOpcode::GetUserData, IrType::U32);
read.AddArgument(&reg);
return read;
};
const auto equal = [&](IrValue& left, IrValue& right) -> IrValue& {
auto& compare = make(IrOpcode::IEqual32, IrType::Bool);
compare.AddArgument(&left);
compare.AddArgument(&right);
return compare;
};
constexpr std::uint32_t Additions = 64;
IrValue* address = &userData(0);
for (std::uint32_t index = 0; index < Additions; ++index) {
auto& sum = make(IrOpcode::IAdd32, IrType::U32);
sum.AddArgument(address);
sum.AddArgument(&constant(4));
address = &sum;
}
auto& load = make(IrOpcode::LoadBufferU32, IrType::U32);
load.AddArgument(address);
auto& pair = make(IrOpcode::CompositeConstructU32x2, IrType::U32x2);
pair.AddArgument(&userData(1));
pair.AddArgument(&load);
auto& low = make(IrOpcode::CompositeExtractU32x2, IrType::U32);
low.AddArgument(&pair);
low.AddArgument(&constant(0));
static std::uint32_t word = 5;
const auto wordAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(&word));
auto& handle = make(IrOpcode::CompositeConstructU64, IrType::U64);
handle.AddArgument(&userData(2));
handle.AddArgument(&userData(3));
auto& read = make(IrOpcode::LoadAddressU32, IrType::U32);
read.SetFlags(MemoryFlags{0, 0});
read.AddArgument(&handle);
read.AddArgument(&constant(0));
IrProgram program;
auto& resources = program.Resources();
resources.srtPlanComplete = true;
resources.resourceTrackingComplete = true;
MemoryInfo memory;
memory.kind = ResourceKind::ScalarAddress;
resources.memoryInfo.push_back(memory);
resources.controlFlow.push_back({&equal(load, constant(0)), {1, 2}, {}, {}});
resources.controlFlow.push_back({&equal(low, constant(7)), {3, 4}, {}, {}});
resources.controlFlow.push_back({&equal(read, constant(5)), {5, 6}, {}, {}});
auto& choice = make(IrOpcode::SelectU32, IrType::U32);
choice.AddArgument(&equal(userData(4), constant(0)));
choice.AddArgument(&load);
choice.AddArgument(&constant(1));
resources.controlFlow.push_back({&equal(choice, constant(1)), {7, 8}, {}, {}});
const auto plan = ResourceMaterializer{}.ExtractPlan(program);
const auto copied = std::ranges::count_if(plan.valueStorage, [](const IrValue& value) { return value.Opcode() == IrOpcode::IAdd32; });
require(copied == 0, "plan pruning: the address computation under a load the evaluator cannot read was copied");
require(plan.controlFlow[0].condition == nullptr && plan.controlFlow[3].condition == nullptr, "plan pruning: a condition the evaluator can never read was kept");
require(plan.controlFlow[1].condition != nullptr && plan.controlFlow[2].condition != nullptr, "plan pruning: a condition the evaluator can read was dropped");
const auto copiedLoad = std::ranges::find_if(plan.valueStorage, [](const IrValue& value) { return value.Opcode() == IrOpcode::LoadBufferU32; });
require(copiedLoad != plan.valueStorage.end() && copiedLoad->ArgumentCount() == 0u, "plan pruning: the unevaluable load was not kept as a leaf");
require(std::ranges::all_of(plan.valueStorage, [](const IrValue& value) { return !value.HasUses() && value.OperandUses().empty() && value.Arguments().capacity() == value.ArgumentCount(); }), "plan compaction: a plan value keeps use lists or spare operand capacity");
const std::array<std::uint32_t, 4> data{0x100u, 7u, static_cast<std::uint32_t>(wordAddress), static_cast<std::uint32_t>(wordAddress >> 32u)};
SrtRuntime runtime{data};
Detail::Evaluator evaluator(plan, runtime);
std::uint32_t result = 0;
require(evaluator.Evaluate(Detail::IrNode(plan.controlFlow[1].condition), result) && result == 1u, "plan pruning: a condition read through an extracted pair was not evaluated");
require(evaluator.Evaluate(Detail::IrNode(plan.controlFlow[2].condition), result) && result == 1u, "plan pruning: a condition on a raw read lost its handle");
}
struct PlanWalk {
std::vector<ShaderRecompiler::DescriptorValue> results;
std::vector<std::uint32_t> flat;
std::vector<std::uint8_t> active;
};
PlanWalk walkPlan(const ShaderRecompiler::IrResourcePlan& plan, std::span<const std::uint32_t> sources, std::span<const std::uint8_t> cleanFlatSlots, const ShaderRecompiler::SrtRuntime& runtime) {
PlanWalk walk;
ShaderRecompiler::SrtWalker{}.EvaluateRuntimeSources(plan, sources, runtime, walk.results, walk.flat, cleanFlatSlots, walk.active);
return walk;
}
bool readFive(void*, std::uint64_t, std::uint32_t* value) {
*value = 5u;
return true;
}
void verifyPlanRetention() {
using namespace ShaderRecompiler;
std::vector<std::unique_ptr<IrValue>> values;
std::uint32_t ids = 0;
const auto make = [&](IrOpcode opcode, IrType type) -> IrValue& {
values.push_back(std::make_unique<IrValue>(opcode, type, ids++));
return *values.back();
};
const auto constant = [&](std::uint32_t value) -> IrValue& {
auto& immediate = make(IrOpcode::Void, IrType::U32);
immediate.SetImmediateU32(value);
return immediate;
};
const auto userData = [&](std::uint32_t index) -> IrValue& {
auto& reg = make(IrOpcode::Void, IrType::ScalarReg);
reg.SetRegister({RegisterBank::Scalar, index});
auto& read = make(IrOpcode::GetUserData, IrType::U32);
read.AddArgument(&reg);
return read;
};
const auto binary = [&](IrOpcode opcode, IrType type, IrValue& left, IrValue& right) -> IrValue& {
auto& value = make(opcode, type);
value.AddArgument(&left);
value.AddArgument(&right);
return value;
};
constexpr std::uint32_t Chain = 64;
IrValue* address = &userData(4);
for (std::uint32_t step = 0; step < Chain; ++step) address = &binary(IrOpcode::IAdd32, IrType::U32, *address, constant(1));
auto& load = make(IrOpcode::LoadBufferU32, IrType::U32);
load.AddArgument(address);
auto& low = binary(IrOpcode::CompositeExtractU32x2, IrType::U32, binary(IrOpcode::CompositeConstructU32x2, IrType::U32x2, userData(1), load), constant(0));
auto& mask = binary(IrOpcode::IEqual32, IrType::Bool, userData(5), constant(0));
auto& choice = make(IrOpcode::SelectU32, IrType::U32);
choice.AddArgument(&mask);
choice.AddArgument(&userData(6));
choice.AddArgument(&load);
auto& offset = binary(IrOpcode::ReadFirstLane, IrType::U32, choice, mask);
auto& read = binary(IrOpcode::LoadAddressU32, IrType::U32, binary(IrOpcode::CompositeConstructU64, IrType::U64, userData(2), userData(3)), offset);
read.SetFlags(MemoryFlags{0, 0});
IrProgram program;
auto& resources = program.Resources();
resources.srtPlanComplete = true;
resources.resourceTrackingComplete = true;
MemoryInfo memory;
memory.kind = ResourceKind::ScalarAddress;
resources.memoryInfo.push_back(memory);
for (std::uint32_t source = 0; source < 2u; ++source) {
DescriptorSource descriptor;
descriptor.dwordCount = 4;
for (std::uint32_t dword = 0; dword < 4u; ++dword) descriptor.dwords[dword] = &userData(10u + source * 4u + dword);
resources.descriptorSources.push_back(descriptor);
BufferResource buffer;
buffer.source = source;
resources.info.buffers.push_back(buffer);
}
resources.controlFlow.push_back({&binary(IrOpcode::IEqual32, IrType::Bool, low, constant(7)), {1, 2}, {0}, {}});
resources.controlFlow.push_back({&binary(IrOpcode::IEqual32, IrType::Bool, read, constant(5)), {3, 4}, {}, {}});
resources.controlFlow.push_back({nullptr, {}, {}, {}});
resources.controlFlow.push_back({nullptr, {}, {}, {}});
resources.controlFlow.push_back({nullptr, {}, {1}, {}});
const auto plan = ResourceMaterializer{}.ExtractPlan(program);
constexpr std::size_t Read = 40;
require(plan.valueStorage.Size() == Read, ("plan retention: the plan keeps " + std::to_string(plan.valueStorage.Size()) + " values; its conditions and descriptor sources read " + std::to_string(Read)).c_str());
require(std::ranges::none_of(plan.valueStorage, [](const IrValue& value) { return value.Opcode() == IrOpcode::IAdd32; }), "plan retention: the address under a vector load was copied");
const auto copiedLoad = std::ranges::find_if(plan.valueStorage, [](const IrValue& value) { return value.Opcode() == IrOpcode::LoadBufferU32; });
require(copiedLoad != plan.valueStorage.end() && copiedLoad->ArgumentCount() == 0u, "plan retention: the vector load was not kept as a leaf");
std::array<std::uint32_t, 18> userWords{};
userWords[1] = 7u;
userWords[2] = 0x1000u;
userWords[6] = 8u;
for (std::uint32_t word = 10; word < userWords.size(); ++word) userWords[word] = 0x100u + word;
SrtRuntime runtime{userWords};
runtime.readMemory = readFive;
runtime.readSpecializationMemory = readFive;
const auto copiedWalk = walkPlan(plan, plan.materializationSources, plan.cleanFlatSlots, runtime);
const std::array<std::uint32_t, 2> sources{0u, 1u};
const auto wholeWalk = walkPlan(program.Resources(), sources, {}, runtime);
require(wholeWalk.active == std::vector<std::uint8_t>{1u, 0u}, "plan retention: the program's walk did not decide both conditions");
require(copiedWalk.active == wholeWalk.active && copiedWalk.results == wholeWalk.results && copiedWalk.flat == wholeWalk.flat, "plan retention: the plan's walk differs from the program's");
}
void verifyPlanRetentionChains() {
using namespace ShaderRecompiler;
std::vector<std::size_t> kept;
std::vector<std::size_t> conditions;
for (const std::uint32_t chain : {1u, 16u, 64u}) {
std::vector<std::uint32_t> code{0xf4080600u, 0xfa000060u, 0xbf8cc07fu, 0x7e140300u};
code.insert(code.end(), chain, 0x4a141481u);
code.insert(code.end(), {0x34141482u, 0xe0301000u, 0x80060b0au, 0xbf8c0000u, 0x7e28050bu, 0xf4000540u, 0x28000000u, 0xf4080700u, 0xfa000020u, 0xbf8c0000u, 0xbf078015u, 0xbf840002u, 0xe0700000u, 0x80070000u, 0xbf810000u});
const std::array<std::uint32_t, 2> userData{0x10000u, 0u};
const auto plan = GetResourcePlan(FrontendRequest(code, userData));
kept.push_back(plan->valueStorage.Size());
conditions.push_back(static_cast<std::size_t>(std::ranges::count_if(plan->controlFlow, [](const ResourceBlock& block) { return block.condition != nullptr; })));
}
require(conditions.front() != 0u, "plan retention: the program's plan keeps no branch condition, so the test does not reach the condition's values");
require(kept[0] == kept[1] && kept[1] == kept[2], ("plan retention: the plan keeps " + std::to_string(kept[0]) + ", " + std::to_string(kept[1]) + " and " + std::to_string(kept[2]) + " values for vector chains of 1, 16 and 64 steps before the scalar load its branch condition reads; it must keep only what the condition's evaluation can reach").c_str());
}
void verifyPlanCopyAnswers() {
using namespace ShaderRecompiler;
std::vector<std::unique_ptr<IrValue>> values;
std::vector<std::unique_ptr<IrBlock>> blocks;
std::uint32_t ids = 0;
const auto make = [&](IrOpcode opcode, IrType type) -> IrValue& {
values.push_back(std::make_unique<IrValue>(opcode, type, ids++));
return *values.back();
};
const auto constant = [&](std::uint32_t value) -> IrValue& {
auto& immediate = make(IrOpcode::Void, IrType::U32);
immediate.SetImmediateU32(value);
return immediate;
};
const auto userData = [&](std::uint32_t index) -> IrValue& {
auto& reg = make(IrOpcode::Void, IrType::ScalarReg);
reg.SetRegister({RegisterBank::Scalar, index});
auto& read = make(IrOpcode::GetUserData, IrType::U32);
read.AddArgument(&reg);
return read;
};
const auto binary = [&](IrOpcode opcode, IrType type, IrValue& left, IrValue& right) -> IrValue& {
auto& value = make(opcode, type);
value.AddArgument(&left);
value.AddArgument(&right);
return value;
};
const auto vectorLoad = [&](IrValue& address) -> IrValue& {
auto& load = make(IrOpcode::LoadBufferU32, IrType::U32);
load.AddArgument(&address);
return load;
};
const auto rawRead = [&](IrValue& low, IrValue& high, IrValue& offset) -> IrValue& {
auto& read = binary(IrOpcode::LoadAddressU32, IrType::U32, binary(IrOpcode::CompositeConstructU64, IrType::U64, low, high), offset);
read.SetFlags(MemoryFlags{0, 0});
return read;
};
const auto walks = [&](IrProgram& program, IrValue& read, const std::array<std::uint32_t, 14>& userWords, std::uint8_t expected, const char* what) {
auto& resources = program.Resources();
resources.srtPlanComplete = true;
resources.resourceTrackingComplete = true;
MemoryInfo memory;
memory.kind = ResourceKind::ScalarAddress;
resources.memoryInfo.push_back(memory);
DescriptorSource descriptor;
descriptor.dwordCount = 4;
for (std::uint32_t dword = 0; dword < 4u; ++dword) descriptor.dwords[dword] = &userData(10u + dword);
resources.descriptorSources.push_back(descriptor);
BufferResource buffer;
resources.info.buffers.push_back(buffer);
resources.controlFlow.push_back({&binary(IrOpcode::IEqual32, IrType::Bool, read, constant(5)), {1, 2}, {}, {}});
resources.controlFlow.push_back({nullptr, {}, {}, {}});
resources.controlFlow.push_back({nullptr, {}, {0}, {}});
const auto plan = ResourceMaterializer{}.ExtractPlan(program);
SrtRuntime runtime{userWords};
runtime.readMemory = readFive;
runtime.readSpecializationMemory = readFive;
const std::array<std::uint32_t, 1> sources{0u};
const auto whole = walkPlan(resources, sources, {}, runtime);
const auto taped = walkPlan(plan, plan.materializationSources, plan.cleanFlatSlots, runtime);
const auto interpreted = walkPlan(plan, sources, {}, runtime);
require(whole.active == std::vector<std::uint8_t>{expected}, what);
require(taped.active == whole.active && taped.results == whole.results, what);
require(interpreted.active == whole.active && interpreted.results == whole.results, what);
};
std::array<std::uint32_t, 14> userWords{};
for (std::uint32_t word = 10; word < userWords.size(); ++word) userWords[word] = 0x100u + word;
IrProgram phiProgram;
auto& shared = userData(1);
const auto low = [&](IrValue& hidden) -> IrValue& {
return binary(IrOpcode::CompositeExtractU32x2, IrType::U32, binary(IrOpcode::CompositeConstructU32x2, IrType::U32x2, shared, hidden), constant(0));
};
auto& left = *blocks.emplace_back(std::make_unique<IrBlock>(1u));
auto& right = *blocks.emplace_back(std::make_unique<IrBlock>(2u));
auto& phi = make(IrOpcode::Phi, IrType::U32);
phi.AddPhiOperand(&left, &low(vectorLoad(userData(4))));
phi.AddPhiOperand(&right, &low(vectorLoad(userData(5))));
userWords[1] = 8u;
userWords[2] = 0x1000u;
walks(phiProgram, rawRead(userData(2), userData(3), phi), userWords, 1u, "plan copy: a phi whose incoming values differ below a vector load resolved in the plan, not in the program");
IrProgram rootProgram;
auto& mask = binary(IrOpcode::IEqual32, IrType::Bool, userData(5), constant(0));
auto& choice = make(IrOpcode::SelectU32, IrType::U32);
choice.AddArgument(&mask);
choice.AddArgument(&userData(2));
choice.AddArgument(&vectorLoad(rawRead(userData(8), userData(9), constant(0))));
userWords[1] = 0u;
userWords[2] = 0u;
walks(rootProgram, rawRead(binary(IrOpcode::ReadFirstLane, IrType::U32, choice, mask), userData(3), constant(0)), userWords, 0u, "plan copy: a read from a null base loaded from memory only below a vector load counted as a null root in the plan, not in the program");
}
void verifySignedMinimum() {
using namespace ShaderRecompiler;
IrResourcePlan plan;
@@ -947,6 +1261,72 @@ void verifyPureFlatSlots() {
require(Detail::ComputePureFlatSlots(plan) == Pure{0, 0, 0, 0}, "pure flat slots: an incomplete plan was classified");
}
void verifyLaneMaskGuardedDescriptorSources() {
using namespace ShaderRecompiler;
constexpr std::uint32_t Format8888UNorm = 56;
constexpr std::uint32_t Type2D = 9;
constexpr std::uint32_t Zero = 0x00000000u;
constexpr std::uint32_t NegativeZero = 0x80000000u;
constexpr std::uint32_t One = 0x3f800000u;
struct alignas(256) Texture { std::array<std::uint8_t, 256> bytes{}; };
static Texture texture;
const auto base = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(texture.bytes.data()));
const std::array<std::uint32_t, 8> valid{static_cast<std::uint32_t>(base >> 8u), static_cast<std::uint32_t>((base >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 3u << 14u, 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u};
const std::array<std::uint32_t, 8> invalid{0x43700000u, 0x43070000u, 0x3b888889u, 0x3bf2b9d6u, 0x3b088889u, 0x3b72b9d6u, 0x3f7f7777u, 0x3f7f0d46u};
const std::array<std::uint32_t, 4> sampler{0u, 0u, 0u, 0u};
std::array<std::uint32_t, 4> output{};
const auto outputAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(output.data()));
const std::array<std::uint32_t, 4> outputV{static_cast<std::uint32_t>(outputAddress), static_cast<std::uint32_t>((outputAddress >> 32u) & 0xffffu), 16u, 0xfacu};
std::array<std::uint32_t, 32> srt{};
const auto fill = [&](const std::array<std::uint32_t, 8>& image, std::uint32_t mode) {
std::copy(image.begin(), image.end(), srt.begin());
std::copy(sampler.begin(), sampler.end(), srt.begin() + 8);
std::copy(outputV.begin(), outputV.end(), srt.begin() + 16);
srt[20] = mode;
};
const auto srtAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(srt.data()));
const std::array<std::uint32_t, 2> userData{static_cast<std::uint32_t>(srtAddress), static_cast<std::uint32_t>(srtAddress >> 32u)};
const std::array<std::uint32_t, 4> capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess};
const std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
RecompileRequest request{};
const std::vector<std::uint32_t> code{0xf4000400u, 0xfa000050u, 0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xd40d006au, 0x00002080u, 0xbf860006u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u};
request.shader = {ShaderStage::Compute, 0x24000u, code, 0, {}};
request.context.waveSize = 32;
request.context.userDataBaseRegister = 0;
request.context.userData = userData;
request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
request.target.vulkanVersion = 0x00401000u;
request.target.spirvVersion = 0x00010300u;
request.target.subgroupSize = 32;
request.target.supportedCapabilities = capabilities;
request.target.supportedExtensions = extensions;
request.target.bdaAbiVersion = BdaAbi::Version;
request.target.fragmentShaderBarycentricEnabled = false;
request.layout.pushConstantSizeBytes = 128;
fill(invalid, Zero);
const auto plan = GetResourcePlan(request);
require(plan->info.images.size() == 1u && plan->info.samplers.size() == 1u, "lane-mask guarded sources: the image and its sampler were not planned");
require(!plan->controlFlow.empty() && std::ranges::any_of(plan->controlFlow, [](const ResourceBlock& block) { return block.condition != nullptr; }), "lane-mask guarded sources: the VCC branch around the image kept no condition");
for (const auto mode : {Zero, NegativeZero}) {
fill(invalid, mode);
AgcDriver::ShaderMemory memory({});
const auto capture = memory.Capture(request);
require(capture->snapshot.images.size() == 1u && std::ranges::all_of(capture->snapshot.images[0].dwords, [](std::uint32_t word) { return word == 0u; }), "lane-mask guarded sources: the T# behind a branch no lane takes was not left null");
}
fill(invalid, One);
{
AgcDriver::ShaderMemory memory({});
expectFailure([&] { static_cast<void>(memory.Capture(request)); }, "runtime image descriptor is invalid", "lane-mask guarded sources: an invalid T# behind a branch the lanes may take was accepted");
}
fill(valid, One);
{
AgcDriver::ShaderMemory memory({});
const auto capture = memory.Capture(request);
require(capture->snapshot.images.size() == 1u && std::equal(valid.begin(), valid.end(), capture->snapshot.images[0].dwords.begin()), "lane-mask guarded sources: the T# behind a branch the lanes may take was not read");
}
}
void verifyGuardedDescriptorSources() {
using namespace ShaderRecompiler;
constexpr std::uint32_t Format8888UNorm = 56;
@@ -1250,6 +1630,7 @@ constexpr std::uint32_t TableFormat8888UNorm = 56;
constexpr std::uint32_t TableFormat32UInt = 20;
constexpr std::uint32_t TableFormat16SInt = 12;
constexpr std::uint32_t TableFormat11_11_10UInt = 34;
constexpr std::uint32_t TableFormat32_32UInt = 62;
constexpr std::uint32_t TableType2D = 9;
constexpr std::uint32_t TableType3D = 10;
@@ -2875,7 +3256,17 @@ void verifyImageTableRegistration() {
const auto* storeTable = TableBinding(*store.result, DescriptorRole::ImageStorageTable);
require((storeFault & Abi::PoisonFlag) == 0u && storeTable != nullptr && storeTable->count != 0u && std::all_of(storeTable->imageWritten.begin(), storeTable->imageWritten.end(), [](bool written) { return written; }), "image table registration: the store's table was not bound as written storage elements");
require(TableBinding(*store.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*store.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: a store-only table bound sampled elements");
const std::array<std::pair<std::uint32_t, const char*>, 3> unsupportedStores{{{0xf0240f08u, "image_store_mip"}, {0xf0280f08u, "image_store_pck"}, {0xf0440108u, "image_atomic_add"}}};
auto atomicCode = storeCode;
atomicCode[8] = 0xf0440108u;
const auto atomic = CompileTable(TableRequest(atomicCode, userData));
const auto& atomicPlan = *atomic.capture->plan;
require(atomicPlan.info.images.size() == 1u && atomicPlan.info.images[0].table == 0u && atomicPlan.info.images[0].tableOperation == TableOperation::Supported, "image table registration: a table T# feeding image_atomic_add was not a supported table operation");
const auto& atomicMap = TableMap(*atomic.result);
const auto atomicFault = atomicMap[Abi::TableHeader(0) + Abi::TableFault];
const auto* atomicTable = TableBinding(*atomic.result, DescriptorRole::ImageStorageTable);
require((atomicFault & Abi::PoisonFlag) == 0u && atomicTable != nullptr && atomicTable->count != 0u && atomicTable->imageAtomic.size() == atomicTable->count && std::all_of(atomicTable->imageWritten.begin(), atomicTable->imageWritten.end(), [](bool written) { return written; }) && std::all_of(atomicTable->imageAtomic.begin(), atomicTable->imageAtomic.end(), [](bool atomic) { return atomic; }), "image table registration: the image_atomic_add table was not bound as atomic storage elements");
require(TableBinding(*atomic.result, DescriptorRole::ImageTable) == nullptr || TableBinding(*atomic.result, DescriptorRole::ImageTable)->count == 0u, "image table registration: an atomic-only table bound sampled elements");
const std::array<std::pair<std::uint32_t, const char*>, 2> unsupportedStores{{{0xf0240f08u, "image_store_mip"}, {0xf0280f08u, "image_store_pck"}}};
for (const auto& [instruction, name] : unsupportedStores) {
auto code = storeCode;
code[8] = instruction;
@@ -3064,6 +3455,18 @@ void verifyPointerImageTable() {
require(!TableTouched(narrowTable.regions, root + (Palette + 32u) / 4u, 32u) && !TableTouched(narrowTable.regions, root + Committed / 4u, Reserved - Committed), "pointer image table: a record outside the key domain was read");
const std::vector<std::pair<std::uint64_t, std::uint64_t>> narrowReads{{base + Palette, 32u}, {base + Palette + 2u * 32u, 32u}, {base + Palette + 255u * 32u, 32u}, {base + Palette + 0x102u * 32u, 32u}};
require(narrowSnapshot.readRanges == narrowReads && narrowTable.result->imageTableReadRanges.size() == 1u && narrowTable.result->imageTableReadRanges[0] == narrowReads, "pointer image table: consumed ranges include skipped keys or the key-domain buffer");
auto phiCode = maskedCode;
phiCode.erase(phiCode.begin() + 8, phiCode.begin() + 10);
phiCode.insert(phiCode.begin() + 8, {
0xbf068010u, 0xbf850003u, 0xf4200444u, 0x20000000u, 0xbf820002u, 0xf4200444u, 0x20000000u
});
const auto phiRequest = request(phiCode, 0x36000u);
const auto phiPlan = GetResourcePlan(phiRequest);
require(phiPlan->info.images.size() == 1u && phiPlan->info.images[0].table == 0u, "pointer image table: a pointer read through a phi was not planned as a table");
const auto phiTable = CompileTable(phiRequest);
require(phiTable.capture->snapshot.tables.tables.at(0).codes == snapshot.codes, "pointer image table: a pointer read through a phi captured different records");
keys = {};
#ifdef _WIN32
VirtualFree(block, 0, MEM_RELEASE);
@@ -3281,7 +3684,7 @@ void verifySelectedRecordDirectOperations() {
FillTableSrt(guest, heap.data(), 32u, 4u);
guest.srt[16] = 2u;
const auto userData = TableUserData(guest);
const std::array<std::tuple<const char*, std::uint32_t, std::uint32_t>, 4> operations{{{"image_store", 0xf0200f08u, TableFormat32UInt}, {"image_atomic_add", 0xf0440108u, TableFormat32UInt}, {"image_load_pck", 0xf0080f08u, TableFormat32UInt}, {"image_load_by2", 0xf1080308u, TableFormat16UNorm}}};
const std::array<std::tuple<const char*, std::uint32_t, std::uint32_t>, 5> operations{{{"image_store", 0xf0200f08u, TableFormat32UInt}, {"image_atomic_add", 0xf0440108u, TableFormat32UInt}, {"image_atomic64_add", 0xf0440308u, TableFormat32_32UInt}, {"image_load_pck", 0xf0080f08u, TableFormat32UInt}, {"image_load_by2", 0xf1080308u, TableFormat16UNorm}}};
for (const auto& [name, word0, format] : operations) {
const auto what = std::string("selected record direct operations: ") + name;
heap[2] = TableT(textures[1].bytes.data(), format, TableType2D);
@@ -3289,11 +3692,11 @@ void verifySelectedRecordDirectOperations() {
const auto request = TableRequest(code, userData);
const auto plan = GetResourcePlan(request);
require(plan->info.images.size() == 1u && plan->info.images[0].table == NoTable && !plan->descriptorSources.at(plan->info.images[0].source).tableColumn.has_value(), (what + " through a T# whose record the CPU evaluates was planned through an image table").c_str());
const auto compiled = CompileTable(request);
require(TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2])), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str());
const auto compiled = word0 == 0xf0440308u ? TableCompiled{} : CompileTable(request);
require(word0 == 0xf0440308u || (TableBinding(*compiled.result, DescriptorRole::ImageTable) == nullptr && compiled.result->imageTableFaults == 0u && TableCovered(compiled.regions, &heap[2], sizeof(heap[2]))), (what + " through a T# whose record the CPU evaluates was not bound from its record").c_str());
const auto wholeCode = WithImageOperation(TableWholeCode, word0, false);
const auto whole = GetResourcePlan(TableRequest(wholeCode, userData));
require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str());
require(whole->info.images.size() == 1u && whole->info.images[0].table == 0u && whole->info.images[0].tableOperation == (word0 == 0xf0200f08u || word0 == 0xf0440108u ? TableOperation::Supported : TableOperation::Unsupported), (what + " through a T# keyed per lane left the image table or has the wrong table operation").c_str());
}
const auto mixedCode = WithImageOperation(TableSrtKeyCode, 0xf0200f08u, true);
const auto mixed = GetResourcePlan(TableRequest(mixedCode, userData));
@@ -3467,6 +3870,45 @@ void verifyPointerChainImageTable() {
static_cast<void>(ValidateAndOptimizeSpirv(compiled.result->spirv, request.target.vulkanVersion, request.target.spirvVersion));
#endif
store(links[1].data() + 4u, address(descriptors[1].data()));
auto phiCode = code;
phiCode.erase(phiCode.begin() + 15, phiCode.begin() + 17);
phiCode.insert(phiCode.begin() + 15, {
0xbf068010u, 0xbf850003u, 0xf4240b02u, 0x24000008u, 0xbf820002u, 0xf4240b02u, 0x24000008u
});
auto phiRequest = TableRequest(phiCode, userData);
phiRequest.shader.codeAddress = 0x38000u;
const auto phiPlan = GetResourcePlan(phiRequest);
require(phiPlan->info.images.size() == 1u && phiPlan->info.images[0].table == 0u, "pointer chain image table: a pointer read through a phi was not planned as a table");
const auto phiTable = CompileTable(phiRequest);
require(phiTable.capture->snapshot.tables.tables.at(0).codes == snapshot.codes, "pointer chain image table: a pointer read through a phi captured different records");
auto divergentCode = code;
divergentCode.erase(divergentCode.begin() + 15, divergentCode.begin() + 17);
divergentCode.insert(divergentCode.begin() + 15, {
0xbf068010u, 0xbf850003u, 0xf4240b02u, 0x24000008u, 0xbf820002u, 0xf4240b02u, 0x24000018u
});
auto divergentRequest = TableRequest(divergentCode, userData);
divergentRequest.shader.codeAddress = 0x39000u;
expectFailure([&] { static_cast<void>(GetResourcePlan(divergentRequest)); }, "GetImageResource dword 0 is not a valid runtime value", "pointer chain image table: a phi reading different table offsets was planned as a table");
auto nonTableCode = code;
nonTableCode.erase(nonTableCode.begin() + 15, nonTableCode.begin() + 17);
nonTableCode.insert(nonTableCode.begin() + 15, {
0xbf068010u, 0xbf850003u, 0xf4240b02u, 0x24000008u, 0xbf820002u, 0xbe960380u, 0xbe970380u
});
auto nonTableRequest = TableRequest(nonTableCode, userData);
nonTableRequest.shader.codeAddress = 0x3a000u;
expectFailure([&] { static_cast<void>(GetResourcePlan(nonTableRequest)); }, "GetImageResource dword 0 is not a valid runtime value", "pointer chain image table: a phi with a non-table branch was planned as a table");
auto armOffsetCode = code;
armOffsetCode.erase(armOffsetCode.begin() + 14, armOffsetCode.begin() + 17);
armOffsetCode.insert(armOffsetCode.begin() + 14, {
0xbf068010u, 0xbf850004u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xbf820003u, 0x8f128411u, 0xf4240b02u, 0x24000008u
});
auto armOffsetRequest = TableRequest(armOffsetCode, userData);
armOffsetRequest.shader.codeAddress = 0x3b000u;
expectFailure([&] { static_cast<void>(GetResourcePlan(armOffsetRequest)); }, "GetImageResource dword 0 is not a valid runtime value", "pointer chain image table: a phi whose arms compute their own table offset was planned as a table");
}
void verifyWaterfallImageTable() {
@@ -4842,7 +5284,7 @@ void verifyPixelRequestSerialization() {
value.posZ, value.posW, value.frontFace, value.frontFaceAllBits, value.ancillary, value.sampleShading,
value.noPerspective, value.linearCentroid, value.pixelKillEnable, value.depthExportEnable,
value.sampleMaskExportEnable, value.earlyZ, value.executeOnNoop, value.conservativeZExport, value.orderedPixelShader,
value.targetOutputMode, value.targetExportMapping, value.alphaToCoverage, value.stencilRefExportEnable, value.rasterPrimitive, value.layerExported, value.ditheredAlphaToCoverage, value.reversedBlend);
value.targetOutputMode, value.targetExportMapping, value.alphaToCoverage, value.stencilRefExportEnable, value.rasterPrimitive, value.layerExported, value.ditheredAlphaToCoverage, value.reversedBlend, value.targetMerge);
};
const std::array<std::uint32_t, 1> code{0xbf810000u};
const std::array<std::uint32_t, 3> userData{0x12345678u, 0xabcdef01u, 0x87654321u};
@@ -4890,6 +5332,7 @@ void verifyPixelRequestSerialization() {
pixel.stencilRefExportEnable = true;
pixel.ditheredAlphaToCoverage = true;
pixel.reversedBlend = 0x570001u;
pixel.targetMerge = {1u, 0x0ff0u, 1u, {0x100u, 0x2u, 0x3u, 0x4u, 0x5u, 0x6u, 0x7u, 0x8u}};
const std::array<std::array<std::uint8_t, 8>, 3> mappings{{
{0x00u, 0xe4u, 0xc6u, 0x1bu, 0xffu, 0x80u, 0x55u, 0xaau},
{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u},
@@ -4922,6 +5365,22 @@ void verifyPixelRequestSerialization() {
withoutReversedBlend.context.pixel->reversedBlend = 0u;
RecompileCacheKey::Build(withoutReversedBlend, replayKey);
require(key == replayKey && RecompileCacheKey::ContextHash(request) == RecompileCacheKey::ContextHash(withoutReversedBlend), "the reversed-order blend code changed the static shader identity");
auto movedMerge = request;
movedMerge.context.pixel->targetMerge.descriptor[0] = 0x200u;
RecompileCacheKey::Build(movedMerge, replayKey);
require(key == replayKey, "the merged target's descriptor changed the static shader identity");
auto depthWritingMerge = request;
depthWritingMerge.context.pixel->targetMerge.depthStencilWrites = true;
RecompileCacheKey::Build(depthWritingMerge, replayKey);
require(key != replayKey, "the merge draw's depth or stencil writes did not change the static shader identity");
auto depthTestingMerge = request;
depthTestingMerge.context.pixel->targetMerge.depthStencilTests = true;
RecompileCacheKey::Build(depthTestingMerge, replayKey);
require(key != replayKey, "the merge draw's depth or stencil tests did not change the static shader identity");
auto withoutMerge = request;
withoutMerge.context.pixel->targetMerge = {};
RecompileCacheKey::Build(withoutMerge, replayKey);
require(key != replayKey, "a color target merge did not change the static shader identity");
auto withoutCoverage = request;
withoutCoverage.context.pixel->alphaToCoverage = false;
RecompileCacheKey::Build(withoutCoverage, replayKey);
@@ -4944,7 +5403,7 @@ void verifyPixelRequestSerialization() {
minimal.context.pixel->layerExported = true;
minimal.context.pixel->ditheredAlphaToCoverage = true;
const auto encoded = serializer.Serialize(minimal);
require(requestPrefix(encoded, 8u) == "NVNQQSwAAAA=", "new requests did not use serialization version 44");
require(requestPrefix(encoded, 8u) == "NVNQQS4AAAA=", "new requests did not use serialization version 46");
const std::array<std::uint32_t, 2> fallbacks{0x40u, 0x1ac4u};
auto withFallbacks = minimal;
withFallbacks.context.linkedCallFallbacks = fallbacks;
@@ -5062,7 +5521,7 @@ void verifyPixelRequestSerialization() {
for (std::size_t bytes = 0; bytes < 45u; ++bytes) {
expectFailure([&] { static_cast<void>(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export, dither or front-face flag, quad pixel mask or quad sample mask was accepted");
}
for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQS0AAAA="}) {
for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQS8AAAA="}) {
expectFailure([&] { static_cast<void>(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted");
}
static constexpr std::string_view version18Request{
@@ -5153,6 +5612,7 @@ void verifyVertexInfoVersion19() {
require(offset + 4u <= plain.size() && flagged[offset] == 0xefu && flagged[offset + 1u] == 0xcdu && flagged[offset + 2u] == 0xabu && flagged[offset + 3u] == 0x89u, "PA_CL_VS_OUT_CNTL is not the four bytes the version 20 vertex info adds");
require(std::equal(plain.begin() + offset + 4u, plain.end(), flagged.begin() + offset + 4u), "PA_CL_VS_OUT_CNTL moved the fields after it");
auto version20Vertex = flagged;
version20Vertex.resize(version20Vertex.size() - sizeof(std::uint32_t));
version20Vertex.pop_back();
version20Vertex.pop_back();
version20Vertex.pop_back();
@@ -5165,6 +5625,7 @@ void verifyVertexInfoVersion19() {
const auto layered = serializer.Deserialize(serializer.Serialize(vertex));
require(layered.request.context.vertex.has_value() && layered.request.context.vertex->paClVsOutCntl == vertex.context.vertex->paClVsOutCntl, "a layered vertex info lost its clip distances or render target index");
auto legacy = plain;
legacy.resize(legacy.size() - sizeof(std::uint32_t));
legacy.pop_back();
legacy.pop_back();
legacy.pop_back();
@@ -5204,16 +5665,39 @@ void verifyVertexInfoVersion19() {
while (allBitsOffset < plainPixel.size() && plainPixel[allBitsOffset] == allBitsBytes[allBitsOffset]) ++allBitsOffset;
require(allBitsOffset == flagOffset + 2u, "the front-face all-bits flag does not follow the dithered coverage flag");
require(blendOffset == allBitsOffset + 18u, "the reversed-order blend code is not serialized after the front-face all-bits flag, quad pixel mask and quad sample masks");
constexpr std::size_t sampleMaskBytes = 16u;
constexpr std::size_t iterationBytes = 35u;
constexpr std::size_t sampleOrderBytes = 16u;
constexpr std::size_t mergeBytes = 40u;
constexpr std::size_t sampleMaskBytes = 16u;
const auto orderOffset = blendOffset + 4u + iterationBytes;
const auto mergeOffset = static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes);
auto merging = pixel;
merging.context.pixel->targetMerge = {3u, 0xf00fu, 2u, {1u, 2u, 3u, 4u, 5u, 6u, 7u, 0x89abcdefu}, true, true};
const auto mergingBytes = decodeBase64(serializer.Serialize(merging));
require(mergingBytes.size() == plainPixel.size() && std::equal(plainPixel.begin(), plainPixel.begin() + mergeOffset, mergingBytes.begin()) && mergingBytes[mergeOffset] == 3u && mergingBytes[mergeOffset + 1] == 0x0fu && mergingBytes[mergeOffset + 2] == 0xf0u && mergingBytes[mergeOffset + 5] == 2u && mergingBytes[mergeOffset + 6] == 1u && mergingBytes[mergeOffset + 34] == 0xefu && mergingBytes[mergeOffset + 37] == 0x89u && mergingBytes[mergeOffset + 38] == 1u && mergingBytes[mergeOffset + 39] == 1u && std::equal(plainPixel.begin() + mergeOffset + static_cast<std::ptrdiff_t>(mergeBytes), plainPixel.end(), mergingBytes.begin() + mergeOffset + static_cast<std::ptrdiff_t>(mergeBytes)), "the color target merge is not the 40 bytes the version 46 pixel info adds after the sample order");
require(serializer.Deserialize(serializer.Serialize(merging)).request.context.pixel->targetMerge == merging.context.pixel->targetMerge, "a serialized request lost its color target merge");
// A version 43 request: no color target merge, and no fused arithmetic flag, pixel function LDS flag or storage size at the end.
const auto version43Bytes = [&](std::vector<std::uint8_t> bytes) {
require(bytes.end() - bytes.begin() > mergeOffset + static_cast<std::ptrdiff_t>(mergeBytes) && std::equal(plainPixel.begin() + mergeOffset, plainPixel.begin() + mergeOffset + static_cast<std::ptrdiff_t>(mergeBytes), bytes.begin() + mergeOffset), "a replay fixture does not hold the default color target merge after the sample order");
bytes.erase(bytes.begin() + mergeOffset, bytes.begin() + mergeOffset + static_cast<std::ptrdiff_t>(mergeBytes));
bytes.resize(bytes.size() - sizeof(std::uint32_t) - 2u);
return bytes;
};
auto version42 = version43Bytes(plainPixel);
version42[4] = 42u;
const auto replay42 = serializer.Deserialize(encodeBase64(version42));
require(replay42.request.context.pixel.has_value() && replay42.request.context.pixel->targetMerge == TargetMerge{} && serializer.Serialize(replay42.request) == encodedPixel, "a version 42 request read a color target merge or misread the fields after it");
auto version39 = version42;
version39.pop_back();
version39.erase(version39.begin() + static_cast<std::ptrdiff_t>(orderOffset), version39.begin() + mergeOffset);
version39[4] = 39u;
const auto replay39 = serializer.Deserialize(encodeBase64(version39));
require(replay39.request.context.pixel.has_value() && replay39.request.context.pixel->targetMerge == TargetMerge{} && replay39.request.context.pixel->sampleOrder == std::array<std::uint32_t, 4>{} && serializer.Serialize(replay39.request) == encodedPixel, "a version 39 request read a sample order or color target merge, or misread the fields after them");
auto ordered = pixel;
ordered.context.pixel->sampleOrder = {0x76543210u, 0x76543210u, 0x76543201u, 0x76543201u};
const auto orderedBytes = decodeBase64(serializer.Serialize(ordered));
require(orderedBytes.size() == plainPixel.size() && std::equal(plainPixel.begin(), plainPixel.begin() + static_cast<std::ptrdiff_t>(orderOffset), orderedBytes.begin()) && orderedBytes[orderOffset] == 0x10u && orderedBytes[orderOffset + 11u] == 0x76u && orderedBytes[orderOffset + 12u] == 0x01u && std::equal(plainPixel.begin() + static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes), plainPixel.end(), orderedBytes.begin() + static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes)), "the sample order is not the four words the version 42 pixel info adds after the stencil operation value export flag");
auto orderedVersion34 = orderedBytes;
orderedVersion34.pop_back();
auto orderedVersion34 = version43Bytes(orderedBytes);
orderedVersion34.pop_back();
orderedVersion34.pop_back();
orderedVersion34.pop_back();
@@ -5221,16 +5705,15 @@ void verifyVertexInfoVersion19() {
orderedVersion34[4] = 34u;
const auto orderedReplay34 = serializer.Deserialize(encodeBase64(orderedVersion34));
require(orderedReplay34.request.context.pixel.has_value() && orderedReplay34.request.context.pixel->sampleOrder == std::array<std::uint32_t, 4>{} && serializer.Serialize(orderedReplay34.request) == encodedPixel, "a version 34 request read a sample order, or misread the fields after its pixel iteration fields");
auto orderedVersion30 = orderedBytes;
orderedVersion30.resize(orderedVersion30.size() - 5u);
auto orderedVersion30 = version43Bytes(orderedBytes);
orderedVersion30.resize(orderedVersion30.size() - 4u);
orderedVersion30.erase(orderedVersion30.begin() + static_cast<std::ptrdiff_t>(blendOffset + 4u), orderedVersion30.begin() + static_cast<std::ptrdiff_t>(orderOffset + sampleOrderBytes));
orderedVersion30[4] = 30u;
const auto orderedReplay30 = serializer.Deserialize(encodeBase64(orderedVersion30));
require(orderedReplay30.request.context.pixel.has_value() && orderedReplay30.request.context.pixel->sampleOrder == std::array<std::uint32_t, 4>{} && serializer.Serialize(orderedReplay30.request) == encodedPixel, "a version 30 request read a sample order, or misread the fields after its reversed-order blend code");
auto orderedSpanned = ordered;
orderedSpanned.target.spannedWaves = true;
auto orderedVersion32 = decodeBase64(serializer.Serialize(orderedSpanned));
orderedVersion32.pop_back();
auto orderedVersion32 = version43Bytes(decodeBase64(serializer.Serialize(orderedSpanned)));
orderedVersion32.pop_back();
require(orderedVersion32.size() > 2u && orderedVersion32.back() == 0u && orderedVersion32[orderedVersion32.size() - 2u] == 0u && orderedVersion32[orderedVersion32.size() - 3u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag");
orderedVersion32.pop_back();
@@ -5251,8 +5734,7 @@ void verifyVertexInfoVersion19() {
sampledPixel.context.pixel->quadSampleMask = {0x1u, 0x2u, 0x4u, 0x8u};
const auto sampledBytes = decodeBase64(serializer.Serialize(sampledPixel));
require(std::equal(maskedBytes.begin(), maskedBytes.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u), sampledBytes.begin()) && sampledBytes[maskOffset + 1u] == 0x1u && sampledBytes[maskOffset + 5u] == 0x2u && sampledBytes[maskOffset + 9u] == 0x4u && sampledBytes[maskOffset + 13u] == 0x8u, "the quad sample masks are not the four words after the quad pixel mask");
auto version30 = sampledBytes;
version30.pop_back();
auto version30 = version43Bytes(sampledBytes);
version30.pop_back();
version30.pop_back();
version30.pop_back();
@@ -5263,8 +5745,7 @@ void verifyVertexInfoVersion19() {
require(replay30.request.context.pixel.has_value() && !replay30.request.context.pixel->iteratesSamples && replay30.request.context.pixel->positionLocation == PixelPositionLocation::Center && replay30.request.context.pixel->sampleLocations == std::array<std::uint8_t, 32>{} && decodeBase64(serializer.Serialize(replay30.request)) == sampledBytes, "a version 30 request read pixel iteration, a position location or sample locations, or misread the fields after its reversed-order blend code");
auto spanned = sampledPixel;
spanned.target.spannedWaves = true;
auto version32 = decodeBase64(serializer.Serialize(spanned));
version32.pop_back();
auto version32 = version43Bytes(decodeBase64(serializer.Serialize(spanned)));
version32.pop_back();
require(version32.size() > 2u && version32.back() == 0u && version32[version32.size() - 2u] == 0u && version32[version32.size() - 3u] == 1u, "the spanned-waves flag is not the byte before the depth remap flag");
version32.pop_back();
@@ -5273,15 +5754,13 @@ void verifyVertexInfoVersion19() {
version32[4] = 32u;
const auto replay32 = serializer.Deserialize(encodeBase64(version32));
require(replay32.request.context.pixel.has_value() && !replay32.request.context.pixel->iteratesSamples && replay32.request.context.pixel->sampleLocations == std::array<std::uint8_t, 32>{} && replay32.request.target.spannedWaves && serializer.Serialize(replay32.request) == serializer.Serialize(spanned), "a version 32 request read pixel iteration or sample locations, or lost its spanned-waves flag");
auto version29 = sampledBytes;
version29.pop_back();
auto version29 = version43Bytes(sampledBytes);
version29.pop_back();
version29.pop_back();
version29.pop_back();
version29.pop_back();
version29.erase(version29.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes), version29.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
auto version36 = sampledBytes;
version36.pop_back();
auto version36 = version43Bytes(sampledBytes);
version36.pop_back();
version36.erase(version36.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes - 1u), version36.begin() + static_cast<std::ptrdiff_t>(maskOffset + 1u + sampleMaskBytes + 4u + iterationBytes + sampleOrderBytes));
version36[4] = 36u;
@@ -5290,8 +5769,7 @@ void verifyVertexInfoVersion19() {
version29[4] = 29u;
const auto replay29 = serializer.Deserialize(encodeBase64(version29));
require(replay29.request.context.pixel.has_value() && replay29.request.context.pixel->quadSampleMask == std::array<std::uint16_t, 4>{0x1u, 0x2u, 0x4u, 0x8u} && replay29.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay29.request)) == sampledBytes, "a version 29 request read a reversed-order blend code or misread the fields after the quad sample masks");
auto version28 = sampledBytes;
version28.pop_back();
auto version28 = version43Bytes(sampledBytes);
version28.pop_back();
version28.pop_back();
version28.pop_back();
@@ -5300,8 +5778,7 @@ void verifyVertexInfoVersion19() {
version28[4] = 28u;
const auto replay28 = serializer.Deserialize(encodeBase64(version28));
require(replay28.request.context.pixel.has_value() && replay28.request.context.pixel->quadPixelMask == 0x2u && replay28.request.context.pixel->quadSampleMask == std::array<std::uint16_t, 4>{0xffffu, 0xffffu, 0xffffu, 0xffffu} && replay28.request.context.pixel->reversedBlend == 0u && decodeBase64(serializer.Serialize(replay28.request)) == maskedBytes, "a version 28 request read quad sample masks or a reversed-order blend code, or misread the fields after them");
auto version27 = maskedBytes;
version27.pop_back();
auto version27 = version43Bytes(maskedBytes);
version27.pop_back();
version27.pop_back();
version27.pop_back();
@@ -5310,8 +5787,7 @@ void verifyVertexInfoVersion19() {
version27[4] = 27u;
const auto replay27 = serializer.Deserialize(encodeBase64(version27));
require(replay27.request.context.pixel.has_value() && replay27.request.context.pixel->quadPixelMask == 0xfu && replay27.request.context.pixel->reversedBlend == 0u && serializer.Serialize(replay27.request) == encodedPixel, "a version 27 request read a quad pixel mask or reversed-order blend code, or misread the fields after it");
auto legacyPixel = plainPixel;
legacyPixel.pop_back();
auto legacyPixel = version43Bytes(plainPixel);
legacyPixel.pop_back();
legacyPixel.pop_back();
legacyPixel.pop_back();
@@ -5366,9 +5842,11 @@ void verifyDepthRemapRequests() {
const auto plainBytes = decodeBase64(plain);
const auto remappedBytes = decodeBase64(remapped);
constexpr std::size_t pointTailBytes = 13u;
require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 8u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 8 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - 1 - pointTailBytes, plainBytes.end(), remappedBytes.end() - 1 - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags, the point tail and the fused arithmetic flag of a vertex request");
require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 13u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 13 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - 6 - pointTailBytes, plainBytes.end(), remappedBytes.end() - 6 - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags, the point tail, the fused arithmetic flag, the pixel function LDS flag and the pixel function storage size of a vertex request");
for (const std::uint8_t legacyVersion : {32u, 34u}) {
auto legacyBytes = remappedBytes;
legacyBytes.resize(legacyBytes.size() - sizeof(std::uint32_t));
legacyBytes.pop_back();
legacyBytes.pop_back();
legacyBytes.pop_back();
legacyBytes[4] = legacyVersion;
@@ -5378,6 +5856,8 @@ void verifyDepthRemapRequests() {
require(serializer.Serialize(legacy.request) == plain, ("a version " + std::to_string(legacyVersion) + " vertex request misread the fields before the depth remap").c_str());
}
auto version35 = remappedBytes;
version35.resize(version35.size() - sizeof(std::uint32_t));
version35.pop_back();
version35.pop_back();
version35.pop_back();
version35[4] = 35u;
@@ -5427,8 +5907,8 @@ void verifyMergedHullConfiguration() {
require(!serializer.Deserialize(separate).request.graphics->tessellation->mergedHull, "a separate hull configuration was read as merged");
constexpr std::size_t pointTailBytes = 13u;
auto legacy = decodeBase64(merged);
require(legacy.size() > 54u + pointTailBytes && legacy[4] == 44u, "a merged hull request did not use serialization version 44");
legacy.resize(legacy.size() - 1u - pointTailBytes);
require(legacy.size() > 54u + pointTailBytes && legacy[4] == 46u, "a merged hull request did not use serialization version 46");
legacy.resize(legacy.size() - sizeof(std::uint32_t) - 2u - pointTailBytes);
legacy.pop_back();
require(legacy[legacy.size() - 52u] == 1u, "the merged hull flag is not the last byte of the tessellation configuration");
legacy.erase(legacy.end() - 52);
@@ -7554,8 +8034,19 @@ void verifyFmaskImageMode() {
const auto modes = ResourceMaterializer::RuntimeImageModes(image);
const auto selected = ResourceMaterializer::RuntimeImageMode(image, descriptor, modes);
require(modes[selected].numericClass == IrTextureNumericClass::Uint && modes[selected].packedFormat == IrBufferFormat::Invalid && !modes[selected].constantSwizzle, "FMASK did not select a bound integer image mode");
for (std::uint32_t format = 156u; format <= 168u; ++format) {
if (format == 166u) continue;
for (const auto format : {156u, 157u, 158u, 159u, 160u, 161u, 163u, 165u}) {
descriptor.dwords[1] = format << 20u;
const auto expanded = ResourceMaterializer::RuntimeImageMode(image, descriptor, modes);
require(expanded != selected && modes[expanded].numericClass == IrTextureNumericClass::Uint && modes[expanded].conversionFormat == FmaskExpansion && modes[expanded].dimension == RdnaImageDimension::Dim2D, "an FMASK format of up to 8 samples did not select the FMASK expansion mode");
}
require(std::ranges::count_if(modes, [](const ImageResource& mode) { return mode.conversionFormat == FmaskExpansion; }) == 1, "FMASK expansion needs exactly one runtime image mode");
for (const auto format : {161u, 166u}) {
descriptor.dwords[1] = format << 20u;
descriptor.dwords[3] = 0x91800005u;
expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(image, descriptor, modes)); }, "FMASK destination channel selector is not implemented", "an FMASK load selecting Y was accepted");
}
descriptor.dwords[3] = 0x91800004u;
for (const auto format : {162u, 164u, 167u, 168u}) {
descriptor.dwords[1] = format << 20u;
expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(image, descriptor, modes)); }, "FMASK format is not implemented", "an unsupported FMASK format was accepted");
}
@@ -7613,6 +8104,37 @@ void verifyVolumeArrayLoadMode() {
expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(store, descriptor, ResourceMaterializer::RuntimeImageModes(store))); }, "incompatible with the static runtime image interface", "2D-array image atomic on a 3D T# was accepted");
}
void verifyMultisamplePlaneReadMode() {
using namespace ShaderRecompiler;
const DescriptorValue descriptor{{0x02754d00u, 0xc3800000u, 0x00cbc167u, 0xe1b20facu, 0u, 0x00700020u, 0u, 0u}, 8u};
ImageResource image{};
image.resourceClass = ImageResourceClass::Sampled;
image.dimension = RdnaImageDimension::Dim2D;
const auto modes = ResourceMaterializer::RuntimeImageModes(image);
const auto selected = ResourceMaterializer::RuntimeImageMode(image, descriptor, modes);
require(modes[selected].dimension == RdnaImageDimension::Dim2D && !modes[selected].cube && modes[selected].numericClass == IrTextureNumericClass::Float, "a 2D read of a 2D MSAA T# (PPSA18216's) did not select a 2D runtime mode");
for (const auto& [word1, word3, word5, numericClass] : std::array<std::tuple<std::uint32_t, std::uint32_t, std::uint32_t, IrTextureNumericClass>, 4>{{{0xc0100000u, 0xe1b20facu, 0x00700020u, IrTextureNumericClass::Float}, {0xc0500000u, 0xe1810facu, 0x00700010u, IrTextureNumericClass::Uint}, {0xc0c00000u, 0xe1b20facu, 0x00700020u, IrTextureNumericClass::Sint}, {0xc0d00000u, 0xe1830facu, 0x00700030u, IrTextureNumericClass::Float}}}) {
const DescriptorValue narrow{{0x02754d00u, word1, 0x00cbc167u, word3, 0u, word5, 0u, 0u}, 8u};
const auto mode = ResourceMaterializer::RuntimeImageMode(image, narrow, modes);
require(modes[mode].dimension == RdnaImageDimension::Dim2D && !modes[mode].cube && modes[mode].numericClass == numericClass, "a 2D read of a 2D MSAA T# with 8-bit or 16-bit texels did not select a 2D runtime mode");
}
const DescriptorValue depthBits{{0x02754d00u, 0xc0b00000u, 0x00cbc167u, 0xe1820facu, 0u, 0x00700020u, 0u, 0u}, 8u};
expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(image, depthBits, modes)); }, "incompatible with the static runtime image interface", "a 2D read of a 16_UINT SW_64KB_Z_X 2D MSAA T#, read as depth bits, was accepted");
const auto refused = [&](const ImageResource& access, const char* what) {
expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(access, descriptor, ResourceMaterializer::RuntimeImageModes(access))); }, "incompatible with the static runtime image interface", what);
};
auto other = image;
other.cubeFaceCompatible = false;
refused(other, "a 2D read of a 2D MSAA T# through a mip, gather, bias, gradient or offset access was accepted");
other = image;
other.depthCompare = true;
refused(other, "a 2D depth comparison of a 2D MSAA T# was accepted");
other = image;
other.resourceClass = ImageResourceClass::Storage;
other.written = true;
refused(other, "a 2D storage access to a 2D MSAA T# was accepted");
}
void verifyVolumeBiasTableRejection() {
using namespace ShaderRecompiler;
namespace Abi = ImageTableAbi;
@@ -7656,6 +8178,85 @@ void verifyVolumeBiasTableRejection() {
}
}
void verifySamplerTableDeduplication() {
using namespace ShaderRecompiler;
namespace Abi = ImageTableAbi;
SamplerResource sampler0{};
sampler0.table = 0u;
SamplerResource sampler1{};
sampler1.table = 1u;
ShaderInfo info{};
info.samplers.push_back(sampler0);
info.samplers.push_back(sampler1);
ResourceSnapshot snapshot{};
snapshot.tables.tables.resize(2u);
const DescriptorValue s0{{0x00000600u, 0x00fff000u, 0x06f03fa0u, 0u}, 4u};
const DescriptorValue s1{{0x00000700u, 0x00fff000u, 0x06f03fa0u, 0u}, 4u};
const DescriptorValue s2{{0x00000800u, 0x00fff000u, 0x06f03fa0u, 0u}, 4u};
std::vector<std::uint32_t> codes(300u);
for (std::size_t i = 0; i < 300u; ++i) codes[i] = static_cast<std::uint32_t>(i % 3u);
for (std::uint32_t table = 0u; table < 2u; ++table) {
auto& column = snapshot.tables.tables[table];
column.records = 300u;
column.keys = 300u;
column.codes = SharedValues<std::uint32_t>(codes);
column.words = SharedValues<DescriptorValue>(std::vector<DescriptorValue>{s0, s1, s2});
}
const auto resolved = ResourceMaterializer::ResolveImageTables(info, snapshot);
require(resolved.samplers.size() == (3u + 1u) * Abi::SamplerEntryElements, "sampler table deduplication: duplicate sampler entries across tables were not deduplicated");
for (std::uint32_t table = 0u; table < 2u; ++table) {
const auto mapStart = resolved.map.at(Abi::TableHeader(table) + Abi::TableMapStart);
for (std::uint32_t key = 0; key < 300u; ++key) {
const auto record = resolved.map.at(mapStart + key);
const auto element = resolved.map.at(record + Abi::SamplerRecordElement);
require(element == (key % 3u) * Abi::SamplerEntryElements, "sampler table deduplication: record points to the wrong deduplicated sampler element");
require((resolved.map.at(record + Abi::SamplerRecordFlags) & Abi::PoisonFlag) == 0u, "sampler table deduplication: deduplicated sampler was poisoned");
}
}
constexpr std::uint32_t MaxUnique = Abi::SamplerElementBudget / Abi::SamplerEntryElements;
std::vector<DescriptorValue> budgetWords;
budgetWords.reserve(MaxUnique + 1u);
for (std::uint32_t i = 0; i < MaxUnique + 1u; ++i) {
budgetWords.push_back(DescriptorValue{{0x00000600u, 0x00fff000u, 0x06f03fa0u, i + 1u}, 4u});
}
std::vector<std::uint32_t> budgetCodes(MaxUnique + 2u);
for (std::uint32_t i = 0; i < MaxUnique; ++i) budgetCodes[i] = i;
budgetCodes[MaxUnique] = 0u;
budgetCodes[MaxUnique + 1u] = MaxUnique;
ShaderInfo budgetInfo{};
SamplerResource budgetSampler{};
budgetSampler.table = 0u;
budgetInfo.samplers.push_back(budgetSampler);
ResourceSnapshot budgetSnapshot{};
budgetSnapshot.tables.tables.resize(1u);
auto& budgetColumn = budgetSnapshot.tables.tables[0];
budgetColumn.records = static_cast<std::uint32_t>(budgetCodes.size());
budgetColumn.keys = static_cast<std::uint32_t>(budgetCodes.size());
budgetColumn.codes = SharedValues<std::uint32_t>(budgetCodes);
budgetColumn.words = SharedValues<DescriptorValue>(budgetWords);
const auto budgetResolved = ResourceMaterializer::ResolveImageTables(budgetInfo, budgetSnapshot);
require(budgetResolved.samplers.size() == Abi::SamplerElementBudget, "sampler table budget: valid unique samplers did not fill budget");
const auto budgetMapStart = budgetResolved.map.at(Abi::TableHeader(0) + Abi::TableMapStart);
const auto dupRecord = budgetResolved.map.at(budgetMapStart + MaxUnique);
const auto dupFlags = budgetResolved.map.at(dupRecord + Abi::SamplerRecordFlags);
require((dupFlags & Abi::PoisonFlag) == 0u, "sampler table budget: duplicate of existing sampler was poisoned when budget is full");
require(budgetResolved.map.at(dupRecord + Abi::SamplerRecordElement) == 0u, "sampler table budget: duplicate sampler did not reuse existing element");
const auto overflowRecord = budgetResolved.map.at(budgetMapStart + MaxUnique + 1u);
const auto overflowFlags = budgetResolved.map.at(overflowRecord + Abi::SamplerRecordFlags);
require((overflowFlags & Abi::PoisonFlag) != 0u, "sampler table budget: exceeding sampler budget did not poison record");
require(budgetResolved.poison.at(overflowFlags & ~Abi::PoisonFlag).reason == Abi::PoisonReason::Budget, "sampler table budget: poisoned reason is not Budget");
const auto outsideRecord = budgetResolved.map.at(Abi::TableHeader(0) + Abi::TableOutsideCode);
const auto outsideFlags = budgetResolved.map.at(outsideRecord + Abi::SamplerRecordFlags);
require((outsideFlags & Abi::PoisonFlag) != 0u, "sampler table budget: TableOutsideCode did not poison when budget is full");
require(budgetResolved.poison.at(outsideFlags & ~Abi::PoisonFlag).reason == Abi::PoisonReason::Budget, "sampler table budget: TableOutsideCode poisoned reason is not Budget");
}
void verifyDynamicStorageMipChain() {
using namespace ShaderRecompiler;
ImageResource image{};
@@ -7820,7 +8421,12 @@ int main(int argc, char** argv) {
verifySignedMinimum();
verifyMayEvaluate();
verifyPureFlatSlots();
verifyPlanConditionPruning();
verifyPlanRetention();
verifyPlanRetentionChains();
verifyPlanCopyAnswers();
verifyGuardedDescriptorSources();
verifyLaneMaskGuardedDescriptorSources();
verifySrtTape();
verifyImageTableKeys();
verifyImageTables();
@@ -7874,7 +8480,9 @@ int main(int argc, char** argv) {
verifyCubeDepthCompareMode();
verifyVolumeArrayLoadMode();
verifyArrayVolumeStoreMode();
verifyMultisamplePlaneReadMode();
verifyVolumeBiasTableRejection();
verifySamplerTableDeduplication();
verifyDynamicStorageMipChain();
verifySrgbDepthCompareModes();
verifyQuadMaskNeighbourReads();
core/libs/prx/libSceAgcDriver/tests/SpirvModule.cpp
+106
View File
@@ -1,5 +1,7 @@
#include "SpirvBackend/SpirvModule.hpp"
#include "SpirvBackend/SpirvSpecialization.hpp"
#include "Recompiler.hpp"
#include <algorithm>
#include <array>
#include <cstdio>
#include <exception>
@@ -298,6 +300,92 @@ void testIdsBeyondTheBoundAreRejected() {
check(rejected, "a module whose ids reach its bound was specialized");
}
void testPrivateCompositeTableLoadFolding() {
for (const auto storageClass : {spv::StorageClassPrivate, spv::StorageClassFunction}) {
for (const auto accessOp : {spv::OpAccessChain, spv::OpInBoundsAccessChain}) {
const auto build = [&](bool dynamicIndex, bool storeThroughChain) {
SpirvModule module;
module.EmitCapability(spv::CapabilityShader);
const auto typeVoid = module.Type(spv::OpTypeVoid);
const auto typeBool = module.Type(spv::OpTypeBool);
const auto typeU32 = module.Type(spv::OpTypeInt, 32u, 0u);
const auto falseVal = module.Constant(spv::OpConstantFalse, typeBool);
const auto zero = module.Constant(spv::OpConstant, typeU32, 0u);
const auto one = module.Constant(spv::OpConstant, typeU32, 1u);
const auto three = module.Constant(spv::OpConstant, typeU32, 3u);
const auto sixteen = module.Constant(spv::OpConstant, typeU32, 16u);
const auto twenty = module.Constant(spv::OpConstant, typeU32, 20u);
const auto thirty = module.Constant(spv::OpConstant, typeU32, 30u);
const auto typeArray = module.Type(spv::OpTypeArray, typeU32, three);
const auto initial = module.Constant(spv::OpConstantComposite, typeArray, std::vector<std::uint32_t>{sixteen, twenty, thirty});
const auto typeInputPtr = module.Type(spv::OpTypePointer, spv::StorageClassInput, typeU32);
const auto typeTablePtr = module.Type(spv::OpTypePointer, storageClass, typeArray);
const auto typeElementPtr = module.Type(spv::OpTypePointer, storageClass, typeU32);
const auto typePrivatePtr = module.Type(spv::OpTypePointer, spv::StorageClassPrivate, typeU32);
const auto typeFunction = module.Type(spv::OpTypeFunction, typeVoid);
const auto input = module.DefineGlobalVariable(typeInputPtr, spv::StorageClassInput);
const auto table = storageClass == spv::StorageClassPrivate
? module.DefineInitializedGlobalVariable(typeTablePtr, spv::StorageClassPrivate, initial)
: module.AllocateId();
const auto output = module.DefineGlobalVariable(typePrivatePtr, spv::StorageClassPrivate);
const auto mainId = module.AllocateId();
const auto lane = module.AllocateId();
const auto rawCond = module.AllocateId();
const auto cond = module.AllocateId();
const auto baseIndex = module.AllocateId();
const auto index = module.AllocateId();
const auto element = module.AllocateId();
const auto loaded = module.AllocateId();
const auto sum = module.AllocateId();
module.AddFunction(spv::OpFunction, typeVoid, mainId, spv::FunctionControlMaskNone, typeFunction);
module.AddFunction(spv::OpLabel, module.AllocateId());
if (storageClass == spv::StorageClassFunction) module.AddFunction(spv::OpVariable, typeTablePtr, table, spv::StorageClassFunction, initial);
module.AddFunction(spv::OpLoad, typeU32, lane, input);
if (storeThroughChain) {
const auto writePtr = module.AllocateId();
module.AddFunction(accessOp, typeElementPtr, writePtr, table, zero);
module.AddFunction(spv::OpStore, writePtr, lane);
}
module.AddFunction(spv::OpUGreaterThan, typeBool, rawCond, lane, zero);
module.AddFunction(spv::OpLogicalOr, typeBool, cond, rawCond, falseVal);
module.AddFunction(spv::OpIAdd, typeU32, baseIndex, one, one);
module.AddFunction(spv::OpSelect, typeU32, index, cond, baseIndex, baseIndex);
module.AddFunction(accessOp, typeElementPtr, element, table, dynamicIndex ? lane : index);
module.AddFunction(spv::OpLoad, typeU32, loaded, element);
module.AddFunction(spv::OpIAdd, typeU32, sum, lane, loaded);
module.AddFunction(spv::OpStore, output, sum);
module.AddFunction(spv::OpReturn);
module.AddFunction(spv::OpFunctionEnd);
return std::pair{ShaderRecompiler::SpecializeSpirv(module.Finalize()), thirty};
};
const auto [folded, expectedConstant] = build(false, false);
const auto foldedCounts = opcodeCounts(folded);
check(!foldedCounts.contains(accessOp), "a constant-index composite table access chain was not removed");
check(!foldedCounts.contains(spv::OpLogicalOr), "a right-false OpLogicalOr was not simplified");
check(!foldedCounts.contains(spv::OpSelect), "an identical-arm OpSelect was not simplified");
check(foldedCounts.contains(spv::OpLoad) && foldedCounts.at(spv::OpLoad) == 1u, "a constant-index composite table load was not folded");
bool addedThirty = false;
for (std::size_t cursor = 5; cursor < folded.size(); cursor += folded[cursor] >> 16u) {
if ((folded[cursor] >> 16u) == 0u) break;
if ((folded[cursor] & 0xffffu) == spv::OpIAdd && folded[cursor + 4u] == expectedConstant) addedThirty = true;
}
check(addedThirty, "a constant-index composite table load did not fold to its element constant");
const auto [dynamic, unusedDynamic] = build(true, false);
static_cast<void>(unusedDynamic);
const auto dynamicCounts = opcodeCounts(dynamic);
check(dynamicCounts.contains(accessOp) && dynamicCounts.at(accessOp) == 1u, "a dynamic-index composite table access chain was removed");
check(dynamicCounts.contains(spv::OpLoad) && dynamicCounts.at(spv::OpLoad) == 2u, "a dynamic-index composite table load was folded");
const auto [mutated, unusedMutated] = build(false, true);
static_cast<void>(unusedMutated);
const auto mutatedCounts = opcodeCounts(mutated);
check(mutatedCounts.contains(accessOp) && mutatedCounts.at(accessOp) == 2u, "a mutated composite table access chain was removed");
check(mutatedCounts.contains(spv::OpLoad) && mutatedCounts.at(spv::OpLoad) == 2u, "a mutated composite table load was folded despite a store through an access chain");
}
}
}
void testBlockBallotsOfOnePredicateShareOneBallot() {
SpirvModule module;
const auto typeBool = module.Type(spv::OpTypeBool);
@@ -356,10 +444,27 @@ void testBlockBallotsOfOnePredicateShareOneBallot() {
check(ballotSources.contains(afterLabel), "a ballot in a later block reused one from an earlier block");
}
void testInternedSpirvSharesEqualWords() {
const std::vector<std::uint32_t> words{spv::MagicNumber, 0x00010400u, 0u, 7u, 0u, (2u << 16u) | spv::OpCapability, spv::CapabilityShader};
auto changed = words;
changed.back() = spv::CapabilityInt64;
const auto first = InternSpirv(words);
const auto second = InternSpirv(words);
const auto other = InternSpirv(changed);
check(std::ranges::equal(first.Words(), words) && std::ranges::equal(other.Words(), changed), "interned SPIR-V changed its words");
check(first.data() == second.data(), "two equal specialized modules kept separate copies of their words");
check(first.data() != other.data(), "different SPIR-V was interned into one copy");
auto edited = second;
edited[6] = spv::CapabilityFloat64;
const auto again = InternSpirv(words);
check(edited[6] == spv::CapabilityFloat64 && first[6] == spv::CapabilityShader && again.data() == first.data(), "a write to one holder of interned SPIR-V changed the others");
}
int main() {
testFoldingLeavesNoDeadBlocksOrConstants();
testIdsBeyondTheBoundAreRejected();
testStructExtractsAreNotShuffled();
testPrivateCompositeTableLoadFolding();
testRepeatedTypesShareOneId();
testOperandShapesDoNotCollide();
testOperandKindsAgree();
@@ -371,6 +476,7 @@ int main() {
testIdsStayUniqueBeyondCacheCapacity();
testEmittedWordsAreStable();
testBlockBallotsOfOnePredicateShareOneBallot();
testInternedSpirvSharesEqualWords();
if (failures != 0) {
std::fprintf(stderr, "%d check(s) failed\n", failures);
return 1;
core/libs/prx/libSceAgcDriver/tests/SpirvOptimizer.cpp
+92
View File
@@ -396,6 +396,7 @@ std::optional<std::pair<std::uint32_t, std::uint32_t>> storedLaneFunction(const
switch (opcode) {
case spv::OpFunction:
case spv::OpLabel:
case spv::OpVariable:
case spv::OpReturn:
case spv::OpFunctionEnd:
break;
@@ -936,6 +937,95 @@ OpStore %slot2 %second
check(opcodeCount(acrossCall, spv::OpGroupNonUniformBallot) == 2, "a ballot was shared across a function call");
}
std::string privateTableModuleSource(const std::string& body, bool functionTable = false) {
const std::string storage = functionTable ? "Function" : "Private";
const std::string globalTable = functionTable ? "" : "%table = OpVariable %tableArray Private %initial\n";
const std::string localTable = functionTable ? "%table = OpVariable %tableArray Function %initial\n" : "";
return R"(OpCapability Shader
OpMemoryModel Logical GLSL450
OpEntryPoint GLCompute %main "main" %index
OpExecutionMode %main LocalSize 8 1 1
OpDecorate %index BuiltIn LocalInvocationIndex
%void = OpTypeVoid
%bool = OpTypeBool
%uint = OpTypeInt 32 0
%false = OpConstantFalse %bool
%zero = OpConstant %uint 0
%one = OpConstant %uint 1
%three = OpConstant %uint 3
%sixteen = OpConstant %uint 16
%twenty = OpConstant %uint 20
%thirty = OpConstant %uint 30
%array = OpTypeArray %uint %three
%initial = OpConstantComposite %array %sixteen %twenty %thirty
%input = OpTypePointer Input %uint
%tableArray = OpTypePointer )" + storage + R"( %array
%tableElement = OpTypePointer )" + storage + R"( %uint
%private = OpTypePointer Private %uint
%function = OpTypeFunction %void
%index = OpVariable %input Input
)" + globalTable + R"(%output = OpVariable %private Private
%main = OpFunction %void None %function
%entry = OpLabel
)" + localTable + R"(%lane = OpLoad %uint %index
)" + body + R"(OpReturn
OpFunctionEnd
)";
}
void testSpecializedPrivateCompositeTableLoad() {
for (const bool functionTable : {false, true}) {
for (const bool inBounds : {false, true}) {
const auto accessName = inBounds ? "OpInBoundsAccessChain" : "OpAccessChain";
const auto accessOp = inBounds ? spv::OpInBoundsAccessChain : spv::OpAccessChain;
const std::string label = std::string(accessName) + (functionTable ? " (Function)" : " (Private)");
try {
const auto folded = assemble(privateTableModuleSource(
"%rawCond = OpUGreaterThan %bool %lane %zero\n"
"%cond = OpLogicalOr %bool %rawCond %false\n"
"%baseOffset = OpIAdd %uint %one %one\n"
"%offset = OpSelect %uint %cond %baseOffset %baseOffset\n"
"%element = " + std::string(accessName) + " %tableElement %table %offset\n"
"%loaded = OpLoad %uint %element\n"
"%sum = OpIAdd %uint %lane %loaded\n"
"OpStore %output %sum\n", functionTable));
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(folded, Vulkan11, Spirv13, false, false));
const auto specializedFolded = ShaderRecompiler::SpecializeSpirv(folded);
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(specializedFolded, Vulkan11, Spirv13, false, false));
check(opcodeCount(specializedFolded, accessOp) == 0u, label + ": constant-index table access chain was not removed");
check(opcodeCount(specializedFolded, spv::OpLogicalOr) == 0u, label + ": right-false OpLogicalOr was not simplified");
check(opcodeCount(specializedFolded, spv::OpSelect) == 0u, label + ": identical-arm OpSelect was not simplified");
check(opcodeCount(specializedFolded, spv::OpLoad) == 1u, label + ": constant-index table load was not folded");
check(storedLaneFunction(specializedFolded) == std::optional<std::pair<std::uint32_t, std::uint32_t>>({1u, 30u}), label + ": constant-index table load folded to the wrong value");
const auto dynamic = assemble(privateTableModuleSource(
"%element = " + std::string(accessName) + " %tableElement %table %lane\n"
"%loaded = OpLoad %uint %element\n"
"OpStore %output %loaded\n", functionTable));
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(dynamic, Vulkan11, Spirv13, false, false));
const auto specializedDynamic = ShaderRecompiler::SpecializeSpirv(dynamic);
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(specializedDynamic, Vulkan11, Spirv13, false, false));
check(opcodeCount(specializedDynamic, accessOp) == 1u, label + ": dynamic-index table access chain was removed");
check(opcodeCount(specializedDynamic, spv::OpLoad) == 2u, label + ": dynamic-index table load was folded");
const auto mutated = assemble(privateTableModuleSource(
"%writeElement = " + std::string(accessName) + " %tableElement %table %zero\n"
"OpStore %writeElement %lane\n"
"%readElement = " + std::string(accessName) + " %tableElement %table %one\n"
"%loaded = OpLoad %uint %readElement\n"
"OpStore %output %loaded\n", functionTable));
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(mutated, Vulkan11, Spirv13, false, false));
const auto specializedMutated = ShaderRecompiler::SpecializeSpirv(mutated);
static_cast<void>(ShaderRecompiler::ValidateAndOptimizeSpirv(specializedMutated, Vulkan11, Spirv13, false, false));
check(opcodeCount(specializedMutated, accessOp) == 2u, label + ": mutated table access chain was removed");
check(opcodeCount(specializedMutated, spv::OpLoad) == 2u, label + ": mutated table load was folded despite a store through an access chain");
} catch (const std::exception& error) {
check(false, label + " table specialization: " + error.what());
}
}
}
}
}
int main() {
@@ -953,6 +1043,7 @@ int main() {
testFoldedConstantUsedByUndecodedInstructionIsKept();
testSharedConstantsAreNotCopiedForUndecodedUsers();
testSharedBooleanConstantsKeepTheirValues();
testSpecializedPrivateCompositeTableLoad();
testBallotsMergedIntoOneBlockShareOneBallot(optimizationEnabled);
if (failures != 0) {
std::fprintf(stderr, "%d optimizer check(s) failed\n", failures);
@@ -961,3 +1052,4 @@ int main() {
std::puts("SPIR-V optimizer checks passed");
return 0;
}
core/libs/prx/libSceAgcDriver/tests/SubgroupBarrier.cpp
+56 -6
View File
@@ -5,10 +5,12 @@
#include <cstdint>
#include <cstdio>
#include <exception>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
namespace {
@@ -34,6 +36,10 @@ constexpr std::array<std::uint32_t, 19> BranchedLdsStep{
0xbf8cc07fu, 0xbf8a0000u, 0xbf810000u,
};
constexpr std::array<std::uint32_t, 5> LdsHalfStore{0x34040082u, 0xd87c0400u, 0x00000102u, 0xbf8cc07fu, 0xbf810000u};
constexpr std::array<std::uint32_t, 5> LdsAtomicAdd{0x34040082u, 0xd8000400u, 0x00000102u, 0xbf8cc07fu, 0xbf810000u};
constexpr std::array<std::uint32_t, 5> LdsAtomicMax{0x34040082u, 0xd84c0400u, 0x00000102u, 0xbf8cc07fu, 0xbf810000u};
struct Barriers {
std::uint32_t subgroupControl = 0;
std::uint32_t subgroupMemory = 0;
@@ -43,14 +49,19 @@ void require(bool condition, const std::string& message) {
if (!condition) throw std::runtime_error(message);
}
Barriers recompile(std::uint32_t threads, bool waveLdsMemoryBarriers) {
struct MemoryBarriers {
std::uint32_t workgroup = 0;
std::uint32_t device = 0;
};
ShaderRecompiler::RecompileResult recompileProgram(std::span<const std::uint32_t> code, std::uint32_t threads, bool waveLdsMemoryBarriers, bool deviceScopeLdsMemoryBarriers) {
using namespace ShaderRecompiler;
const std::array<std::uint32_t, 2> capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform};
const std::array<std::string_view, 1> extensions{"SPV_KHR_storage_buffer_storage_class"};
RecompileRequest request{};
request.shader = {ShaderStage::Compute, 0x20000u, BranchedLdsStep, 0, {}};
request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}};
request.context.waveSize = 32;
request.context.compute = ShaderComputeStageInfo{{threads, 1u, 1u}, 256u, {false, false, false}, false, 1u};
request.context.compute = ShaderComputeStageInfo{{threads, 1u, 1u}, 512u, {false, false, false}, false, 1u};
request.target.vulkanVersion = 0x00403000u;
request.target.spirvVersion = 0x00010600u;
request.target.subgroupSize = 32;
@@ -61,15 +72,42 @@ Barriers recompile(std::uint32_t threads, bool waveLdsMemoryBarriers) {
request.target.maxWorkgroupInvocations = 1024;
request.target.maxWorkgroupSharedMemoryBytes = 49152;
request.target.waveLdsMemoryBarriers = waveLdsMemoryBarriers;
request.target.deviceScopeLdsMemoryBarriers = deviceScopeLdsMemoryBarriers;
request.layout = {0, 0, 0, 128};
request.useCache = false;
const auto result = Recompile(request);
const auto& words = result.spirv.Words();
return Recompile(request);
}
std::unordered_map<std::uint32_t, std::uint32_t> constantsOf(std::span<const std::uint32_t> words) {
std::unordered_map<std::uint32_t, std::uint32_t> constants;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
if (static_cast<spv::Op>(words[at] & 0xffffu) == spv::OpConstant && (words[at] >> 16u) == 4u) constants[words[at + 2]] = words[at + 3];
}
return constants;
}
MemoryBarriers memoryBarriers(std::span<const std::uint32_t> code, bool deviceScopeLdsMemoryBarriers) {
const auto result = recompileProgram(code, 128, false, deviceScopeLdsMemoryBarriers);
const auto& words = result.spirv.Words();
const auto constants = constantsOf(words);
MemoryBarriers barriers;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
if (static_cast<spv::Op>(words[at] & 0xffffu) != spv::OpMemoryBarrier) continue;
const auto scope = constants.find(words[at + 1]);
require(scope != constants.end(), "a memory barrier scope is not a 32-bit constant");
if (scope->second == spv::ScopeWorkgroup) ++barriers.workgroup;
if (scope->second == spv::ScopeDevice) ++barriers.device;
}
return barriers;
}
Barriers recompile(std::uint32_t threads, bool waveLdsMemoryBarriers) {
const auto result = recompileProgram(BranchedLdsStep, threads, waveLdsMemoryBarriers, false);
const auto& words = result.spirv.Words();
const auto constants = constantsOf(words);
Barriers barriers;
for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) {
const auto op = static_cast<spv::Op>(words[at] & 0xffffu);
if (op == spv::OpConstant && (words[at] >> 16u) == 4u) constants[words[at + 2]] = words[at + 3];
if (op != spv::OpControlBarrier && op != spv::OpMemoryBarrier) continue;
const auto scope = constants.find(words[at + 1]);
require(scope != constants.end(), "a barrier scope is not a 32-bit constant");
@@ -83,6 +121,17 @@ Barriers recompile(std::uint32_t threads, bool waveLdsMemoryBarriers) {
void verifyDriverQuirk() {
for (const auto& item : Cases) {
require(AgcDriver::SubgroupControlBarrierUsable(item.driver) == item.usable, "driver " + std::to_string(static_cast<int>(item.driver)) + ": expected subgroup control barriers to be " + (item.usable ? "usable" : "unusable"));
require(AgcDriver::WorkgroupScopeFenceUsable(item.driver) == item.usable, "driver " + std::to_string(static_cast<int>(item.driver)) + ": expected workgroup-scope fences to be " + (item.usable ? "usable" : "unusable"));
}
}
void verifyLdsMemoryBarrierScope() {
const std::array<std::pair<const char*, std::span<const std::uint32_t>>, 3> programs{{{"ds_write_b16", LdsHalfStore}, {"ds_add_u32", LdsAtomicAdd}, {"ds_max_f32", LdsAtomicMax}}};
for (const auto& [name, code] : programs) {
const auto workgroup = memoryBarriers(code, false);
require(workgroup.workgroup == 1 && workgroup.device == 0, std::string(name) + " did not order its LDS update with one workgroup-scope OpMemoryBarrier");
const auto device = memoryBarriers(code, true);
require(device.workgroup == 0 && device.device == 1, std::string(name) + " with device-scope LDS memory barriers did not order its LDS update with one device-scope OpMemoryBarrier");
}
}
@@ -101,6 +150,7 @@ int main() {
try {
verifyDriverQuirk();
verifyEmittedBarriers();
verifyLdsMemoryBarrierScope();
std::puts("subgroup barrier tests passed");
return 0;
} catch (const std::exception& error) {
core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp
+18
View File
@@ -63,6 +63,7 @@ void srgbTwinTests() {
Require(SrgbTwinFormat(context, VK_FORMAT_R8_UNORM) == VK_FORMAT_UNDEFINED, "a device that decodes 8_SRGB in the shader must not give R8 an sRGB twin");
Require(SrgbTwinFormat(context, VK_FORMAT_R8G8B8A8_UNORM) == VK_FORMAT_R8G8B8A8_SRGB && SrgbTwinFormat(context, VK_FORMAT_BC7_UNORM_BLOCK) == VK_FORMAT_BC7_SRGB_BLOCK, "formats without a shader decode must keep their sRGB twins");
Require(SrgbTwinFormat(context, VK_FORMAT_R32_SFLOAT) == VK_FORMAT_UNDEFINED, "formats without an sRGB counterpart have no twin");
Require(SrgbTwinFormat(context, VK_FORMAT_R8G8B8A8_SNORM) == VK_FORMAT_R8G8B8A8_SRGB, "8_8_8_8 SNORM must read its bytes through the sRGB twin");
}
template<typename TAction>
@@ -223,6 +224,22 @@ void packedFloatDccClearTests() {
Require(TextureClearKeys(resource, 1024) == DccKeys::Clear1111, "packed float texture discarded its white fast-clear keys");
}
void fillContractTests() {
std::array<std::uint32_t, 4> texels;
texels.fill(0xdeadbeefu);
auto span = std::as_writable_bytes(std::span(texels));
Require(!FillDccClear(VK_FORMAT_R8G8B8A8_UNORM, DccKeys::ClearRegister, false, span), "a register clear encoded a texel");
for (const auto texel : texels) Require(texel == 0u, "a declined clear left its span untouched");
texels.fill(0xdeadbeefu);
Require(!FillDccClear(VK_FORMAT_R8G8B8A8_UNORM, DccKeys::Uncompressed, false, span), "an uncompressed code encoded a texel");
for (const auto texel : texels) Require(texel == 0u, "a declined clear left its span untouched");
std::array<std::byte, 10> partial;
partial.fill(std::byte{0x5a});
Require(FillDccClear(VK_FORMAT_B10G11R11_UFLOAT_PACK32, DccKeys::Clear1111, false, partial), "packed float fast clear was not encoded");
const std::array<std::byte, 4> white{std::byte{0xc0}, std::byte{0x03}, std::byte{0x1e}, std::byte{0x78}};
for (std::size_t index = 0; index < partial.size(); ++index) Require(partial[index] == (index < 8u ? white[index % 4u] : std::byte{0}), "a clear fill over a partial trailing element did not zero the tail");
}
}
void RunTextureFormatTests() {
@@ -287,5 +304,6 @@ void RunTextureFormatTests() {
convertedDccClearTests();
singleDccKeysTests();
packedFloatDccClearTests();
fillContractTests();
gcnBufferFormatTests();
}
core/libs/prx/libSceAgcDriver/tests/VectorWindow.cpp
+36
View File
@@ -40,6 +40,13 @@ constexpr std::uint32_t SampleV8V2S0S8 = 0x00400802u;
constexpr std::uint32_t AddV8V1V8 = 0x4a101101u;
constexpr std::uint32_t BufferStoreDword = 0xe0700000u;
constexpr std::uint32_t StoreV8WithS12 = 0x80030800u;
constexpr std::uint32_t AndM0GroupXOne = 0x877c8104u;
constexpr std::uint32_t MovreldV2V0 = 0x7e048500u;
constexpr std::uint32_t MoveS16Two = 0xbe900382u;
constexpr std::uint32_t BranchForwardTwo = 0xbf820002u;
constexpr std::uint32_t SubtractS16One = 0x80908110u;
constexpr std::uint32_t CompareS16NotZero = 0xbf078010u;
constexpr std::uint32_t BranchBackFourOnScc = 0xbf85fffcu;
struct Shape {
std::size_t functionArrays = 0;
@@ -306,6 +313,35 @@ int main(int argc, char** argv) {
return shape.functionArrays == 0 && shape.accessChains == 0;
});
passed &= run("a sampler dword spilled into a window register with ordinary writes keeps its definition past a relative write", [] {
const std::array<std::uint32_t, 16> userData{};
const std::array<std::uint32_t, 14> spilled{AndM0GroupXOne, MoveV2Zero, MoveV3Zero, MoveV4One, WritelaneV4, FromS8Lane0, MovreldV2V0, ReadlaneS8, FromV4Lane0, ImageSampleLz2d, SampleV8V2S0S8, BufferStoreDword, StoreV8WithS12, EndProgram};
return Scan(RecompileProgram(spilled, userData)).functionArrays == 1;
});
passed &= run("a sampler dword spilled into a window register and overwritten by an ordinary write before its v_readlane stays a run-time value", [] {
const std::array<std::uint32_t, 16> userData{};
const std::array<std::uint32_t, 14> overwritten{AndM0GroupXOne, MoveV2Zero, MoveV3Zero, WritelaneV4, FromS8Lane0, MovreldV2V0, MoveV4One, ReadlaneS8, FromV4Lane0, ImageSampleLz2d, SampleV8V2S0S8, BufferStoreDword, StoreV8WithS12, EndProgram};
try {
RecompileProgram(overwritten, userData);
} catch (const std::exception& error) {
return std::string_view(error.what()).find("GetSamplerResource dword 0 is not a valid runtime value; chain: ReadLane(LoadVectorWindow(Immediate), Immediate)") != std::string_view::npos;
}
return false;
});
passed &= run("a sampler dword spilled into a window register before a loop entered at its condition keeps its definition past the loop's relative write", [] {
const std::array<std::uint32_t, 16> userData{};
const std::array<std::uint32_t, 18> code{AndM0GroupXOne, MoveV2Zero, MoveV3Zero, WritelaneV4, FromS8Lane0, MoveS16Two, BranchForwardTwo, MovreldV2V0, SubtractS16One, CompareS16NotZero, BranchBackFourOnScc, ReadlaneS8, FromV4Lane0, ImageSampleLz2d, SampleV8V2S0S8, BufferStoreDword, StoreV8WithS12, EndProgram};
return Scan(RecompileProgram(code, userData)).functionArrays == 1;
});
passed &= run("a sampler dword spilled into a window register with ordinary writes before a loop entered at its condition keeps its definition past the loop's relative write", [] {
const std::array<std::uint32_t, 16> userData{};
const std::array<std::uint32_t, 19> code{AndM0GroupXOne, MoveV2Zero, MoveV3Zero, MoveV4One, WritelaneV4, FromS8Lane0, MoveS16Two, BranchForwardTwo, MovreldV2V0, SubtractS16One, CompareS16NotZero, BranchBackFourOnScc, ReadlaneS8, FromV4Lane0, ImageSampleLz2d, SampleV8V2S0S8, BufferStoreDword, StoreV8WithS12, EndProgram};
return Scan(RecompileProgram(code, userData)).functionArrays == 1;
});
passed &= run("a sampler dword spilled into a relatively read register keeps its definition through v_readlane", [] {
const std::array<std::uint32_t, 14> code{MoveM0GroupX, MoveV2Zero, MoveV3Zero, WritelaneV4, FromS8Lane0, MovrelsV1V4, ReadlaneS8, FromV4Lane0, ImageSampleLz2d, SampleV8V2S0S8, AddV8V1V8, BufferStoreDword, StoreV8WithS12, EndProgram};
const std::array<std::uint32_t, 16> userData{};
core/libs/prx/libSceAgcDriver/tests/execution/BcBlockCopy.cpp
+14 -6
View File
@@ -35,6 +35,7 @@ constexpr std::uint32_t Format32x2Uint = 62;
constexpr std::uint32_t Format32x4Uint = 75;
constexpr std::uint32_t FormatBc1Unorm = 169;
constexpr std::uint32_t FormatBc7Unorm = 181;
constexpr std::uint32_t ForceDegamma = 1u << 20u;
constexpr std::uint32_t Type2D = 9;
constexpr std::uint32_t ClampEdge = 2;
constexpr std::size_t BlockBytes = 1u << 20u;
@@ -130,8 +131,8 @@ std::array<std::uint32_t, 8> TextureDescriptor(const void* data, const Shape& sh
};
}
std::array<std::uint32_t, 4> SamplerDescriptor() {
return {ClampEdge | (ClampEdge << 3u) | (ClampEdge << 6u), 0u, 0u, 0u};
std::array<std::uint32_t, 4> SamplerDescriptor(std::uint32_t forced) {
return {ClampEdge | (ClampEdge << 3u) | (ClampEdge << 6u) | forced, 0u, 0u, 0u};
}
ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData) {
@@ -167,12 +168,12 @@ void Settle(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::uint64_t
AgcDriver::GuestMemory::CollectWritesUncached(reinterpret_cast<std::uintptr_t>(texels), bytes);
}
std::vector<std::uint32_t> Sample(AgcDriver::VulkanDevice& device, std::uint8_t* texels, const Shape& shape) {
std::vector<std::uint32_t> Sample(AgcDriver::VulkanDevice& device, std::uint8_t* texels, const Shape& shape, std::uint32_t forced = 0u) {
std::vector<std::uint32_t> userData(24, 0u);
const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size_bytes()));
const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size_bytes()));
const auto texture = TextureDescriptor(texels, shape);
const auto sampler = SamplerDescriptor();
const auto sampler = SamplerDescriptor(forced);
std::copy(input.begin(), input.end(), userData.begin());
std::copy(output.begin(), output.end(), userData.begin() + 4);
std::copy(texture.begin(), texture.end(), userData.begin() + 8);
@@ -189,12 +190,12 @@ std::vector<std::uint32_t> Sample(AgcDriver::VulkanDevice& device, std::uint8_t*
return {Output.begin(), Output.end()};
}
std::vector<std::uint32_t> Reference(AgcDriver::VulkanDevice& device, const Case& test, std::uint32_t value) {
std::vector<std::uint32_t> Reference(AgcDriver::VulkanDevice& device, const Case& test, std::uint32_t value, std::uint32_t forced = 0u) {
GuestBlock block;
auto* texels = block.Data();
Store(device, texels, test, value);
Settle(device, texels, SurfaceBytes(texels, test.blocks));
return Sample(device, texels, test.compressed);
return Sample(device, texels, test.compressed, forced);
}
void RequireSame(const std::vector<std::uint32_t>& sampled, const std::vector<std::uint32_t>& expected, const std::string& what) {
@@ -208,7 +209,11 @@ void PendingBlocksCase(AgcDriver::VulkanDevice& device, const Case& test) {
constexpr std::uint32_t Second = 0x9bdf6840u;
const auto first = Reference(device, test, First);
const auto second = Reference(device, test, Second);
const auto forcedFirst = Reference(device, test, First, ForceDegamma);
const auto forcedSecond = Reference(device, test, Second, ForceDegamma);
Require(first != second, test.name + ": the two block patterns sample alike");
Require(forcedFirst != first && forcedSecond != second, test.name + ": a sampler that forces sRGB decoding samples the stored blocks like a plain one");
Require(forcedFirst != forcedSecond, test.name + ": the two block patterns sample alike through a sampler that forces sRGB decoding");
GuestBlock block;
auto* texels = block.Data();
const auto bytes = SurfaceBytes(texels, test.compressed);
@@ -222,6 +227,9 @@ void PendingBlocksCase(AgcDriver::VulkanDevice& device, const Case& test) {
RequireSame(pending, second, test.name + ": sample of pending blocks");
Store(device, texels, test, First);
RequireSame(Sample(device, texels, test.compressed), first, test.name + ": sample of blocks stored again");
Store(device, texels, test, Second);
RequireSame(Sample(device, texels, test.compressed, ForceDegamma), forcedSecond, test.name + ": sRGB-forced sample of pending blocks");
Require(AgcDriver::Graphics::StorageTexture::FindPending(reinterpret_cast<std::uintptr_t>(texels), bytes) != nullptr, test.name + ": the sRGB-forced sample stored the pending blocks to guest memory");
}
}
core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp
+1
View File
@@ -149,6 +149,7 @@ void Run(AgcDriver::VulkanDevice& device) {
{0, 0, 0, 128}
};
request.useCache = false;
request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false};
const auto result = ShaderRecompiler::Recompile(request);
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
device.WaitIdle();
core/libs/prx/libSceAgcDriver/tests/execution/BufferFormat.cpp
+93
View File
@@ -171,6 +171,96 @@ void Check() {
}
}
constexpr std::array<std::uint32_t, 5> SubwordWord3{0x31005facu, 0x31006facu, 0x3100bfacu, 0x3100cfacu, 0x31012facu};
constexpr const char* SubwordNames[16] = {
"buffer_load_format_x 8_UINT +1",
"buffer_load_format_x 8_UINT +2",
"buffer_load_format_x 8_UINT +3",
"buffer_load_format_x 8_SINT +1",
"buffer_load_format_x 8_SINT +2",
"buffer_load_format_x 8_SINT +3",
"buffer_load_format_x 16_UINT +1",
"buffer_load_format_x 16_UINT +2",
"buffer_load_format_x 16_UINT +3",
"buffer_load_format_x 16_SINT +1",
"buffer_load_format_x 16_SINT +2",
"buffer_load_format_x 16_SINT +3",
"buffer_load_format_xy 8_8_UINT +1 x",
"buffer_load_format_xy 8_8_UINT +1 y",
"buffer_load_format_xy 8_8_UINT +2 x",
"buffer_load_format_xy 8_8_UINT +2 y",
};
std::vector<std::uint32_t> SubwordCode() {
std::vector<std::uint32_t> code{0x34060087u, 0x34500085u};
std::uint32_t vdata = 10u;
const auto emitLoad = [&](std::uint32_t op, std::uint32_t srsrc, std::uint32_t offset, std::uint32_t words) {
code.push_back(0xe0001000u | (op << 18u) | offset);
code.push_back(0x80000028u | (srsrc << 16u) | (vdata << 8u));
vdata += words;
};
for (std::uint32_t offset = 1u; offset <= 3u; ++offset) emitLoad(0u, 2u, offset, 1u);
for (std::uint32_t offset = 1u; offset <= 3u; ++offset) emitLoad(0u, 3u, offset, 1u);
for (std::uint32_t offset = 1u; offset <= 3u; ++offset) emitLoad(0u, 4u, offset, 1u);
for (std::uint32_t offset = 1u; offset <= 3u; ++offset) emitLoad(0u, 5u, offset, 1u);
for (std::uint32_t offset = 1u; offset <= 2u; ++offset) emitLoad(1u, 6u, offset, 2u);
code.push_back(0xbf8c3f70u);
for (std::uint32_t index = 0u; index < 16u; ++index) {
code.push_back(0xe0701000u | (index * 4u));
code.push_back(0x80010003u | ((10u + index) << 8u));
}
code.push_back(0xbf810000u);
return code;
}
void RunSubwordOffsets(AgcDriver::VulkanDevice& device) {
for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]);
Output.fill(0xdeadbeefu);
std::vector<std::uint32_t> userData(8u + 4u * SubwordWord3.size(), 0u);
const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u), 0x31016facu);
std::copy(output.begin(), output.end(), userData.begin() + 4);
for (std::size_t index = 0; index < SubwordWord3.size(); ++index) {
const auto descriptor = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u), SubwordWord3[index]);
std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 8 + 4 * index);
}
static const auto program = SubwordCode();
const std::span<const std::uint32_t> code(program);
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
const auto result = ShaderRecompiler::Recompile(request);
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
device.WaitIdle();
}
void CheckSubwordOffsets() {
for (std::uint32_t tid = 0; tid < Threads; ++tid) {
const std::uint32_t word = Rows[tid][0];
const auto u8 = [&](std::uint32_t offset) { return (word >> (offset * 8u)) & 0xffu; };
const auto s8 = [&](std::uint32_t offset) { return static_cast<std::uint32_t>(static_cast<std::int32_t>(static_cast<std::int8_t>(u8(offset)))); };
const auto u16 = [&](std::uint32_t offset) { return (word >> (offset * 8u)) & 0xffffu; };
const auto s16 = [&](std::uint32_t offset) { return static_cast<std::uint32_t>(static_cast<std::int32_t>(static_cast<std::int16_t>(u16(offset)))); };
const std::array<std::uint32_t, 16> expected{
u8(1u), u8(2u), u8(3u),
s8(1u), s8(2u), s8(3u),
u16(1u), u16(2u), u16(3u),
s16(1u), s16(2u), s16(3u),
u8(1u), u8(2u),
u8(2u), u8(3u),
};
for (std::uint32_t index = 0; index < expected.size(); ++index) {
Expect(tid, Output[tid * Results + index], expected[index], SubwordNames[index]);
}
}
}
}
int main() {
@@ -179,6 +269,8 @@ int main() {
if (!device) return VulkanTestSkipped;
Run(*device);
Check();
RunSubwordOffsets(*device);
CheckSubwordOffsets();
std::puts("buffer format tests passed");
return 0;
} catch (const std::exception& error) {
@@ -186,3 +278,4 @@ int main() {
return 1;
}
}
core/libs/prx/libSceAgcDriver/tests/execution/ComputeFsTableCalls.cpp
+6
View File
@@ -1,6 +1,7 @@
#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include "SceShaders.hpp"
#include "VulkanTestDevice.hpp"
#include <array>
@@ -21,6 +22,8 @@ constexpr std::uint32_t Threads = 64;
constexpr std::uint32_t Untouched = 0xcafebabeu;
constexpr std::uint32_t Marker = 0x55u;
constexpr std::uint64_t FsTable = 0xfe0040000ull;
constexpr std::uint64_t DriverMemory = 0xfe0000000ull;
constexpr std::size_t DriverMemoryBytes = 2u << 20u;
alignas(256) constexpr std::array<std::uint32_t, 32> Caller{
0x34040082u, 0x7e0202ffu, 0x00000055u, 0xbeea03ffu, 0xe0040000u, 0xbeeb038fu, 0xf4080235u, 0xfa000000u, 0xbf8cc07fu, 0xf4240384u,
@@ -84,6 +87,9 @@ void CheckTable() {
Require(AgcDriver::GuestMemory::Accessible(reinterpret_cast<const void*>(table[2]), sizeof(routine)), "the FS routine is not readable");
std::memcpy(routine.data(), reinterpret_cast<const void*>(table[2]), sizeof(routine));
Require(routine[0] == 0xbe80200eu, "the FS routine does not start with s_setpc_b64 s[14:15]");
#ifndef _WIN32
if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) Require(AgcDriver::GuestMemory::Watched(DriverMemory, DriverMemoryBytes), "the AGC driver memory at 0xfe0000000 is not write-watched, so a refused import copies it on every address-based build");
#endif
}
}
core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp
+12 -2
View File
@@ -3,6 +3,7 @@
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "SceShaders.hpp"
#include "VulkanTestDevice.hpp"
#include <algorithm>
#include <array>
#include <cstdint>
#include <cstdio>
@@ -32,10 +33,14 @@ alignas(256) constexpr std::array<std::uint32_t, 64> HoistedCaller{
0x7e020280u, 0xf4080400u, 0xfa000000u, 0xbf8cc07fu, 0xbf068203u, 0xbf850001u, 0x7e020280u, 0x8f148402u,
0xf4280208u, 0x28000000u, 0xbf8cc07fu, 0xbe8e0308u, 0xbe8f0309u, 0xbf068103u, 0xbe8e210eu, 0x34040082u,
0xe0701000u, 0x80010102u, 0xbf810000u, 0xbf9f0000u, 0x30306c73u, 0x00000040u, 0x00000008u};
alignas(256) constexpr std::array<std::uint32_t, 64> SliceCaller{
0x7e020280u, 0xf40c0600u, 0xfa000010u, 0x8f148402u, 0xbf8cc07fu, 0xf428020eu, 0x28000000u, 0xbf8cc07fu,
0xbe8e0308u, 0xbe8f0309u, 0xbf068103u, 0xbe8e210eu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u,
0xbf9f0000u, 0x30306c73u, 0x00000040u, 0x00000008u};
alignas(256) constexpr std::array<std::uint32_t, 3> SelectSixteen{0x85159091u, 0x7e020215u, 0xbe80200eu};
alignas(256) constexpr std::array<std::uint32_t, 3> SelectThirtyTwo{0x8515a0a1u, 0x7e020215u, 0xbe80200eu};
alignas(16) std::array<std::uint32_t, 8> Table{};
alignas(16) std::array<std::uint32_t, 4> Descriptors{};
alignas(16) std::array<std::uint32_t, 12> Descriptors{};
alignas(256) std::array<std::uint32_t, Threads> Output{};
void Fill() {
@@ -43,7 +48,10 @@ void Fill() {
const auto second = reinterpret_cast<std::uintptr_t>(SelectThirtyTwo.data());
Table = {static_cast<std::uint32_t>(first), static_cast<std::uint32_t>(first >> 32u), 0u, 0u, static_cast<std::uint32_t>(second), static_cast<std::uint32_t>(second >> 32u), 0u, 0u};
const auto table = reinterpret_cast<std::uintptr_t>(Table.data());
Descriptors = {static_cast<std::uint32_t>(table), static_cast<std::uint32_t>((table >> 32u) & 0xffffu) | (16u << 16u), 2u, 0x31016facu};
const std::array<std::uint32_t, 4> descriptor{static_cast<std::uint32_t>(table), static_cast<std::uint32_t>((table >> 32u) & 0xffffu) | (16u << 16u), 2u, 0x31016facu};
Descriptors = {};
std::copy(descriptor.begin(), descriptor.end(), Descriptors.begin());
std::copy(descriptor.begin(), descriptor.end(), Descriptors.begin() + 8);
}
std::array<std::uint32_t, 8> UserData(std::uint32_t index, bool scc) {
@@ -149,6 +157,8 @@ int main() {
Check(1u, true, wave32, 33u, "the second record's callee with SCC 1");
Check(0u, false, wave32, 16u, "the first record's callee through a descriptor loaded before a branch", HoistedCaller.data());
Check(1u, true, wave32, 33u, "the second record's callee through a descriptor loaded before a branch", HoistedCaller.data());
Check(0u, false, wave32, 16u, "the first record's callee through the second half of an s_load_dwordx8", SliceCaller.data());
Check(1u, true, wave32, 33u, "the second record's callee through the second half of an s_load_dwordx8", SliceCaller.data());
const auto log = Dispatch(2u, false, wave32);
Require(log.find("called address 0x0 at program counter") != std::string::npos && log.find("which no linked callee of that call matches") != std::string::npos,
std::string("a record past the table did not report an unlinked call") + (wave32 ? " (wave32)" : " (wave64)") + ": " + log);
core/libs/prx/libSceAgcDriver/tests/execution/CopiedWriterLabelOrder.cpp
+270
View File
@@ -0,0 +1,270 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "Recompiler.hpp"
#include "VulkanTestDevice.hpp"
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <mutex>
#include <span>
#include <string>
#include <thread>
#include <vector>
namespace {
using AgcDriver::Graphics::Recorder;
using AgcDriver::Graphics::Require;
constexpr std::uint32_t Threads = 32;
constexpr std::size_t BlockBytes = 65536;
constexpr std::uint32_t RecordBytes = 16;
constexpr std::size_t StoredOffset = 8;
constexpr std::size_t CopiedRecordOffset = 0x1004;
constexpr std::size_t DirectRecordOffset = 0x1000;
constexpr std::uint32_t CpuValue = 0xc0001000u;
constexpr std::uint32_t GpuValue = 1;
constexpr std::uint32_t LabelValue = 1;
alignas(256) constexpr std::array<std::uint32_t, 6> StoreCode{
0x7e000281u, 0x7e020288u, 0xe0701000u, 0x80000001u, 0xbf810000u, 0xbf810000u,
};
class GuestBlock {
public:
GuestBlock() {
#ifdef _WIN32
block = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
#else
block = static_cast<std::uint8_t*>(std::aligned_alloc(BlockBytes, BlockBytes));
#endif
Require(block != nullptr, "copied writer label order: cannot allocate a guest block");
std::memset(block, 0, BlockBytes);
GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true);
}
~GuestBlock() {
GuestAllocations::Mutation().Remove(block);
#ifdef _WIN32
VirtualFree(block, 0, MEM_RELEASE);
#else
std::free(block);
#endif
}
GuestBlock(const GuestBlock&) = delete;
GuestBlock& operator=(const GuestBlock&) = delete;
volatile std::uint32_t* Word(std::size_t offset) const { return reinterpret_cast<volatile std::uint32_t*>(block + offset); }
std::uint64_t Address() const { return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(block)); }
private:
std::uint8_t* block = nullptr;
};
std::string Hex(std::uint32_t value) {
char text[16];
std::snprintf(text, sizeof(text), "0x%08x", value);
return text;
}
void RecordDispatch(AgcDriver::VulkanDevice& device, std::uint64_t record) {
std::vector<std::uint32_t> userData(16, 0u);
const std::array<std::uint32_t, 4> descriptor{static_cast<std::uint32_t>(record), static_cast<std::uint32_t>((record >> 32u) & 0xffffu), RecordBytes, 0x31016facu};
std::copy(descriptor.begin(), descriptor.end(), userData.begin());
const std::span<const std::uint32_t> code(StoreCode);
const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1};
ShaderRecompiler::RecompileRequest request{
{ShaderRecompiler::ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
{32, 0, userData, compute, std::nullopt, std::nullopt, memory},
device.Target(),
{0, 0, 0, 128}
};
request.useCache = false;
const auto result = ShaderRecompiler::Recompile(request);
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
}
bool Drains() {
return std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr;
}
VkDeviceSize StorageAlignment(AgcDriver::VulkanDevice& device) {
return device.graphicsContext().limits.minStorageBufferOffsetAlignment;
}
std::array<std::byte, 4> LabelBytes(std::uint32_t value) {
std::array<std::byte, 4> bytes{};
std::memcpy(bytes.data(), &value, sizeof(value));
return bytes;
}
void WaitForGpu(AgcDriver::VulkanDevice& device, std::uint64_t epoch) {
if (device.CanWaitUnlocked()) device.WaitRecorded(epoch);
else std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
std::uint64_t SubmitLocked(AgcDriver::VulkanDevice& device) {
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
return device.SubmitAndEpoch();
}
int LabelPass(AgcDriver::VulkanDevice& device, const GuestBlock& records, std::size_t recordOffset, const GuestBlock& labels, std::size_t labelOffset, std::uint64_t stamp, std::uint64_t writeBacks, int expectedReason, const std::string& what) {
volatile std::uint32_t* stored = records.Word(recordOffset + StoredOffset);
volatile std::uint32_t* label = labels.Word(labelOffset);
*stored = CpuValue;
*label = 0;
RecordDispatch(device, records.Address() + recordOffset);
const auto pending = Recorder::PendingWriteBackCompletions();
if (writeBacks != 0 && pending == 0 && recordOffset % StorageAlignment(device) == 0) {
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.WaitIdle();
return VulkanTestSkipped;
}
Require(pending == writeBacks, what + ": " + std::to_string(pending) + " write-back completions are pending after the dispatch, expected " + std::to_string(writeBacks));
const auto bytes = LabelBytes(LabelValue);
int reason = 0;
std::uint64_t epoch = 0;
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
reason = device.WriteLabelOnGpu(labels.Address() + labelOffset, bytes, stamp, 0, false);
if (reason == 3) {
device.WaitIdle();
return 1;
}
Require(reason == expectedReason, what + ": the label after the dispatch was handled with reason " + std::to_string(reason) + ", expected " + std::to_string(expectedReason));
if (reason >= 1 && reason <= 4) {
if (reason != 1) device.WaitIdle();
AgcDriver::GuestMemory::Write(labels.Address() + labelOffset, bytes, 4);
} else if (reason == 5) {
Require(Recorder::PendingWriteBackCompletions() == 1 && Recorder::PendingCompletionStores() == 1, what + ": the label behind the write-back drained the device or is not a pending completion store");
const std::array<std::byte, 16> wide{};
Require(device.WriteLabelOnGpu(labels.Address() + labelOffset + 0x100, wide, stamp, 0, false) == 0, what + ": a 16-byte store away from the copied writer and the pending completion store was not recorded on the GPU");
epoch = device.SubmitAndEpoch();
} else {
device.SubmitRecorded(true);
}
}
if (reason == 5) {
WaitForGpu(device, epoch);
Require(*label == 0, what + ": the label is visible before the batch that writes back the dispatch's record was reaped");
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.ReapRecorded(epoch);
Require(Recorder::PendingCompletionStores() == 0 && Recorder::PendingWriteBackCompletions() == 0, what + ": completions outlived the reap of their batch");
}
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
while (*label != LabelValue) {
Require(std::chrono::steady_clock::now() < deadline, what + ": the label never landed");
std::this_thread::sleep_for(std::chrono::microseconds(100));
}
const std::uint32_t seen = *stored;
Require(seen == GpuValue, what + ": the label is visible but the record holds " + Hex(seen) + ", the dispatch stored " + Hex(GpuValue));
*stored = CpuValue;
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.WaitIdle();
}
const std::uint32_t kept = *stored;
Require(kept == CpuValue, what + ": the record holds " + Hex(kept) + " after the drain, a write-back landed over the value the CPU wrote after the label (" + Hex(CpuValue) + ")");
Require(Recorder::PendingWriteBackCompletions() == 0, what + ": write-back completions outlived the drain");
return 0;
}
void StreamOrderPass(AgcDriver::VulkanDevice& device, const GuestBlock& copied, const GuestBlock& direct, const GuestBlock& labels, std::uint64_t stamp) {
constexpr std::uint32_t First = 0x11u;
constexpr std::uint32_t Second = 0x22u;
const std::uint64_t address = labels.Address() + 0x4000;
volatile std::uint32_t* label = labels.Word(0x4000);
*label = 0;
*copied.Word(CopiedRecordOffset + StoredOffset) = CpuValue;
RecordDispatch(device, copied.Address() + CopiedRecordOffset);
Require(Recorder::PendingWriteBackCompletions() == 1, "stream order: the copied dispatch has no pending write-back");
const auto copiedEpoch = SubmitLocked(device);
RecordDispatch(device, direct.Address() + DirectRecordOffset);
std::uint64_t firstEpoch = 0;
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
Require(Recorder::PendingWriteBackCompletions() == 1, "stream order: the copied writer was written back before the first label");
const int reason = device.WriteLabelOnGpu(address, LabelBytes(First), stamp, 0, false);
Require(reason == 5, "stream order: the first label behind the copied writer was handled with reason " + std::to_string(reason) + ", expected 5");
firstEpoch = device.SubmitAndEpoch();
}
WaitForGpu(device, firstEpoch);
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.ReapRecorded(copiedEpoch);
Require(Recorder::PendingWriteBackCompletions() == 0, "stream order: the copied writer was not written back by the reap of its batch");
Require(Recorder::PendingCompletionStores() == 1, "stream order: the first label's completion store is not pending after the copied writer's reap");
Require(*label == 0, "stream order: the first label landed before its batch was reaped");
}
RecordDispatch(device, direct.Address() + DirectRecordOffset);
std::uint64_t secondEpoch = 0;
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
const int reason = device.WriteLabelOnGpu(address, LabelBytes(Second), stamp + 1, 0, false);
Require(reason == 5, "stream order: a later label to the address of a pending completion store was handled with reason " + std::to_string(reason) + ", expected 5");
secondEpoch = device.SubmitAndEpoch();
Require(secondEpoch > firstEpoch, "stream order: the second label went into the first label's batch");
}
WaitForGpu(device, secondEpoch);
const std::uint32_t early = *label;
Require(early != Second, "stream order: the later label is visible while the earlier completion store to its address has not landed");
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.ReapRecorded(firstEpoch);
Require(*label == First, "stream order: the label holds " + Hex(*label) + " after the first label's batch was reaped, expected " + Hex(First));
device.ReapRecorded(secondEpoch);
const std::uint32_t last = *label;
Require(last == Second, "stream order: the label holds " + Hex(last) + " after the reap, the later label stored " + Hex(Second));
Require(Recorder::PendingCompletionStores() == 0 && Recorder::PendingCompletionLabels() == 0, "stream order: completion stores outlived the reap of their batches");
device.WaitIdle();
}
}
int main() {
try {
#ifdef _WIN32
_putenv_s("APS5_CPU_COPIES", "1");
_putenv_s("APS5_NO_OPPORTUNISTIC_REAP", "1");
#else
setenv("APS5_CPU_COPIES", "1", 1);
setenv("APS5_NO_OPPORTUNISTIC_REAP", "1", 1);
#endif
const GuestBlock copied;
const GuestBlock direct;
const GuestBlock labels;
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
const int copiedPass = LabelPass(*device, copied, CopiedRecordOffset, labels, 0x2000, 1, 1, Drains() ? 2 : 5, "copied writer");
if (copiedPass == VulkanTestSkipped) {
std::printf("skipped, the device reports a minStorageBufferOffsetAlignment of %llu, so the dispatch binds its record at +0x%zx in place and nothing is written back; the test needs an alignment above 4\n",
static_cast<unsigned long long>(StorageAlignment(*device)), CopiedRecordOffset);
return VulkanTestSkipped;
}
if (copiedPass != 0 || LabelPass(*device, direct, DirectRecordOffset, labels, 0x3000, 2, 0, 0, "direct writer") != 0) {
std::puts("skipped, the device imports no host memory");
return VulkanTestSkipped;
}
if (!Drains()) StreamOrderPass(*device, copied, direct, labels, 3);
std::puts("copied writer label order tests passed");
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
core/libs/prx/libSceAgcDriver/tests/execution/DccKeyPresync.cpp
+21
View File
@@ -234,6 +234,22 @@ void RunKnownKeys(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
RequireSampledClear(scenario, "a dispatch behind a pending fill of the keys");
}
void RunOneKeyReadPerLookup(AgcDriver::VulkanDevice& device, const Scenario& scenario, const char* what) {
Reset(scenario);
const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
auto prepared = device.PrepareDispatch(sample, {}, Threads, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()));
const auto before = AgcDriver::Graphics::SampledLookupKeyScans();
{
std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
device.Dispatch(sample, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(SampleSecondTexelCode.data()), std::move(prepared));
device.WaitIdle();
}
const auto scans = AgcDriver::Graphics::SampledLookupKeyScans() - before;
char message[192];
std::snprintf(message, sizeof(message), "%s scanned the keys of its sampled texture %llu times with nothing written between the scans, expected at most once", what, static_cast<unsigned long long>(scans));
Require(scans <= 1u, message);
}
void RunNoWriter(AgcDriver::VulkanDevice& device, const Scenario& scenario) {
const auto sample = Compile(device, SampleSecondTexelCode, scenario.sampleData);
const auto before = AgcDriver::VulkanDevice::PresyncCounts();
@@ -273,6 +289,11 @@ int main() {
RunLockedSync(*device, control);
RunPresync(*device, changed);
RunNoWriter(*device, changed);
GuestBlock single;
GuestBlock singleOutput;
const auto reads = Prepare(single.Data(), singleOutput.Data());
RunOneKeyReadPerLookup(*device, reads, "a dispatch's first lookup");
RunOneKeyReadPerLookup(*device, reads, "a dispatch's repeated lookup");
RunKnownKeys(*device, fills);
std::puts("DCC key pre-sync tests passed");
return 0;
core/libs/prx/libSceAgcDriver/tests/execution/DepthStorage.cpp
+28 -2
View File
@@ -254,9 +254,17 @@ void Run(Context context, std::uint8_t* block) {
layeredStencil.depthOrLastArray = 1;
ExpectFailure([&] { static_cast<void>(DepthSurfaceStorage(context, {}, layeredStencil, 0)); }, "storage image access to depth/stencil surface", "a two-layer 2D array view of the stencil plane was accepted");
layeredDepth.depthOrLastArray = 1;
ExpectFailure([&] { static_cast<void>(DepthSurfaceStorage(context, {}, layeredDepth, 0)); }, "with a layer pitch of", "a two-layer 2D array view of the depth plane with a layer pitch other than the depth slice size was accepted");
const auto layerPitch = static_cast<std::uint64_t>(DescribeSurface(layeredDepth).GuestLayerOffset(1));
const auto sliceBytes = DepthSliceBytes({Side, Side}, 4u);
Require(layerPitch != sliceBytes, "the test's layered depth view has the depth slice size as its layer pitch");
const auto layered = DepthSurfaceStorage(context, {}, layeredDepth, 0);
Require(layered != nullptr && layered->ImageLayers() == 2, "a two-layer 2D array view over the depth plane's guest layer pitch was not taken as a storage image");
submit(recorder);
readBack(*layered);
Require(CountFloats(block + layerPitch, 0.f) == Side * Side, "the two-layer 2D array view did not take the layer that only overlaps the depth slice from guest memory");
Require(CountFloats(block, 0.5f) == Side * Side, "the two-layer 2D array view did not receive the layer the depth surface holds");
layeredDepth.baseArray = 1;
ExpectFailure([&] { static_cast<void>(DepthSurfaceStorage(context, {}, layeredDepth, 0)); }, "with a layer pitch of", "a 2D array view of layer 1 of the depth plane with a layer pitch other than the depth slice size was accepted");
Require(DepthSurfaceStorage(context, {}, layeredDepth, 0) != nullptr, "a 2D array view from layer 1 of the depth plane was not taken as a storage image");
const auto own = StorageTexture::Create(context, *context.detiler, PlaneResource(address, Format32Float), 0);
const std::array<std::uint32_t, 4> eighth{std::bit_cast<std::uint32_t>(0.125f), std::bit_cast<std::uint32_t>(0.125f), std::bit_cast<std::uint32_t>(0.125f), std::bit_cast<std::uint32_t>(0.125f)};
Require(own->FillClear(eighth, StorageTexture::WholeImage, refusal), refusal != nullptr ? refusal : "the storage image could not be filled");
@@ -267,6 +275,24 @@ void Run(Context context, std::uint8_t* block) {
submit(recorder);
readBack(*reread);
Require(CountFloats(block, 0.125f) == Side * Side, "a storage image written over the depth plane did not reach the depth surface");
const float overlapping = 0.375f;
for (std::size_t i = 0; i < Side * Side; ++i) std::memcpy(block + layerPitch + i * sizeof(overlapping), &overlapping, sizeof(overlapping));
auto swizzled = PlaneResource(address, Format32Float);
swizzled.tileMode = TextureTileMode::kStandard64KB;
const auto tiled = DepthSurfaceStorage(context, {}, swizzled, 0);
Require(tiled != nullptr, "a 64 KiB depth plane has no storage image");
const std::array<std::uint32_t, 4> pendingFill{std::bit_cast<std::uint32_t>(0.75f), std::bit_cast<std::uint32_t>(0.75f), std::bit_cast<std::uint32_t>(0.75f), std::bit_cast<std::uint32_t>(0.75f)};
Require(tiled->FillClear(pendingFill, StorageTexture::WholeImage, refusal), refusal != nullptr ? refusal : "the 64 KiB depth plane's storage image could not be filled");
submit(recorder);
auto spanning = layeredDepth;
spanning.baseArray = 0;
spanning.depthOrLastArray = 3;
const auto spanningView = DepthSurfaceStorage(context, {}, spanning, 0);
Require(spanningView != nullptr && spanningView->ImageLayers() == 4, "a four-layer 2D array view over a 64 KiB depth plane was not taken as a storage image");
Require(CountFloats(block, 0.75f) == Side * Side, "the pending depth result was not stored into guest memory before the layered view was created");
submit(recorder);
readBack(*spanningView);
Require(CountFloats(block + layerPitch, overlapping) == Side * Side, "the layer that only overlaps the depth surface did not hold the guest bytes that were there");
Require(!DepthSurfaceStoreBack(context, *StorageTexture::Create(context, *context.detiler, PlaneResource(address, Format32Float, Side / 2), 0)), "a storage image of another extent over the depth memory was stored into the depth surface");
}
core/libs/prx/libSceAgcDriver/tests/execution/DirectWriteCompletion.cpp
+266
View File
@@ -0,0 +1,266 @@
#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include "prx/libc/include/GuestArena.hpp"
#include "prx/libc/include/GuestWriteWatch.hpp"
#include "VulkanTestDevice.hpp"
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <iostream>
#include <memory>
#include <mutex>
#include <span>
#include <stdexcept>
#include <thread>
#include <vector>
namespace {
using AgcDriver::Graphics::Require;
using AgcDriver::Graphics::StorageTexture;
constexpr std::uint32_t Side = 256;
constexpr std::uint32_t Format32UInt = 20;
constexpr std::uint32_t TileR64KBX = 0x1b;
constexpr std::uint32_t Type2D = 9;
constexpr std::size_t Block = 0x10000;
constexpr std::size_t SurfaceBytes = 4 * Block;
constexpr std::size_t SettledImportBytes = 64 * Block;
constexpr std::uint32_t Untouched = 0xdeadbeefu;
constexpr std::uint32_t Stored = 0x12345678u;
constexpr std::uint32_t Filled = 0x9abcdef0u;
constexpr std::uint32_t Written = 0x0badf00du;
alignas(256) constexpr std::array<std::uint32_t, 7> StoreCode{
0x7e040300, 0x7e060280, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000,
};
alignas(256) constexpr std::array<std::uint32_t, 11> FillCode{
0xd7460004, 0x04010c08, 0x7e000204, 0x7e020205, 0x7e040206, 0x7e060207, 0xe01c2000, 0x80000004, 0xe01c2000, 0x80020004, 0xbf810000,
};
std::uint64_t AddressOf(const void* data) {
return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
}
std::array<std::uint32_t, 8> TextureDescriptor(const std::uint32_t* texels) {
const auto address = AddressOf(texels);
return {
static_cast<std::uint32_t>(address >> 8u),
static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u),
((Side - 1u) >> 2u) | ((Side - 1u) << 14u),
0xfacu | (TileR64KBX << 20u) | (Type2D << 28u),
0u,
0u,
0u,
0u,
};
}
void Dispatch(std::vector<std::uint32_t>& words, std::span<const std::uint32_t> code, std::uint32_t threads, std::uint32_t resource, std::span<const std::uint32_t> userData, std::uint32_t groups) {
const auto registers = [&](const std::uint32_t first, const std::span<const std::uint32_t> values) {
words.push_back(0xc0007600u | (static_cast<std::uint32_t>(values.size()) << 16u));
words.push_back(first);
words.insert(words.end(), values.begin(), values.end());
};
const std::array<std::uint32_t, 3> shape{threads, 1, 1};
registers(0x207, shape);
const auto address = AddressOf(code.data());
const std::array<std::uint32_t, 2> program{static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u)};
registers(0x20c, program);
const std::array<std::uint32_t, 1> layout{resource};
registers(0x213, layout);
registers(0x240, userData);
words.insert(words.end(), {0xc0031500u, groups, 1, 1, 0x41u});
}
void Store(std::vector<std::uint32_t>& words, const std::uint32_t* texels) {
std::array<std::uint32_t, 16> userData{};
const auto texture = TextureDescriptor(texels);
std::copy(texture.begin(), texture.end(), userData.begin() + 8);
Dispatch(words, StoreCode, 8, static_cast<std::uint32_t>(userData.size()) << 1u, userData, 1);
}
void Fill(std::vector<std::uint32_t>& words, const std::uint32_t* texels, const std::uint32_t* copied) {
const auto address = AddressOf(texels);
const auto other = AddressOf(copied);
const std::array<std::uint32_t, 12> userData{
static_cast<std::uint32_t>(address),
static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (16u << 16u),
static_cast<std::uint32_t>(SurfaceBytes / 16u),
0x4bfacu,
Filled,
Filled,
Filled,
Filled,
static_cast<std::uint32_t>(other),
static_cast<std::uint32_t>((other >> 32u) & 0xffffu) | (16u << 16u),
static_cast<std::uint32_t>(SurfaceBytes / 16u),
0x4bfacu,
};
Dispatch(words, FillCode, 64, (static_cast<std::uint32_t>(userData.size()) << 1u) | 0x80u, userData, static_cast<std::uint32_t>(SurfaceBytes / 1024u));
}
class Surface {
public:
Surface() {
#ifdef _WIN32
region = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, Block));
if (region != nullptr) GuestArena::GuestArenaCommit_nid_postfix(region, SurfaceBytes, PAGE_READWRITE, SurfaceBytes);
#else
void* mapped = mmap(nullptr, SurfaceBytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (mapped != MAP_FAILED) {
const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1};
if (aligned != begin) munmap(mapped, aligned - begin);
if (aligned != begin + Block) munmap(reinterpret_cast<void*>(aligned + SurfaceBytes), begin + Block - aligned);
region = reinterpret_cast<std::uint8_t*>(aligned);
GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(region, SurfaceBytes);
}
#endif
Require(region != nullptr, "direct write completion: cannot allocate the guest surface");
texels = reinterpret_cast<std::uint32_t*>(region);
std::fill(texels, texels + SurfaceBytes / 4u, Untouched);
GuestAllocations::Mutation().Add(region, SurfaceBytes, true, true, true);
}
~Surface() {
GuestAllocations::Mutation().Remove(region);
#ifdef _WIN32
GuestArena::GuestArenaReset_nid_postfix(region, SurfaceBytes);
GuestArena::GuestArenaRelease_nid_postfix(region, SurfaceBytes);
#else
munmap(region, SurfaceBytes);
GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(region, SurfaceBytes);
#endif
}
Surface(const Surface&) = delete;
Surface& operator=(const Surface&) = delete;
std::uint32_t* Texels() const { return texels; }
private:
std::uint8_t* region = nullptr;
std::uint32_t* texels = nullptr;
};
enum class Outcome { Kept, Dropped, Missed };
void WriteOnceRecorded(const Surface& surface) {
const auto address = AddressOf(surface.Texels());
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
for (;;) {
{
std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex());
if (StorageTexture::FindPending(address, SurfaceBytes) != nullptr) {
surface.Texels()[0] = Written;
return;
}
}
Require(std::chrono::steady_clock::now() < deadline, "direct write completion: the image store was never recorded");
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
Outcome StoreAfterFill(const Surface& surface, bool cpuWrite) {
std::vector<std::uint32_t> copied(SurfaceBytes / 4u, Untouched);
GuestAllocations::Mutation().Add(copied.data(), SurfaceBytes, true, true, true);
struct Unregister {
void* data;
~Unregister() { GuestAllocations::Mutation().Remove(data); }
} unregister{copied.data()};
std::vector<std::uint32_t> words;
Fill(words, surface.Texels(), copied.data());
Store(words, surface.Texels());
alignas(8) static std::uint32_t gate = 0;
gate = 0;
if (cpuWrite) {
const auto at = AddressOf(&gate);
words.insert(words.end(), {0xc0053c00u, 0x13u, static_cast<std::uint32_t>(at), static_cast<std::uint32_t>(at >> 32u), 1u, 0xffffffffu, 0x19u});
}
Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
AgcDriver::Submit(&packet, 0);
if (cpuWrite) {
WriteOnceRecorded(surface);
std::atomic_ref<std::uint32_t>(gate).store(1);
}
AgcDriverWaitIdle_nid_postfix();
std::vector<std::byte> landed(SurfaceBytes);
AgcDriver::GuestMemory::Read(AddressOf(copied.data()), landed);
const auto address = AddressOf(surface.Texels());
std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex());
const auto image = StorageTexture::FindPending(address, SurfaceBytes);
if (image == nullptr) return Outcome::Missed;
const bool current = image->Refresh();
const bool pending = StorageTexture::FindPending(address, SurfaceBytes) == image;
StorageTexture::FlushPending(address, SurfaceBytes, nullptr, "direct write completion fixture");
return current && pending ? Outcome::Kept : Outcome::Dropped;
}
}
int main() {
try {
{
const auto device = OpenVulkanTestDevice();
if (!device) return VulkanTestSkipped;
}
struct ShutdownGuard {
~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
} shutdown;
{
Surface probe;
Require(AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(probe.Texels()))).guestBytes == SurfaceBytes, "direct write completion: unexpected surface size");
}
{
constexpr std::size_t Runs = SettledImportBytes / SurfaceBytes - 1;
std::vector<std::unique_ptr<Surface>> surfaces;
for (std::size_t index = 0; index <= 2 * Runs + 1; ++index) surfaces.push_back(std::make_unique<Surface>());
static_cast<void>(StoreAfterFill(*surfaces[0], false));
std::size_t kept = 0;
std::size_t dropped = 0;
for (std::size_t run = 1; run <= Runs; ++run) {
const auto outcome = StoreAfterFill(*surfaces[run], false);
kept += outcome == Outcome::Kept;
dropped += outcome == Outcome::Dropped;
}
AgcDriverWaitIdle_nid_postfix();
std::printf("[direct write completion] %zu of %zu runs held the image results pending, %zu dropped them\n", kept + dropped, Runs, dropped);
Require(kept + dropped != 0, "direct write completion: no run held the image results pending");
Require(dropped == 0, "direct write completion: the completion of a buffer write dropped image results recorded after it");
static_cast<void>(StoreAfterFill(*surfaces[Runs + 1], false));
std::size_t keptOverWrite = 0;
std::size_t lostWrites = 0;
for (std::size_t run = Runs + 2; run <= 2 * Runs + 1; ++run) {
keptOverWrite += StoreAfterFill(*surfaces[run], true) == Outcome::Kept;
lostWrites += surfaces[run]->Texels()[0] != Written;
}
AgcDriverWaitIdle_nid_postfix();
std::printf("[direct write completion] %zu of %zu runs with a CPU write before the completion kept the image results, %zu lost the write\n", keptOverWrite, Runs, lostWrites);
Require(keptOverWrite == 0, "direct write completion: image results were kept over a CPU write made before the completion");
Require(lostWrites == 0, "direct write completion: a CPU write made before the completion was lost");
}
AgcDriverShutdown_nid_postfix();
std::puts("direct write completion tests passed");
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheReuse.cpp
+22 -4
View File
@@ -183,6 +183,20 @@ void StageDataHitsRefreshTheirWords(bool libraries) {
Require(BrightestColorByte() == 255, mode + ": the draw of constant 1.0 wrote " + std::to_string(BrightestColorByte()) + ", expected 255");
}
void RefreshedStageDataVerifies() {
Registered(TriangleVertexHeader, TriangleVertexCode.data(), sizeof(TriangleVertexCode), 2);
Registered(ConstantPixelHeader, ConstantPixelCode.data(), sizeof(ConstantPixelCode), 1);
ColorMemory.fill(std::byte{0});
PixelConstant[0] = 0.25f;
const auto cached = SubmitConstantDraws(3);
Require(BrightestColorByte() == 64, "draw-entry verification: the initial constant did not render");
PixelConstant[0] = 1.0f;
const auto refreshed = SubmitConstantDraws(1);
Require(refreshed.verifyHits - cached.verifyHits == 1 && refreshed.stageDataHits - cached.stageDataHits == 1, "draw-entry verification: the refreshed constant did not exercise a verified stage data hit");
Require(refreshed.verifyMismatches == cached.verifyMismatches, "draw-entry verification: refreshed stage data differed from a fresh capture");
Require(BrightestColorByte() == 255, "draw-entry verification: the refreshed constant did not render");
}
void RetainedPixelCacheTransitions() {
auto commands = Draws(32.0f, VertexCode.data(), 1, PixelCode.data(), 1, {});
Append(commands, 0x76, {0x00c, std::bit_cast<std::uint32_t>(1.0f)});
@@ -239,10 +253,14 @@ int main() {
libraries = device->GraphicsPipelineLibraries();
}
if (!libraries && std::getenv("APS5_NO_GPL") == nullptr) return VulkanTestSkipped;
ThirdDrawHitsTheCache(libraries);
UserDataDrawsReuseTheirShape(libraries);
StageDataHitsRefreshTheirWords(libraries);
RetainedPixelCacheTransitions();
if (std::getenv("APS5_VERIFY_DRAW_ENTRIES") != nullptr) {
RefreshedStageDataVerifies();
} else {
ThirdDrawHitsTheCache(libraries);
UserDataDrawsReuseTheirShape(libraries);
StageDataHitsRefreshTheirWords(libraries);
RetainedPixelCacheTransitions();
}
AgcDriverShutdown_nid_postfix();
Require(AgcDriver::Graphics::PipelineLibraryWorkers() == 0, "the driver's shutdown left the pipeline library worker running");
std::cout << "A repeated draw reused the cached draw " << (libraries ? "with" : "without") << " graphics pipeline libraries\n";
core/libs/prx/libSceAgcDriver/tests/execution/FmaskImageReads.cpp
+78 -10
View File
@@ -32,9 +32,9 @@ std::array<std::uint32_t, 4> Buffer(const void* data, std::uint32_t bytes) {
return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>(address >> 32u) & 0xffffu, bytes, 0x31016facu};
}
std::array<std::uint32_t, 8> Image(bool layered, std::uint32_t pipeXor, std::uint32_t swizzle = 4u) {
std::array<std::uint32_t, 8> Image(bool layered, std::uint32_t pipeXor, std::uint32_t swizzle = 4u, std::uint32_t format = 166u) {
const auto address = reinterpret_cast<std::uintptr_t>(Texels.data()) + pipeXor;
return {static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u) | (166u << 20u) | (3u << 30u), 31u | ((Height - 1u) << 14u), swizzle | (24u << 20u) | ((layered ? 13u : 9u) << 28u), layered ? 2u | (1u << 16u) : 0u, 0u, 0u, 0u};
return {static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u) | (format << 20u) | (3u << 30u), 31u | ((Height - 1u) << 14u), swizzle | (24u << 20u) | ((layered ? 13u : 9u) << 28u), layered ? 2u | (1u << 16u) : 0u, 0u, 0u, 0u};
}
std::uint32_t Mapping(std::uint32_t x, std::uint32_t y, std::uint32_t layer, std::uint32_t round) {
@@ -56,15 +56,16 @@ std::uint64_t EquationOffset(const TextureSwizzleEquation& equation, std::uint32
return offset;
}
void Fill(bool layered, std::uint32_t pipeXor, std::uint32_t round) {
template<typename TValue>
void FillWith(bool layered, std::uint32_t pipeXor, std::uint32_t surfaceFormat, std::uint32_t bytes, TValue valueAt) {
auto words = Image(layered, pipeXor);
words[1] = (words[1] & ~(0x1ffu << 20u)) | (20u << 20u);
words[1] = (words[1] & ~(0x1ffu << 20u)) | (surfaceFormat << 20u);
const auto resource = DecodeTextureResource(words);
const auto geometry = DescribeSurface(resource);
Require(geometry.guestBytes <= Texels.size(), "FMASK fixture exceeds its allocation");
const auto& mip = geometry.mips.at(0);
const auto block = ThinBlockLayout(resource.tileMode, 4u);
const auto* equation = FindTextureSwizzleEquation(24u, 4u);
const auto block = ThinBlockLayout(resource.tileMode, bytes);
const auto* equation = FindTextureSwizzleEquation(24u, bytes);
Require(equation != nullptr, "FMASK fixture has no Z swizzle equation");
std::ranges::fill(Texels, std::byte{0});
for (std::uint32_t layer = 0; layer < geometry.layers; ++layer) {
@@ -73,14 +74,18 @@ void Fill(bool layered, std::uint32_t pipeXor, std::uint32_t round) {
auto offset = geometry.GuestLayerOffset(layer) + mip.tiledOffset;
if (mip.tail) offset += EquationOffset(*equation, x + mip.tailX, y + mip.tailY, layer) ^ pipeXor;
else offset += (static_cast<std::uint64_t>(y / block[2]) * mip.blocksPerRow + x / block[1]) * block[0] + (EquationOffset(*equation, x, y, layer) ^ pipeXor);
const auto value = Mapping(x, y, layer, round);
Require(offset + sizeof(value) <= Texels.size(), "FMASK fixture addresses outside its allocation");
std::memcpy(Texels.data() + offset, &value, sizeof(value));
const std::uint32_t value = valueAt(x, y, layer);
Require(offset + bytes <= Texels.size(), "FMASK fixture addresses outside its allocation");
std::memcpy(Texels.data() + offset, &value, bytes);
}
}
}
}
void Fill(bool layered, std::uint32_t pipeXor, std::uint32_t round) {
FillWith(layered, pipeXor, 20u, 4u, [round](std::uint32_t x, std::uint32_t y, std::uint32_t layer) { return Mapping(x, y, layer, round); });
}
ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, std::span<const std::uint32_t> image, bool registration) {
std::vector<std::uint32_t> userData(16u);
const auto input = Buffer(Input.data(), sizeof(Input));
@@ -140,6 +145,68 @@ void Run(AgcDriver::VulkanDevice& device, bool layered, bool registration, std::
}
}
struct NarrowFmask {
std::uint32_t format;
std::uint32_t surfaceFormat;
std::uint32_t bytes;
std::uint32_t samples;
std::uint32_t fragments;
std::uint32_t sampleBits;
std::uint32_t identity;
std::uint32_t expandedIdentity;
};
constexpr std::array<NarrowFmask, 8> NarrowFormats{{
{156u, 5u, 1u, 2u, 1u, 1u, 0x02u, 0x80u},
{157u, 5u, 1u, 4u, 1u, 1u, 0x0eu, 0x8880u},
{158u, 5u, 1u, 8u, 1u, 1u, 0xfeu, 0x88888880u},
{159u, 5u, 1u, 2u, 2u, 1u, 0x02u, 0x10u},
{160u, 5u, 1u, 4u, 2u, 2u, 0xa4u, 0x8810u},
{161u, 5u, 1u, 4u, 4u, 2u, 0xe4u, 0x3210u},
{163u, 11u, 2u, 8u, 2u, 2u, 0xaaa4u, 0x88888810u},
{165u, 20u, 4u, 8u, 4u, 4u, 0x44443210u, 0x88883210u},
}};
std::uint32_t Stored(const NarrowFmask& fmask, std::uint32_t x, std::uint32_t y, std::uint32_t layer, std::uint32_t round) {
if (round == 0u) return fmask.identity;
const auto value = (x * 0x9e3779b1u) ^ (y * 0x85ebca77u) ^ (layer * 0xc2b2ae3du) ^ (fmask.format * 0x27d4eb2fu);
return fmask.bytes == 4u ? value : value & ((1u << (fmask.bytes * 8u)) - 1u);
}
std::uint32_t Expanded(const NarrowFmask& fmask, std::uint32_t stored) {
std::uint32_t expanded = 0;
for (std::uint32_t sample = 0; sample < fmask.samples; ++sample) {
const auto fragment = (stored >> (sample * fmask.sampleBits)) & ((1u << fmask.sampleBits) - 1u);
expanded |= (fragment >= fmask.fragments ? 8u : fragment) << (sample * 4u);
}
return expanded;
}
void RunNarrow(AgcDriver::VulkanDevice& device, const NarrowFmask& fmask, bool layered, bool registration) {
auto code = Code;
if (layered) code[4] = 0xf0001f28u;
Require(Expanded(fmask, fmask.identity) == fmask.expandedIdentity, "FMASK " + std::to_string(fmask.format) + " reference does not expand PAL's identity value");
for (std::uint32_t round = 0; round < 2u; ++round) {
FillWith(layered, 0u, fmask.surfaceFormat, fmask.bytes, [&](std::uint32_t x, std::uint32_t y, std::uint32_t layer) { return Stored(fmask, x, y, layer, round); });
for (std::uint32_t lane = 0; lane < Threads; ++lane) {
Input[lane * 4u] = (lane * 37u + 11u) % Width;
Input[lane * 4u + 1u] = (lane * 23u + 5u) % Height;
Input[lane * 4u + 2u] = layered ? lane & 1u : 0u;
Input[lane * 4u + 3u] = 0u;
}
Output.fill(Sentinel);
const auto shader = Compile(device, code, Image(layered, 0u, 4u, fmask.format), registration);
device.Dispatch(shader, 1u, 1u, 1u);
device.WaitIdle();
for (std::uint32_t lane = 0; lane < Threads; ++lane) {
const auto stored = Stored(fmask, Input[lane * 4u], Input[lane * 4u + 1u], layered ? Input[lane * 4u + 2u] + 1u : 0u, round);
const std::array expected{Expanded(fmask, stored), 0u, 0u, 0u};
for (std::uint32_t component = 0; component < 4u; ++component) Require(Output[lane * 4u + component] == expected[component], "FMASK " + std::to_string(fmask.format) + " load did not expand its mapping: round " + std::to_string(round) + ", lane " + std::to_string(lane) + ", component " + std::to_string(component) + ", read " + std::to_string(Output[lane * 4u + component]) + ", expected " + std::to_string(expected[component]));
}
Require(std::all_of(Output.begin() + Threads * 4u, Output.end(), [](auto value) { return value == Sentinel; }), "FMASK load wrote outside its output");
}
}
}
int main() {
@@ -149,9 +216,10 @@ int main() {
for (const bool registration : {false, true}) {
for (const bool layered : {false, true}) {
for (const auto pipeXor : {0u, 0xf00u}) Run(*device, layered, registration, pipeXor);
for (const auto& fmask : NarrowFormats) RunNarrow(*device, fmask, layered, registration);
}
}
std::cout << "FMASK image reads passed: 40 dispatches, 5120 component checks, 8 compression refusals\n";
std::cout << "FMASK image reads passed: 104 dispatches, 13312 component checks, 8 compression refusals\n";
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp
+1
View File
@@ -684,6 +684,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const std
{0, 0, 0, 128}
};
request.useCache = false;
request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false};
const auto result = ShaderRecompiler::Recompile(request);
device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(code.data()));
device.WaitIdle();
core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp
+1
View File
@@ -504,6 +504,7 @@ ShaderRecompiler::RecompileResult Compile(std::uint32_t waveSize, const ShaderRe
{0, 0, 0, 128}
};
request.useCache = false;
request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false};
return ShaderRecompiler::Recompile(request);
}
Loaded 100 of 410 files, more files were not shown because too many files have changed in this diff. Show more