fix(libSceVideoOut): register and change tiled PS4 display buffers in the flip test

sceVideoOutRegisterBuffers registers a PS4 buffer with tiling mode 0 as a
GCN display buffer with its pitch padded to 128 pixels and refuses tiling
modes above 1, so video_out_flip_ps4 now checks both instead of expecting
tiled PS4 buffers to throw.

sceVideoOutSubmitChangeBufferAttribute2 on a PS4 buffer set computes the
same padded pitch as the registration. Before, a PS4 set changed to tiling
mode 0 kept a pitch of 0 and failed the 128-pixel padding check. The test
changes a PS4 set to tiled, checks its GCN description, and changes it back
to linear.

core/libs/prx/libSceVideoOut/src/Buffer.cpp

@@ -83,6 +83,11 @@ uint64_t Ps5PixelFormat(uint32_t pixelFormat) {
     throw std::runtime_error(text);
 }
 
+uint32_t Ps4TiledPitch(uint32_t tilingMode, uint32_t width, uint32_t pitchInPixel) {
+    if (tilingMode != 0) return 0;
+    return pitchInPixel != 0 ? pitchInPixel : (width + 127u) / 128u * 128u;
+}
+
 }
 
 extern "C" {
@@ -209,7 +214,7 @@ int APS5_VABI sceVideoOutRegisterBuffers(int handle, int bufferIndexStart, void*
     group.attribute.width = attribute->width;
     group.attribute.height = attribute->height;
     group.attribute.pitch_in_pixel = attribute->tiling_mode == 0 ? 0 : attribute->pitch_in_pixel;
-    group.ps4Pitch = attribute->tiling_mode == 0 ? (attribute->pitch_in_pixel != 0 ? attribute->pitch_in_pixel : (attribute->width + 127u) / 128u * 128u) : 0;
+    group.ps4Pitch = Ps4TiledPitch(attribute->tiling_mode, attribute->width, attribute->pitch_in_pixel);
     group.category = VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED;
     group.occupied = true;
     group.ps4 = true;
@@ -242,6 +247,7 @@ int APS5_VABI sceVideoOutSubmitChangeBufferAttribute2(int handle, int setIndex,
     AgcDriverCheckGuestMemory_nid_postfix(attribute, sizeof(*attribute), alignof(VideoOutBufferAttribute2));
     auto group = cfg->groups[setIndex];
     group.attribute = *attribute;
+    if (group.ps4) group.ps4Pitch = Ps4TiledPitch(attribute->tiling_mode, attribute->width, attribute->pitch_in_pixel);
     for (int i = 0; i < VIDEO_OUT_BUFFER_NUM_MAX; ++i) {
         if (cfg->buffers[i].groupIndex != setIndex) continue;
         const auto display = DescribeVideoOutBuffer(cfg->buffers[i], group);

core/libs/prx/libSceVideoOut/tests/Flip.cpp

@@ -153,20 +153,39 @@ void testPs4Buffers() {
     tiledChange.width = 64;
     tiledChange.height = 64;
     tiledChange.pixel_format = 0x8000000000000000ull;
-    check(expectFailure([&] { sceVideoOutSubmitChangeBufferAttribute2(handle, 0, &tiledChange, nullptr); }).find("only supported for linear and PS4 buffers") != std::string::npos, "a PS4 buffer set was changed to tiled");
+    check(sceVideoOutSubmitChangeBufferAttribute2(handle, 0, &tiledChange, nullptr) == 0, "a PS4 buffer set was not changed to tiled");
+    {
+        const auto cfg = VideoOutDriver::Get().GetConfig(handle);
+        std::lock_guard lock(cfg->mutex);
+        const auto display = DescribeVideoOutBuffer(cfg->buffers[1], cfg->groups[0]);
+        check(display.tilingMode == AgcDriver::DisplayTilingGcn && display.pitchInPixel == 128 && display.alignment == 4096, "the PS4 buffer set changed to tiled is not a GCN display buffer padded to 128 pixels");
+    }
+    auto linearChange = tiledChange;
+    linearChange.tiling_mode = 1;
+    linearChange.pitch_in_pixel = 64;
+    check(sceVideoOutSubmitChangeBufferAttribute2(handle, 0, &linearChange, nullptr) == 0, "a PS4 buffer set was not changed back to linear");
     void* const odd[1] = {storage.data() + 65536 + 256};
     check(expectFailure([&] { sceVideoOutRegisterBuffers(handle, 3, odd, 1, &attribute); }).find("4 KiB alignment") != std::string::npos, "a PS4 buffer was accepted below 4 KiB alignment");
     void* const fourth[1] = {storage.data() + 65536};
-    auto tiled = attribute;
-    tiled.tiling_mode = 0;
-    tiled.pitch_in_pixel = 0;
-    check(expectFailure([&] { sceVideoOutRegisterBuffers(handle, 3, fourth, 1, &tiled); }).find("tiled display buffers are not implemented") != std::string::npos, "a PS4 tiled buffer was accepted");
     auto tenBit = attribute;
     tenBit.pixel_format = 0x88060000u;
     check(expectFailure([&] { sceVideoOutRegisterBuffers(handle, 3, fourth, 1, &tenBit); }).find("unsupported PS4 pixel format 0x88060000") != std::string::npos, "an unknown PS4 pixel format was accepted");
     auto withOption = attribute;
     withOption.option = 1;
     check(expectFailure([&] { sceVideoOutRegisterBuffers(handle, 3, fourth, 1, &withOption); }).find("VIDEO_OUT_ERROR_INVALID_OPTION") != std::string::npos, "a PS4 buffer option was accepted");
+    auto unknownTiling = attribute;
+    unknownTiling.tiling_mode = 2;
+    check(expectFailure([&] { sceVideoOutRegisterBuffers(handle, 3, fourth, 1, &unknownTiling); }).find("VIDEO_OUT_ERROR_INVALID_TILING_MODE") != std::string::npos, "a PS4 buffer with tiling mode 2 was accepted");
+    auto tiled = attribute;
+    tiled.tiling_mode = 0;
+    tiled.pitch_in_pixel = 0;
+    check(sceVideoOutRegisterBuffers(handle, 3, fourth, 1, &tiled) == 2, "the PS4 tiled buffer set index is not 2");
+    {
+        const auto cfg = VideoOutDriver::Get().GetConfig(handle);
+        std::lock_guard lock(cfg->mutex);
+        const auto display = DescribeVideoOutBuffer(cfg->buffers[3], cfg->groups[2]);
+        check(cfg->groups[2].attribute.tiling_mode == 0 && display.tilingMode == AgcDriver::DisplayTilingGcn && display.pitchInPixel == 128, "the PS4 tiled buffer is not a GCN display buffer padded to 128 pixels");
+    }
     std::array<std::uint8_t, sizeof(VideoOutFlipStatus)> raw{};
     raw.fill(0xaa);
     check(sceVideoOutGetFlipStatus(handle, reinterpret_cast<VideoOutFlipStatus*>(raw.data())) == 0, "the PS4 flip status failed");