Merge pull request #13203 from EagleBearerYS/feat/libSceVdecCore-sourced-b268-3

feat(libSceVdecCore): implement the decode exports from shadPS4

core/libs/prx/libSceVdecCore/Export.cpp

@@ -4,12 +4,18 @@
 #include "prx/libSceVdecCore/include/Api.hpp"
 #include "prx/libSceVdecCore/include/Decoder.hpp"
 #include <algorithm>
+#include <atomic>
+#include <condition_variable>
 #include <cstdint>
 #include <cstddef>
+#include <cstring>
+#include <deque>
+#include <limits>
 #include <memory>
 #include <mutex>
 #include <stdexcept>
 #include <string>
+#include <thread>
 #include <unordered_map>
 #include <vector>
 #include "SceTypes.hpp"
@@ -17,12 +23,17 @@
 
 extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
 }
 
 namespace {
 
 constexpr int VdecCoreErrorInvalidArgument = static_cast<int>(0x80C00001);
 constexpr int VdecCoreErrorInvalidCodecType = static_cast<int>(0x80C00003);
+constexpr int VdecCoreErrorNoPicture = static_cast<int>(0x80C00010);
+constexpr int VdecCoreErrorFatal = static_cast<int>(0x80C00012);
+constexpr int VdecCoreErrorDecodeRunning = static_cast<int>(0x80C00018);
+constexpr std::int64_t VdecCorePtsInvalid = -1;
 constexpr std::uint32_t CodecHevc = 4;
 constexpr std::uint32_t MaxCodecType = 4;
 constexpr std::uint32_t MaxDecoderType = 8;
@@ -31,12 +42,15 @@ constexpr std::uint32_t DefaultFrameWidth = 1920;
 constexpr std::uint32_t DefaultFrameHeight = 1080;
 constexpr std::uint32_t DefaultHorizontalAlign = 256;
 constexpr std::uint32_t FrameHeightAlign = 16;
+constexpr std::uint32_t FrameWidthAlign = 16;
+constexpr std::uint32_t OutputPitchAlign = 64;
 constexpr std::uint32_t FrameBufferPadding = 0x1000;
 constexpr std::uint32_t FrameBufferGranularity = 0x10000;
 constexpr std::uint32_t FrameBufferAlignment = 256;
 constexpr std::uint64_t InstanceSize = 0x40000;
 constexpr std::uint64_t ComputeSize = 0x10000;
 constexpr std::uint32_t ComputeAlignment = 0x10000;
+constexpr std::uint64_t MaxAccessUnitSize = static_cast<std::uint64_t>(std::numeric_limits<int>::max() - AV_INPUT_BUFFER_PADDING_SIZE - 1);
 
 struct Geometry {
     std::uint32_t width;
@@ -46,15 +60,52 @@ struct Geometry {
     std::uint32_t frameBufferSize;
 };
 
+struct Command {
+    enum class Type {
+        Packet,
+        Reset,
+        Flush,
+    };
+    Type type = Type::Packet;
+    AVPacket* packet = nullptr;
+    std::shared_ptr<std::atomic<bool>> done;
+};
+
 struct Decoder {
     std::uint32_t codecType = 0;
     Geometry geometry{};
     AVCodecContext* context = nullptr;
+    SwsContext* scaler = nullptr;
+    AVFrame* decodedFrame = nullptr;
     std::vector<VdecCoreMemoryBlock> mappedBlocks;
     std::vector<void*> frameBufferSlots;
+    std::uint32_t currentSlot = 0;
+    std::vector<std::uint8_t> fallbackFrameBuffer;
+    std::int64_t lastPts = VdecCorePtsInvalid;
+    std::int64_t lastDts = VdecCorePtsInvalid;
+    std::uint64_t lastUserData = 0;
+    std::mutex mutex;
+    std::condition_variable inputReady;
+    std::condition_variable outputReady;
+    std::deque<Command> commands;
+    std::deque<AVFrame*> frames;
+    bool busy = false;
+    std::jthread worker;
 
     ~Decoder() {
+        {
+            std::lock_guard lock(mutex);
+            worker.request_stop();
+        }
+        inputReady.notify_all();
+        if (worker.joinable()) worker.join();
+        for (auto& command : commands) av_packet_free(&command.packet);
+        commands.clear();
+        for (auto* frame : frames) av_frame_free(&frame);
+        frames.clear();
+        av_frame_free(&decodedFrame);
         avcodec_free_context(&context);
+        sws_freeContext(scaler);
     }
 };
 
@@ -102,14 +153,142 @@ std::unordered_map<const void*, std::unique_ptr<Decoder>>::iterator Find(Decoder
     return found;
 }
 
+Decoder& Lookup(const void* decoder, const char* caller) {
+    auto& state = State();
+    std::lock_guard lock(state.mutex);
+    return *Find(state, decoder, caller)->second;
+}
+
+void DrainFrames(Decoder& decoder) {
+    while (true) {
+        AVFrame* frame = av_frame_alloc();
+        if (frame == nullptr) break;
+        if (avcodec_receive_frame(decoder.context, frame) != 0) {
+            av_frame_free(&frame);
+            break;
+        }
+        std::lock_guard lock(decoder.mutex);
+        decoder.frames.push_back(frame);
+    }
+}
+
+void RunWorker(std::stop_token stop, Decoder* decoder) {
+    while (!stop.stop_requested()) {
+        Command command;
+        {
+            std::unique_lock lock(decoder->mutex);
+            decoder->inputReady.wait(lock, [&] { return stop.stop_requested() || !decoder->commands.empty(); });
+            if (stop.stop_requested()) break;
+            command = std::move(decoder->commands.front());
+            decoder->commands.pop_front();
+            decoder->busy = true;
+        }
+        switch (command.type) {
+        case Command::Type::Packet: {
+            int result = avcodec_send_packet(decoder->context, command.packet);
+            if (result == AVERROR_EOF) {
+                avcodec_flush_buffers(decoder->context);
+                result = avcodec_send_packet(decoder->context, command.packet);
+            }
+            if (result == AVERROR(EAGAIN)) {
+                DrainFrames(*decoder);
+                avcodec_send_packet(decoder->context, command.packet);
+            }
+            av_packet_free(&command.packet);
+            DrainFrames(*decoder);
+            break;
+        }
+        case Command::Type::Flush:
+            avcodec_send_packet(decoder->context, nullptr);
+            DrainFrames(*decoder);
+            avcodec_flush_buffers(decoder->context);
+            command.done->store(true, std::memory_order_release);
+            break;
+        case Command::Type::Reset:
+            avcodec_flush_buffers(decoder->context);
+            {
+                std::lock_guard lock(decoder->mutex);
+                for (auto* frame : decoder->frames) av_frame_free(&frame);
+                decoder->frames.clear();
+                decoder->currentSlot = 0;
+            }
+            command.done->store(true, std::memory_order_release);
+            break;
+        }
+        std::lock_guard lock(decoder->mutex);
+        decoder->busy = !decoder->commands.empty();
+        decoder->outputReady.notify_all();
+    }
+}
+
+void RunCommand(Decoder& decoder, Command::Type type, std::uint32_t* count) {
+    auto done = std::make_shared<std::atomic<bool>>(false);
+    std::unique_lock lock(decoder.mutex);
+    decoder.commands.push_back({type, nullptr, done});
+    decoder.inputReady.notify_one();
+    decoder.outputReady.wait(lock, [&] { return done->load(std::memory_order_acquire); });
+    if (count != nullptr) *count = static_cast<std::uint32_t>(decoder.frames.size());
+}
+
+AVFrame* ConvertNv12(Decoder& decoder, const AVFrame& frame) {
+    AVFrame* converted = av_frame_alloc();
+    if (converted == nullptr) throw std::runtime_error("VdecCore: cannot allocate an NV12 frame");
+    converted->format = AV_PIX_FMT_NV12;
+    converted->width = frame.width;
+    converted->height = frame.height;
+    converted->pts = frame.pts;
+    converted->pkt_dts = frame.pkt_dts;
+    converted->opaque = frame.opaque;
+    if (av_frame_get_buffer(converted, 0) < 0) {
+        av_frame_free(&converted);
+        throw std::runtime_error("VdecCore: cannot allocate an NV12 frame");
+    }
+    decoder.scaler = sws_getCachedContext(decoder.scaler, frame.width, frame.height, static_cast<AVPixelFormat>(frame.format), frame.width, frame.height, AV_PIX_FMT_NV12, SWS_BILINEAR, nullptr, nullptr, nullptr);
+    if (decoder.scaler == nullptr || sws_scale(decoder.scaler, frame.data, frame.linesize, 0, frame.height, converted->data, converted->linesize) != frame.height) {
+        av_frame_free(&converted);
+        throw std::runtime_error("VdecCore: cannot convert a decoded picture to NV12");
+    }
+    return converted;
+}
+
+void CopyNv12(std::uint8_t* destination, const AVFrame& frame) {
+    const auto width = static_cast<std::uint32_t>(frame.width);
+    const auto height = static_cast<std::uint32_t>(frame.height);
+    const std::size_t pitch = AlignUp(width, OutputPitchAlign);
+    const std::size_t alignedHeight = AlignUp(height, FrameHeightAlign);
+    std::uint8_t* luma = destination;
+    std::uint8_t* chroma = destination + pitch * alignedHeight;
+    const auto lumaRow = [&](std::size_t row) { return frame.data[0] + row * static_cast<std::size_t>(frame.linesize[0]); };
+    const auto chromaRow = [&](std::size_t row) { return frame.data[1] + row * static_cast<std::size_t>(frame.linesize[1]); };
+    for (std::size_t row = 0; row < height; ++row) std::memcpy(luma + row * pitch, lumaRow(row), width);
+    for (std::size_t row = 0; row < height / 2; ++row) std::memcpy(chroma + row * pitch, chromaRow(row), width);
+    for (std::size_t row = height; row < alignedHeight && height > 0; ++row) std::memcpy(luma + row * pitch, lumaRow(height - 1), width);
+    for (std::size_t row = height / 2; row < alignedHeight / 2 && height / 2 > 0; ++row) std::memcpy(chroma + row * pitch, chromaRow(height / 2 - 1), width);
+}
+
+std::uint64_t PictureTime(std::int64_t decoded, std::int64_t last) {
+    return static_cast<std::uint64_t>(decoded != AV_NOPTS_VALUE ? decoded : last);
+}
+
 }
 
 extern "C" {
 
+std::uint64_t VdecCoreMaxAccessUnitSize_nid_no_patch() {
+    return MaxAccessUnitSize;
+}
+
+std::int32_t VdecCoreAccessUnitLength_nid_no_patch(std::uint64_t auSize) {
+    if (auSize == 0 || auSize > MaxAccessUnitSize) return -1;
+    return static_cast<std::int32_t>(auSize);
+}
+
 std::uint32_t VdecCoreFrameBufferSlots_nid_no_patch(const void* decoder, void** slots, std::uint32_t capacity) {
     auto& state = State();
     std::lock_guard lock(state.mutex);
-    const auto& found = Find(state, decoder, "VdecCoreFrameBufferSlots")->second->frameBufferSlots;
+    auto& target = *Find(state, decoder, "VdecCoreFrameBufferSlots")->second;
+    std::lock_guard decoderLock(target.mutex);
+    const auto& found = target.frameBufferSlots;
     const auto copied = std::min<std::size_t>(capacity, found.size());
     std::copy_n(found.begin(), copied, slots);
     return static_cast<std::uint32_t>(found.size());
@@ -132,6 +311,8 @@ int APS5_VABI sceVdecCoreCreateDecoder(const VdecCoreConfigInfo* config, const V
     created->codecType = config->codecType;
     created->geometry = FrameGeometry(*config);
     OpenCodec(*created);
+    created->decodedFrame = av_frame_alloc();
+    created->worker = std::jthread(RunWorker, created.get());
     void* handle = created.get();
     auto& state = State();
     std::lock_guard lock(state.mutex);
@@ -158,13 +339,126 @@ int APS5_VABI sceVdecCoreFinalizeComputeResource() {
     return 0;
 }
 
-int APS5_VABI sceVdecCoreFlushDecodeOutput() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreFlushDecodeOutput(void* decoder, std::uint32_t* count, void*) {
+    if (decoder == nullptr) return VdecCoreErrorInvalidArgument;
+    RunCommand(Lookup(decoder, __func__), Command::Type::Flush, count);
     return 0;
 }
 
-int APS5_VABI sceVdecCoreGetDecodeOutput() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreGetDecodeOutput(void* decoder, VdecCoreFrameBuffer* frameBuffer, std::uint32_t, VdecCoreOutputInfo* output, void*) {
+    if (decoder == nullptr || output == nullptr) return VdecCoreErrorInvalidArgument;
+    auto& target = Lookup(decoder, __func__);
+    AVFrame* frame = nullptr;
+    {
+        std::lock_guard lock(target.mutex);
+        if (target.frames.empty()) {
+            output->frameWidth = target.geometry.width;
+            output->framePitch = AlignUp(target.geometry.width, OutputPitchAlign);
+            output->frameHeight = target.geometry.height;
+            output->pictureCount = 0;
+            output->frameBuffer = nullptr;
+            output->frameFormat = 0;
+            output->reserved0 = 0;
+            output->pictures[0].frameBuffer = nullptr;
+            output->pictures[1].frameBuffer = nullptr;
+            output->lumaBuffer = nullptr;
+            output->chromaBuffer = nullptr;
+            return VdecCoreErrorNoPicture;
+        }
+        frame = target.frames.front();
+        target.frames.pop_front();
+    }
+    struct Release {
+        AVFrame*& frame;
+        ~Release() { av_frame_free(&frame); }
+    } release{frame};
+    if (frame->format != AV_PIX_FMT_NV12) {
+        AVFrame* converted = ConvertNv12(target, *frame);
+        av_frame_free(&frame);
+        frame = converted;
+    }
+    const auto width = static_cast<std::uint32_t>(frame->width);
+    const auto height = static_cast<std::uint32_t>(frame->height);
+    const std::uint32_t alignedWidth = AlignUp(width, FrameWidthAlign);
+    const std::uint32_t pitch = AlignUp(width, OutputPitchAlign);
+    const std::uint32_t alignedHeight = AlignUp(height, FrameHeightAlign);
+    const std::uint32_t pictureSize = pitch * alignedHeight * 3 / 2;
+    const std::uint32_t sarWidth = frame->sample_aspect_ratio.num > 0 ? static_cast<std::uint32_t>(frame->sample_aspect_ratio.num) : 1;
+    const std::uint32_t sarHeight = frame->sample_aspect_ratio.den > 0 ? static_cast<std::uint32_t>(frame->sample_aspect_ratio.den) : 1;
+    std::uint8_t* destination = nullptr;
+    std::uint64_t capacity = 0;
+    std::int64_t lastPts = 0;
+    std::int64_t lastDts = 0;
+    {
+        std::lock_guard lock(target.mutex);
+        lastPts = target.lastPts;
+        lastDts = target.lastDts;
+        if (frameBuffer != nullptr && frameBuffer->frameBuffer != nullptr) {
+            destination = static_cast<std::uint8_t*>(frameBuffer->frameBuffer);
+            capacity = frameBuffer->frameBufferSize > 0 ? frameBuffer->frameBufferSize : target.geometry.frameBufferSize;
+        } else if (!target.frameBufferSlots.empty()) {
+            destination = static_cast<std::uint8_t*>(target.frameBufferSlots[target.currentSlot % target.frameBufferSlots.size()]);
+            ++target.currentSlot;
+            capacity = target.geometry.frameBufferSize;
+        } else {
+            const std::uint64_t requested = frameBuffer != nullptr && frameBuffer->frameBufferSize > 0 ? frameBuffer->frameBufferSize : target.geometry.frameBufferSize;
+            capacity = std::max<std::uint64_t>(std::uint64_t{pictureSize} + sizeof(VdecCoreHwPictureInfo), requested);
+            if (target.fallbackFrameBuffer.size() < capacity) target.fallbackFrameBuffer.resize(static_cast<std::size_t>(capacity));
+            destination = target.fallbackFrameBuffer.data();
+        }
+    }
+    if (pictureSize > capacity) throw std::runtime_error("VdecCore: a decoded picture is larger than its frame buffer");
+    CopyNv12(destination, *frame);
+    const std::uint64_t pts = PictureTime(frame->pts, lastPts);
+    const std::uint64_t dts = PictureTime(frame->pkt_dts, lastDts);
+    const auto userData = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(frame->opaque));
+    if (capacity - pictureSize >= sizeof(VdecCoreHwPictureInfo)) {
+        auto* info = reinterpret_cast<VdecCoreHwPictureInfo*>(destination + capacity - sizeof(VdecCoreHwPictureInfo));
+        std::memset(info, 0, sizeof(*info));
+        info->codecType = target.codecType == CodecHevc ? 1 : 0;
+        info->userData = userData;
+        info->pts = pts;
+        info->dts = dts;
+        info->valid = 1;
+        info->width = alignedWidth;
+        info->height = alignedHeight;
+        info->cropLeft = 0;
+        info->cropRight = pitch > width ? (pitch - width) / 2 : 0;
+        info->cropTop = 0;
+        info->cropBottom = alignedHeight > height ? (alignedHeight - height) / 2 : 0;
+        info->sarWidth = sarWidth;
+        info->sarHeight = sarHeight;
+        info->flags = 0x01;
+        if (info->cropRight > 0 || info->cropBottom > 0) info->flags |= 0x02;
+    }
+    output->frameWidth = alignedWidth;
+    output->framePitch = pitch;
+    output->frameHeight = alignedHeight;
+    output->pictureCount = 1;
+    output->frameBuffer = destination;
+    output->frameFormat = 0;
+    output->reserved0 = 0;
+    output->pictures[0].flags = target.codecType;
+    output->pictures[0].reserved0 = 0;
+    output->pictures[0].pts = pts;
+    output->pictures[0].dts = dts;
+    output->pictures[0].userData = userData;
+    output->pictures[0].pictureInfo = nullptr;
+    output->pictures[0].frameBuffer = destination;
+    output->pictures[0].frameBufferSize = pictureSize;
+    output->pictures[0].framePitch = pitch;
+    output->pictures[0].reserved1 = 0;
+    output->pictures[1].frameBuffer = nullptr;
+    output->aspectRatio = 1;
+    output->sarWidth = sarWidth;
+    output->videoFormat = 5;
+    output->colourPrimaries = 1;
+    output->transferCharacteristics = 1;
+    output->framePitchInBytes = pitch;
+    output->lumaBuffer = destination;
+    output->chromaBuffer = destination + static_cast<std::size_t>(pitch) * alignedHeight;
+    output->videoFullRangeFlag = frame->color_range == AVCOL_RANGE_JPEG ? 1 : 0;
+    output->colourDescriptionPresentFlag = 1;
     return 0;
 }
 
@@ -179,6 +473,7 @@ int APS5_VABI sceVdecCoreMapMemoryBlock(void* decoder, const VdecCoreMemoryBlock
     std::lock_guard lock(state.mutex);
     auto& target = *Find(state, decoder, __func__)->second;
     if (blocks == nullptr || count == 0) return 0;
+    std::lock_guard decoderLock(target.mutex);
     for (std::uint32_t index = 0; index < count; ++index) {
         const VdecCoreMemoryBlock block = blocks[index];
         target.mappedBlocks.push_back(block);
@@ -223,13 +518,42 @@ int APS5_VABI sceVdecCoreQueryInstanceSize(const VdecCoreConfigInfo* config, std
     return 0;
 }
 
-int APS5_VABI sceVdecCoreResetDecoder() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreResetDecoder(void* decoder, void*) {
+    if (decoder == nullptr) return VdecCoreErrorInvalidArgument;
+    RunCommand(Lookup(decoder, __func__), Command::Type::Reset, nullptr);
     return 0;
 }
 
-int APS5_VABI sceVdecCoreSetDecodeInput() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreSetDecodeInput(void* decoder, const VdecCoreInputData* input, void*, std::uint32_t* status) {
+    if (decoder == nullptr || input == nullptr) return VdecCoreErrorInvalidArgument;
+    auto& target = Lookup(decoder, __func__);
+    if (status != nullptr) {
+        status[0] = 1;
+        status[1] = 1;
+    }
+    const std::int32_t length = VdecCoreAccessUnitLength_nid_no_patch(input->auSize);
+    if (input->auData == nullptr || length < 0) return VdecCoreErrorInvalidArgument;
+    const std::int64_t pts = input->pts == VdecCorePtsInvalid ? AV_NOPTS_VALUE : input->pts;
+    const std::int64_t dts = input->dts == VdecCorePtsInvalid ? AV_NOPTS_VALUE : input->dts;
+    AVPacket* packet = av_packet_alloc();
+    if (packet == nullptr) return VdecCoreErrorFatal;
+    if (av_new_packet(packet, length) < 0) {
+        av_packet_free(&packet);
+        return VdecCoreErrorFatal;
+    }
+    std::memcpy(packet->data, input->auData, static_cast<std::size_t>(length));
+    packet->pts = pts;
+    packet->dts = dts;
+    packet->opaque = reinterpret_cast<void*>(static_cast<std::uintptr_t>(input->userData));
+    {
+        std::lock_guard lock(target.mutex);
+        target.lastPts = input->pts;
+        target.lastDts = input->dts;
+        target.lastUserData = input->userData;
+        target.commands.push_back({Command::Type::Packet, packet, nullptr});
+        target.busy = true;
+    }
+    target.inputReady.notify_one();
     return 0;
 }
 
@@ -238,8 +562,12 @@ int APS5_VABI sceVdecCoreSetDecodeOutputSw() {
     return 0;
 }
 
-int APS5_VABI sceVdecCoreSyncDecode() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreSyncDecode(void* decoder, std::uint32_t* count) {
+    if (decoder == nullptr) return VdecCoreErrorInvalidArgument;
+    auto& target = Lookup(decoder, __func__);
+    std::unique_lock lock(target.mutex);
+    target.outputReady.wait(lock, [&] { return !target.frames.empty() || (!target.busy && target.commands.empty()); });
+    if (count != nullptr) *count = static_cast<std::uint32_t>(target.frames.size());
     return 0;
 }
 
@@ -248,13 +576,20 @@ int APS5_VABI sceVdecCoreSyncDecodeOutputSw() {
     return 0;
 }
 
-int APS5_VABI sceVdecCoreSyncDecodeWptr() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI sceVdecCoreSyncDecodeWptr(void* decoder, std::uint32_t* count, void*, void*) {
+    return sceVdecCoreSyncDecode(decoder, count);
 }
 
-int APS5_VABI sceVdecCoreTrySyncDecode() {
-    NotImplemented_nid_no_patch(__func__);
+int APS5_VABI sceVdecCoreTrySyncDecode(void* decoder, std::uint32_t* count) {
+    if (decoder == nullptr) return VdecCoreErrorInvalidArgument;
+    auto& target = Lookup(decoder, __func__);
+    std::lock_guard lock(target.mutex);
+    if (!target.frames.empty()) {
+        if (count != nullptr) *count = static_cast<std::uint32_t>(target.frames.size());
+        return 0;
+    }
+    if (count != nullptr) *count = 0;
+    if (target.busy || !target.commands.empty()) return VdecCoreErrorDecodeRunning;
     return 0;
 }
 
@@ -263,9 +598,8 @@ int APS5_VABI sceVdecCoreTrySyncDecodeOutputSw() {
     return 0;
 }
 
-int APS5_VABI sceVdecCoreTrySyncDecodeWptr() {
-    NotImplemented_nid_no_patch(__func__);
-    return 0;
+int APS5_VABI sceVdecCoreTrySyncDecodeWptr(void* decoder, std::uint32_t* count, void*, void*) {
+    return sceVdecCoreTrySyncDecode(decoder, count);
 }
 
 }

core/libs/prx/libSceVdecCore/include/Api.hpp

@@ -74,6 +74,86 @@ struct VdecCoreMemoryBlock {
 
 static_assert(sizeof(VdecCoreMemoryBlock) == 0x18);
 
+struct VdecCoreFrameBuffer {
+    void* frameBuffer;
+    std::uint64_t frameBufferSize;
+};
+
+static_assert(sizeof(VdecCoreFrameBuffer) == 0x10);
+
+struct VdecCoreInputData {
+    const void* auData;
+    std::uint64_t auSize;
+    std::int64_t pts;
+    std::int64_t dts;
+    std::uint64_t userData;
+    std::uint32_t flags;
+    std::uint32_t streamId;
+};
+
+static_assert(sizeof(VdecCoreInputData) == 0x30);
+
+struct VdecCorePicture {
+    std::uint32_t flags;
+    std::uint32_t reserved0;
+    std::uint64_t pts;
+    std::uint64_t dts;
+    std::uint64_t userData;
+    void* pictureInfo;
+    void* frameBuffer;
+    std::uint32_t frameBufferSize;
+    std::uint32_t framePitch;
+    std::uint64_t reserved1;
+};
+
+static_assert(sizeof(VdecCorePicture) == 0x40);
+
+struct VdecCoreHwPictureInfo {
+    std::uint32_t codecType;
+    std::uint32_t reserved0;
+    std::uint64_t userData;
+    std::uint64_t pts;
+    std::uint64_t dts;
+    std::uint32_t valid;
+    std::uint32_t reserved1;
+    std::uint32_t width;
+    std::uint32_t height;
+    std::uint32_t cropLeft;
+    std::uint32_t cropRight;
+    std::uint32_t cropTop;
+    std::uint32_t cropBottom;
+    std::uint32_t sarWidth;
+    std::uint32_t sarHeight;
+    std::uint32_t reserved2;
+    std::uint32_t flags;
+    std::uint8_t reserved3[0x400 - 0x50];
+};
+
+static_assert(sizeof(VdecCoreHwPictureInfo) == 0x400);
+
+struct VdecCoreOutputInfo {
+    std::uint32_t frameWidth;
+    std::uint32_t framePitch;
+    std::uint32_t frameHeight;
+    std::uint32_t pictureCount;
+    void* frameBuffer;
+    std::uint32_t frameFormat;
+    std::uint32_t reserved0;
+    VdecCorePicture pictures[2];
+    std::uint32_t aspectRatio;
+    std::uint32_t sarWidth;
+    std::uint32_t videoFormat;
+    std::uint32_t colourPrimaries;
+    std::uint32_t transferCharacteristics;
+    std::uint32_t framePitchInBytes;
+    void* lumaBuffer;
+    void* chromaBuffer;
+    std::uint32_t videoFullRangeFlag;
+    std::uint32_t colourDescriptionPresentFlag;
+};
+
+static_assert(sizeof(VdecCoreOutputInfo) == 0xd0);
+
 extern "C" {
 
 int APS5_VABI sceVdecCoreQueryMemorySizeSvp();
@@ -82,21 +162,21 @@ int APS5_VABI sceVdecCoreAnalyzeStream();
 int APS5_VABI sceVdecCoreCreateDecoder(const VdecCoreConfigInfo* config, const VdecCoreMemoryInfo* memory, void* computeQueue, void** decoder);
 int APS5_VABI sceVdecCoreDeleteDecoder(void* decoder);
 int APS5_VABI sceVdecCoreFinalizeComputeResource();
-int APS5_VABI sceVdecCoreFlushDecodeOutput();
-int APS5_VABI sceVdecCoreGetDecodeOutput();
+int APS5_VABI sceVdecCoreFlushDecodeOutput(void* decoder, std::uint32_t* count, void* reserved);
+int APS5_VABI sceVdecCoreGetDecodeOutput(void* decoder, VdecCoreFrameBuffer* frameBuffer, std::uint32_t flags, VdecCoreOutputInfo* output, void* reserved);
 int APS5_VABI sceVdecCoreInitializeComputeResource();
 int APS5_VABI sceVdecCoreMapMemoryBlock(void* decoder, const VdecCoreMemoryBlock* blocks, std::uint32_t count);
 int APS5_VABI sceVdecCoreQueryComputeResourceInfo(VdecCoreComputeResourceInfo* info);
 int APS5_VABI sceVdecCoreQueryFrameBufferInfo(const VdecCoreConfigInfo* config, VdecCoreFrameBufferInfo* info);
 int APS5_VABI sceVdecCoreQueryInstanceSize(const VdecCoreConfigInfo* config, std::uint64_t* instanceSize, std::uint64_t* computeSize, std::uint64_t* reservedSize);
-int APS5_VABI sceVdecCoreResetDecoder();
-int APS5_VABI sceVdecCoreSetDecodeInput();
+int APS5_VABI sceVdecCoreResetDecoder(void* decoder, void* resetParam);
+int APS5_VABI sceVdecCoreSetDecodeInput(void* decoder, const VdecCoreInputData* input, void* reserved, std::uint32_t* status);
 int APS5_VABI sceVdecCoreSetDecodeOutputSw();
-int APS5_VABI sceVdecCoreSyncDecode();
+int APS5_VABI sceVdecCoreSyncDecode(void* decoder, std::uint32_t* count);
 int APS5_VABI sceVdecCoreSyncDecodeOutputSw();
-int APS5_VABI sceVdecCoreSyncDecodeWptr();
-int APS5_VABI sceVdecCoreTrySyncDecode();
+int APS5_VABI sceVdecCoreSyncDecodeWptr(void* decoder, std::uint32_t* count, void* writePointer, void* readPointer);
+int APS5_VABI sceVdecCoreTrySyncDecode(void* decoder, std::uint32_t* count);
 int APS5_VABI sceVdecCoreTrySyncDecodeOutputSw();
-int APS5_VABI sceVdecCoreTrySyncDecodeWptr();
+int APS5_VABI sceVdecCoreTrySyncDecodeWptr(void* decoder, std::uint32_t* count, void* writePointer, void* readPointer);
 
 }

core/libs/prx/libSceVdecCore/include/Decoder.hpp

@@ -5,5 +5,7 @@
 extern "C" {
 
 std::uint32_t VdecCoreFrameBufferSlots_nid_no_patch(const void* decoder, void** slots, std::uint32_t capacity);
+std::uint64_t VdecCoreMaxAccessUnitSize_nid_no_patch();
+std::int32_t VdecCoreAccessUnitLength_nid_no_patch(std::uint64_t auSize);
 
 }

core/libs/tests/GuestVdecCore.cpp

@@ -1,10 +1,14 @@
 #include "prx/libSceVdecCore/include/Api.hpp"
 #include "prx/libSceVdecCore/include/Decoder.hpp"
+#include "H264Fixture.hpp"
 
+#include <algorithm>
 #include <array>
+#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
 #include <vector>
 
 #ifdef _WIN32
@@ -17,6 +21,15 @@ namespace {
 
 constexpr int InvalidArgument = static_cast<int>(0x80C00001);
 constexpr int InvalidCodecType = static_cast<int>(0x80C00003);
+constexpr int NoPicture = static_cast<int>(0x80C00010);
+constexpr int DecodeRunning = static_cast<int>(0x80C00018);
+constexpr std::int64_t PtsInvalid = -1;
+constexpr std::uint64_t SlotSize = 0x10000;
+constexpr std::uint32_t OutputPitch = 64;
+constexpr std::array<std::uint64_t, 6> VisiblePictureHashes{
+    0x5871fdc85dfa274eull, 0x26ce4afb974cffc4ull, 0xd2e739633733d5baull,
+    0x54feb3d290e028fdull, 0xfa1342a4ba1819c8ull, 0x870125dd7f079a4cull,
+};
 constexpr std::uint32_t CodecAvc = 0;
 constexpr std::uint32_t CodecHevc = 4;
 constexpr std::uint32_t DecoderVdec2 = 2;
@@ -174,6 +187,322 @@ void CheckMapMemoryBlock() {
     Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the mapped decoder");
 }
 
+void* CreateAvc(std::int32_t width) {
+    const VdecCoreConfigInfo config = Config(width, static_cast<std::int32_t>(H264Fixture::Height), 256, 0);
+    void* decoder = nullptr;
+    Require(sceVdecCoreCreateDecoder(&config, nullptr, nullptr, &decoder) == 0 && decoder != nullptr, "create an AVC decoder");
+    return decoder;
+}
+
+const std::vector<std::vector<std::uint8_t>>& Units() {
+    static const std::vector<std::vector<std::uint8_t>> units = H264Fixture::accessUnits(H264Fixture::Stream.data(), H264Fixture::Stream.size(), false);
+    Require(units.size() == H264Fixture::PictureHashes.size(), "the fixture's access units");
+    return units;
+}
+
+void Feed(void* decoder, std::size_t unit, std::int64_t pts, std::int64_t dts, std::uint64_t userData) {
+    const auto& data = Units()[unit];
+    const VdecCoreInputData input{data.data(), data.size(), pts, dts, userData, 0, 0};
+    std::array<std::uint32_t, 2> status{};
+    Require(sceVdecCoreSetDecodeInput(decoder, &input, nullptr, status.data()) == 0, "set an access unit as decode input");
+    Require(status[0] == 1 && status[1] == 1, "the input status of an accepted access unit");
+}
+
+void FeedAll(void* decoder, std::int64_t ptsBase, std::int64_t dtsBase, std::uint64_t userBase) {
+    for (std::size_t unit = 0; unit < Units().size(); ++unit) {
+        const auto offset = static_cast<std::int64_t>(unit);
+        Feed(decoder, unit, ptsBase == PtsInvalid ? PtsInvalid : ptsBase + offset, dtsBase == PtsInvalid ? PtsInvalid : dtsBase + offset, userBase + unit);
+    }
+}
+
+std::uint32_t Flush(void* decoder) {
+    std::uint32_t count = 0xffffffffu;
+    Require(sceVdecCoreFlushDecodeOutput(decoder, &count, nullptr) == 0, "flush the decode output");
+    return count;
+}
+
+VdecCoreOutputInfo Output(void* decoder, VdecCoreFrameBuffer* frameBuffer) {
+    VdecCoreOutputInfo output{};
+    Require(sceVdecCoreGetDecodeOutput(decoder, frameBuffer, 0, &output, nullptr) == 0, "get a decoded picture");
+    return output;
+}
+
+std::uint64_t VisibleHash(const std::uint8_t* buffer, std::uint32_t pitch, std::uint32_t frameHeight) {
+    std::uint64_t hash = 0xcbf29ce484222325ull;
+    const auto mix = [&](const std::uint8_t* row) {
+        for (std::uint32_t column = 0; column < H264Fixture::Width; ++column) hash = (hash ^ row[column]) * 0x100000001b3ull;
+    };
+    for (std::uint32_t row = 0; row < H264Fixture::Height; ++row) mix(buffer + static_cast<std::size_t>(row) * pitch);
+    const auto* chroma = buffer + static_cast<std::size_t>(pitch) * frameHeight;
+    for (std::uint32_t row = 0; row < H264Fixture::Height / 2; ++row) mix(chroma + static_cast<std::size_t>(row) * pitch);
+    return hash;
+}
+
+bool ExtendsLastRows(const std::uint8_t* buffer, std::uint32_t pitch, std::uint32_t frameHeight) {
+    const auto* chroma = buffer + static_cast<std::size_t>(pitch) * frameHeight;
+    const auto* lastLuma = buffer + static_cast<std::size_t>(pitch) * (H264Fixture::Height - 1);
+    const auto* lastChroma = chroma + static_cast<std::size_t>(pitch) * (H264Fixture::Height / 2 - 1);
+    for (std::uint32_t row = H264Fixture::Height; row < frameHeight; ++row) {
+        if (std::memcmp(buffer + static_cast<std::size_t>(row) * pitch, lastLuma, H264Fixture::Width) != 0) return false;
+    }
+    for (std::uint32_t row = H264Fixture::Height / 2; row < frameHeight / 2; ++row) {
+        if (std::memcmp(chroma + static_cast<std::size_t>(row) * pitch, lastChroma, H264Fixture::Width) != 0) return false;
+    }
+    return true;
+}
+
+const VdecCoreHwPictureInfo& HwInfo(const VdecCoreOutputInfo& output, std::uint64_t allocatedSize) {
+    return *reinterpret_cast<const VdecCoreHwPictureInfo*>(static_cast<const std::uint8_t*>(output.frameBuffer) + allocatedSize - sizeof(VdecCoreHwPictureInfo));
+}
+
+void RequirePicture(const VdecCoreOutputInfo& output, std::size_t picture, std::uint64_t allocatedSize, std::uint64_t pts, std::uint64_t userData, const char* message) {
+    const std::uint32_t height = H264Fixture::CodedHeight;
+    auto* buffer = static_cast<const std::uint8_t*>(output.frameBuffer);
+    Require(output.pictureCount == 1 && buffer != nullptr && output.frameFormat == 0 && output.reserved0 == 0, message);
+    Require(output.frameWidth == H264Fixture::Width && output.framePitch == OutputPitch && output.frameHeight == height && output.framePitchInBytes == OutputPitch, message);
+    Require(output.lumaBuffer == buffer && output.chromaBuffer == buffer + static_cast<std::size_t>(OutputPitch) * height, message);
+    const VdecCorePicture& first = output.pictures[0];
+    Require(first.flags == 0 && first.reserved0 == 0 && first.pictureInfo == nullptr && first.frameBuffer == buffer && first.frameBufferSize == OutputPitch * height * 3 / 2 && first.framePitch == OutputPitch && first.reserved1 == 0, message);
+    Require(output.pictures[1].frameBuffer == nullptr, message);
+    Require(first.pts == pts && first.userData == userData, message);
+    Require(output.aspectRatio == 1 && output.sarWidth == 1 && output.videoFormat == 5 && output.colourPrimaries == 1 && output.transferCharacteristics == 1 && output.videoFullRangeFlag == 0 && output.colourDescriptionPresentFlag == 1, message);
+    Require(VisibleHash(buffer, output.framePitch, output.frameHeight) == VisiblePictureHashes[picture], message);
+    Require(ExtendsLastRows(buffer, output.framePitch, output.frameHeight), message);
+    const VdecCoreHwPictureInfo& info = HwInfo(output, allocatedSize);
+    Require(info.codecType == 0 && info.userData == userData && info.pts == pts && info.dts == first.dts && info.valid == 1, message);
+    Require(info.width == H264Fixture::Width && info.height == height && info.cropLeft == 0 && info.cropRight == 0 && info.cropTop == 0 && info.cropBottom == (height - H264Fixture::Height) / 2, message);
+    Require(info.sarWidth == 1 && info.sarHeight == 1 && info.flags == 0x03, message);
+}
+
+void CheckAccessUnitLength() {
+    const std::uint64_t limit = VdecCoreMaxAccessUnitSize_nid_no_patch();
+    Require(limit > 0 && limit < 0x7fffffffull, "the largest access unit leaves room for the packet padding in an int");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(0) < 0, "an empty access unit has no length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(1) == 1, "a one-byte access unit keeps its length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(limit) == static_cast<std::int32_t>(limit), "the largest access unit keeps its length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(limit + 1) < 0, "an access unit one past the packet limit has no length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(0x7fffffffull) < 0, "an access unit of the largest int has no length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch((1ull << 32) + 16) < 0, "an access unit whose low 32 bits look small has no length");
+    Require(VdecCoreAccessUnitLength_nid_no_patch(~0ull) < 0, "the largest 64-bit size has no length");
+}
+
+void CheckDecodeArguments() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    const auto& unit = Units()[0];
+    VdecCoreInputData input{unit.data(), unit.size(), 0, 0, 0, 0, 0};
+    std::array<std::uint32_t, 2> status{7, 7};
+    Require(sceVdecCoreSetDecodeInput(nullptr, &input, nullptr, status.data()) == InvalidArgument, "set input without a decoder");
+    Require(sceVdecCoreSetDecodeInput(decoder, nullptr, nullptr, status.data()) == InvalidArgument, "set input without input data");
+    Require(status[0] == 7 && status[1] == 7, "an input refused for a missing decoder or input leaves the status alone");
+    input.auData = nullptr;
+    Require(sceVdecCoreSetDecodeInput(decoder, &input, nullptr, status.data()) == InvalidArgument, "set input without access unit data");
+    Require(status[0] == 1 && status[1] == 1, "the status is written before the access unit is checked");
+    status = {7, 7};
+    input.auData = unit.data();
+    input.auSize = 0;
+    Require(sceVdecCoreSetDecodeInput(decoder, &input, nullptr, status.data()) == InvalidArgument, "set input with an empty access unit");
+    Require(status[0] == 1 && status[1] == 1, "the status is written before the access unit size is checked");
+    const std::uint64_t limit = VdecCoreMaxAccessUnitSize_nid_no_patch();
+    const std::array<std::uint64_t, 4> oversized{1ull << 31, limit + 1, (1ull << 32) + unit.size(), ~0ull};
+    for (const auto size : oversized) {
+        status = {7, 7};
+        input.auSize = size;
+        Require(sceVdecCoreSetDecodeInput(decoder, &input, nullptr, status.data()) == InvalidArgument, "set input with an access unit past the packet limit");
+        Require(status[0] == 1 && status[1] == 1, "the status is written before an oversized access unit is refused");
+    }
+    input.auSize = unit.size();
+    Require(sceVdecCoreSetDecodeInput(decoder, &input, nullptr, nullptr) == 0, "set input without a status");
+
+    std::uint32_t count = 7;
+    std::uint64_t pointers[2] = {7, 7};
+    VdecCoreOutputInfo output{};
+    VdecCoreFrameBuffer frameBuffer{};
+    Require(sceVdecCoreResetDecoder(nullptr, nullptr) == InvalidArgument, "reset without a decoder");
+    Require(sceVdecCoreSyncDecode(nullptr, &count) == InvalidArgument, "sync without a decoder");
+    Require(sceVdecCoreSyncDecodeWptr(nullptr, &count, &pointers[0], &pointers[1]) == InvalidArgument, "write-pointer sync without a decoder");
+    Require(sceVdecCoreTrySyncDecode(nullptr, &count) == InvalidArgument, "try-sync without a decoder");
+    Require(sceVdecCoreTrySyncDecodeWptr(nullptr, &count, &pointers[0], &pointers[1]) == InvalidArgument, "write-pointer try-sync without a decoder");
+    Require(sceVdecCoreGetDecodeOutput(nullptr, &frameBuffer, 0, &output, nullptr) == InvalidArgument, "get output without a decoder");
+    Require(sceVdecCoreGetDecodeOutput(decoder, &frameBuffer, 0, nullptr, nullptr) == InvalidArgument, "get output without output info");
+    Require(sceVdecCoreFlushDecodeOutput(nullptr, &count, nullptr) == InvalidArgument, "flush without a decoder");
+    Require(count == 7 && pointers[0] == 7 && pointers[1] == 7, "refused calls leave their outputs alone");
+
+    Require(sceVdecCoreSyncDecode(decoder, nullptr) == 0, "sync without a count");
+    Require(sceVdecCoreFlushDecodeOutput(decoder, nullptr, nullptr) == 0, "flush without a count");
+    std::uint32_t held = 7;
+    Require(sceVdecCoreTrySyncDecode(decoder, nullptr) == 0 && sceVdecCoreTrySyncDecode(decoder, &held) == 0 && held == 1, "try-sync after a flush counts the held picture");
+    Require(sceVdecCoreResetDecoder(decoder, nullptr) == 0, "reset");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the argument decoder");
+}
+
+void CheckEmptyQueue() {
+    void* decoder = CreateAvc(100);
+    std::uint32_t count = 7;
+    Require(sceVdecCoreTrySyncDecode(decoder, &count) == 0 && count == 0, "try-sync on an idle decoder");
+    count = 7;
+    Require(sceVdecCoreSyncDecode(decoder, &count) == 0 && count == 0, "sync on an idle decoder");
+    VdecCoreOutputInfo output;
+    std::memset(&output, 0xab, sizeof(output));
+    Require(sceVdecCoreGetDecodeOutput(decoder, nullptr, 0, &output, nullptr) == NoPicture, "get output from an empty queue");
+    Require(output.frameWidth == 100 && output.framePitch == 128 && output.frameHeight == H264Fixture::Height, "an empty queue reports the maximum width, its 64-byte pitch and the maximum height");
+    Require(output.pictureCount == 0 && output.frameBuffer == nullptr && output.frameFormat == 0 && output.reserved0 == 0, "an empty queue reports no picture");
+    Require(output.pictures[0].frameBuffer == nullptr && output.pictures[1].frameBuffer == nullptr && output.lumaBuffer == nullptr && output.chromaBuffer == nullptr, "an empty queue reports no buffers");
+    Require(output.pictures[0].pts == 0xababababababababull && output.aspectRatio == 0xababababu && output.colourDescriptionPresentFlag == 0xababababu, "an empty queue leaves the other fields alone");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the idle decoder");
+}
+
+void CheckDecodeIntoSlots() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    std::vector<std::uint8_t> memory(SlotSize * 4);
+    const VdecCoreMemoryBlock block{memory.data(), memory.size(), 256, 0};
+    Require(sceVdecCoreMapMemoryBlock(decoder, &block, 1) == 0, "map four frame buffer slots");
+    const std::vector<void*> slots = Slots(decoder);
+    Require(slots.size() == 4, "four slots");
+
+    std::size_t pictures = 0;
+    std::vector<std::uint64_t> dts;
+    const auto take = [&](std::uint32_t available) {
+        for (std::uint32_t index = 0; index < available; ++index) {
+            const VdecCoreOutputInfo output = Output(decoder, nullptr);
+            Require(pictures < H264Fixture::PictureHashes.size(), "more pictures than access units");
+            Require(output.frameBuffer == slots[pictures % slots.size()], "pictures go to the mapped slots in turn");
+            const auto unit = H264Fixture::DisplayOrderUnits[pictures];
+            RequirePicture(output, pictures, SlotSize, 1000 + unit, 3000 + unit, "a picture decoded into a slot");
+            dts.push_back(output.pictures[0].dts);
+            ++pictures;
+        }
+    };
+    for (std::size_t unit = 0; unit < Units().size(); ++unit) {
+        Feed(decoder, unit, 1000 + static_cast<std::int64_t>(unit), 2000 + static_cast<std::int64_t>(unit), 3000 + unit);
+        std::uint32_t tried = 0xffffffffu;
+        const int result = sceVdecCoreTrySyncDecode(decoder, &tried);
+        Require(result == 0 || (result == DecodeRunning && tried == 0), "try-sync during a decode");
+        std::uint32_t count = 0xffffffffu;
+        Require(sceVdecCoreSyncDecode(decoder, &count) == 0, "sync after an access unit");
+        take(count);
+    }
+    take(Flush(decoder));
+    Require(pictures == H264Fixture::PictureHashes.size(), "every access unit gives a picture");
+    for (const auto value : dts) Require(value >= 2000 && value < 2000 + Units().size(), "a picture's dts is one of the inputs'");
+    VdecCoreOutputInfo output{};
+    Require(sceVdecCoreGetDecodeOutput(decoder, nullptr, 0, &output, nullptr) == NoPicture, "no picture after the flushed ones");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the slot decoder");
+}
+
+void CheckInvalidPts() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    FeedAll(decoder, PtsInvalid, PtsInvalid, 5000);
+    Require(Flush(decoder) == H264Fixture::PictureHashes.size(), "a flush holds every picture");
+    for (std::size_t picture = 0; picture < H264Fixture::PictureHashes.size(); ++picture) {
+        const VdecCoreOutputInfo output = Output(decoder, nullptr);
+        RequirePicture(output, picture, SlotSize, ~std::uint64_t{0}, 5000 + H264Fixture::DisplayOrderUnits[picture], "a picture without a pts");
+        Require(output.pictures[0].dts == ~std::uint64_t{0}, "a picture without a dts takes the last input's invalid dts");
+    }
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the invalid pts decoder");
+}
+
+void CheckDestinations() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    FeedAll(decoder, 1000, 2000, 3000);
+    Require(Flush(decoder) == H264Fixture::PictureHashes.size(), "a flush holds every picture for the fallback");
+    const VdecCoreOutputInfo fallback = Output(decoder, nullptr);
+    RequirePicture(fallback, 0, SlotSize, 1000, 3000, "a picture in the fallback buffer");
+    const VdecCoreOutputInfo again = Output(decoder, nullptr);
+    Require(again.frameBuffer == fallback.frameBuffer, "the fallback buffer is reused");
+    RequirePicture(again, 1, SlotSize, 1000 + H264Fixture::DisplayOrderUnits[1], 3000 + H264Fixture::DisplayOrderUnits[1], "a second picture in the fallback buffer");
+
+    std::vector<std::uint8_t> slotMemory(SlotSize);
+    const VdecCoreMemoryBlock block{slotMemory.data(), slotMemory.size(), 256, 0};
+    Require(sceVdecCoreMapMemoryBlock(decoder, &block, 1) == 0, "map one slot");
+    std::vector<std::uint8_t> caller(0x2000);
+    VdecCoreFrameBuffer frameBuffer{caller.data(), caller.size()};
+    const VdecCoreOutputInfo supplied = Output(decoder, &frameBuffer);
+    Require(supplied.frameBuffer == caller.data(), "a caller frame buffer comes before the slots");
+    RequirePicture(supplied, 2, caller.size(), 1000 + H264Fixture::DisplayOrderUnits[2], 3000 + H264Fixture::DisplayOrderUnits[2], "a picture in the caller's frame buffer");
+    VdecCoreFrameBuffer unaddressed{nullptr, caller.size()};
+    const VdecCoreOutputInfo slot = Output(decoder, &unaddressed);
+    Require(slot.frameBuffer == slotMemory.data(), "a frame buffer without an address leaves the picture to the slots");
+    RequirePicture(slot, 3, SlotSize, 1000 + H264Fixture::DisplayOrderUnits[3], 3000 + H264Fixture::DisplayOrderUnits[3], "a picture in a slot keeps the slot's size, not the caller's");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the destination decoder");
+}
+
+void CheckOutputStaysInItsDestination() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    FeedAll(decoder, 1000, 2000, 3000);
+    Require(Flush(decoder) == H264Fixture::PictureHashes.size(), "a flush holds every picture for the destination bounds");
+    constexpr std::uint8_t Guard = 0x5a;
+    std::vector<std::uint8_t> memory(SlotSize * 2, Guard);
+    const VdecCoreMemoryBlock block{memory.data(), SlotSize, 256, 0};
+    Require(sceVdecCoreMapMemoryBlock(decoder, &block, 1) == 0, "map one slot followed by guard bytes");
+    VdecCoreFrameBuffer larger{nullptr, SlotSize * 2};
+    const VdecCoreOutputInfo slot = Output(decoder, &larger);
+    Require(slot.frameBuffer == memory.data(), "a frame buffer with no address and a larger size goes to the slot");
+    RequirePicture(slot, 0, SlotSize, 1000, 3000, "a picture in a slot under a larger caller size");
+    Require(std::all_of(memory.begin() + static_cast<std::ptrdiff_t>(SlotSize), memory.end(), [](std::uint8_t byte) { return byte == Guard; }), "a picture in a slot under a larger caller size wrote past the slot");
+    const std::uint32_t pictureSize = OutputPitch * H264Fixture::CodedHeight * 3 / 2;
+    std::vector<std::uint8_t> tight(pictureSize + 0x200, Guard);
+    VdecCoreFrameBuffer small{tight.data(), tight.size()};
+    const VdecCoreOutputInfo caller = Output(decoder, &small);
+    Require(caller.frameBuffer == tight.data(), "a caller frame buffer with room for the picture only");
+    const auto unit = H264Fixture::DisplayOrderUnits[1];
+    Require(caller.pictures[0].pts == 1000 + unit && caller.pictures[0].userData == 3000 + unit, "the picture in a caller frame buffer with room for the picture only");
+    Require(VisibleHash(tight.data(), caller.framePitch, caller.frameHeight) == VisiblePictureHashes[1] && ExtendsLastRows(tight.data(), caller.framePitch, caller.frameHeight), "picture information too large for the room after a picture overwrote the picture");
+    Require(std::all_of(tight.begin() + static_cast<std::ptrdiff_t>(pictureSize), tight.end(), [](std::uint8_t byte) { return byte == Guard; }), "picture information was written into a caller frame buffer with no room for it after the picture");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the bounds decoder");
+}
+
+void CheckFlushResetAndSync() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    std::vector<std::uint8_t> memory(SlotSize * 3);
+    const VdecCoreMemoryBlock block{memory.data(), memory.size(), 256, 0};
+    Require(sceVdecCoreMapMemoryBlock(decoder, &block, 1) == 0, "map three slots");
+    const std::vector<void*> slots = Slots(decoder);
+
+    FeedAll(decoder, 1000, 2000, 3000);
+    const std::uint32_t held = Flush(decoder);
+    Require(held == H264Fixture::PictureHashes.size(), "a flush returns every held picture");
+    std::uint32_t count = 0;
+    Require(sceVdecCoreSyncDecode(decoder, &count) == 0 && count == held, "sync after a flush counts the held pictures");
+    std::uint32_t tried = 0;
+    Require(sceVdecCoreTrySyncDecode(decoder, &tried) == 0 && tried == held, "try-sync after a sync counts the held pictures");
+    std::uint64_t pointers[2] = {7, 7};
+    std::uint32_t forwarded = 0;
+    Require(sceVdecCoreSyncDecodeWptr(decoder, &forwarded, &pointers[0], &pointers[1]) == 0 && forwarded == held, "the write-pointer sync forwards to the sync");
+    forwarded = 0;
+    Require(sceVdecCoreTrySyncDecodeWptr(decoder, &forwarded, &pointers[0], &pointers[1]) == 0 && forwarded == held, "the write-pointer try-sync forwards to the try-sync");
+    Require(pointers[0] == 7 && pointers[1] == 7, "the write-pointer variants leave the pointers alone");
+
+    Require(Output(decoder, nullptr).frameBuffer == slots[0], "the first picture goes to the first slot");
+    Require(Output(decoder, nullptr).frameBuffer == slots[1], "the second picture goes to the second slot");
+
+    FeedAll(decoder, 1000, 2000, 3000);
+    Require(sceVdecCoreResetDecoder(decoder, nullptr) == 0, "reset with queued input");
+    count = 7;
+    Require(sceVdecCoreSyncDecode(decoder, &count) == 0 && count == 0, "a reset drops the queued input and its pictures");
+    tried = 7;
+    Require(sceVdecCoreTrySyncDecode(decoder, &tried) == 0 && tried == 0, "try-sync after a reset");
+    VdecCoreOutputInfo output{};
+    Require(sceVdecCoreGetDecodeOutput(decoder, nullptr, 0, &output, nullptr) == NoPicture, "no picture after a reset");
+    Require(Flush(decoder) == 0, "a flush after a reset holds nothing");
+
+    FeedAll(decoder, 1000, 2000, 3000);
+    Require(Flush(decoder) == H264Fixture::PictureHashes.size(), "the decoder decodes again after a reset");
+    const VdecCoreOutputInfo restarted = Output(decoder, nullptr);
+    Require(restarted.frameBuffer == slots[0], "a reset starts the slots again");
+    RequirePicture(restarted, 0, SlotSize, 1000, 3000, "the first picture after a reset");
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete the flushed decoder");
+}
+
+void CheckDeleteWithQueuedInput() {
+    void* decoder = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    for (int round = 0; round < 8; ++round) FeedAll(decoder, 1000, 2000, 3000);
+    Require(sceVdecCoreDeleteDecoder(decoder) == 0, "delete a decoder with queued input");
+    void* held = CreateAvc(static_cast<std::int32_t>(H264Fixture::Width));
+    FeedAll(held, 1000, 2000, 3000);
+    Require(Flush(held) == H264Fixture::PictureHashes.size(), "a flush before the delete holds every picture");
+    Require(sceVdecCoreDeleteDecoder(held) == 0, "delete a decoder with held pictures");
+}
+
 }
 
 int main() {
@@ -183,5 +512,14 @@ int main() {
     CheckComputeResourceInfo();
     CheckCreateAndDelete();
     CheckMapMemoryBlock();
+    CheckAccessUnitLength();
+    CheckDecodeArguments();
+    CheckEmptyQueue();
+    CheckDecodeIntoSlots();
+    CheckInvalidPts();
+    CheckDestinations();
+    CheckOutputStaysInItsDestination();
+    CheckFlushResetAndSync();
+    CheckDeleteWithQueuedInput();
     return 0;
 }

docs/dev/debt/libSceVdecCore.md

@@ -8,8 +8,9 @@
 | `sceVdecCoreQueryMemorySizeSvp` | `uAsJCITzX5k` |
 
 
-- [sceVdecCoreAnalyzeStream, sceVdecCoreFinalizeComputeResource, sceVdecCoreFlushDecodeOutput, sceVdecCoreGetDecodeOutput, sceVdecCoreInitializeComputeResource, sceVdecCoreResetDecoder, sceVdecCoreSetDecodeInput, sceVdecCoreSetDecodeOutputSw, sceVdecCoreSyncDecode, sceVdecCoreSyncDecodeOutputSw, sceVdecCoreSyncDecodeWptr, sceVdecCoreTrySyncDecode, sceVdecCoreTrySyncDecodeOutputSw and sceVdecCoreTrySyncDecodeWptr](../../../core/libs/prx/libSceVdecCore/Export.cpp) (libSceVdecCore) - names and NIDs are present in the [PS5 3.20 export inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceVdecCore.c), but no credible prototypes were found. The declarations have no parameters solely to resolve known imports; this is not an ABI claim and callers must not call them until signatures are verified
+- [sceVdecCoreAnalyzeStream, sceVdecCoreFinalizeComputeResource, sceVdecCoreInitializeComputeResource, sceVdecCoreSetDecodeOutputSw, sceVdecCoreSyncDecodeOutputSw and sceVdecCoreTrySyncDecodeOutputSw](../../../core/libs/prx/libSceVdecCore/Export.cpp) (libSceVdecCore) - names and NIDs are present in the [PS5 3.20 export inventory](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceVdecCore.c), but no credible prototypes were found. The declarations have no parameters solely to resolve known imports; this is not an ABI claim and callers must not call them until signatures are verified
 
 ### Functional
 
-- [sceVdecCoreQueryInstanceSize, sceVdecCoreQueryFrameBufferInfo, sceVdecCoreQueryComputeResourceInfo, sceVdecCoreCreateDecoder, sceVdecCoreDeleteDecoder and sceVdecCoreMapMemoryBlock](../../../core/libs/prx/libSceVdecCore/Export.cpp) (libSceVdecCore) - console behaviour is unmeasured. The signatures, error codes, 1920x1080 and 16-frame defaults and slot cutting follow [shadPS4's `vdeccore.cpp`](https://github.com/shadps4-emu/shadPS4/blob/0614ce324be092791b758d79f93d9f1ca5b10a81/src/core/libraries/videodec/vdeccore.cpp#L275-L706), and the sizes are shadPS4's constants, not console values: instance 0x40000, compute 0x10000, and a frame buffer of NV12 plus 0x1000 rounded to 64 KiB. CreateDecoder opens an FFmpeg H.264 or HEVC context but starts no decode thread; the memory info and compute queue are ignored, and a handle CreateDecoder did not return, or a codec that is unavailable or fails to open, throws
+- [sceVdecCoreQueryInstanceSize, sceVdecCoreQueryFrameBufferInfo, sceVdecCoreQueryComputeResourceInfo, sceVdecCoreCreateDecoder, sceVdecCoreDeleteDecoder and sceVdecCoreMapMemoryBlock](../../../core/libs/prx/libSceVdecCore/Export.cpp) (libSceVdecCore) - console behaviour is unmeasured. The signatures, error codes, 1920x1080 and 16-frame defaults and slot cutting follow [shadPS4's `vdeccore.cpp`](https://github.com/shadps4-emu/shadPS4/blob/0614ce324be092791b758d79f93d9f1ca5b10a81/src/core/libraries/videodec/vdeccore.cpp#L275-L706), and the sizes are shadPS4's constants, not console values: instance 0x40000, compute 0x10000, and a frame buffer of NV12 plus 0x1000 rounded to 64 KiB. CreateDecoder opens an FFmpeg H.264 or HEVC context and starts the decoder's decode thread, which DeleteDecoder stops and joins; the memory info and compute queue are ignored, and a handle CreateDecoder did not return, or a codec that is unavailable or fails to open, throws
+- [sceVdecCoreResetDecoder, sceVdecCoreSetDecodeInput, sceVdecCoreSyncDecode, sceVdecCoreGetDecodeOutput, sceVdecCoreFlushDecodeOutput, sceVdecCoreSyncDecodeWptr, sceVdecCoreTrySyncDecode and sceVdecCoreTrySyncDecodeWptr](../../../core/libs/prx/libSceVdecCore/Export.cpp) (libSceVdecCore) - console behaviour is unmeasured; they follow [shadPS4's `vdeccore.cpp`](https://github.com/shadps4-emu/shadPS4/blob/0614ce324be092791b758d79f93d9f1ca5b10a81/src/core/libraries/videodec/vdeccore.cpp#L118-L768). Decoding is FFmpeg software decoding on a host thread. GetDecodeOutput's pitch is the width rounded to 64 while CreateDecoder's frame buffer size uses the horizontal alignment (256 by default), as in shadPS4. Without a caller frame buffer or mapped slot a picture goes to host memory the decoder owns, and the empty-queue fields are shadPS4's. Unlike shadPS4, a picture larger than its frame buffer throws, an access unit too large for an FFmpeg packet is refused, and the NV12 copy goes row by row