From eb4d64a3b05476d8c5d3ab5674d69187d19216e2 Mon Sep 17 00:00:00 2001 From: Yossi Yadgar Date: Mon, 19 Oct 2026 10:08:33 +0300 Subject: [PATCH 1/4] fix(libSceVideodec2): frame an access unit whose length prefix looks like a start code --- core/libs/prx/libSceVideodec2/Export.cpp | 36 ++++++++++++++++-------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/core/libs/prx/libSceVideodec2/Export.cpp b/core/libs/prx/libSceVideodec2/Export.cpp index ea5ca36c4e..e5daeed313 100644 --- a/core/libs/prx/libSceVideodec2/Export.cpp +++ b/core/libs/prx/libSceVideodec2/Export.cpp @@ -372,21 +372,35 @@ void OpenCodec(Decoder& decoder) { } } -const std::vector* ToAnnexB(Decoder& decoder, const std::uint8_t* data, std::size_t size) { - if (size >= 4 && data[0] == 0 && data[1] == 0 && (data[2] == 1 || (data[2] == 0 && data[3] == 1))) { - decoder.annexB.assign(data, data + size); - return &decoder.annexB; - } - decoder.annexB.clear(); +bool TilesAsAvcc(const std::uint8_t* data, std::size_t size) { + if (size < 4) return false; for (std::size_t index = 0; index < size;) { - if (size - index < 4) throw std::runtime_error("Videodec2: truncated NAL length"); + if (size - index < 4) return false; const std::size_t length = (static_cast(data[index]) << 24u) | (static_cast(data[index + 1]) << 16u) | (static_cast(data[index + 2]) << 8u) | data[index + 3]; - if (length == 0 || length > size - index - 4) throw std::runtime_error("Videodec2: invalid NAL length"); - static constexpr std::uint8_t startCode[4] = {0, 0, 0, 1}; - decoder.annexB.insert(decoder.annexB.end(), startCode, startCode + 4); - decoder.annexB.insert(decoder.annexB.end(), data + index + 4, data + index + 4 + length); + if (length == 0 || length > size - index - 4) return false; index += 4 + length; } + return true; +} + +bool StartsWithStartCode(const std::uint8_t* data, std::size_t size) { + return size >= 3 && data[0] == 0 && data[1] == 0 && (data[2] == 1 || (size >= 4 && data[2] == 0 && data[3] == 1)); +} + +const std::vector* ToAnnexB(Decoder& decoder, const std::uint8_t* data, std::size_t size) { + if (TilesAsAvcc(data, size)) { + decoder.annexB.clear(); + for (std::size_t index = 0; index < size;) { + const std::size_t length = (static_cast(data[index]) << 24u) | (static_cast(data[index + 1]) << 16u) | (static_cast(data[index + 2]) << 8u) | data[index + 3]; + static constexpr std::uint8_t startCode[4] = {0, 0, 0, 1}; + decoder.annexB.insert(decoder.annexB.end(), startCode, startCode + 4); + decoder.annexB.insert(decoder.annexB.end(), data + index + 4, data + index + 4 + length); + index += 4 + length; + } + return &decoder.annexB; + } + if (!StartsWithStartCode(data, size)) throw std::runtime_error("Videodec2: access unit is neither Annex-B nor length-prefixed"); + decoder.annexB.assign(data, data + size); return &decoder.annexB; } From 43b5f33f80f94ddf3cc5c4363f2f44e6afe98acc Mon Sep 17 00:00:00 2001 From: Yossi Yadgar Date: Mon, 19 Oct 2026 10:08:33 +0300 Subject: [PATCH 2/4] test(libSceVideodec2): decode an AVCC access unit whose length prefix is a start code --- core/libs/tests/GuestVideodec2.cpp | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/core/libs/tests/GuestVideodec2.cpp b/core/libs/tests/GuestVideodec2.cpp index f8d3be0ffc..f483c38881 100644 --- a/core/libs/tests/GuestVideodec2.cpp +++ b/core/libs/tests/GuestVideodec2.cpp @@ -576,6 +576,48 @@ void testDecode(bool lengthPrefixed, bool separateParameters = false) { checkThrows([&] { sceVideodec2DeleteDecoder_nid_postfix(handle); }); } +void testLengthPrefixLooksLikeStartCode() { + auto units = accessUnits(true); + std::vector filler; + constexpr std::size_t fillerLength = 0x101; + for (int shift = 24; shift >= 0; shift -= 8) filler.push_back(static_cast(fillerLength >> shift)); + filler.push_back(0x0c); + filler.insert(filler.end(), fillerLength - 1, 0xff); + units.front().insert(units.front().begin(), filler.begin(), filler.end()); + check(units.front()[0] == 0 && units.front()[1] == 0 && units.front()[2] == 1, "the crafted unit does not begin with start-code bytes"); + + const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 0, 1, 100, 0, static_cast(Width), static_cast(Height), 4, 1}; + DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; + check(sceVideodec2QueryDecoderMemoryInfo_nid_postfix(&config, &memory) == 0, "memory query failed"); + std::uint64_t handle = 0; + check(sceVideodec2CreateDecoder_nid_postfix(&config, &memory, &handle) == 0, "decoder creation failed"); + std::vector buffer(memory.maxFrameBufferSize); + std::size_t pictures = 0; + const auto take = [&](const OutputInfo& output) { + if (!output.isValid) return false; + check(pictures < PictureHashes.size(), "more pictures than access units"); + check(!output.isErrorFrame && output.frameWidth == Width && output.frameHeight == CodedHeight, "unexpected picture geometry"); + check(hashNv12(buffer.data(), output.framePitch, output.frameHeight) == PictureHashes[pictures], "picture " + std::to_string(pictures) + " differs from the reference"); + ++pictures; + return true; + }; + for (std::size_t unit = 0; unit < units.size(); ++unit) { + const InputData input{sizeof(InputData), units[unit].data(), units[unit].size(), 1000 + unit, unit, 0xa0 + unit}; + FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false}; + OutputInfo output{sizeof(OutputInfo)}; + check(sceVideodec2Decode_nid_postfix(handle, &input, &frame, &output) == 0, "decode failed"); + take(output); + } + for (;;) { + FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false}; + OutputInfo output{sizeof(OutputInfo)}; + check(sceVideodec2Flush_nid_postfix(handle, &frame, &output) == 0, "flush failed"); + if (!take(output)) break; + } + check(pictures == PictureHashes.size(), "missing pictures after flush"); + check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed"); +} + std::uint64_t createDecoder(std::uint32_t codec, std::uint32_t profile) { const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, codec, profile, 0, static_cast(Width), static_cast(Height), 4, 1}; DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; @@ -784,6 +826,7 @@ int main() { testFailures(); testDecode(false); testDecode(true); + testLengthPrefixLooksLikeStartCode(); testDecode(false, true); const auto hevc = hevcUnits(HevcStream.data(), HevcStream.size()); testClip(CodecHevc, 1, hevc, HevcHashes); From f4f471f415cd52138e60d3b174bc9c1421720518 Mon Sep 17 00:00:00 2001 From: Yossi Yadgar Date: Mon, 19 Oct 2026 13:53:35 +0300 Subject: [PATCH 3/4] fix(libSceVideodec2): settle an access unit that reads as both Annex-B and AVCC A unit opening with 00 00 01 xx can also tile as AVCC when its first NAL is 0x1xx bytes long, in either direction. Annex-B requires a zero_byte before the first NAL of an access unit, so a unit opening with a 3-byte start code is taken as AVCC unless the decoder has already seen unambiguous Annex-B units that open that way. A unit opening with 00 00 00 01 that also tiles keeps the framing of the last unambiguous unit, Annex-B when there is none. Media Player Classic PS5 sends Annex-B units with 4-byte start codes and leaves exactly the colliding samples in AVCC, so the decoder's last framing alone would choose wrong there. --- core/libs/prx/libSceVideodec2/Export.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceVideodec2/Export.cpp b/core/libs/prx/libSceVideodec2/Export.cpp index e5daeed313..6350f757cd 100644 --- a/core/libs/prx/libSceVideodec2/Export.cpp +++ b/core/libs/prx/libSceVideodec2/Export.cpp @@ -304,6 +304,8 @@ struct Decoder { std::deque ready; std::vector annexB; std::vector parameterSets; + bool lengthPrefixed = false; + bool shortStartCodes = false; bool flushing = false; std::array vp9References{}; Vp9PictureInfo vp9Colors{}; @@ -388,7 +390,18 @@ bool StartsWithStartCode(const std::uint8_t* data, std::size_t size) { } const std::vector* ToAnnexB(Decoder& decoder, const std::uint8_t* data, std::size_t size) { - if (TilesAsAvcc(data, size)) { + const bool tiles = TilesAsAvcc(data, size); + const bool opensWithStartCode = StartsWithStartCode(data, size); + if (!tiles && !opensWithStartCode) throw std::runtime_error("Videodec2: access unit is neither Annex-B nor length-prefixed"); + const bool shortStartCode = opensWithStartCode && data[2] == 1; + bool lengthPrefixed = tiles; + if (tiles && opensWithStartCode) { + lengthPrefixed = shortStartCode ? !decoder.shortStartCodes : decoder.lengthPrefixed; + } else { + decoder.lengthPrefixed = tiles; + if (shortStartCode) decoder.shortStartCodes = true; + } + if (lengthPrefixed) { decoder.annexB.clear(); for (std::size_t index = 0; index < size;) { const std::size_t length = (static_cast(data[index]) << 24u) | (static_cast(data[index + 1]) << 16u) | (static_cast(data[index + 2]) << 8u) | data[index + 3]; @@ -399,7 +412,6 @@ const std::vector* ToAnnexB(Decoder& decoder, const std::uint8_t* } return &decoder.annexB; } - if (!StartsWithStartCode(data, size)) throw std::runtime_error("Videodec2: access unit is neither Annex-B nor length-prefixed"); decoder.annexB.assign(data, data + size); return &decoder.annexB; } From a23a24c1e7537dc746c794994aa2c3546c1b9775 Mon Sep 17 00:00:00 2001 From: Yossi Yadgar Date: Mon, 19 Oct 2026 13:53:35 +0300 Subject: [PATCH 4/4] test(libSceVideodec2): decode units whose start code and length prefix collide Move the AVCC collision to the second unit, add an AVCC unit opening 00 00 01 23 among Annex-B units, and an Annex-B stream with 3-byte start codes whose SPS and PPS unit, padded with trailing zero bytes, also reads as one length prefix. --- core/libs/tests/GuestVideodec2.cpp | 76 ++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/core/libs/tests/GuestVideodec2.cpp b/core/libs/tests/GuestVideodec2.cpp index f483c38881..a38a94c9c6 100644 --- a/core/libs/tests/GuestVideodec2.cpp +++ b/core/libs/tests/GuestVideodec2.cpp @@ -576,16 +576,7 @@ void testDecode(bool lengthPrefixed, bool separateParameters = false) { checkThrows([&] { sceVideodec2DeleteDecoder_nid_postfix(handle); }); } -void testLengthPrefixLooksLikeStartCode() { - auto units = accessUnits(true); - std::vector filler; - constexpr std::size_t fillerLength = 0x101; - for (int shift = 24; shift >= 0; shift -= 8) filler.push_back(static_cast(fillerLength >> shift)); - filler.push_back(0x0c); - filler.insert(filler.end(), fillerLength - 1, 0xff); - units.front().insert(units.front().begin(), filler.begin(), filler.end()); - check(units.front()[0] == 0 && units.front()[1] == 0 && units.front()[2] == 1, "the crafted unit does not begin with start-code bytes"); - +void decodeInOrder(const std::vector>& units) { const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 0, 1, 100, 0, static_cast(Width), static_cast(Height), 4, 1}; DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; check(sceVideodec2QueryDecoderMemoryInfo_nid_postfix(&config, &memory) == 0, "memory query failed"); @@ -618,6 +609,70 @@ void testLengthPrefixLooksLikeStartCode() { check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed"); } +bool readsAsOneLengthPrefix(const std::vector& unit) { + const std::size_t length = (static_cast(unit[0]) << 24u) | (static_cast(unit[1]) << 16u) | (static_cast(unit[2]) << 8u) | unit[3]; + return length + 4 == unit.size(); +} + +std::vector lengthPrefixedFiller(std::size_t length) { + std::vector filler; + for (int shift = 24; shift >= 0; shift -= 8) filler.push_back(static_cast(length >> shift)); + filler.push_back(0x0c); + filler.insert(filler.end(), length - 1, 0xff); + return filler; +} + +void testLengthPrefixLooksLikeStartCode() { + auto units = accessUnits(true); + const auto filler = lengthPrefixedFiller(0x101); + check(!(units[0][0] == 0 && units[0][1] == 0 && (units[0][2] == 1 || units[0][3] == 1)), "the first length-prefixed unit already begins with start-code bytes"); + units[1].insert(units[1].begin(), filler.begin(), filler.end()); + check(units[1][0] == 0 && units[1][1] == 0 && units[1][2] == 1, "the crafted unit does not begin with start-code bytes"); + decodeInOrder(units); + + auto mixed = accessUnits(false); + const auto mixedFiller = lengthPrefixedFiller(0x123); + mixed[1] = accessUnits(true)[1]; + mixed[1].insert(mixed[1].begin(), mixedFiller.begin(), mixedFiller.end()); + check(mixed[0][2] == 0 && mixed[0][3] == 1 && mixed[1][0] == 0 && mixed[1][1] == 0 && mixed[1][2] == 1, "the length-prefixed unit does not follow an Annex-B unit with a four-byte start code"); + decodeInOrder(mixed); +} + +void testStartCodeLooksLikeLengthPrefix() { + auto units = accessUnits(false); + for (auto& unit : units) { + check(unit[0] == 0 && unit[1] == 0 && unit[2] == 0 && unit[3] == 1, "an access unit does not open with a four-byte start code"); + unit.erase(unit.begin()); + } + auto& first = units.front(); + std::size_t sequence = 0; + std::size_t afterPicture = 0; + bool picture = false; + for (std::size_t index = 0; index + 3 < first.size(); ++index) { + if (first[index] != 0 || first[index + 1] != 0 || first[index + 2] != 1) continue; + const auto type = first[index + 3] & 0x1fu; + if (picture) { + afterPicture = first[index - 1] == 0 ? index - 1 : index; + break; + } + if (type == 7 && sequence == 0) sequence = index; + if (type == 8 && sequence != 0) picture = true; + index += 2; + } + check(sequence != 0 && afterPicture > sequence, "the first access unit has no SPS and PPS before another NAL"); + const std::vector delimiter(first.begin(), first.begin() + static_cast(first[sequence - 1] == 0 ? sequence - 1 : sequence)); + check(delimiter.size() >= 4 && delimiter[2] == 1 && !readsAsOneLengthPrefix(delimiter), "the access unit delimiter is not an unambiguous unit with a three-byte start code"); + std::vector parameters(first.begin() + static_cast(sequence), first.begin() + static_cast(afterPicture)); + first.erase(first.begin(), first.begin() + static_cast(afterPicture)); + if (first[2] == 0) first.erase(first.begin()); + const std::size_t padded = 4 + ((std::size_t{1} << 8u) | parameters[3]); + check(parameters.size() < padded, "the parameter sets are too long to pad"); + parameters.resize(padded, 0); + check(readsAsOneLengthPrefix(parameters), "the padded parameter sets do not read as one length-prefixed NAL"); + units.insert(units.begin(), {delimiter, parameters}); + decodeInOrder(units); +} + std::uint64_t createDecoder(std::uint32_t codec, std::uint32_t profile) { const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, codec, profile, 0, static_cast(Width), static_cast(Height), 4, 1}; DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; @@ -827,6 +882,7 @@ int main() { testDecode(false); testDecode(true); testLengthPrefixLooksLikeStartCode(); + testStartCodeLooksLikeLengthPrefix(); testDecode(false, true); const auto hevc = hevcUnits(HevcStream.data(), HevcStream.size()); testClip(CodecHevc, 1, hevc, HevcHashes);