fix(libSceAudiodec): report a decode error when FFmpeg cannot open an M4AAC layout

An ADTS header names its object type, sampling index and channel
configuration, and the decoder reopens FFmpeg with an AudioSpecificConfig
built from them. A layout FFmpeg refuses, such as the reserved sampling
index 13, threw out of sceAudiodecDecode into guest code. PPSA08427 ended
on such a throw after a few minutes on the title screen, on a build from
before the current layout handling.

- A reopen that FFmpeg refuses leaves the decoder without a context and
  the access unit is a decode error (M4AAC_RESULT_DECODE_ERROR,
  ERROR_API_FAIL); the raw layout search skips a refused layout.
- A decoder without a context decodes nothing until a header names a
  layout that opens; sceAudiodecClearContext leaves it as it is.
- The open failure names the codec, the AudioSpecificConfig and FFmpeg's
  error, and is logged on stderr.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

core/libs/prx/libSceAudiodec/src/Codecs.cpp

@@ -1,4 +1,5 @@
 #include "prx/libSceAudiodec/src/Codecs.hpp"
+#include "prx/libc/include/General.hpp"
 
 #include "libatrac9.h"
 extern "C" {
@@ -8,6 +9,7 @@ extern "C" {
 
 #include <algorithm>
 #include <cmath>
+#include <cstdio>
 #include <cstring>
 #include <memory>
 #include <stdexcept>
@@ -47,10 +49,13 @@ public:
 
     ~FfmpegDecoder() override { avcodec_free_context(&context); }
 
-    void Reset() override { avcodec_flush_buffers(context); }
+    void Reset() override {
+        if (context) avcodec_flush_buffers(context);
+    }
 
 protected:
     DecodeResult decodePacket(const std::uint8_t* data, std::size_t size, std::uint8_t* pcm, std::size_t pcmSize) {
+        if (!context) return {DecodeStatus::InvalidData};
         const std::unique_ptr<AVPacket, PacketDeleter> packet(av_packet_alloc());
         const std::unique_ptr<AVFrame, FrameDeleter> frame(av_frame_alloc());
         if (!packet || !frame) throw std::bad_alloc();
@@ -85,10 +90,16 @@ protected:
         return result;
     }
 
-    void reopen(std::vector<std::uint8_t> replacement) {
+    bool tryReopen(std::vector<std::uint8_t> replacement) {
         avcodec_free_context(&context);
         extradata = std::move(replacement);
-        open();
+        try {
+            open();
+            return true;
+        } catch (const std::runtime_error& error) {
+            APS5_LOG_ERR("%s", error.what());
+            return false;
+        }
     }
 
 private:
@@ -102,7 +113,18 @@ private:
             std::memcpy(context->extradata, extradata.data(), extradata.size());
             context->extradata_size = static_cast<int>(extradata.size());
         }
-        if (avcodec_open2(context, codec, nullptr) < 0) throw std::runtime_error("libSceAudiodec: cannot open an FFmpeg decoder");
+        if (const int status = avcodec_open2(context, codec, nullptr); status < 0) {
+            avcodec_free_context(&context);
+            char reason[AV_ERROR_MAX_STRING_SIZE]{};
+            av_strerror(status, reason, sizeof(reason));
+            std::string config;
+            for (const auto byte : extradata) {
+                char hex[4];
+                std::snprintf(hex, sizeof(hex), "%02x", byte);
+                config += hex;
+            }
+            throw std::runtime_error("libSceAudiodec: cannot open the FFmpeg " + std::string(avcodec_get_name(id)) + " decoder" + (config.empty() ? "" : " (config " + config + ")") + ": " + reason);
+        }
     }
 
     std::int32_t wordSize;
@@ -186,14 +208,14 @@ public:
         if (layout.channelConfiguration == 0) {
             if (!layouts.empty()) {
                 layouts.clear();
-                reopen({});
+                if (!tryReopen({})) return {DecodeStatus::InvalidData};
             }
             return decodePacket(data, frameBytes, pcm, pcmSize);
         }
         if (layouts.empty() || layouts.front() != layout) {
             layouts = {layout};
             sbr = highEfficiency ? Sbr::Implicit : Sbr::Absent;
-            reopen(audioSpecificConfig(layout, sbr, 0));
+            if (!tryReopen(audioSpecificConfig(layout, sbr, 0))) return {DecodeStatus::InvalidData};
         }
         DecodeResult result = decodeAccessUnit(data + headerBytes, frameBytes - headerBytes, pcm, pcmSize);
         result.consumed = frameBytes;
@@ -203,18 +225,18 @@ public:
 private:
     DecodeResult decodeAccessUnit(const std::uint8_t* data, std::size_t size, std::uint8_t* pcm, std::size_t pcmSize) {
         for (std::size_t index = 0; index < layouts.size(); ++index) {
-            if (index != 0) reopen(audioSpecificConfig(layouts[index], sbr, 0));
+            if (index != 0 && !tryReopen(audioSpecificConfig(layouts[index], sbr, 0))) continue;
             DecodeResult result = decodePacket(data, size, pcm, pcmSize);
             if (result.status == DecodeStatus::InvalidData) continue;
             layouts = {layouts[index]};
             if (sbr == Sbr::Implicit && layouts.front().channelConfiguration == 1 && result.channels > 1) {
                 sbr = Sbr::WithoutParametricStereo;
-                reopen(audioSpecificConfig(layouts.front(), sbr, result.sampleRate));
+                if (!tryReopen(audioSpecificConfig(layouts.front(), sbr, result.sampleRate))) return {DecodeStatus::InvalidData};
                 result = decodePacket(data, size, pcm, pcmSize);
             }
             return result;
         }
-        if (layouts.size() > 1) reopen(audioSpecificConfig(layouts.front(), sbr, 0));
+        if (layouts.size() > 1) tryReopen(audioSpecificConfig(layouts.front(), sbr, 0));
         return {DecodeStatus::InvalidData};
     }
 

core/libs/tests/GuestAudiodec.cpp

@@ -259,6 +259,25 @@ void TestAac(bool adts, std::uint32_t maxChannels = 2) {
     Require(sceAudiodecDeleteDecoder(handle) == 0);
 }
 
+void TestAacUnopenableLayout() {
+    AudiodecParamM4aac param{sizeof(param), 1, 1, 3, 2, 0};
+    AudiodecM4aacInfo info{sizeof(info), 0, 0, 0, 0};
+    AudiodecCtrl ctrl{&param, &info, nullptr, nullptr};
+    const std::int32_t handle = sceAudiodecCreateDecoder(&ctrl, 3);
+    Require(handle > 0);
+    Streams streams;
+    const std::size_t length = AdtsLength(AAC_ADTS);
+    std::vector<std::uint8_t> reserved(AAC_ADTS, AAC_ADTS + length);
+    reserved[2] = static_cast<std::uint8_t>((reserved[2] & 0xC3) | 13 << 2);
+    Require(Decode(handle, ctrl, streams, reserved.data(), reserved.size()) == errorApiFail && info.i_result == -4);
+    Require(streams.au.ui_au_size == 0 && streams.pcm.ui_pcm_size == 0);
+    Require(Decode(handle, ctrl, streams, reserved.data(), reserved.size()) == errorApiFail && info.i_result == -4);
+    Require(sceAudiodecClearContext(handle) == 0);
+    Require(Decode(handle, ctrl, streams, AAC_ADTS, length) == 0);
+    Require(streams.au.ui_au_size == length && streams.pcm.ui_pcm_size == 1024 * 2 * 2 && info.i_result == 0);
+    Require(sceAudiodecDeleteDecoder(handle) == 0);
+}
+
 void TestAacRaw51(std::uint32_t maxChannels) {
     AudiodecParamM4aac param{sizeof(param), 1, 2, 3, maxChannels, 0};
     AudiodecM4aacInfo info{sizeof(info), 0, 0, 0, 0};
@@ -469,6 +488,7 @@ int main() {
             TestHeAac(HEAAC_STEREO, sizeof(HEAAC_STEREO), 2, adts, highEfficiency);
         }
     }
+    TestAacUnopenableLayout();
     TestMp3();
     TestAt9();
     for (std::uint32_t type = 1; type <= 3; ++type) Require(sceAudiodecTermLibrary(type) == 0);