From a668165c4b293887e0495d0f5774c67fe0460397 Mon Sep 17 00:00:00 2001 From: battleblob1 Date: Wed, 21 Oct 2026 12:58:36 +0300 Subject: [PATCH 1/3] fix(recompiler): accept optional wave bodies of several blocks in stencil replay MaskReplaySafety accepted an EXEC- or VCC-skipped body only when it was a single block branching straight to the merge. Bodies with their own branches (s_cbranch_execz around nested control flow) were refused. optionalBody now takes the single-entry region of blocks reached from the body before the merge: every block is entered only from the branch or the region, none returns, nothing in it is exported or discarded, its values leave only through merge phis, and every incoming value of a merge phi equals the skipped value on inactive lanes. inactiveEquals follows the region's own phis for that. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/MaskedSelect.cpp | 55 +++++++++++++++++++ core/shader/recompiler/StencilReplay.cpp | 45 +++++++++++---- 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index a77f88d58f..d73bb2d266 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -1192,6 +1192,61 @@ int main() { passed &= run("an optional wave body cannot change helper coordinates for later implicit samples", [&] { return !optionalPixelBody(true); }); passed &= run("an optional wave body cannot change helper coordinates for LOD queries", [&] { return !optionalPixelBody(true, IrOpcode::ImageQueryLod); }); + const auto optionalPixelRegion = [&](bool unmaskedArm) { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& zero = b.constant(0u); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& body = b.newBlock(); + auto& left = b.newBlock(); + auto& right = b.newBlock(); + auto& join = b.newBlock(); + auto& merge = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&left); + body.AddBranch(&right); + left.AddBranch(&join); + right.AddBranch(&join); + join.AddBranch(&merge); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.block = &body; + auto& uniformBit = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.emit(IrOpcode::GetUserData, IrType::U32, {}), &zero}); + auto& bodyInfo = b.program.Metadata().blockInfo[1u]; + bodyInfo.terminator.kind = TerminatorKind::ConditionalBranch; + bodyInfo.terminator.condition = BranchCondition::SccNonZero; + bodyInfo.terminator.trueBlock = left.Id(); + bodyInfo.terminator.falseBlock = right.Id(); + bodyInfo.condition = &uniformBit; + for (const unsigned index : {2u, 3u, 4u}) { + b.program.Metadata().blockInfo[index].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[index].terminator.trueBlock = index == 4u ? merge.Id() : join.Id(); + } + b.block = &left; + auto& leftValue = b.select(predicate, b.constant(42u), zero); + b.block = &right; + auto& rightValue = unmaskedArm ? b.constant(7u) : b.select(predicate, b.constant(43u), zero); + b.block = &join; + auto& joined = b.phi(IrType::U32, left, leftValue); + joined.AddPhiOperand(&right, &rightValue); + b.block = &merge; + auto& merged = b.phi(IrType::U32, entry, zero); + merged.AddPhiOperand(&join, &joined); + b.keep(merged); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }; + + passed &= run("an optional wave body spanning several blocks preserves inactive values through its own phis", [&] { return optionalPixelRegion(false); }); + passed &= run("an optional wave body spanning several blocks cannot write an inactive lane on any of its paths", [&] { return !optionalPixelRegion(true); }); + passed &= run("varying attribute sources cannot be treated as uniform any-active predicates", [&] { for (const auto opcode : {IrOpcode::GetAttribute, IrOpcode::GetInterpolationParameter, IrOpcode::GetInterpolationParameterF16, IrOpcode::GetTessellationAttribute}) { Builder b; diff --git a/core/shader/recompiler/StencilReplay.cpp b/core/shader/recompiler/StencilReplay.cpp index 30bc43b765..dfb8f37f6b 100644 --- a/core/shader/recompiler/StencilReplay.cpp +++ b/core/shader/recompiler/StencilReplay.cpp @@ -438,6 +438,9 @@ private: value = value->Resolve(); if (sameValue(value, other)) return true; if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, depth + 1u); + if (value->IsPhi() && region.contains(value->Parent()) && value->ArgumentCount() != 0u && value->ArgumentCount() == value->PhiBlockCount()) { + return std::all_of(value->Arguments().begin(), value->Arguments().end(), [&](const IrValue* argument) { return inactiveEquals(argument, other, predicate, depth + 1u); }); + } return false; } @@ -482,26 +485,43 @@ private: } bool optionalBody(const IrBlock* branch, const IrBlock* body, const IrBlock* merge, const IrValue* predicate) { - const auto& bodyInfo = program.Metadata().blockInfo[blockIndices.at(body)]; - if (bodyInfo.terminator.kind != TerminatorKind::Branch || indices.at(bodyInfo.terminator.trueBlock) != blockIndices.at(merge)) return false; + region.clear(); + if (body == merge) return false; + std::vector pending{body}; + while (!pending.empty()) { + const IrBlock* block = pending.back(); + pending.pop_back(); + if (block == merge || !region.insert(block).second) continue; + if (block == branch || region.size() > 64u || program.Metadata().blockInfo[blockIndices.at(block)].terminator.kind == TerminatorKind::Return) return false; + for (const auto* successor : block->Successors()) pending.push_back(successor); + } + for (const auto* block : region) { + for (const auto* predecessor : block->Predecessors()) { + if (block == body ? predecessor != branch : !region.contains(predecessor)) return false; + } + } std::set merged; for (const auto* instruction : merge->Instructions()) { if (!instruction->IsPhi()) continue; - if (instruction->ArgumentCount() != 2u || instruction->PhiBlockCount() != 2u) return false; + if (instruction->ArgumentCount() < 2u || instruction->ArgumentCount() != instruction->PhiBlockCount()) return false; const IrValue* skipped = nullptr; - const IrValue* taken = nullptr; - for (std::size_t index = 0; index < 2u; ++index) { + std::vector taken; + for (std::size_t index = 0; index < instruction->PhiBlockCount(); ++index) { if (instruction->PhiBlock(index) == branch) skipped = instruction->Argument(index)->Resolve(); - if (instruction->PhiBlock(index) == body) taken = instruction->Argument(index)->Resolve(); + else if (region.contains(instruction->PhiBlock(index))) taken.push_back(instruction->Argument(index)->Resolve()); + else return false; } - if (!skipped || !taken || !inactiveEquals(taken, skipped, predicate) || quadObserved(instruction)) return false; + if (!skipped || taken.empty() || quadObserved(instruction)) return false; + for (const auto* value : taken) if (!inactiveEquals(value, skipped, predicate)) return false; merged.insert(instruction); } - for (const auto* instruction : body->Instructions()) { - if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false; - for (const auto& use : instruction->OperandUses()) { - if (use.user->Parent() == body || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue; - return false; + for (const auto* block : region) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false; + for (const auto& use : instruction->OperandUses()) { + if (region.contains(use.user->Parent()) || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue; + return false; + } } } return !merged.empty(); @@ -553,6 +573,7 @@ private: std::map indices; std::map blockIndices; std::vector> allPaths; + std::set region; }; std::optional> Values(const IrProgram& program, const IrValue* value, std::set& visiting, unsigned depth, std::optional component = {}, const IrValue* active = nullptr) { From 9f6dd5b6ed6c8635d4981bcbf44471f0b4492939 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Fri, 23 Oct 2026 22:50:38 +0530 Subject: [PATCH 2/3] fix(shader): pair structurally equivalent whole-quad evaluations IrProgram does not intern values, so titles that evaluate the same whole-quad ballot twice emit distinct WqmU64 nodes for the low and high mask words. maskStable and helpers paired those words by object identity and rejected the split form. Compare the two evaluations structurally instead: same-block ballots must agree on their source tree and must not be separated by an execution-mask change, and every newly accepted node is recorded as restricted. Assassins Creed Shadows (PPSA20397) hits this in shader 0x407325e400. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../libSceAgcDriver/tests/MaskedSelect.cpp | 48 ++++++++++++++ core/shader/recompiler/StencilReplay.cpp | 66 ++++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index d73bb2d266..47f6971970 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -1074,6 +1074,54 @@ int main() { return !StencilReplaySafe(b.program); }); + const auto wholeQuadEval = [](Builder& b, IrValue& live) -> IrValue& { + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live}); + auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})}); + auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})}); + return b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})}); + }; + + const auto splitQuads = [&](Builder& b, IrValue& live) -> IrValue& { + auto& wqmLow = wholeQuadEval(b, live); + auto& wqmHigh = wholeQuadEval(b, live); + return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + }; + + passed &= run("stencil replay pairs structurally equivalent whole-quad evaluations for split low and high words", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + exportUnder(b, splitQuads(b, b.logicalAnd(pixelPredicate(b), helperPredicate(b, false))), true, true); + return StencilReplaySafe(b.program); + }); + + passed &= run("stencil replay refuses to pair whole-quad evaluations separated by an execution-mask change", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& live = b.logicalAnd(pixelPredicate(b), helperPredicate(b, false)); + auto& wqmLow = wholeQuadEval(b, live); + b.emit(IrOpcode::SetExecLo, IrType::Void, {&b.constant(0xffffffffu)}); + auto& wqmHigh = wholeQuadEval(b, live); + auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + exportUnder(b, guard, true, true); + return !StencilReplaySafe(b.program); + }); + + passed &= run("stencil replay refuses to pair whole-quad evaluations whose predicates differ only in a constant", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& helperLow = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast(StageInputKind::HelperInvocation)), &b.constant(0u)}); + auto& lowLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperLow, &b.constant(0u)})); + auto& wqmLow = wholeQuadEval(b, lowLive); + auto& helperHigh = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast(StageInputKind::HelperInvocation)), &b.constant(0u)}); + auto& highLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperHigh, &b.constant(1u)})); + auto& wqmHigh = wholeQuadEval(b, highLive); + auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + exportUnder(b, guard, true, true); + return !StencilReplaySafe(b.program); + }); + const auto execCopy = [](Builder& b, IrValue& low, IrValue& high) -> IrValue& { auto& active = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); auto& lanes = b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(1u)})}); diff --git a/core/shader/recompiler/StencilReplay.cpp b/core/shader/recompiler/StencilReplay.cpp index dfb8f37f6b..04815b6ec4 100644 --- a/core/shader/recompiler/StencilReplay.cpp +++ b/core/shader/recompiler/StencilReplay.cpp @@ -132,6 +132,60 @@ private: if (value && !value->HasImmediate()) restricted.insert(value); } + void restrictSource(const IrValue* value, unsigned depth) { + value = value->Resolve(); + if (depth > 64u || value->HasImmediate()) return; + restrict(value); + for (const auto* argument : value->Arguments()) restrictSource(argument, depth + 1u); + } + + bool execEquivalent(const IrValue* left, const IrValue* right) const { + const IrBlock* block = left->Parent(); + if (block == nullptr || block != right->Parent()) return false; + bool between = false; + for (const IrValue* instruction : block->Instructions()) { + if (instruction == left || instruction == right) { + if (between) return true; + between = true; + continue; + } + if (!between) continue; + switch (instruction->Opcode()) { + case IrOpcode::SetExec: + case IrOpcode::SetExecLo: + case IrOpcode::SetExecHi: + case IrOpcode::Discard: + case IrOpcode::Sendmsg: + case IrOpcode::SetThreadBitScalarRegister: + case IrOpcode::SetScalarMaskTag: + return false; + default: + break; + } + } + return false; + } + + bool equivalentSource(const IrValue* left, const IrValue* right, unsigned depth) { + if (depth > 64u) return false; + left = left->Resolve(); + right = right->Resolve(); + if (left == right || sameValue(left, right)) return true; + if (left->HasImmediate() || right->HasImmediate()) return left->HasImmediate() && right->HasImmediate() && left->Type() == right->Type() && left->ImmediateU64() == right->ImmediateU64(); + if (left->Opcode() != right->Opcode() || left->ArgumentCount() != right->ArgumentCount()) return false; + if (left->Opcode() == IrOpcode::Ballot && !execEquivalent(left, right)) return false; + for (std::size_t index = 0; index < left->ArgumentCount(); ++index) if (!equivalentSource(left->Argument(index), right->Argument(index), depth + 1u)) return false; + return true; + } + + bool equivalentWqm(const IrValue* left, const IrValue* right, unsigned depth) { + left = left->Resolve(); + right = right->Resolve(); + if (left == right) return true; + if (left->Opcode() != IrOpcode::WqmU64 || right->Opcode() != IrOpcode::WqmU64 || left->ArgumentCount() != 1u || right->ArgumentCount() != 1u) return false; + return equivalentSource(left->Argument(0), right->Argument(0), depth + 1u); + } + std::optional own(const IrValue* value) { value = value->Resolve(); if (value->Opcode() != IrOpcode::INotEqual32) return std::nullopt; @@ -189,7 +243,8 @@ private: valid = (maskStable(pair(0u, 0u), depth + 1u) && maskStable(pair(1u, 1u), depth + 1u)) || (waveSize != 32u && maskStable(pair(0u, 1u), depth + 1u) && maskStable(pair(1u, 0u), depth + 1u)); } else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) { const IrValue* wqm = mask.low->Argument(0)->Resolve(); - const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u)); + const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr; + const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth)); if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) { const IrValue* source = wqm->Argument(0)->Resolve(); valid = source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u && maskStable(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u); @@ -197,7 +252,11 @@ private: const auto provenance = helpers(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u); valid = provenance == HelperMask::RealOnly || provenance == HelperMask::QuadStable; } - if (valid) { restrict(wqm); restrict(source); } + if (valid) { + restrict(wqm); + restrict(source); + if (highWqm != wqm) restrictSource(highWqm, 0u); + } } } else if (mask.low->IsPhi() && mask.low->ArgumentCount() != 0u && mask.low->ArgumentCount() == mask.low->PhiBlockCount() && (waveSize == 32u || (mask.high->IsPhi() && mask.high->Parent() == mask.low->Parent() && mask.high->ArgumentCount() == mask.high->PhiBlockCount()))) { valid = true; @@ -288,7 +347,8 @@ private: } } else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) { const IrValue* wqm = mask.low->Argument(0)->Resolve(); - const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u)); + const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr; + const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth)); if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) { const IrValue* source = wqm->Argument(0)->Resolve(); if (source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u) { From 4a590355437c82397525a67d235ba008d761e7b5 Mon Sep 17 00:00:00 2001 From: akkadbakkad89 Date: Fri, 23 Oct 2026 22:50:33 +0530 Subject: [PATCH 3/3] fix(recompiler): exempt proven control conditions and own-bit ballots in stencil replay MaskReplaySafety counted every use reaching a conditional branch condition as a quad observation, including EXEC/VCC guards that control() itself already proves replay-safe, and every Ballot regardless of how its bits are read. Nested optional regions (a merged phi re-used as the predicate of an inner execz) and ballots consumed only through the own-lane funnel (word extract, lane<32 select, lane&31 shift, &1 isolate) were refused even though neither crosses lane boundaries; the same reasoning exempts explicit-lod samples, which do not compute implicit quad derivatives. The check now runs conditional branches through a memoized controlProven() with a visiting set, exempts those condition uses in quadObserved, walks past own-bit ballots into the isolated per-lane bools, and treats explicit-lod samples as lane-local in both quadObserved and helperObserved. optionalBody now uses a local region set so nested control proofs do not corrupt the outer walk. Anything not matching these shapes is refused as before. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../libSceAgcDriver/tests/MaskedSelect.cpp | 108 ++++++++++++++++++ core/shader/recompiler/StencilReplay.cpp | 93 +++++++++++++-- 2 files changed, 191 insertions(+), 10 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index 47f6971970..84be667ce8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -1295,6 +1295,114 @@ int main() { passed &= run("an optional wave body spanning several blocks preserves inactive values through its own phis", [&] { return optionalPixelRegion(false); }); passed &= run("an optional wave body spanning several blocks cannot write an inactive lane on any of its paths", [&] { return !optionalPixelRegion(true); }); + const auto nestedOptionalGuard = [&](bool unsafeInner, bool ballotEscape) { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& base = helperPredicate(b, false); + auto& body = b.newBlock(); + auto& merge = b.newBlock(); + auto& inner = b.newBlock(); + auto& join = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&merge); + merge.AddBranch(&inner); + merge.AddBranch(&join); + inner.AddBranch(&join); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id(); + b.block = &body; + auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base}); + b.block = &merge; + auto& merged = b.phi(IrType::Bool, entry, base); + merged.AddPhiOperand(&body, &taken); + auto& innerCondition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&merged}); + auto& mergeInfo = b.program.Metadata().blockInfo[2u]; + mergeInfo.terminator.kind = TerminatorKind::ConditionalBranch; + mergeInfo.terminator.condition = BranchCondition::ExecZero; + mergeInfo.terminator.trueBlock = join.Id(); + mergeInfo.terminator.falseBlock = inner.Id(); + mergeInfo.condition = &innerCondition; + b.program.Metadata().blockInfo[3u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[3u].terminator.trueBlock = join.Id(); + b.block = &inner; + auto& written = b.select(merged, b.constant(42u), b.constant(0u)); + if (unsafeInner) b.emit(IrOpcode::Discard, IrType::Void, {}); + b.block = &join; + auto& joined = b.phi(IrType::U32, merge, b.constant(0u)); + joined.AddPhiOperand(&inner, &written); + if (ballotEscape) { + const auto [low, high] = b.ballotWords(merged); + b.keep(b.threadBit(*low, *high)); + b.keep(b.add(*low, 1u)); + } + b.keep(joined); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }; + + passed &= run("an optional wave body merge may guard another optional wave body", [&] { return nestedOptionalGuard(false, false); }); + passed &= run("an optional wave body merge cannot guard a body with side effects", [&] { return !nestedOptionalGuard(true, false); }); + passed &= run("an optional wave body merge cannot feed a ballot escaping the lane's own bit", [&] { return !nestedOptionalGuard(false, true); }); + + passed &= run("an optional wave body merge may feed an own-lane ballot read", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& base = helperPredicate(b, false); + auto& body = b.newBlock(); + auto& merge = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&merge); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id(); + b.block = &body; + auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base}); + b.block = &merge; + auto& merged = b.phi(IrType::Bool, entry, base); + merged.AddPhiOperand(&body, &taken); + b.keep(b.ballotBit(merged)); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }); + + passed &= run("an active mask may flag a sample with an explicit lod", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + MemoryInfo memory{}; + memory.kind = ResourceKind::Image; + memory.imageSampleFlags = RdnaImageSampleFlagLevelZero; + b.program.Resources().memoryInfo.push_back(memory); + auto& active = b.ballotBit(b.truth()); + auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&b.constant(0u)}); + auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {}); + auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {}); + MemoryFlags flags{0u, 0u}; + std::uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address, &active}, bits); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }); + passed &= run("varying attribute sources cannot be treated as uniform any-active predicates", [&] { for (const auto opcode : {IrOpcode::GetAttribute, IrOpcode::GetInterpolationParameter, IrOpcode::GetInterpolationParameterF16, IrOpcode::GetTessellationAttribute}) { Builder b; diff --git a/core/shader/recompiler/StencilReplay.cpp b/core/shader/recompiler/StencilReplay.cpp index 04815b6ec4..b72ffe0e3e 100644 --- a/core/shader/recompiler/StencilReplay.cpp +++ b/core/shader/recompiler/StencilReplay.cpp @@ -108,7 +108,7 @@ public: } for (const auto& info : program.Metadata().blockInfo) { if (info.terminator.kind != TerminatorKind::ConditionalBranch) continue; - if (!control(info)) return false; + if (!controlProven(info)) return false; } for (const auto& [value, mask] : ownMasks) if (helperObserved(value)) return false; for (const auto* block : program.BlockOrder()) { @@ -493,27 +493,88 @@ private: return left == right || (left->HasImmediate() && right->HasImmediate() && left->Type() == IrType::U32 && right->Type() == IrType::U32 && left->ImmediateU32() == right->ImmediateU32()); } - bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, unsigned depth = 0) { + bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, const std::set& region, unsigned depth = 0) { if (depth > 32u) return false; value = value->Resolve(); if (sameValue(value, other)) return true; - if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, depth + 1u); + if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, region, depth + 1u); if (value->IsPhi() && region.contains(value->Parent()) && value->ArgumentCount() != 0u && value->ArgumentCount() == value->PhiBlockCount()) { - return std::all_of(value->Arguments().begin(), value->Arguments().end(), [&](const IrValue* argument) { return inactiveEquals(argument, other, predicate, depth + 1u); }); + return std::all_of(value->Arguments().begin(), value->Arguments().end(), [&](const IrValue* argument) { return inactiveEquals(argument, other, predicate, region, depth + 1u); }); } return false; } - bool quadObserved(const IrValue* value) const { + bool explicitLodSample(const IrValue* value) const { + if (value->Opcode() == IrOpcode::ImageSampleExplicitLod) return true; + if (value->Opcode() != IrOpcode::ImageSampleRaw && value->Opcode() != IrOpcode::ImageGatherRaw) return false; + const auto index = value->Flags().index; + return index < program.Resources().memoryInfo.size() && ImageSampleExplicitLod(program.Resources().memoryInfo[index].imageSampleFlags, program.Resources().stage); + } + + bool ownBitShifted(const IrValue* user, std::size_t operand, std::vector& terminals) const { + if (user->Opcode() != IrOpcode::ShiftRightLogical32 || user->ArgumentCount() != 2u || operand != 0u) return false; + const IrValue* amount = user->Argument(1)->Resolve(); + if (amount->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* lane = Other(amount, 31u); + if (lane == nullptr || lane->Opcode() != IrOpcode::LaneId) return false; + for (const auto& use : user->OperandUses()) { + const IrValue* isolated = use.user; + if (isolated->Opcode() != IrOpcode::BitwiseAnd32 || Other(isolated, 1u) != user) return false; + terminals.push_back(isolated); + } + return true; + } + + bool ownBitSelected(const IrValue* user, std::size_t operand, unsigned half, std::vector& terminals) const { + if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || operand != half + 1u) return false; + const IrValue* condition = user->Argument(0)->Resolve(); + if (condition->Opcode() != IrOpcode::ULessThan32 || condition->ArgumentCount() != 2u || condition->Argument(0)->Resolve()->Opcode() != IrOpcode::LaneId || !Immediate(condition->Argument(1), 32u)) return false; + for (const auto& use : user->OperandUses()) if (!ownBitShifted(use.user, use.operand, terminals)) return false; + return true; + } + + bool ownBitBallot(const IrValue* ballot, std::vector& terminals) const { + if (ballot->Opcode() != IrOpcode::Ballot || ballot->ArgumentCount() != 1u) return false; + for (const auto& use : ballot->OperandUses()) { + const IrValue* extract = use.user; + if (extract->Opcode() != IrOpcode::CompositeExtractU32x4 || extract->ArgumentCount() != 2u || extract->Argument(0)->Resolve() != ballot) return false; + const IrValue* index = extract->Argument(1)->Resolve(); + if (!index->HasImmediate() || index->ImmediateU32() >= (waveSize == 32u ? 1u : 2u)) return false; + const auto half = index->ImmediateU32(); + for (const auto& euse : extract->OperandUses()) { + const IrValue* word = euse.user; + if (word->Opcode() == IrOpcode::BitwiseAnd32) { + for (const auto& muse : word->OperandUses()) { + const bool read = waveSize == 64u ? ownBitSelected(muse.user, muse.operand, half, terminals) : ownBitShifted(muse.user, muse.operand, terminals); + if (!read) return false; + } + } else { + const bool read = waveSize == 64u ? ownBitSelected(word, euse.operand, half, terminals) : ownBitShifted(word, euse.operand, terminals); + if (!read) return false; + } + } + } + return true; + } + + bool quadObserved(const IrValue* value) { std::set visited; std::vector pending{value}; while (!pending.empty()) { const IrValue* current = pending.back(); pending.pop_back(); if (!visited.insert(current).second) continue; - for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current) return true; + for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current && !controlProven(info)) return true; for (const auto& use : current->OperandUses()) { const auto opcode = use.user->Opcode(); + if (opcode == IrOpcode::Ballot) { + std::vector terminals; + if (ownBitBallot(use.user, terminals)) { + pending.insert(pending.end(), terminals.begin(), terminals.end()); + continue; + } + } + if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; } if (ImageOpcodeInfoOf(opcode).needsSampler || opcode == IrOpcode::Ballot || opcode == IrOpcode::WqmU64 || opcode == IrOpcode::ReadLane || opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::DppMoveU32 || opcode == IrOpcode::DppUpdateU32 || opcode == IrOpcode::SwizzleU32 || opcode == IrOpcode::PermuteU32 || opcode == IrOpcode::BpermuteU32 || opcode == IrOpcode::Permlane16U32 || opcode == IrOpcode::WriteLane || SharedAccessOf(opcode) != SharedAccess::None) return true; if (opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue; pending.push_back(use.user); @@ -536,6 +597,7 @@ private: } for (const auto& use : current->OperandUses()) { const auto opcode = use.user->Opcode(); + if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; } if (ImageOpcodeInfoOf(opcode).needsSampler) return true; if (restricted.contains(use.user) || opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue; pending.push_back(use.user); @@ -545,8 +607,8 @@ private: } bool optionalBody(const IrBlock* branch, const IrBlock* body, const IrBlock* merge, const IrValue* predicate) { - region.clear(); if (body == merge) return false; + std::set region; std::vector pending{body}; while (!pending.empty()) { const IrBlock* block = pending.back(); @@ -571,8 +633,9 @@ private: else if (region.contains(instruction->PhiBlock(index))) taken.push_back(instruction->Argument(index)->Resolve()); else return false; } - if (!skipped || taken.empty() || quadObserved(instruction)) return false; - for (const auto* value : taken) if (!inactiveEquals(value, skipped, predicate)) return false; + if (!skipped || taken.empty()) return false; + if (quadObserved(instruction)) return false; + for (const auto* value : taken) if (!inactiveEquals(value, skipped, predicate, region)) return false; merged.insert(instruction); } for (const auto* block : region) { @@ -587,6 +650,15 @@ private: return !merged.empty(); } + bool controlProven(const BlockInfo& info) { + if (const auto found = controlResults.find(info.id); found != controlResults.end()) return found->second; + if (!controlVisiting.insert(info.id).second) return false; + const bool result = control(info); + controlVisiting.erase(info.id); + controlResults[info.id] = result; + return result; + } + bool control(const BlockInfo& info) { const auto* branch = program.BlockOrder()[indices.at(info.id)]; const auto* trueBlock = program.BlockOrder()[indices.at(info.terminator.trueBlock)]; @@ -633,7 +705,8 @@ private: std::map indices; std::map blockIndices; std::vector> allPaths; - std::set region; + std::map controlResults; + std::set controlVisiting; }; std::optional> Values(const IrProgram& program, const IrValue* value, std::set& visiting, unsigned depth, std::optional component = {}, const IrValue* active = nullptr) {