diff --git a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp index a77f88d58f..84be667ce8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp @@ -1074,6 +1074,54 @@ int main() { return !StencilReplaySafe(b.program); }); + const auto wholeQuadEval = [](Builder& b, IrValue& live) -> IrValue& { + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live}); + auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); + auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})}); + auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})}); + return b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})}); + }; + + const auto splitQuads = [&](Builder& b, IrValue& live) -> IrValue& { + auto& wqmLow = wholeQuadEval(b, live); + auto& wqmHigh = wholeQuadEval(b, live); + return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + }; + + passed &= run("stencil replay pairs structurally equivalent whole-quad evaluations for split low and high words", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + exportUnder(b, splitQuads(b, b.logicalAnd(pixelPredicate(b), helperPredicate(b, false))), true, true); + return StencilReplaySafe(b.program); + }); + + passed &= run("stencil replay refuses to pair whole-quad evaluations separated by an execution-mask change", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& live = b.logicalAnd(pixelPredicate(b), helperPredicate(b, false)); + auto& wqmLow = wholeQuadEval(b, live); + b.emit(IrOpcode::SetExecLo, IrType::Void, {&b.constant(0xffffffffu)}); + auto& wqmHigh = wholeQuadEval(b, live); + auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + exportUnder(b, guard, true, true); + return !StencilReplaySafe(b.program); + }); + + passed &= run("stencil replay refuses to pair whole-quad evaluations whose predicates differ only in a constant", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& helperLow = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast(StageInputKind::HelperInvocation)), &b.constant(0u)}); + auto& lowLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperLow, &b.constant(0u)})); + auto& wqmLow = wholeQuadEval(b, lowLive); + auto& helperHigh = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast(StageInputKind::HelperInvocation)), &b.constant(0u)}); + auto& highLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperHigh, &b.constant(1u)})); + auto& wqmHigh = wholeQuadEval(b, highLive); + auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)})); + exportUnder(b, guard, true, true); + return !StencilReplaySafe(b.program); + }); + const auto execCopy = [](Builder& b, IrValue& low, IrValue& high) -> IrValue& { auto& active = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()}); auto& lanes = b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(1u)})}); @@ -1192,6 +1240,169 @@ int main() { passed &= run("an optional wave body cannot change helper coordinates for later implicit samples", [&] { return !optionalPixelBody(true); }); passed &= run("an optional wave body cannot change helper coordinates for LOD queries", [&] { return !optionalPixelBody(true, IrOpcode::ImageQueryLod); }); + const auto optionalPixelRegion = [&](bool unmaskedArm) { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& zero = b.constant(0u); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& body = b.newBlock(); + auto& left = b.newBlock(); + auto& right = b.newBlock(); + auto& join = b.newBlock(); + auto& merge = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&left); + body.AddBranch(&right); + left.AddBranch(&join); + right.AddBranch(&join); + join.AddBranch(&merge); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.block = &body; + auto& uniformBit = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.emit(IrOpcode::GetUserData, IrType::U32, {}), &zero}); + auto& bodyInfo = b.program.Metadata().blockInfo[1u]; + bodyInfo.terminator.kind = TerminatorKind::ConditionalBranch; + bodyInfo.terminator.condition = BranchCondition::SccNonZero; + bodyInfo.terminator.trueBlock = left.Id(); + bodyInfo.terminator.falseBlock = right.Id(); + bodyInfo.condition = &uniformBit; + for (const unsigned index : {2u, 3u, 4u}) { + b.program.Metadata().blockInfo[index].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[index].terminator.trueBlock = index == 4u ? merge.Id() : join.Id(); + } + b.block = &left; + auto& leftValue = b.select(predicate, b.constant(42u), zero); + b.block = &right; + auto& rightValue = unmaskedArm ? b.constant(7u) : b.select(predicate, b.constant(43u), zero); + b.block = &join; + auto& joined = b.phi(IrType::U32, left, leftValue); + joined.AddPhiOperand(&right, &rightValue); + b.block = &merge; + auto& merged = b.phi(IrType::U32, entry, zero); + merged.AddPhiOperand(&join, &joined); + b.keep(merged); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }; + + passed &= run("an optional wave body spanning several blocks preserves inactive values through its own phis", [&] { return optionalPixelRegion(false); }); + passed &= run("an optional wave body spanning several blocks cannot write an inactive lane on any of its paths", [&] { return !optionalPixelRegion(true); }); + + const auto nestedOptionalGuard = [&](bool unsafeInner, bool ballotEscape) { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& base = helperPredicate(b, false); + auto& body = b.newBlock(); + auto& merge = b.newBlock(); + auto& inner = b.newBlock(); + auto& join = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&merge); + merge.AddBranch(&inner); + merge.AddBranch(&join); + inner.AddBranch(&join); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id(); + b.block = &body; + auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base}); + b.block = &merge; + auto& merged = b.phi(IrType::Bool, entry, base); + merged.AddPhiOperand(&body, &taken); + auto& innerCondition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&merged}); + auto& mergeInfo = b.program.Metadata().blockInfo[2u]; + mergeInfo.terminator.kind = TerminatorKind::ConditionalBranch; + mergeInfo.terminator.condition = BranchCondition::ExecZero; + mergeInfo.terminator.trueBlock = join.Id(); + mergeInfo.terminator.falseBlock = inner.Id(); + mergeInfo.condition = &innerCondition; + b.program.Metadata().blockInfo[3u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[3u].terminator.trueBlock = join.Id(); + b.block = &inner; + auto& written = b.select(merged, b.constant(42u), b.constant(0u)); + if (unsafeInner) b.emit(IrOpcode::Discard, IrType::Void, {}); + b.block = &join; + auto& joined = b.phi(IrType::U32, merge, b.constant(0u)); + joined.AddPhiOperand(&inner, &written); + if (ballotEscape) { + const auto [low, high] = b.ballotWords(merged); + b.keep(b.threadBit(*low, *high)); + b.keep(b.add(*low, 1u)); + } + b.keep(joined); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }; + + passed &= run("an optional wave body merge may guard another optional wave body", [&] { return nestedOptionalGuard(false, false); }); + passed &= run("an optional wave body merge cannot guard a body with side effects", [&] { return !nestedOptionalGuard(true, false); }); + passed &= run("an optional wave body merge cannot feed a ballot escaping the lane's own bit", [&] { return !nestedOptionalGuard(false, true); }); + + passed &= run("an optional wave body merge may feed an own-lane ballot read", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + auto& entry = *b.block; + auto& predicate = pixelPredicate(b); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate}); + auto& base = helperPredicate(b, false); + auto& body = b.newBlock(); + auto& merge = b.newBlock(); + entry.AddBranch(&body); + entry.AddBranch(&merge); + body.AddBranch(&merge); + auto& entryInfo = b.program.Metadata().blockInfo[0u]; + entryInfo.terminator.kind = TerminatorKind::ConditionalBranch; + entryInfo.terminator.condition = BranchCondition::ExecZero; + entryInfo.terminator.trueBlock = merge.Id(); + entryInfo.terminator.falseBlock = body.Id(); + entryInfo.condition = &condition; + b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch; + b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id(); + b.block = &body; + auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base}); + b.block = &merge; + auto& merged = b.phi(IrType::Bool, entry, base); + merged.AddPhiOperand(&body, &taken); + b.keep(b.ballotBit(merged)); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }); + + passed &= run("an active mask may flag a sample with an explicit lod", [&] { + Builder b; + b.program.Resources().stage = IrShaderStage::Pixel; + MemoryInfo memory{}; + memory.kind = ResourceKind::Image; + memory.imageSampleFlags = RdnaImageSampleFlagLevelZero; + b.program.Resources().memoryInfo.push_back(memory); + auto& active = b.ballotBit(b.truth()); + auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&b.constant(0u)}); + auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {}); + auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {}); + MemoryFlags flags{0u, 0u}; + std::uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address, &active}, bits); + exportUnder(b, b.truth(), true, true); + return StencilReplaySafe(b.program); + }); + passed &= run("varying attribute sources cannot be treated as uniform any-active predicates", [&] { for (const auto opcode : {IrOpcode::GetAttribute, IrOpcode::GetInterpolationParameter, IrOpcode::GetInterpolationParameterF16, IrOpcode::GetTessellationAttribute}) { Builder b; diff --git a/core/shader/recompiler/StencilReplay.cpp b/core/shader/recompiler/StencilReplay.cpp index 30bc43b765..b72ffe0e3e 100644 --- a/core/shader/recompiler/StencilReplay.cpp +++ b/core/shader/recompiler/StencilReplay.cpp @@ -108,7 +108,7 @@ public: } for (const auto& info : program.Metadata().blockInfo) { if (info.terminator.kind != TerminatorKind::ConditionalBranch) continue; - if (!control(info)) return false; + if (!controlProven(info)) return false; } for (const auto& [value, mask] : ownMasks) if (helperObserved(value)) return false; for (const auto* block : program.BlockOrder()) { @@ -132,6 +132,60 @@ private: if (value && !value->HasImmediate()) restricted.insert(value); } + void restrictSource(const IrValue* value, unsigned depth) { + value = value->Resolve(); + if (depth > 64u || value->HasImmediate()) return; + restrict(value); + for (const auto* argument : value->Arguments()) restrictSource(argument, depth + 1u); + } + + bool execEquivalent(const IrValue* left, const IrValue* right) const { + const IrBlock* block = left->Parent(); + if (block == nullptr || block != right->Parent()) return false; + bool between = false; + for (const IrValue* instruction : block->Instructions()) { + if (instruction == left || instruction == right) { + if (between) return true; + between = true; + continue; + } + if (!between) continue; + switch (instruction->Opcode()) { + case IrOpcode::SetExec: + case IrOpcode::SetExecLo: + case IrOpcode::SetExecHi: + case IrOpcode::Discard: + case IrOpcode::Sendmsg: + case IrOpcode::SetThreadBitScalarRegister: + case IrOpcode::SetScalarMaskTag: + return false; + default: + break; + } + } + return false; + } + + bool equivalentSource(const IrValue* left, const IrValue* right, unsigned depth) { + if (depth > 64u) return false; + left = left->Resolve(); + right = right->Resolve(); + if (left == right || sameValue(left, right)) return true; + if (left->HasImmediate() || right->HasImmediate()) return left->HasImmediate() && right->HasImmediate() && left->Type() == right->Type() && left->ImmediateU64() == right->ImmediateU64(); + if (left->Opcode() != right->Opcode() || left->ArgumentCount() != right->ArgumentCount()) return false; + if (left->Opcode() == IrOpcode::Ballot && !execEquivalent(left, right)) return false; + for (std::size_t index = 0; index < left->ArgumentCount(); ++index) if (!equivalentSource(left->Argument(index), right->Argument(index), depth + 1u)) return false; + return true; + } + + bool equivalentWqm(const IrValue* left, const IrValue* right, unsigned depth) { + left = left->Resolve(); + right = right->Resolve(); + if (left == right) return true; + if (left->Opcode() != IrOpcode::WqmU64 || right->Opcode() != IrOpcode::WqmU64 || left->ArgumentCount() != 1u || right->ArgumentCount() != 1u) return false; + return equivalentSource(left->Argument(0), right->Argument(0), depth + 1u); + } + std::optional own(const IrValue* value) { value = value->Resolve(); if (value->Opcode() != IrOpcode::INotEqual32) return std::nullopt; @@ -189,7 +243,8 @@ private: valid = (maskStable(pair(0u, 0u), depth + 1u) && maskStable(pair(1u, 1u), depth + 1u)) || (waveSize != 32u && maskStable(pair(0u, 1u), depth + 1u) && maskStable(pair(1u, 0u), depth + 1u)); } else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) { const IrValue* wqm = mask.low->Argument(0)->Resolve(); - const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u)); + const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr; + const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth)); if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) { const IrValue* source = wqm->Argument(0)->Resolve(); valid = source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u && maskStable(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u); @@ -197,7 +252,11 @@ private: const auto provenance = helpers(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u); valid = provenance == HelperMask::RealOnly || provenance == HelperMask::QuadStable; } - if (valid) { restrict(wqm); restrict(source); } + if (valid) { + restrict(wqm); + restrict(source); + if (highWqm != wqm) restrictSource(highWqm, 0u); + } } } else if (mask.low->IsPhi() && mask.low->ArgumentCount() != 0u && mask.low->ArgumentCount() == mask.low->PhiBlockCount() && (waveSize == 32u || (mask.high->IsPhi() && mask.high->Parent() == mask.low->Parent() && mask.high->ArgumentCount() == mask.high->PhiBlockCount()))) { valid = true; @@ -288,7 +347,8 @@ private: } } else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) { const IrValue* wqm = mask.low->Argument(0)->Resolve(); - const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u)); + const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr; + const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth)); if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) { const IrValue* source = wqm->Argument(0)->Resolve(); if (source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u) { @@ -433,24 +493,88 @@ private: return left == right || (left->HasImmediate() && right->HasImmediate() && left->Type() == IrType::U32 && right->Type() == IrType::U32 && left->ImmediateU32() == right->ImmediateU32()); } - bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, unsigned depth = 0) { + bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, const std::set& region, unsigned depth = 0) { if (depth > 32u) return false; value = value->Resolve(); if (sameValue(value, other)) return true; - if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, depth + 1u); + if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, region, depth + 1u); + if (value->IsPhi() && region.contains(value->Parent()) && value->ArgumentCount() != 0u && value->ArgumentCount() == value->PhiBlockCount()) { + return std::all_of(value->Arguments().begin(), value->Arguments().end(), [&](const IrValue* argument) { return inactiveEquals(argument, other, predicate, region, depth + 1u); }); + } return false; } - bool quadObserved(const IrValue* value) const { + bool explicitLodSample(const IrValue* value) const { + if (value->Opcode() == IrOpcode::ImageSampleExplicitLod) return true; + if (value->Opcode() != IrOpcode::ImageSampleRaw && value->Opcode() != IrOpcode::ImageGatherRaw) return false; + const auto index = value->Flags().index; + return index < program.Resources().memoryInfo.size() && ImageSampleExplicitLod(program.Resources().memoryInfo[index].imageSampleFlags, program.Resources().stage); + } + + bool ownBitShifted(const IrValue* user, std::size_t operand, std::vector& terminals) const { + if (user->Opcode() != IrOpcode::ShiftRightLogical32 || user->ArgumentCount() != 2u || operand != 0u) return false; + const IrValue* amount = user->Argument(1)->Resolve(); + if (amount->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* lane = Other(amount, 31u); + if (lane == nullptr || lane->Opcode() != IrOpcode::LaneId) return false; + for (const auto& use : user->OperandUses()) { + const IrValue* isolated = use.user; + if (isolated->Opcode() != IrOpcode::BitwiseAnd32 || Other(isolated, 1u) != user) return false; + terminals.push_back(isolated); + } + return true; + } + + bool ownBitSelected(const IrValue* user, std::size_t operand, unsigned half, std::vector& terminals) const { + if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || operand != half + 1u) return false; + const IrValue* condition = user->Argument(0)->Resolve(); + if (condition->Opcode() != IrOpcode::ULessThan32 || condition->ArgumentCount() != 2u || condition->Argument(0)->Resolve()->Opcode() != IrOpcode::LaneId || !Immediate(condition->Argument(1), 32u)) return false; + for (const auto& use : user->OperandUses()) if (!ownBitShifted(use.user, use.operand, terminals)) return false; + return true; + } + + bool ownBitBallot(const IrValue* ballot, std::vector& terminals) const { + if (ballot->Opcode() != IrOpcode::Ballot || ballot->ArgumentCount() != 1u) return false; + for (const auto& use : ballot->OperandUses()) { + const IrValue* extract = use.user; + if (extract->Opcode() != IrOpcode::CompositeExtractU32x4 || extract->ArgumentCount() != 2u || extract->Argument(0)->Resolve() != ballot) return false; + const IrValue* index = extract->Argument(1)->Resolve(); + if (!index->HasImmediate() || index->ImmediateU32() >= (waveSize == 32u ? 1u : 2u)) return false; + const auto half = index->ImmediateU32(); + for (const auto& euse : extract->OperandUses()) { + const IrValue* word = euse.user; + if (word->Opcode() == IrOpcode::BitwiseAnd32) { + for (const auto& muse : word->OperandUses()) { + const bool read = waveSize == 64u ? ownBitSelected(muse.user, muse.operand, half, terminals) : ownBitShifted(muse.user, muse.operand, terminals); + if (!read) return false; + } + } else { + const bool read = waveSize == 64u ? ownBitSelected(word, euse.operand, half, terminals) : ownBitShifted(word, euse.operand, terminals); + if (!read) return false; + } + } + } + return true; + } + + bool quadObserved(const IrValue* value) { std::set visited; std::vector pending{value}; while (!pending.empty()) { const IrValue* current = pending.back(); pending.pop_back(); if (!visited.insert(current).second) continue; - for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current) return true; + for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current && !controlProven(info)) return true; for (const auto& use : current->OperandUses()) { const auto opcode = use.user->Opcode(); + if (opcode == IrOpcode::Ballot) { + std::vector terminals; + if (ownBitBallot(use.user, terminals)) { + pending.insert(pending.end(), terminals.begin(), terminals.end()); + continue; + } + } + if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; } if (ImageOpcodeInfoOf(opcode).needsSampler || opcode == IrOpcode::Ballot || opcode == IrOpcode::WqmU64 || opcode == IrOpcode::ReadLane || opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::DppMoveU32 || opcode == IrOpcode::DppUpdateU32 || opcode == IrOpcode::SwizzleU32 || opcode == IrOpcode::PermuteU32 || opcode == IrOpcode::BpermuteU32 || opcode == IrOpcode::Permlane16U32 || opcode == IrOpcode::WriteLane || SharedAccessOf(opcode) != SharedAccess::None) return true; if (opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue; pending.push_back(use.user); @@ -473,6 +597,7 @@ private: } for (const auto& use : current->OperandUses()) { const auto opcode = use.user->Opcode(); + if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; } if (ImageOpcodeInfoOf(opcode).needsSampler) return true; if (restricted.contains(use.user) || opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue; pending.push_back(use.user); @@ -482,31 +607,58 @@ private: } bool optionalBody(const IrBlock* branch, const IrBlock* body, const IrBlock* merge, const IrValue* predicate) { - const auto& bodyInfo = program.Metadata().blockInfo[blockIndices.at(body)]; - if (bodyInfo.terminator.kind != TerminatorKind::Branch || indices.at(bodyInfo.terminator.trueBlock) != blockIndices.at(merge)) return false; + if (body == merge) return false; + std::set region; + std::vector pending{body}; + while (!pending.empty()) { + const IrBlock* block = pending.back(); + pending.pop_back(); + if (block == merge || !region.insert(block).second) continue; + if (block == branch || region.size() > 64u || program.Metadata().blockInfo[blockIndices.at(block)].terminator.kind == TerminatorKind::Return) return false; + for (const auto* successor : block->Successors()) pending.push_back(successor); + } + for (const auto* block : region) { + for (const auto* predecessor : block->Predecessors()) { + if (block == body ? predecessor != branch : !region.contains(predecessor)) return false; + } + } std::set merged; for (const auto* instruction : merge->Instructions()) { if (!instruction->IsPhi()) continue; - if (instruction->ArgumentCount() != 2u || instruction->PhiBlockCount() != 2u) return false; + if (instruction->ArgumentCount() < 2u || instruction->ArgumentCount() != instruction->PhiBlockCount()) return false; const IrValue* skipped = nullptr; - const IrValue* taken = nullptr; - for (std::size_t index = 0; index < 2u; ++index) { + std::vector taken; + for (std::size_t index = 0; index < instruction->PhiBlockCount(); ++index) { if (instruction->PhiBlock(index) == branch) skipped = instruction->Argument(index)->Resolve(); - if (instruction->PhiBlock(index) == body) taken = instruction->Argument(index)->Resolve(); + else if (region.contains(instruction->PhiBlock(index))) taken.push_back(instruction->Argument(index)->Resolve()); + else return false; } - if (!skipped || !taken || !inactiveEquals(taken, skipped, predicate) || quadObserved(instruction)) return false; + if (!skipped || taken.empty()) return false; + if (quadObserved(instruction)) return false; + for (const auto* value : taken) if (!inactiveEquals(value, skipped, predicate, region)) return false; merged.insert(instruction); } - for (const auto* instruction : body->Instructions()) { - if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false; - for (const auto& use : instruction->OperandUses()) { - if (use.user->Parent() == body || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue; - return false; + for (const auto* block : region) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false; + for (const auto& use : instruction->OperandUses()) { + if (region.contains(use.user->Parent()) || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue; + return false; + } } } return !merged.empty(); } + bool controlProven(const BlockInfo& info) { + if (const auto found = controlResults.find(info.id); found != controlResults.end()) return found->second; + if (!controlVisiting.insert(info.id).second) return false; + const bool result = control(info); + controlVisiting.erase(info.id); + controlResults[info.id] = result; + return result; + } + bool control(const BlockInfo& info) { const auto* branch = program.BlockOrder()[indices.at(info.id)]; const auto* trueBlock = program.BlockOrder()[indices.at(info.terminator.trueBlock)]; @@ -553,6 +705,8 @@ private: std::map indices; std::map blockIndices; std::vector> allPaths; + std::map controlResults; + std::set controlVisiting; }; std::optional> Values(const IrProgram& program, const IrValue* value, std::set& visiting, unsigned depth, std::optional component = {}, const IrValue* active = nullptr) {