mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-28 00:35:47 +00:00
Merge pull request #9317 from akkadbakkad89/fix/stencil-replay-quad-observations
fix(recompiler): exempt proven control conditions and own-bit ballots in stencil replay
This commit is contained in:
2 files changed
+385
-20
No files matched your search
@@ -1074,6 +1074,54 @@ int main() {
|
||||
return !StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
const auto wholeQuadEval = [](Builder& b, IrValue& live) -> IrValue& {
|
||||
auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&live});
|
||||
auto& allLow = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
|
||||
auto& allHigh = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
|
||||
auto& low = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allLow, &b.constant(0u)})});
|
||||
auto& high = b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&allHigh, &b.constant(1u)})});
|
||||
return b.emit(IrOpcode::WqmU64, IrType::U64, {&b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&low, &high})});
|
||||
};
|
||||
|
||||
const auto splitQuads = [&](Builder& b, IrValue& live) -> IrValue& {
|
||||
auto& wqmLow = wholeQuadEval(b, live);
|
||||
auto& wqmHigh = wholeQuadEval(b, live);
|
||||
return b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
|
||||
};
|
||||
|
||||
passed &= run("stencil replay pairs structurally equivalent whole-quad evaluations for split low and high words", [&] {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
exportUnder(b, splitQuads(b, b.logicalAnd(pixelPredicate(b), helperPredicate(b, false))), true, true);
|
||||
return StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
passed &= run("stencil replay refuses to pair whole-quad evaluations separated by an execution-mask change", [&] {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
auto& live = b.logicalAnd(pixelPredicate(b), helperPredicate(b, false));
|
||||
auto& wqmLow = wholeQuadEval(b, live);
|
||||
b.emit(IrOpcode::SetExecLo, IrType::Void, {&b.constant(0xffffffffu)});
|
||||
auto& wqmHigh = wholeQuadEval(b, live);
|
||||
auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
|
||||
exportUnder(b, guard, true, true);
|
||||
return !StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
passed &= run("stencil replay refuses to pair whole-quad evaluations whose predicates differ only in a constant", [&] {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
auto& helperLow = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast<std::uint32_t>(StageInputKind::HelperInvocation)), &b.constant(0u)});
|
||||
auto& lowLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperLow, &b.constant(0u)}));
|
||||
auto& wqmLow = wholeQuadEval(b, lowLive);
|
||||
auto& helperHigh = b.emit(IrOpcode::GetBuiltin, IrType::U32, {&b.constant(static_cast<std::uint32_t>(StageInputKind::HelperInvocation)), &b.constant(0u)});
|
||||
auto& highLive = b.logicalAnd(pixelPredicate(b), b.emit(IrOpcode::IEqual32, IrType::Bool, {&helperHigh, &b.constant(1u)}));
|
||||
auto& wqmHigh = wholeQuadEval(b, highLive);
|
||||
auto& guard = b.threadBit(b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmLow, &b.constant(0u)}), b.emit(IrOpcode::CompositeExtractU64, IrType::U32, {&wqmHigh, &b.constant(1u)}));
|
||||
exportUnder(b, guard, true, true);
|
||||
return !StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
const auto execCopy = [](Builder& b, IrValue& low, IrValue& high) -> IrValue& {
|
||||
auto& active = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.truth()});
|
||||
auto& lanes = b.emit(IrOpcode::CompositeConstructU64, IrType::U64, {&b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(0u)}), &b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&active, &b.constant(1u)})});
|
||||
@@ -1192,6 +1240,169 @@ int main() {
|
||||
passed &= run("an optional wave body cannot change helper coordinates for later implicit samples", [&] { return !optionalPixelBody(true); });
|
||||
passed &= run("an optional wave body cannot change helper coordinates for LOD queries", [&] { return !optionalPixelBody(true, IrOpcode::ImageQueryLod); });
|
||||
|
||||
const auto optionalPixelRegion = [&](bool unmaskedArm) {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
auto& entry = *b.block;
|
||||
auto& predicate = pixelPredicate(b);
|
||||
auto& zero = b.constant(0u);
|
||||
auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
|
||||
auto& body = b.newBlock();
|
||||
auto& left = b.newBlock();
|
||||
auto& right = b.newBlock();
|
||||
auto& join = b.newBlock();
|
||||
auto& merge = b.newBlock();
|
||||
entry.AddBranch(&body);
|
||||
entry.AddBranch(&merge);
|
||||
body.AddBranch(&left);
|
||||
body.AddBranch(&right);
|
||||
left.AddBranch(&join);
|
||||
right.AddBranch(&join);
|
||||
join.AddBranch(&merge);
|
||||
auto& entryInfo = b.program.Metadata().blockInfo[0u];
|
||||
entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
|
||||
entryInfo.terminator.condition = BranchCondition::ExecZero;
|
||||
entryInfo.terminator.trueBlock = merge.Id();
|
||||
entryInfo.terminator.falseBlock = body.Id();
|
||||
entryInfo.condition = &condition;
|
||||
b.block = &body;
|
||||
auto& uniformBit = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.emit(IrOpcode::GetUserData, IrType::U32, {}), &zero});
|
||||
auto& bodyInfo = b.program.Metadata().blockInfo[1u];
|
||||
bodyInfo.terminator.kind = TerminatorKind::ConditionalBranch;
|
||||
bodyInfo.terminator.condition = BranchCondition::SccNonZero;
|
||||
bodyInfo.terminator.trueBlock = left.Id();
|
||||
bodyInfo.terminator.falseBlock = right.Id();
|
||||
bodyInfo.condition = &uniformBit;
|
||||
for (const unsigned index : {2u, 3u, 4u}) {
|
||||
b.program.Metadata().blockInfo[index].terminator.kind = TerminatorKind::Branch;
|
||||
b.program.Metadata().blockInfo[index].terminator.trueBlock = index == 4u ? merge.Id() : join.Id();
|
||||
}
|
||||
b.block = &left;
|
||||
auto& leftValue = b.select(predicate, b.constant(42u), zero);
|
||||
b.block = &right;
|
||||
auto& rightValue = unmaskedArm ? b.constant(7u) : b.select(predicate, b.constant(43u), zero);
|
||||
b.block = &join;
|
||||
auto& joined = b.phi(IrType::U32, left, leftValue);
|
||||
joined.AddPhiOperand(&right, &rightValue);
|
||||
b.block = &merge;
|
||||
auto& merged = b.phi(IrType::U32, entry, zero);
|
||||
merged.AddPhiOperand(&join, &joined);
|
||||
b.keep(merged);
|
||||
exportUnder(b, b.truth(), true, true);
|
||||
return StencilReplaySafe(b.program);
|
||||
};
|
||||
|
||||
passed &= run("an optional wave body spanning several blocks preserves inactive values through its own phis", [&] { return optionalPixelRegion(false); });
|
||||
passed &= run("an optional wave body spanning several blocks cannot write an inactive lane on any of its paths", [&] { return !optionalPixelRegion(true); });
|
||||
|
||||
const auto nestedOptionalGuard = [&](bool unsafeInner, bool ballotEscape) {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
auto& entry = *b.block;
|
||||
auto& predicate = pixelPredicate(b);
|
||||
auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
|
||||
auto& base = helperPredicate(b, false);
|
||||
auto& body = b.newBlock();
|
||||
auto& merge = b.newBlock();
|
||||
auto& inner = b.newBlock();
|
||||
auto& join = b.newBlock();
|
||||
entry.AddBranch(&body);
|
||||
entry.AddBranch(&merge);
|
||||
body.AddBranch(&merge);
|
||||
merge.AddBranch(&inner);
|
||||
merge.AddBranch(&join);
|
||||
inner.AddBranch(&join);
|
||||
auto& entryInfo = b.program.Metadata().blockInfo[0u];
|
||||
entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
|
||||
entryInfo.terminator.condition = BranchCondition::ExecZero;
|
||||
entryInfo.terminator.trueBlock = merge.Id();
|
||||
entryInfo.terminator.falseBlock = body.Id();
|
||||
entryInfo.condition = &condition;
|
||||
b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch;
|
||||
b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id();
|
||||
b.block = &body;
|
||||
auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base});
|
||||
b.block = &merge;
|
||||
auto& merged = b.phi(IrType::Bool, entry, base);
|
||||
merged.AddPhiOperand(&body, &taken);
|
||||
auto& innerCondition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&merged});
|
||||
auto& mergeInfo = b.program.Metadata().blockInfo[2u];
|
||||
mergeInfo.terminator.kind = TerminatorKind::ConditionalBranch;
|
||||
mergeInfo.terminator.condition = BranchCondition::ExecZero;
|
||||
mergeInfo.terminator.trueBlock = join.Id();
|
||||
mergeInfo.terminator.falseBlock = inner.Id();
|
||||
mergeInfo.condition = &innerCondition;
|
||||
b.program.Metadata().blockInfo[3u].terminator.kind = TerminatorKind::Branch;
|
||||
b.program.Metadata().blockInfo[3u].terminator.trueBlock = join.Id();
|
||||
b.block = &inner;
|
||||
auto& written = b.select(merged, b.constant(42u), b.constant(0u));
|
||||
if (unsafeInner) b.emit(IrOpcode::Discard, IrType::Void, {});
|
||||
b.block = &join;
|
||||
auto& joined = b.phi(IrType::U32, merge, b.constant(0u));
|
||||
joined.AddPhiOperand(&inner, &written);
|
||||
if (ballotEscape) {
|
||||
const auto [low, high] = b.ballotWords(merged);
|
||||
b.keep(b.threadBit(*low, *high));
|
||||
b.keep(b.add(*low, 1u));
|
||||
}
|
||||
b.keep(joined);
|
||||
exportUnder(b, b.truth(), true, true);
|
||||
return StencilReplaySafe(b.program);
|
||||
};
|
||||
|
||||
passed &= run("an optional wave body merge may guard another optional wave body", [&] { return nestedOptionalGuard(false, false); });
|
||||
passed &= run("an optional wave body merge cannot guard a body with side effects", [&] { return !nestedOptionalGuard(true, false); });
|
||||
passed &= run("an optional wave body merge cannot feed a ballot escaping the lane's own bit", [&] { return !nestedOptionalGuard(false, true); });
|
||||
|
||||
passed &= run("an optional wave body merge may feed an own-lane ballot read", [&] {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
auto& entry = *b.block;
|
||||
auto& predicate = pixelPredicate(b);
|
||||
auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&predicate});
|
||||
auto& base = helperPredicate(b, false);
|
||||
auto& body = b.newBlock();
|
||||
auto& merge = b.newBlock();
|
||||
entry.AddBranch(&body);
|
||||
entry.AddBranch(&merge);
|
||||
body.AddBranch(&merge);
|
||||
auto& entryInfo = b.program.Metadata().blockInfo[0u];
|
||||
entryInfo.terminator.kind = TerminatorKind::ConditionalBranch;
|
||||
entryInfo.terminator.condition = BranchCondition::ExecZero;
|
||||
entryInfo.terminator.trueBlock = merge.Id();
|
||||
entryInfo.terminator.falseBlock = body.Id();
|
||||
entryInfo.condition = &condition;
|
||||
b.program.Metadata().blockInfo[1u].terminator.kind = TerminatorKind::Branch;
|
||||
b.program.Metadata().blockInfo[1u].terminator.trueBlock = merge.Id();
|
||||
b.block = &body;
|
||||
auto& taken = b.emit(IrOpcode::SelectU1, IrType::Bool, {&predicate, &b.truth(), &base});
|
||||
b.block = &merge;
|
||||
auto& merged = b.phi(IrType::Bool, entry, base);
|
||||
merged.AddPhiOperand(&body, &taken);
|
||||
b.keep(b.ballotBit(merged));
|
||||
exportUnder(b, b.truth(), true, true);
|
||||
return StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
passed &= run("an active mask may flag a sample with an explicit lod", [&] {
|
||||
Builder b;
|
||||
b.program.Resources().stage = IrShaderStage::Pixel;
|
||||
MemoryInfo memory{};
|
||||
memory.kind = ResourceKind::Image;
|
||||
memory.imageSampleFlags = RdnaImageSampleFlagLevelZero;
|
||||
b.program.Resources().memoryInfo.push_back(memory);
|
||||
auto& active = b.ballotBit(b.truth());
|
||||
auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&b.constant(0u)});
|
||||
auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {});
|
||||
auto& sampler = b.emit(IrOpcode::GetSamplerResource, IrType::SamplerResource, {});
|
||||
MemoryFlags flags{0u, 0u};
|
||||
std::uint64_t bits = 0;
|
||||
std::memcpy(&bits, &flags, sizeof(flags));
|
||||
b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address, &active}, bits);
|
||||
exportUnder(b, b.truth(), true, true);
|
||||
return StencilReplaySafe(b.program);
|
||||
});
|
||||
|
||||
passed &= run("varying attribute sources cannot be treated as uniform any-active predicates", [&] {
|
||||
for (const auto opcode : {IrOpcode::GetAttribute, IrOpcode::GetInterpolationParameter, IrOpcode::GetInterpolationParameterF16, IrOpcode::GetTessellationAttribute}) {
|
||||
Builder b;
|
||||
|
||||
@@ -108,7 +108,7 @@ public:
|
||||
}
|
||||
for (const auto& info : program.Metadata().blockInfo) {
|
||||
if (info.terminator.kind != TerminatorKind::ConditionalBranch) continue;
|
||||
if (!control(info)) return false;
|
||||
if (!controlProven(info)) return false;
|
||||
}
|
||||
for (const auto& [value, mask] : ownMasks) if (helperObserved(value)) return false;
|
||||
for (const auto* block : program.BlockOrder()) {
|
||||
@@ -132,6 +132,60 @@ private:
|
||||
if (value && !value->HasImmediate()) restricted.insert(value);
|
||||
}
|
||||
|
||||
void restrictSource(const IrValue* value, unsigned depth) {
|
||||
value = value->Resolve();
|
||||
if (depth > 64u || value->HasImmediate()) return;
|
||||
restrict(value);
|
||||
for (const auto* argument : value->Arguments()) restrictSource(argument, depth + 1u);
|
||||
}
|
||||
|
||||
bool execEquivalent(const IrValue* left, const IrValue* right) const {
|
||||
const IrBlock* block = left->Parent();
|
||||
if (block == nullptr || block != right->Parent()) return false;
|
||||
bool between = false;
|
||||
for (const IrValue* instruction : block->Instructions()) {
|
||||
if (instruction == left || instruction == right) {
|
||||
if (between) return true;
|
||||
between = true;
|
||||
continue;
|
||||
}
|
||||
if (!between) continue;
|
||||
switch (instruction->Opcode()) {
|
||||
case IrOpcode::SetExec:
|
||||
case IrOpcode::SetExecLo:
|
||||
case IrOpcode::SetExecHi:
|
||||
case IrOpcode::Discard:
|
||||
case IrOpcode::Sendmsg:
|
||||
case IrOpcode::SetThreadBitScalarRegister:
|
||||
case IrOpcode::SetScalarMaskTag:
|
||||
return false;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool equivalentSource(const IrValue* left, const IrValue* right, unsigned depth) {
|
||||
if (depth > 64u) return false;
|
||||
left = left->Resolve();
|
||||
right = right->Resolve();
|
||||
if (left == right || sameValue(left, right)) return true;
|
||||
if (left->HasImmediate() || right->HasImmediate()) return left->HasImmediate() && right->HasImmediate() && left->Type() == right->Type() && left->ImmediateU64() == right->ImmediateU64();
|
||||
if (left->Opcode() != right->Opcode() || left->ArgumentCount() != right->ArgumentCount()) return false;
|
||||
if (left->Opcode() == IrOpcode::Ballot && !execEquivalent(left, right)) return false;
|
||||
for (std::size_t index = 0; index < left->ArgumentCount(); ++index) if (!equivalentSource(left->Argument(index), right->Argument(index), depth + 1u)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool equivalentWqm(const IrValue* left, const IrValue* right, unsigned depth) {
|
||||
left = left->Resolve();
|
||||
right = right->Resolve();
|
||||
if (left == right) return true;
|
||||
if (left->Opcode() != IrOpcode::WqmU64 || right->Opcode() != IrOpcode::WqmU64 || left->ArgumentCount() != 1u || right->ArgumentCount() != 1u) return false;
|
||||
return equivalentSource(left->Argument(0), right->Argument(0), depth + 1u);
|
||||
}
|
||||
|
||||
std::optional<ReplayMask> own(const IrValue* value) {
|
||||
value = value->Resolve();
|
||||
if (value->Opcode() != IrOpcode::INotEqual32) return std::nullopt;
|
||||
@@ -189,7 +243,8 @@ private:
|
||||
valid = (maskStable(pair(0u, 0u), depth + 1u) && maskStable(pair(1u, 1u), depth + 1u)) || (waveSize != 32u && maskStable(pair(0u, 1u), depth + 1u) && maskStable(pair(1u, 0u), depth + 1u));
|
||||
} else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) {
|
||||
const IrValue* wqm = mask.low->Argument(0)->Resolve();
|
||||
const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u));
|
||||
const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr;
|
||||
const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth));
|
||||
if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) {
|
||||
const IrValue* source = wqm->Argument(0)->Resolve();
|
||||
valid = source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u && maskStable(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u);
|
||||
@@ -197,7 +252,11 @@ private:
|
||||
const auto provenance = helpers(ReplayMask{source->Argument(0)->Resolve(), source->Argument(1)->Resolve()}, depth + 1u);
|
||||
valid = provenance == HelperMask::RealOnly || provenance == HelperMask::QuadStable;
|
||||
}
|
||||
if (valid) { restrict(wqm); restrict(source); }
|
||||
if (valid) {
|
||||
restrict(wqm);
|
||||
restrict(source);
|
||||
if (highWqm != wqm) restrictSource(highWqm, 0u);
|
||||
}
|
||||
}
|
||||
} else if (mask.low->IsPhi() && mask.low->ArgumentCount() != 0u && mask.low->ArgumentCount() == mask.low->PhiBlockCount() && (waveSize == 32u || (mask.high->IsPhi() && mask.high->Parent() == mask.low->Parent() && mask.high->ArgumentCount() == mask.high->PhiBlockCount()))) {
|
||||
valid = true;
|
||||
@@ -288,7 +347,8 @@ private:
|
||||
}
|
||||
} else if (mask.low->Opcode() == IrOpcode::CompositeExtractU64 && mask.low->ArgumentCount() == 2u && Immediate(mask.low->Argument(1), 0u)) {
|
||||
const IrValue* wqm = mask.low->Argument(0)->Resolve();
|
||||
const bool paired = waveSize == 32u || (mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u && mask.high->Argument(0)->Resolve() == wqm && Immediate(mask.high->Argument(1), 1u));
|
||||
const IrValue* highWqm = mask.high->Opcode() == IrOpcode::CompositeExtractU64 && mask.high->ArgumentCount() == 2u ? mask.high->Argument(0)->Resolve() : nullptr;
|
||||
const bool paired = waveSize == 32u || (highWqm != nullptr && Immediate(mask.high->Argument(1), 1u) && equivalentWqm(wqm, highWqm, depth));
|
||||
if (paired && wqm->Opcode() == IrOpcode::WqmU64 && wqm->ArgumentCount() == 1u) {
|
||||
const IrValue* source = wqm->Argument(0)->Resolve();
|
||||
if (source->Opcode() == IrOpcode::CompositeConstructU64 && source->ArgumentCount() == 2u) {
|
||||
@@ -433,24 +493,88 @@ private:
|
||||
return left == right || (left->HasImmediate() && right->HasImmediate() && left->Type() == IrType::U32 && right->Type() == IrType::U32 && left->ImmediateU32() == right->ImmediateU32());
|
||||
}
|
||||
|
||||
bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, unsigned depth = 0) {
|
||||
bool inactiveEquals(const IrValue* value, const IrValue* other, const IrValue* predicate, const std::set<const IrBlock*>& region, unsigned depth = 0) {
|
||||
if (depth > 32u) return false;
|
||||
value = value->Resolve();
|
||||
if (sameValue(value, other)) return true;
|
||||
if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, depth + 1u);
|
||||
if ((value->Opcode() == IrOpcode::SelectU32 || value->Opcode() == IrOpcode::SelectF32 || value->Opcode() == IrOpcode::SelectU1) && value->ArgumentCount() == 3u && (value->Argument(0)->Resolve() == predicate->Resolve() || GuardImplies(program, value->Argument(0), predicate))) return inactiveEquals(value->Argument(2), other, predicate, region, depth + 1u);
|
||||
if (value->IsPhi() && region.contains(value->Parent()) && value->ArgumentCount() != 0u && value->ArgumentCount() == value->PhiBlockCount()) {
|
||||
return std::all_of(value->Arguments().begin(), value->Arguments().end(), [&](const IrValue* argument) { return inactiveEquals(argument, other, predicate, region, depth + 1u); });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool quadObserved(const IrValue* value) const {
|
||||
bool explicitLodSample(const IrValue* value) const {
|
||||
if (value->Opcode() == IrOpcode::ImageSampleExplicitLod) return true;
|
||||
if (value->Opcode() != IrOpcode::ImageSampleRaw && value->Opcode() != IrOpcode::ImageGatherRaw) return false;
|
||||
const auto index = value->Flags<MemoryFlags>().index;
|
||||
return index < program.Resources().memoryInfo.size() && ImageSampleExplicitLod(program.Resources().memoryInfo[index].imageSampleFlags, program.Resources().stage);
|
||||
}
|
||||
|
||||
bool ownBitShifted(const IrValue* user, std::size_t operand, std::vector<const IrValue*>& terminals) const {
|
||||
if (user->Opcode() != IrOpcode::ShiftRightLogical32 || user->ArgumentCount() != 2u || operand != 0u) return false;
|
||||
const IrValue* amount = user->Argument(1)->Resolve();
|
||||
if (amount->Opcode() != IrOpcode::BitwiseAnd32) return false;
|
||||
const IrValue* lane = Other(amount, 31u);
|
||||
if (lane == nullptr || lane->Opcode() != IrOpcode::LaneId) return false;
|
||||
for (const auto& use : user->OperandUses()) {
|
||||
const IrValue* isolated = use.user;
|
||||
if (isolated->Opcode() != IrOpcode::BitwiseAnd32 || Other(isolated, 1u) != user) return false;
|
||||
terminals.push_back(isolated);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ownBitSelected(const IrValue* user, std::size_t operand, unsigned half, std::vector<const IrValue*>& terminals) const {
|
||||
if (user->Opcode() != IrOpcode::SelectU32 || user->ArgumentCount() != 3u || operand != half + 1u) return false;
|
||||
const IrValue* condition = user->Argument(0)->Resolve();
|
||||
if (condition->Opcode() != IrOpcode::ULessThan32 || condition->ArgumentCount() != 2u || condition->Argument(0)->Resolve()->Opcode() != IrOpcode::LaneId || !Immediate(condition->Argument(1), 32u)) return false;
|
||||
for (const auto& use : user->OperandUses()) if (!ownBitShifted(use.user, use.operand, terminals)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ownBitBallot(const IrValue* ballot, std::vector<const IrValue*>& terminals) const {
|
||||
if (ballot->Opcode() != IrOpcode::Ballot || ballot->ArgumentCount() != 1u) return false;
|
||||
for (const auto& use : ballot->OperandUses()) {
|
||||
const IrValue* extract = use.user;
|
||||
if (extract->Opcode() != IrOpcode::CompositeExtractU32x4 || extract->ArgumentCount() != 2u || extract->Argument(0)->Resolve() != ballot) return false;
|
||||
const IrValue* index = extract->Argument(1)->Resolve();
|
||||
if (!index->HasImmediate() || index->ImmediateU32() >= (waveSize == 32u ? 1u : 2u)) return false;
|
||||
const auto half = index->ImmediateU32();
|
||||
for (const auto& euse : extract->OperandUses()) {
|
||||
const IrValue* word = euse.user;
|
||||
if (word->Opcode() == IrOpcode::BitwiseAnd32) {
|
||||
for (const auto& muse : word->OperandUses()) {
|
||||
const bool read = waveSize == 64u ? ownBitSelected(muse.user, muse.operand, half, terminals) : ownBitShifted(muse.user, muse.operand, terminals);
|
||||
if (!read) return false;
|
||||
}
|
||||
} else {
|
||||
const bool read = waveSize == 64u ? ownBitSelected(word, euse.operand, half, terminals) : ownBitShifted(word, euse.operand, terminals);
|
||||
if (!read) return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool quadObserved(const IrValue* value) {
|
||||
std::set<const IrValue*> visited;
|
||||
std::vector<const IrValue*> pending{value};
|
||||
while (!pending.empty()) {
|
||||
const IrValue* current = pending.back();
|
||||
pending.pop_back();
|
||||
if (!visited.insert(current).second) continue;
|
||||
for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current) return true;
|
||||
for (const auto& info : program.Metadata().blockInfo) if (info.condition && info.condition->Resolve() == current && !controlProven(info)) return true;
|
||||
for (const auto& use : current->OperandUses()) {
|
||||
const auto opcode = use.user->Opcode();
|
||||
if (opcode == IrOpcode::Ballot) {
|
||||
std::vector<const IrValue*> terminals;
|
||||
if (ownBitBallot(use.user, terminals)) {
|
||||
pending.insert(pending.end(), terminals.begin(), terminals.end());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; }
|
||||
if (ImageOpcodeInfoOf(opcode).needsSampler || opcode == IrOpcode::Ballot || opcode == IrOpcode::WqmU64 || opcode == IrOpcode::ReadLane || opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::DppMoveU32 || opcode == IrOpcode::DppUpdateU32 || opcode == IrOpcode::SwizzleU32 || opcode == IrOpcode::PermuteU32 || opcode == IrOpcode::BpermuteU32 || opcode == IrOpcode::Permlane16U32 || opcode == IrOpcode::WriteLane || SharedAccessOf(opcode) != SharedAccess::None) return true;
|
||||
if (opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue;
|
||||
pending.push_back(use.user);
|
||||
@@ -473,6 +597,7 @@ private:
|
||||
}
|
||||
for (const auto& use : current->OperandUses()) {
|
||||
const auto opcode = use.user->Opcode();
|
||||
if (ImageOpcodeInfoOf(opcode).needsSampler && explicitLodSample(use.user)) { pending.push_back(use.user); continue; }
|
||||
if (ImageOpcodeInfoOf(opcode).needsSampler) return true;
|
||||
if (restricted.contains(use.user) || opcode == IrOpcode::SetAttribute || opcode == IrOpcode::Reference || opcode == IrOpcode::ReferenceU32) continue;
|
||||
pending.push_back(use.user);
|
||||
@@ -482,31 +607,58 @@ private:
|
||||
}
|
||||
|
||||
bool optionalBody(const IrBlock* branch, const IrBlock* body, const IrBlock* merge, const IrValue* predicate) {
|
||||
const auto& bodyInfo = program.Metadata().blockInfo[blockIndices.at(body)];
|
||||
if (bodyInfo.terminator.kind != TerminatorKind::Branch || indices.at(bodyInfo.terminator.trueBlock) != blockIndices.at(merge)) return false;
|
||||
if (body == merge) return false;
|
||||
std::set<const IrBlock*> region;
|
||||
std::vector<const IrBlock*> pending{body};
|
||||
while (!pending.empty()) {
|
||||
const IrBlock* block = pending.back();
|
||||
pending.pop_back();
|
||||
if (block == merge || !region.insert(block).second) continue;
|
||||
if (block == branch || region.size() > 64u || program.Metadata().blockInfo[blockIndices.at(block)].terminator.kind == TerminatorKind::Return) return false;
|
||||
for (const auto* successor : block->Successors()) pending.push_back(successor);
|
||||
}
|
||||
for (const auto* block : region) {
|
||||
for (const auto* predecessor : block->Predecessors()) {
|
||||
if (block == body ? predecessor != branch : !region.contains(predecessor)) return false;
|
||||
}
|
||||
}
|
||||
std::set<const IrValue*> merged;
|
||||
for (const auto* instruction : merge->Instructions()) {
|
||||
if (!instruction->IsPhi()) continue;
|
||||
if (instruction->ArgumentCount() != 2u || instruction->PhiBlockCount() != 2u) return false;
|
||||
if (instruction->ArgumentCount() < 2u || instruction->ArgumentCount() != instruction->PhiBlockCount()) return false;
|
||||
const IrValue* skipped = nullptr;
|
||||
const IrValue* taken = nullptr;
|
||||
for (std::size_t index = 0; index < 2u; ++index) {
|
||||
std::vector<const IrValue*> taken;
|
||||
for (std::size_t index = 0; index < instruction->PhiBlockCount(); ++index) {
|
||||
if (instruction->PhiBlock(index) == branch) skipped = instruction->Argument(index)->Resolve();
|
||||
if (instruction->PhiBlock(index) == body) taken = instruction->Argument(index)->Resolve();
|
||||
else if (region.contains(instruction->PhiBlock(index))) taken.push_back(instruction->Argument(index)->Resolve());
|
||||
else return false;
|
||||
}
|
||||
if (!skipped || !taken || !inactiveEquals(taken, skipped, predicate) || quadObserved(instruction)) return false;
|
||||
if (!skipped || taken.empty()) return false;
|
||||
if (quadObserved(instruction)) return false;
|
||||
for (const auto* value : taken) if (!inactiveEquals(value, skipped, predicate, region)) return false;
|
||||
merged.insert(instruction);
|
||||
}
|
||||
for (const auto* instruction : body->Instructions()) {
|
||||
if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false;
|
||||
for (const auto& use : instruction->OperandUses()) {
|
||||
if (use.user->Parent() == body || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue;
|
||||
return false;
|
||||
for (const auto* block : region) {
|
||||
for (const auto* instruction : block->Instructions()) {
|
||||
if (instruction->Opcode() == IrOpcode::SetAttribute || instruction->Opcode() == IrOpcode::Discard) return false;
|
||||
for (const auto& use : instruction->OperandUses()) {
|
||||
if (region.contains(use.user->Parent()) || merged.contains(use.user) || use.user->Opcode() == IrOpcode::Reference || use.user->Opcode() == IrOpcode::ReferenceU32) continue;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return !merged.empty();
|
||||
}
|
||||
|
||||
bool controlProven(const BlockInfo& info) {
|
||||
if (const auto found = controlResults.find(info.id); found != controlResults.end()) return found->second;
|
||||
if (!controlVisiting.insert(info.id).second) return false;
|
||||
const bool result = control(info);
|
||||
controlVisiting.erase(info.id);
|
||||
controlResults[info.id] = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
bool control(const BlockInfo& info) {
|
||||
const auto* branch = program.BlockOrder()[indices.at(info.id)];
|
||||
const auto* trueBlock = program.BlockOrder()[indices.at(info.terminator.trueBlock)];
|
||||
@@ -553,6 +705,8 @@ private:
|
||||
std::map<std::uint32_t, std::size_t> indices;
|
||||
std::map<const IrBlock*, std::size_t> blockIndices;
|
||||
std::vector<std::vector<const IrBlock*>> allPaths;
|
||||
std::map<std::uint32_t, bool> controlResults;
|
||||
std::set<std::uint32_t> controlVisiting;
|
||||
};
|
||||
|
||||
std::optional<std::set<std::uint32_t>> Values(const IrProgram& program, const IrValue* value, std::set<const IrValue*>& visiting, unsigned depth, std::optional<unsigned> component = {}, const IrValue* active = nullptr) {
|
||||
|
||||
Reference in new issue
Block a user