Merge pull request #5390 from hugodearaujo/feat/agc-sparse-unaligned-import

feat(agc): import shared ranges whose backing offset is not 64 KiB aligned

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -2200,7 +2200,7 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
     // pair for a whole group of labels recorded back to back (Recorder::RecordStore orders the
     // stores of a run against each other and against the work before and after, the queued DCC
     // key stores included).
-    const auto recordStore = [&] { recorder.RecordStore(import->buffer, address - import->base, bytes, address); };
+    const auto recordStore = [&] { recorder.RecordStore(import->buffer, Graphics::HostImportOffset(*import, address), bytes, address); };
     // Only a write-back that can land over these bytes (a listed copied writer of the range, as
     // FillBuffer tests) puts the label behind the completions; any other pending completion
     // stores elsewhere. Debug aid: APS5_LABEL_GATE_ALL=1 gates on any completion, as before (a
@@ -2401,7 +2401,7 @@ bool VulkanDevice::FillBuffer(std::uint64_t address, std::size_t bytes, std::spa
         Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
         Graphics::Recorder::CountBarriers(CommandClass::Fill);
     }
-    const auto offset = address - import->base;
+    const auto offset = Graphics::HostImportOffset(*import, address);
     if (uniform) {
         ++uniformFills;
         context.Resolved(&Graphics::DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, import->buffer, offset, bytes, pattern[0]);
@@ -2668,7 +2668,7 @@ VulkanDevice::CopyOutcome VulkanDevice::CopyBuffer(std::uint64_t destination, st
         Graphics::Recorder::CountBarriers(CommandClass::Copy);
     }
     const auto gpuTiming = recorder.BeginGpuTiming(programAddress);
-    const VkBufferCopy region{source - sourceImport->base, destination - destinationImport->base, bytes};
+    const VkBufferCopy region{Graphics::HostImportOffset(*sourceImport, source), Graphics::HostImportOffset(*destinationImport, destination), bytes};
     context.Resolved(&Graphics::DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, sourceImport->buffer, destinationImport->buffer, 1, &region);
     recorder.EndGpuTiming(gpuTiming, bytes);
     // The copied bytes are visible to everything recorded after and to the host, as after a fill.
@@ -4623,7 +4623,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
     const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId);
     if (state->checkpointMarker != nullptr) state->checkpointMarker(commands, reinterpret_cast<const void*>(static_cast<std::uintptr_t>(record.programAddress)));
     if (argumentImport != nullptr) {
-        context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base);
+        context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, Graphics::HostImportOffset(*argumentImport, arguments));
     } else {
         const auto* limit = state->properties.limits.maxComputeWorkGroupCount;
         if (record.x <= limit[0] && record.y <= limit[1] && record.z <= limit[2]) {

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -40,8 +40,13 @@ struct HostImport {
     bool joined = false;
     std::shared_ptr<void> chunk;
     mutable std::uint64_t lastUse = 0;
+    std::uint64_t lead = 0;
 };
 
+inline VkDeviceSize HostImportOffset(const HostImport& import, std::uint64_t address) {
+    return address - import.base + import.lead;
+}
+
 enum class ImportWatch : std::uint8_t { Watch, Unwatch };
 
 struct ImportProbe {
@@ -63,8 +68,8 @@ ImportWatch PrepareImportWatch(const Context& context);
 void SetImportWatch(const Context& context, ImportWatch watch);
 
 // The host import of the registered allocation containing [address, address + bytes), made on demand
-// (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in
-// the import's buffer.
+// (alignment and budget permitting), or null. Bytes at `address` are at `HostImportOffset(*import,
+// address)` in the import's buffer.
 const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
 const HostImport* ExistingHostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
 void SetHostImportWindow(std::uint64_t bytes);

core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp

@@ -234,7 +234,7 @@ void TraceKeyStore(const char* path, std::uint64_t begin, std::size_t count) {
 bool StoreUncompressedOnGpu(const Context& context, Recorder& recorder, std::uint64_t begin, std::size_t count) {
     const auto* import = HostImportFor(context, begin, count);
     if (import == nullptr) return false;
-    const VkDeviceSize first = begin - import->base;
+    const VkDeviceSize first = HostImportOffset(*import, begin);
     const VkDeviceSize last = first + count;
     const VkDeviceSize fillBegin = std::min((first + 3) & ~VkDeviceSize{3}, last);
     const VkDeviceSize fillEnd = std::max(last & ~VkDeviceSize{3}, fillBegin);

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -258,7 +258,7 @@ void recordCmaskClear(const Context& context, Recorder& recorder, const HostImpo
     recorder.FlushKeyStoresOverlapping(address, bytes);
     const auto commands = recorder.Commands();
     recorder.Keep(resident);
-    context.detiler->DispatchCmaskClear(commands, import.buffer, address - import.base, bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0);
+    context.detiler->DispatchCmaskClear(commands, import.buffer, HostImportOffset(import, address), bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0);
     recorder.NotePendingWrite(address, bytes);
     proveCmask(address, bytes, GuestMemory::MarkWritten(address, bytes));
 }
@@ -968,7 +968,7 @@ const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address
     const auto drawWriters = DrawCopiedWriters();
     if (std::any_of(drawWriters->begin(), drawWriters->end(), overlaps)) return nullptr;
     const auto* import = HostImportFor(context, address, bytes);
-    if (import == nullptr || (address - import->base) % alignment != 0) return nullptr;
+    if (import == nullptr || HostImportOffset(*import, address) % alignment != 0) return nullptr;
     recorder->NotePendingRead(address, bytes, Recorder::ReadKind::DrawInput);
     if (behindGpuWrite != nullptr && behind) *behindGpuWrite = true;
     return import;
@@ -1150,7 +1150,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
         }
         if (import != nullptr) {
             inputs.indexHandle = import->buffer;
-            inputs.indexOffset = draw.indexAddress - import->base;
+            inputs.indexOffset = HostImportOffset(*import, draw.indexAddress);
             inputs.inPlaceReads.emplace_back(draw.indexAddress, draw.indexAddress + indexBytes);
         } else {
             inputs.indices = std::move(copy.buffer);
@@ -1219,7 +1219,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
         RequireNoMultisampleTarget(begin, bytes, "a vertex buffer read");
         if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range], &inputs.vertexInputBehindGpuWrite)) {
             rangeHandles.push_back(import->buffer);
-            rangeOffsets.push_back(begin - import->base);
+            rangeOffsets.push_back(HostImportOffset(*import, begin));
             inputs.inPlaceReads.emplace_back(begin, end);
             continue;
         }
@@ -1385,8 +1385,8 @@ void recordDrawCommands(const Context& context, VkCommandBuffer commands, const
         else context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance);
     } else if (indirect->path == IndirectDrawPath::Gpu) {
         if (args->countIndirect) {
-            if (draw.indexed) context.Function<PFN_vkCmdDrawIndexedIndirectCountKHR>("vkCmdDrawIndexedIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, args->countAddress - indirect->countImport->base, args->count, args->stride);
-            else context.Function<PFN_vkCmdDrawIndirectCountKHR>("vkCmdDrawIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, args->countAddress - indirect->countImport->base, args->count, args->stride);
+            if (draw.indexed) context.Function<PFN_vkCmdDrawIndexedIndirectCountKHR>("vkCmdDrawIndexedIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, HostImportOffset(*indirect->countImport, args->countAddress), args->count, args->stride);
+            else context.Function<PFN_vkCmdDrawIndirectCountKHR>("vkCmdDrawIndirectCountKHR")(commands, argumentBuffer, argumentOffset, indirect->countImport->buffer, HostImportOffset(*indirect->countImport, args->countAddress), args->count, args->stride);
         } else if (args->count <= 1 || context.multiDrawIndirect) {
             if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexedIndirect, "vkCmdDrawIndexedIndirect")(commands, argumentBuffer, argumentOffset, args->count, args->stride);
             else context.Resolved(&DeviceFunctions::cmdDrawIndirect, "vkCmdDrawIndirect")(commands, argumentBuffer, argumentOffset, args->count, args->stride);
@@ -1434,7 +1434,7 @@ bool recordIndirectArguments(const Context& context, VkCommandBuffer commands, R
         if (args->countIndirect) recorder->NotePendingRead(args->countAddress, 4, Recorder::ReadKind::Indirect);
     }
     argumentBuffer = indirect.argumentImport->buffer;
-    argumentOffset = args->arguments - indirect.argumentImport->base;
+    argumentOffset = HostImportOffset(*indirect.argumentImport, args->arguments);
     const bool rewritten = args->vertexRule == Rule::Constant || args->instanceRule == Rule::Constant;
     if (!rewritten) return false;
     const auto bytes = static_cast<std::size_t>(args->RangeBytes());
@@ -1717,7 +1717,7 @@ void captureInputs(const Context& context, Recorder& recorder, VkCommandBuffer c
         }
         const auto* imported = HostImportFor(context, begin, bytes);
         Require(imported != nullptr, "capture input has no host import");
-        addSample(begin, bytes, imported->buffer, begin - imported->base, reinterpret_cast<const std::byte*>(begin));
+        addSample(begin, bytes, imported->buffer, HostImportOffset(*imported, begin), reinterpret_cast<const std::byte*>(begin));
     }
     CaptureTrace::Log("input-capture draw=%llu batch=%llu target=%llx ranges=%zu bytes=%zu", draw, batch, static_cast<unsigned long long>(target), samples.size(), total);
     if (samples.empty()) return;

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -511,6 +511,7 @@ const char* createDmaBufImport(const Context& context, HostImport& entry, int fi
 #ifndef _WIN32
 constexpr std::uint64_t ImportChunkBytes = std::uint64_t{64} << 20u;
 constexpr VkBufferUsageFlags ChunkImportUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
+constexpr std::uint64_t SparseBlockBytes = 65536;
 
 struct ImportChunk {
     VkDevice device = VK_NULL_HANDLE;
@@ -683,13 +684,15 @@ VkMemoryRequirements dmaBufBufferRequirements(const Context& context, const Host
 }
 
 bool createSparseDmaBufImport(const Context& context, HostImport& entry, int file, std::uint64_t offset) {
+    const auto lead = offset % SparseBlockBytes;
+    offset -= lead;
     const bool ownQueue = context.sparseQueue != VK_NULL_HANDLE && context.sparseQueueMutex != nullptr;
     if (!context.sparseBinding || entry.bytes <= ImportChunkBytes || (!ownQueue && !GuestMemory::GpuMutex().HeldByThisThread()) || udmabufDevice() < 0) return false;
     struct stat backing {};
     if (fstat(file, &backing) != 0) throwSparseImport(entry, "fstat of the shared backing", VK_ERROR_INVALID_EXTERNAL_HANDLE);
     const auto chunkTypes = dmaBufBufferRequirements(context, entry, 0, ChunkImportUsage, ImportChunkBytes, nullptr).memoryTypeBits;
     VkBuffer buffer = VK_NULL_HANDLE;
-    const auto requirements = dmaBufBufferRequirements(context, entry, VK_BUFFER_CREATE_SPARSE_BINDING_BIT, SparseImportUsage, entry.bytes, &buffer);
+    const auto requirements = dmaBufBufferRequirements(context, entry, VK_BUFFER_CREATE_SPARSE_BINDING_BIT, SparseImportUsage, lead + entry.bytes, &buffer);
     const auto alignment = requirements.alignment;
     const bool chosen = alignment != 0 && offset % alignment == 0 && ImportChunkBytes % alignment == 0 && (chunkTypes & ~requirements.memoryTypeBits) == 0 && (requirements.memoryTypeBits & chunkTypes) != 0 &&
         (offset + requirements.size + ImportChunkBytes - 1) / ImportChunkBytes * ImportChunkBytes <= static_cast<std::uint64_t>(backing.st_size);
@@ -740,9 +743,10 @@ bool createSparseDmaBufImport(const Context& context, HostImport& entry, int fil
     if (address == 0) fail("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN);
     entry.buffer = buffer;
     entry.memory = VK_NULL_HANDLE;
-    entry.address = address;
+    entry.address = address + lead;
     entry.chunk = std::move(chunks);
     entry.dmaBuf = true;
+    entry.lead = lead;
     return true;
 }
 #endif
@@ -3958,7 +3962,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) {
             // device-local buffer that the CPU fallback replaces (none without a recorder to
             // record the copies: UploadFinish then binds the region in place).
             bool staged = Recorder::Active() != nullptr && stagingEligible(region, addressable);
-            const bool misaligned = !bindableInPlace(region.begin - entry->base, addressable);
+            const bool misaligned = !bindableInPlace(HostImportOffset(*entry, region.begin), addressable);
             if (staged) {
                 const auto allocateStart = std::chrono::steady_clock::now();
                 region.buffer = stagingBuffer(context, static_cast<std::size_t>(region.end - region.begin), gpuCopyUsage(addressable));
@@ -4162,7 +4166,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
             // A staged region (see stagingEligible) is copied out of the import even when aligned;
             // without a recorder to record the copies it binds in place like any other.
             const bool staged = recorder != nullptr && stagingEligible(region, addressable);
-            if (entry != nullptr && !staged && bindableInPlace(region.begin - entry->base, addressable)) {
+            if (entry != nullptr && !staged && bindableInPlace(HostImportOffset(*entry, region.begin), addressable)) {
                 region.direct = entry;
                 region.runCut = runImport;
                 region.snapshot.clear();
@@ -4178,7 +4182,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
                 copyOnGpu = true;
                 region.deviceLocal = staged;
                 region.copySource = entry->buffer;
-                region.copySourceBase = entry->base;
+                region.copySourceBase = entry->base - entry->lead;
             }
         }
         if (region.direct != nullptr) {
@@ -4506,7 +4510,7 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std:
         return std::string(text);
     };
     if (!(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr))) Require(false, describe("exceeds"));
-    const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin;
+    const auto base = region.direct != nullptr ? region.direct->base - region.direct->lead : region.mirror != nullptr ? region.mirror->base : region.begin;
     const auto offset = address - base;
     Require(context.limits.minStorageBufferOffsetAlignment != 0, "no storage buffer offset alignment");
     Require(base % 4 == 0, "a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented");

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -2319,7 +2319,7 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
             batch = std::make_unique<CommandBatch>(context);
             commands = batch->Handle();
         }
-        const auto importOffset = descriptor.baseAddress - import->base;
+        const auto importOffset = HostImportOffset(*import, descriptor.baseAddress);
         if (direct) {
             recordDirectUploadBarrier(commands, layers == nullptr);
             for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) {
@@ -2745,7 +2745,7 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span<
         batch = std::make_unique<CommandBatch>(context);
         commands = batch->Handle();
     }
-    const auto importOffset = descriptor.baseAddress - import.base;
+    const auto importOffset = HostImportOffset(import, descriptor.baseAddress);
     if (direct) {
         recordDirectUploadBarrier(commands, discard);
         for (const auto& window : windows) {
@@ -2878,7 +2878,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
                 if (!shadowed.empty() && shadowed.back().end == guestBegin && shadowed.back().slab == destination->slab) shadowed.back().end = pieceEnd;
                 else shadowed.push_back({guestBegin, pieceEnd, destination->slab});
             } else {
-                importCopies[segment].push_back({scratchOffset, guestBegin - import.base, pieceEnd - guestBegin});
+                importCopies[segment].push_back({scratchOffset, HostImportOffset(import, guestBegin), pieceEnd - guestBegin});
                 if (!imported.empty() && imported.back().second == guestBegin) imported.back().second = pieceEnd;
                 else imported.emplace_back(guestBegin, pieceEnd);
             }
@@ -2973,7 +2973,7 @@ std::uint64_t StorageTexture::writeBackWindows(std::span<const ImportSegment> se
         }
         for (const auto& [begin, end, slab, import] : seeds) {
             // The piece's own slab (a seeded unit is one the piece covers partly).
-            const VkBufferCopy seed{begin - import->base, SlabOffset(*import, *slab, begin), end - begin};
+            const VkBufferCopy seed{HostImportOffset(*import, begin), SlabOffset(*import, *slab, begin), end - begin};
             context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, import->buffer, slab->buffer, 1, &seed);
             NoteShadowSeed(begin, end);
         }
@@ -4792,7 +4792,7 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
                 for (const auto& [from, to] : keep) {
                     const auto begin = std::max(from, segment.begin);
                     const auto end = std::min(to, segment.end);
-                    if (begin < end) copies.push_back({begin - descriptor.baseAddress, begin - segment.import->base, end - begin});
+                    if (begin < end) copies.push_back({begin - descriptor.baseAddress, HostImportOffset(*segment.import, begin), end - begin});
                 }
                 if (!copies.empty()) context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, tiledScratch->Handle(), segment.import->buffer, static_cast<std::uint32_t>(copies.size()), copies.data());
             }

core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp

@@ -67,11 +67,12 @@ Statistics& Stats() {
 
 struct UnitShadow {
     Context context;
-    // The import's range and buffer (taken at creation: a publish under the import table's mutex
-    // must not look them up), and the unit base (the import's base rounded down to a unit).
+    // The import's range, buffer and lead (taken at creation: a publish under the import table's
+    // mutex must not look them up), and the unit base (the import's base rounded down to a unit).
     std::uint64_t importBase = 0;
     std::uint64_t importBytes = 0;
     VkBuffer importBuffer = VK_NULL_HANDLE;
+    std::uint64_t importLead = 0;
     std::uint64_t base = 0;
     // Per unit: 0 = never shadowed, else the tracker generation the slab bytes are current at;
     // whether a slab -> import copy was recorded at that generation.
@@ -162,6 +163,7 @@ std::shared_ptr<UnitShadow> findOrCreate(Shadows& registry, const Context& conte
     shadow->importBase = import.base;
     shadow->importBytes = import.bytes;
     shadow->importBuffer = import.buffer;
+    shadow->importLead = import.lead;
     shadow->base = import.base & ~(UnitBytes - 1);
     const auto units = (import.base + import.bytes - shadow->base + UnitBytes - 1) / UnitBytes;
     shadow->generation.assign(static_cast<std::size_t>(units), 0);
@@ -279,7 +281,7 @@ void verifyPublished(const UnitShadow& shadow, const std::vector<SlabCopies>& co
             RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
             batch.SubmitAndWait();
             host.Invalidate();
-            const auto address = shadow.importBase + region.dstOffset;
+            const auto address = shadow.importBase + region.dstOffset - shadow.importLead;
             Stats().verified.fetch_add(1, std::memory_order_relaxed);
             if (!GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(region.size)) || std::memcmp(reinterpret_cast<const void*>(address), host.Bytes().data(), static_cast<std::size_t>(region.size)) != 0) {
                 Stats().mismatches.fetch_add(1, std::memory_order_relaxed);
@@ -336,7 +338,7 @@ std::size_t publishUnits(const std::shared_ptr<UnitShadow>& shadow, std::uint64_
             }
             const auto [begin, end] = shadow->UnitRange(unit);
             if (end <= begin) continue;
-            const VkBufferCopy region{(unit - slab->firstUnit) * UnitBytes + (begin - shadow->UnitBegin(unit)), begin - shadow->importBase, end - begin};
+            const VkBufferCopy region{(unit - slab->firstUnit) * UnitBytes + (begin - shadow->UnitBegin(unit)), begin - shadow->importBase + shadow->importLead, end - begin};
             if (!copies.empty() && copies.back().slab == slab && copies.back().regions.back().dstOffset + copies.back().regions.back().size == region.dstOffset) {
                 copies.back().regions.back().size += region.size;
                 ranges.back().second = end;
@@ -606,7 +608,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
     std::vector<ShadowRun> result;
     if (runs.empty()) return result;
     const auto importPiece = [&](std::uint64_t begin, std::uint64_t end) {
-        result.push_back({begin, end, import.buffer, surfaceBase + begin - import.base, false, nullptr});
+        result.push_back({begin, end, import.buffer, HostImportOffset(import, surfaceBase + begin), false, nullptr});
         if (countReads) Stats().detiledImportBytes.fetch_add(end - begin, std::memory_order_relaxed);
     };
     std::shared_ptr<UnitShadow> shadow;
@@ -668,7 +670,7 @@ std::vector<ShadowRun> ShadowSources(const Context& context, const HostImport& i
                 slab->lastUse = present;
                 result.push_back({pieceBegin, pieceEnd, slab->buffer, SlabOffset(import, *slab, surfaceBase + pieceBegin), true, slab});
             } else {
-                result.push_back({pieceBegin, pieceEnd, import.buffer, surfaceBase + pieceBegin - import.base, false, nullptr});
+                result.push_back({pieceBegin, pieceEnd, import.buffer, HostImportOffset(import, surfaceBase + pieceBegin), false, nullptr});
             }
             (isFresh ? shadowBytes : importBytes) += pieceEnd - pieceBegin;
         }

core/libs/prx/libSceAgcDriver/tests/HostImportSparseDmaBuf.cpp

@@ -59,6 +59,7 @@ struct MockDevice {
     std::vector<Bind> binds;
     VkResult bindResult = VK_SUCCESS;
     std::uint64_t nextAddress = 0x500000000ull;
+    VkDeviceSize sparseAlignment = Alignment;
 };
 
 MockDevice mock;
@@ -111,8 +112,9 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer buffer, const Vk
 }
 
 VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer buffer, VkMemoryRequirements* requirements) {
-    requirements->size = (mock.sizes.at(buffer) + Alignment - 1) / Alignment * Alignment;
-    requirements->alignment = Alignment;
+    const auto alignment = mock.sparse.contains(buffer) ? mock.sparseAlignment : Alignment;
+    requirements->size = (mock.sizes.at(buffer) + alignment - 1) / alignment * alignment;
+    requirements->alignment = alignment;
     requirements->memoryTypeBits = 1;
 }
 
@@ -305,14 +307,35 @@ void ImportsALargeRangeAsOneSparseBuffer() {
     Expect(memories.size() == chunks, "two parts of the range are bound to the same chunk import");
 }
 
-void UnalignedOffsetIsDecidedBeforeImporting() {
+void UnalignedOffsetStartsAtTheBlockBelow() {
     const auto bytes = RangeBytes();
-    SharedRange range(bytes, 4096, true);
+    constexpr std::uint64_t lead = 4096;
+    SharedRange range(bytes, lead, true);
     const auto context = mockContext();
     mock.binds.clear();
+    const auto address = mock.nextAddress;
+    std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
+    const auto* import = HostImportFor(context, range.Address(), static_cast<std::size_t>(bytes));
+    Expect(import != nullptr, "a range whose backing offset is not 64 KiB aligned was not imported above udmabuf's size limit");
+    if (import == nullptr) return;
+    Expect(mock.sparse.contains(import->buffer) && import->lead == lead, "the import does not start its sparse buffer at the 64 KiB block below the range");
+    Expect(import->address == address + lead && HostImportOffset(*import, range.Address()) == lead, "the import's device address or buffer offset does not point at the range itself");
+    Expect(!mock.binds.empty() && mock.binds.front().resourceOffset == 0 && mock.binds.front().memoryOffset == 0, "the first bind does not map the block below the range");
+    VkDeviceSize bound = 0;
+    for (const auto& bind : mock.binds) bound += bind.size;
+    Expect(bound >= lead + bytes, "the binds do not cover the range after its lead");
+}
+
+void LargerSparseBlockIsDecidedBeforeImporting() {
+    const auto bytes = RangeBytes();
+    SharedRange range(bytes, Alignment, true);
+    const auto context = mockContext();
+    mock.binds.clear();
+    mock.sparseAlignment = 2 * Alignment;
     const auto memories = mock.memories.size();
     std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex());
     const auto* import = HostImportFor(context, range.Address(), static_cast<std::size_t>(bytes));
+    mock.sparseAlignment = Alignment;
     Expect(import == nullptr, "a range whose backing offset is not on the sparse block size was imported above udmabuf's size limit");
     Expect(mock.memories.size() == memories && mock.binds.empty(), "a chunk was imported or bound for a range that does not take the sparse path");
 }
@@ -355,7 +378,8 @@ int main() {
         SettlingSurvivesDeviceReplacement(false);
         SettlingSurvivesDeviceReplacement(true);
         ImportsALargeRangeAsOneSparseBuffer();
-        UnalignedOffsetIsDecidedBeforeImporting();
+        UnalignedOffsetStartsAtTheBlockBelow();
+        LargerSparseBlockIsDecidedBeforeImporting();
         FailureAfterTheDecisionThrows(false, VK_SUCCESS, "UDMABUF_CREATE");
         FailureAfterTheDecisionThrows(true, VK_ERROR_DEVICE_LOST, "vkQueueBindSparse");
     } catch (const std::exception& error) {

core/libs/prx/libSceAgcDriver/tests/execution/SparseDmaBufImport.cpp

@@ -1,10 +1,14 @@
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
 #include "VulkanTestDevice.hpp"
 #ifndef _WIN32
 #include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
 #include <fcntl.h>
 #include <sys/mman.h>
 #include <unistd.h>
@@ -17,6 +21,7 @@
 #include <fstream>
 #include <iostream>
 #include <mutex>
+#include <span>
 
 #ifdef _WIN32
 int main() {
@@ -165,8 +170,74 @@ int main() {
             Require(value == 0x5EEDF00Du, "sparse import: the GPU fill did not reach the backing memory across the chunk boundary");
         }
         Require(mapping[boundary - 32769] == 0x11 && mapping[boundary + 32768] == 0x11, "sparse import: the GPU fill reached past its range");
-
         GuestAllocations::Mutation().Remove(mapping);
+
+        constexpr std::uint64_t Lead = 16384;
+        auto* unaligned = mapping + Lead;
+        const auto unalignedBytes = bytes - 65536;
+        std::memset(mapping + boundary - 65536, 0x22, 131072);
+        const auto unalignedAddress = reinterpret_cast<std::uint64_t>(unaligned);
+        device->ImportGuestMemory(Register(unaligned, unalignedBytes), GuestAllocations::GuestAllocationsGeneration_nid_postfix(), false);
+        Require(Imported(device->Device(), unalignedAddress, static_cast<std::size_t>(unalignedBytes)), "sparse import: a range whose backing offset is not 64 KiB aligned was not imported");
+        {
+            std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+            const std::array<std::uint32_t, 4> pattern{0xA11CE5EDu, 0xA11CE5EDu, 0xA11CE5EDu, 0xA11CE5EDu};
+            filled = device->FillBuffer(address + boundary - 32768, 65536, pattern);
+            device->WaitIdle();
+        }
+        Require(filled, "sparse import: the fill across the chunk boundary of the unaligned range was not served by the import");
+        for (std::uint64_t offset = boundary - 32768; offset < boundary + 32768; offset += 4) {
+            std::uint32_t value = 0;
+            std::memcpy(&value, mapping + offset, sizeof(value));
+            Require(value == 0xA11CE5EDu, "sparse import: the GPU fill of the unaligned range missed its guest address");
+        }
+        Require(mapping[boundary - 32769] == 0x22 && mapping[boundary + 32768] == 0x22, "sparse import: the GPU fill of the unaligned range reached past its range");
+
+        if (AgcDriver::Graphics::UnitShadowEnabled()) {
+            using namespace AgcDriver::Graphics;
+            constexpr std::uint64_t Unit = 65536;
+            const auto unitBegin = ((unalignedAddress + Unit - 1) & ~(Unit - 1)) + Unit;
+            auto* const unitBytes = reinterpret_cast<std::uint8_t*>(unitBegin);
+            std::memset(unitBytes - Lead, 0x33, Unit + 2 * Lead);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(mapping, static_cast<std::size_t>(mappingBytes));
+            std::size_t published = 0;
+            {
+                std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex());
+                const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(*device);
+                const auto* import = HostImportFor(context, unalignedAddress, static_cast<std::size_t>(unalignedBytes));
+                Require(import != nullptr, "sparse import: the unaligned range has no import");
+                if (!import->dmaBuf) std::printf("note: the device imported the unaligned range through a host pointer, which has no lead\n");
+                Require(import->lead == (import->dmaBuf ? Lead : 0), "sparse import: the unaligned import does not start its buffer at the block below it");
+                auto* recorder = Recorder::Active();
+                if (recorder != nullptr && AgcDriver::GuestMemory::Watched(unitBegin, Unit)) {
+                    AgcDriver::GuestMemory::CollectWritesUncached(unitBegin, Unit);
+                    const auto destination = ShadowDestinationFor(context, *import, unitBegin, unitBegin + Unit);
+                    Require(destination.has_value(), "sparse import: no unit shadow for a whole unit of the unaligned import");
+                    const auto commands = recorder->Commands();
+                    recorder->Keep(destination->slab);
+                    context.Function<PFN_vkCmdFillBuffer>("vkCmdFillBuffer")(commands, destination->buffer, destination->offset, Unit, 0x5AD0BEEFu);
+                    RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+                    recorder->MarkCovered(VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
+                    const ShadowedRange range{unitBegin, unitBegin + Unit, destination->slab};
+                    MarkShadowed(*import, std::span(&range, 1), AgcDriver::GuestMemory::TrackerGeneration());
+                    published = PublishShadow(unitBegin, Unit, PublishScope::Whole, PublishReason::Hook);
+                    recorder->Sync();
+                }
+            }
+            GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(mapping, static_cast<std::size_t>(mappingBytes));
+            if (published == 0) {
+                std::printf("note: the unaligned import's unit was not shadowed (no recorder or not watched); the publish was not checked\n");
+            } else {
+                for (std::uint64_t offset = 0; offset < Unit; offset += 4) {
+                    std::uint32_t value = 0;
+                    std::memcpy(&value, unitBytes + offset, sizeof(value));
+                    Require(value == 0x5AD0BEEFu, "sparse import: a unit shadow of the unaligned import was not published at its guest address");
+                }
+                for (std::uint64_t offset = 0; offset < Lead; ++offset) Require((unitBytes - Lead)[offset] == 0x33 && unitBytes[Unit + offset] == 0x33, "sparse import: a unit shadow of the unaligned import was published outside its unit");
+            }
+        }
+        GuestAllocations::Mutation().Remove(unaligned);
+
         GuestAllocations::Mutation().Remove(probe);
         std::free(probe);
         std::cout << "AGC sparse dma-buf import tests passed\n";

docs/dev/debt/libSceAgcDriver.md

@@ -83,7 +83,7 @@
 - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the write watch and a settle queue exists: those of 1 MiB or more stay watched, and smaller ones return to the watch once their settle has completed (see Pages under an import window): a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. Excluded blocks keep their write-watch marks as a hint only (`UntrackedHint`): a storage image or a sampled texture snapshot over them compares its bytes again after a mark in its range, a driver store, a mapping change or a recorded GPU write instead of at every collect epoch or lookup, and a mark the driver makes without a CPU store costs one comparison. This preserves CPU edits through byte comparisons but limits snapshot reuse on excluded blocks to draws of one collect epoch and disables unit shadows for them; restoring tracking on import retirement without replacing the backing remains unimplemented.
 - [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) (libSceAgcDriver) whose write protection was resolved while a host import that leaves its range watched was made are compared with the bytes they held before the import and count as stored over only when those differ. A CPU store of the bytes already there within the window cannot be told apart from an import that resolves the write protection without storing, as the NVIDIA driver (610.57.04) does for every imported page, so such a store does not win over pending results. Where host-pointer and dma-buf imports differ in whether they stay watched, a failed host-pointer import and the dma-buf import after it are two windows, and the pages the failed one resolved count as stored over. On Windows, the NVIDIA driver (617.42) marks every page of a host import of private `MEM_WRITE_WATCH` memory written, asynchronously, until a GPU use of the import completes; a 4-byte read of it on the settle queue ends that even while main-queue work that used the import first is still running, and an earlier import of the same pages whose first use runs after that read marks none of them (`agc_import_settle_first_use`, `agc_import_settle_earlier_import`). Such an import stays watched only after a 4-byte read of it on the settle queue has completed inside the window. Each one waits for that read, which keeps its window open for one round trip on the settle queue, so a same-value CPU store from another thread in that time counts as unchanged too. An import made where no settle queue exists is compared instead. An import of under 1 MiB is not settled inside its window, because the settle blocks the importing thread and a title that imports a 64 KiB buffer around every submission lost about 19 % of its frame rate to that wait (Dripl1x, #6060). It is compared until its read, submitted in a batch by a later import lookup under `GpuMutex` while the imports match the registry, has completed; a later lookup then returns its whole blocks to the write watch, each stamped as a CPU store, so whatever was derived from them while they were compared reads as changed once. Its blocks stay compared while storage results are pending over them, and those of an import retired before that stay compared.
 - [Settled dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a dma-buf import of 1 MiB or more is read by a 4-byte copy on the settle queue right after it is made (`settleImport`), so the kernel maps its pages for the GPU there and not in the first submission that uses it (about 2 s for a 3 GiB import on RADV). The import's buffer is shared with the settle queue's family. An import made at map time settles on the guest thread that maps the range, inside its map call, so that thread waits while the kernel maps the pages: 0.47 to 2.2 s for the 3 GiB range of Hades (PPSA03355) and 0.49 to 4.3 s for the 4.5 GiB range of ASTRO BOT (PPSA21567) on RADV. Both titles map these ranges at start-up; a title that maps large ranges during gameplay stalls its mapping thread for as long. An import made from a submission settles under the GPU mutex, so its frame still pays for it; a smaller import, and every import on a device with no settle queue (one without a copy-capable queue besides the main one), pays in its first submission. A range imported through a host pointer, as lavapipe imports shared memory, is not settled
-- [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when its offset in the backing file is not a multiple of the buffer's sparse block size (64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory
+- [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory
 - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view.
 - An [unnormalized S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) maps clamp mode 0 (repeat) on X and Y to clamp-to-edge as required by Vulkan unnormalized coordinates, and assigns the magnification filter when minification and magnification filters differ (since unnormalized coordinates sample at LOD 0). Anisotropy, TRUNC_COORD, MC_COORD_TRUNC and other clamp modes still throw. Secondary LOD bias (LOD_BIAS_SEC, bits [19:14]) is accepted and ignored, matching RDNA2 hardware measurements across explicit and implicit samples. High on Life (PPSA15388, #4736) uses secondary LOD bias and unnormalized coordinates with clamp mode 0 and differing min/mag filters during home base and city rendering.
 - [Background link-time-optimized relink of library pipelines](../../../core/libs/prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp) (libSceAgcDriver) is skipped on every GPU under NVIDIA's proprietary Vulkan driver (`VK_DRIVER_ID_NVIDIA_PROPRIETARY`), so those pipelines keep their fast-linked form. The optimized link loses the device when bound on an RTX 3080 (Ampere, #2866) and an RTX 4060 (Ada, #3434); an RTX 5060 and an RTX 5080 (Blackwell) on the same driver do not, so the scope is wider than the evidence. No standard Vulkan query tells the architecture apart, and the driver version that fixes it is unknown. `APS5_NO_PIPELINE_LTO` disables the relink on every driver.

docs/user/USAGE.md

@@ -115,7 +115,7 @@ On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.d
 
 On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed when the title maps it, not when the title allocates it or first uses its pages. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover the mapped direct memory together with all other committed memory. Otherwise `sceKernelMapDirectMemory` throws `replace guest placeholder with resident direct memory` with the Windows error (`commit direct memory pages` when a page is mapped at a second address, `keep unmapped direct memory` when it is unmapped); enlarge the page file or close other applications.
 
-On Linux, when the Vulkan driver imports dma-buf memory (not the NVIDIA proprietary driver), shared direct memory is imported through `/dev/udmabuf`, and the user who runs the game needs read-write access to it. Many distributions create it as `root:kvm` with mode `0660`: add the user to the `kvm` group and log in again (an ACL such as `setfacl -m u:$USER:rw /dev/udmabuf` lasts until the next reboot). Without access, startup prints `[gpu] open /dev/udmabuf: Permission denied`, these ranges are copied instead of imported, and GPU stores to them through FLAT/GLOBAL addresses fail with `BDA access failed`. A range larger than 64 MiB is imported as one sparse buffer over 64 MiB udmabuf buffers when the device binds sparse memory (RADV, ANV). Without sparse binding, or when the range does not start on the device's sparse block size in the backing file, ranges above udmabuf's `size_limit_mb` (64 MiB by default) are copied as well.
+On Linux, when the Vulkan driver imports dma-buf memory (not the NVIDIA proprietary driver), shared direct memory is imported through `/dev/udmabuf`, and the user who runs the game needs read-write access to it. Many distributions create it as `root:kvm` with mode `0660`: add the user to the `kvm` group and log in again (an ACL such as `setfacl -m u:$USER:rw /dev/udmabuf` lasts until the next reboot). Without access, startup prints `[gpu] open /dev/udmabuf: Permission denied`, these ranges are copied instead of imported, and GPU stores to them through FLAT/GLOBAL addresses fail with `BDA access failed`. A range larger than 64 MiB is imported as one sparse buffer over 64 MiB udmabuf buffers when the device binds sparse memory (RADV, ANV). Without sparse binding, or when the device's sparse block size is larger than 64 KiB, ranges above udmabuf's `size_limit_mb` (64 MiB by default) are copied as well.
 
 Linux: