Fast readbacks

diegolix29 · diegolix29 · commit becdad9e3219 · 2025-06-06T02:55:49.000-06:00
diff --git a/src/video_core/amdgpu/liverpool.cpp b/src/video_core/amdgpu/liverpool.cpp
@@ -694,6 +694,9 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
                 break;
             }
             case PM4ItOpcode::Rewind: {
+                if (!rasterizer) {
+                    break;
+                }
                 const PM4CmdRewind* rewind = reinterpret_cast<const PM4CmdRewind*>(header);
                 while (!rewind->Valid()) {
                     YIELD_GFX();
@@ -873,6 +876,9 @@ Liverpool::Task Liverpool::ProcessCompute(const u32* acb, u32 acb_dwords, u32 vq
             break;
         }
         case PM4ItOpcode::Rewind: {
+            if (!rasterizer) {
+                break;
+            }
             const PM4CmdRewind* rewind = reinterpret_cast<const PM4CmdRewind*>(header);
             while (!rewind->Valid()) {
                 YIELD_ASC(vqid);
diff --git a/src/video_core/buffer_cache/buffer_cache.cpp b/src/video_core/buffer_cache/buffer_cache.cpp
@@ -167,7 +167,10 @@ void BufferCache::DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 si
                     .dstOffset = total_size_bytes,
                     .size = new_size,
                 });
-                total_size_bytes += new_size;
+                // Align up to avoid cache conflicts
+                constexpr u64 align = 64ULL;
+                constexpr u64 mask = ~(align - 1ULL);
+                total_size_bytes += (new_size + align - 1) & mask;
             };
             gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download);
             gpu_modified_ranges.Subtract(device_addr_out, range_size);
@@ -322,16 +325,25 @@ void BufferCache::InlineData(VAddr address, const void* value, u32 num_bytes, bo
     InlineDataBuffer(*buffer, address, value, num_bytes);
 }
 
+void BufferCache::EnsureRegionRegistered(VAddr address, u32 size) {
+    if (!IsRegionRegistered(address, size)) {
+        const BufferId buffer_id = FindBuffer(address, size);
+        (void)buffer_id;
+    }
+}
+
 void BufferCache::CopyBuffer(VAddr dst, VAddr src, u32 num_bytes, bool dst_gds, bool src_gds) {
-    if (!dst_gds && !IsRegionRegistered(dst, num_bytes)) {
-        if (!src_gds && !IsRegionRegistered(src, num_bytes)) {
-            // Both buffers were not transferred to GPU yet. Can safely copy in host memory.
+    if (!src_gds && !IsRegionRegistered(src, num_bytes)) {
+        if (!dst_gds && !IsRegionRegistered(dst, num_bytes)) {
             memcpy(std::bit_cast<void*>(dst), std::bit_cast<void*>(src), num_bytes);
             return;
         }
-        // Without a readback there's nothing we can do with this
-        // Fallback to creating dst buffer on GPU to at least have this data there
+        // Force GPU write to dst even if dst was not previously registered
+        EnsureRegionRegistered(dst, num_bytes); // You'd implement this
+        InlineData(dst, std::bit_cast<void*>(src), num_bytes, dst_gds);
+        return;
     }
+
     if (!src_gds && !IsRegionRegistered(src, num_bytes)) {
         InlineData(dst, std::bit_cast<void*>(src), num_bytes, dst_gds);
         return;
@@ -375,7 +387,7 @@ void BufferCache::CopyBuffer(VAddr dst, VAddr src, u32 num_bytes, bool dst_gds,
             .size = num_bytes,
         },
     };
-    scheduler.EndRendering();
+
     const auto cmdbuf = scheduler.CommandBuffer();
     cmdbuf.pipelineBarrier2(vk::DependencyInfo{
         .dependencyFlags = vk::DependencyFlagBits::eByRegion,
@@ -931,15 +943,16 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, VAddr device_addr,
         const u32 height = std::max(image.info.size.height >> m, 1u);
         const u32 depth =
             image.info.props.is_volume ? std::max(image.info.size.depth >> m, 1u) : 1u;
-        const auto& [mip_size, mip_pitch, mip_height, mip_ofs] = image.info.mips_layout[m];
-        offset += mip_ofs;
-        if (offset + mip_size > max_offset) {
+        const auto& mip = image.info.mips_layout[m];
+        offset += mip.offset;
+        if (offset + mip.size > max_offset) {
             break;
         }
+
         copies.push_back({
             .bufferOffset = offset,
-            .bufferRowLength = static_cast<u32>(mip_pitch),
-            .bufferImageHeight = static_cast<u32>(mip_height),
+            .bufferRowLength = static_cast<u32>(mip.pitch),
+            .bufferImageHeight = static_cast<u32>(mip.height),
             .imageSubresource{
                 .aspectMask = image.aspect_mask & ~vk::ImageAspectFlagBits::eStencil,
                 .mipLevel = m,
@@ -1131,4 +1144,4 @@ void BufferCache::DeleteBuffer(BufferId buffer_id) {
     buffer.is_deleted = true;
 }
 
-} // namespace VideoCore
+} // namespace VideoCore
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
@@ -66,6 +66,7 @@ class BufferCache {
         VAddr end;
         bool has_stream_leap = false;
     };
+
 public:
     explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
                          Vulkan::Rasterizer& rasterizer_, AmdGpu::Liverpool* liverpool,
@@ -100,7 +101,7 @@ class BufferCache {
     /// Invalidates any buffer in the logical page range.
     void InvalidateMemory(VAddr device_addr, u64 size, bool unmap);
 
-        void ReadMemory(VAddr device_addr, u64 size);
+    void ReadMemory(VAddr device_addr, u64 size);
 
     /// Binds host vertex buffers for the current draw.
     void BindVertexBuffers(const Vulkan::GraphicsPipeline& pipeline);
@@ -110,8 +111,8 @@ class BufferCache {
 
     /// Writes a value to GPU buffer. (uses command buffer to temporarily store the data)
     void InlineData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
-
-        void CopyBuffer(VAddr dst, VAddr src, u32 num_bytes, bool dst_gds, bool src_gds);
+    void EnsureRegionRegistered(VAddr address, u32 size);
+    void CopyBuffer(VAddr dst, VAddr src, u32 num_bytes, bool dst_gds, bool src_gds);
 
     /// Writes a value to GPU buffer. (uses staging buffer to temporarily store the data)
     void WriteData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
diff --git a/src/video_core/page_manager.cpp b/src/video_core/page_manager.cpp
@@ -1,10 +1,9 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <thread>
-#include <boost/icl/interval_set.hpp>
+#include <boost/container/small_vector.hpp>
 #include "common/assert.h"
-#include "common/error.h"
+#include "common/debug.h"
 #include "common/signal_context.h"
 #include "common/spin_lock.h"
 #include "core/memory.h"
@@ -16,23 +15,57 @@
 #include <sys/mman.h>
 #include "common/adaptive_mutex.h"
 #ifdef ENABLE_USERFAULTFD
+#include <thread>
 #include <fcntl.h>
 #include <linux/userfaultfd.h>
 #include <poll.h>
 #include <sys/ioctl.h>
+#include "common/error.h"
 #endif
 #else
 #include <windows.h>
+#endif
+
+#ifdef __linux__
+#include "common/adaptive_mutex.h"
+#else
 #include "common/spin_lock.h"
 #endif
 
 namespace VideoCore {
 
+constexpr size_t PAGE_SIZE = 4_KB;
+constexpr size_t PAGE_BITS = 12;
+
 struct PageManager::Impl {
+    struct PageState {
+        u8 num_watchers{};
+
+        Core::MemoryPermission Perm() const noexcept {
+            return num_watchers == 0 ? Core::MemoryPermission::ReadWrite
+                                     : Core::MemoryPermission::Read;
+        }
+
+        template <s32 delta>
+        u8 AddDelta() {
+            if constexpr (delta == 1) {
+                return ++num_watchers;
+            } else {
+                ASSERT_MSG(num_watchers > 0, "Not enough watchers");
+                return --num_watchers;
+            }
+        }
+    };
+
+    struct UpdateProtectRange {
+        VAddr addr;
+        u64 size;
+        Core::MemoryPermission perms;
+    };
+
     static constexpr size_t ADDRESS_BITS = 40;
     static constexpr size_t NUM_ADDRESS_PAGES = 1ULL << (40 - PAGE_BITS);
     inline static Vulkan::Rasterizer* rasterizer;
-
 #ifdef ENABLE_USERFAULTFD
     Impl(Vulkan::Rasterizer* rasterizer_) {
         rasterizer = rasterizer_;
@@ -67,7 +100,8 @@ struct PageManager::Impl {
         ASSERT_MSG(ret != -1, "Uffdio unregister failed");
     }
 
-    void Protect(VAddr address, size_t size, bool allow_write) {
+    void Protect(VAddr address, size_t size, Core::MemoryPermission perms) {
+        bool allow_write = True(perms & Core::MemoryPermission::Write);
         uffdio_writeprotect wp;
         wp.range.start = address;
         wp.range.len = size;
@@ -143,6 +177,7 @@ struct PageManager::Impl {
     }
 
     void Protect(VAddr address, size_t size, Core::MemoryPermission perms) {
+        RENDERER_TRACE;
         auto* memory = Core::Memory::Instance();
         auto& impl = memory->GetAddressSpace();
         impl.Protect(address, size, perms);
@@ -152,96 +187,75 @@ struct PageManager::Impl {
         const auto addr = reinterpret_cast<VAddr>(fault_address);
         if (Common::IsWriteError(context)) {
             return rasterizer->InvalidateMemory(addr, 1);
-        } else {
-            return rasterizer->ReadMemory(addr, 1);
         }
         return false;
     }
-#endif
 
-    template <s32 delta, bool is_read>
+#endif
+    template <s32 delta>
     void UpdatePageWatchers(VAddr addr, u64 size) {
-        std::scoped_lock lk{lock};
-        std::atomic_thread_fence(std::memory_order_acquire);
-
-        size_t page = addr >> PAGE_BITS;
-        auto perms = cached_pages[page].Perms();
-        u64 range_begin = 0;
-        u64 range_bytes = 0;
-
-        const auto release_pending = [&] {
-            if (range_bytes > 0) {
-                Protect(range_begin << PAGE_BITS, range_bytes, perms);
-                range_bytes = 0;
-            }
-        };
-        // Iterate requested pages.
-        const size_t page_end = Common::DivCeil(addr + size, PAGE_SIZE);
-        for (; page != page_end; ++page) {
-            PageState& state = cached_pages[page];
-
-            // Apply the change to the page state.
-            const auto new_count = state.AddDelta<is_read, delta>();
-
-            // If the protection changed flush pending (un)protect action.
-            if (auto new_perms = state.Perms(); new_perms != perms) [[unlikely]] {
-                release_pending();
-                perms = new_perms;
-            }
+        RENDERER_TRACE;
+        boost::container::small_vector<UpdateProtectRange, 16> update_ranges;
+        {
+            std::scoped_lock lk(lock);
+
+            size_t page = addr >> PAGE_BITS;
+            auto perms = cached_pages[page].Perm();
+            u64 range_begin = 0;
+            u64 range_bytes = 0;
+
+            const auto release_pending = [&] {
+                if (range_bytes > 0) {
+                    RENDERER_TRACE;
+                    // Add pending (un)protect action
+                    update_ranges.push_back({range_begin << PAGE_BITS, range_bytes, perms});
+                    range_bytes = 0;
+                }
+            };
+
+            // Iterate requested pages
+            const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE);
+            const u64 aligned_addr = page << PAGE_BITS;
+            const u64 aligned_end = page_end << PAGE_BITS;
+            // ASSERT_MSG(rasterizer->IsMapped(aligned_addr, aligned_end - aligned_addr),
+            //            "Attempted to track non-GPU memory at address {:#x}, size {:#x}.",
+            //            aligned_addr, aligned_end - aligned_addr);
+
+            for (; page != page_end; ++page) {
+                PageState& state = cached_pages[page];
+
+                // Apply the change to the page state
+                const u8 new_count = state.AddDelta<delta>();
+
+                // If the protection changed add pending (un)protect action
+                if (auto new_perms = state.Perm(); new_perms != perms) [[unlikely]] {
+                    release_pending();
+                    perms = new_perms;
+                }
 
-            // If the page must be (un)protected add it to pending range.
-            if ((new_count == 0 && delta < 0) || (new_count == 1 && delta > 0)) {
-                if (range_bytes == 0) {
-                    range_begin = page;
+                // If the page must be (un)protected, add it to the pending range
+                if ((new_count == 0 && delta < 0) || (new_count == 1 && delta > 0)) {
+                    if (range_bytes == 0) {
+                        range_begin = page;
+                    }
+                    range_bytes += PAGE_SIZE;
+                } else {
+                    release_pending();
                 }
-                range_bytes += PAGE_SIZE;
-            } else {
-                release_pending();
             }
-        }
-        release_pending();
-    }
-
-    struct PageState {
-        u8 num_write_watchers : 7;
-        // At the moment only buffer cache can request read watchers.
-        // And buffers cannot overlap, thus only 1 can exist per page.
-        u8 num_read_watchers : 1;
-
-        Core::MemoryPermission WritePerm() const noexcept {
-            return num_write_watchers == 0 ? Core::MemoryPermission::Write
-                                           : Core::MemoryPermission::None;
-        }
 
-        Core::MemoryPermission ReadPerm() const noexcept {
-            return num_read_watchers == 0 ? Core::MemoryPermission::Read
-                                          : Core::MemoryPermission::None;
+            // Add pending (un)protect action
+            release_pending();
         }
 
-        Core::MemoryPermission Perms() const noexcept {
-            return ReadPerm() | WritePerm();
+        // Flush deferred protects
+        for (const auto& range : update_ranges) {
+            Protect(range.addr, range.size, range.perms);
         }
-
-        template <bool is_read, s32 delta>
-        u8 AddDelta() {
-            if constexpr (is_read) {
-                if constexpr (delta == 1) {
-                    return ++num_read_watchers;
-                } else {
-                    return --num_read_watchers;
-                }
-            } else {
-                if constexpr (delta == 1) {
-                    return ++num_write_watchers;
-                } else {
-                    return --num_write_watchers;
-                }
-            }
-        }
-    };
+    }
 
     std::array<PageState, NUM_ADDRESS_PAGES> cached_pages{};
-#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
+#ifdef __linux__
     Common::AdaptiveMutex lock;
 #else
     Common::SpinLock lock;
@@ -263,12 +277,12 @@ void PageManager::OnGpuUnmap(VAddr address, size_t size) {
 
 template <s32 delta, bool is_read>
 void PageManager::UpdatePageWatchers(VAddr addr, u64 size) const {
-    impl->UpdatePageWatchers<delta, is_read>(addr, size);
+    impl->UpdatePageWatchers<delta>(addr, size);
 }
 
 template void PageManager::UpdatePageWatchers<1, true>(VAddr addr, u64 size) const;
 template void PageManager::UpdatePageWatchers<1, false>(VAddr addr, u64 size) const;
 template void PageManager::UpdatePageWatchers<-1, true>(VAddr addr, u64 size) const;
 template void PageManager::UpdatePageWatchers<-1, false>(VAddr addr, u64 size) const;
 
-} // namespace VideoCore
+} // namespace VideoCore
diff --git a/src/video_core/texture_cache/tile_manager.cpp b/src/video_core/texture_cache/tile_manager.cpp
