From b7b8f704a567ec7ecd38f97edf779b0550e2d486 Mon Sep 17 00:00:00 2001 From: Brad Grantham Date: Tue, 19 Nov 2024 10:12:07 -0800 Subject: [PATCH] Add sparse image and buffer support The current trim handling for buffers and images does not support sparse buffers or sparse images. It only allows one image or buffer binding to a single memory range of a single memory object. As a result, it cannot manage sparse resources, which has led to missing asset issues in trim trace for some titles that utilize sparse resources. This commit introduces support for sparse buffers and sparse images (using opaque memory binding only), effectively resolving the missing asset issue. Originally by Ming Zheng --- .../encode/custom_vulkan_encoder_commands.h | 20 ++ framework/encode/vulkan_capture_manager.h | 151 +++++++- framework/encode/vulkan_handle_wrappers.h | 10 + .../vulkan_state_tracker_initializers.h | 10 + framework/encode/vulkan_state_writer.cpp | 331 ++++++++++++++---- framework/graphics/vulkan_resources_util.cpp | 131 +++++++ framework/graphics/vulkan_resources_util.h | 73 +++- 7 files changed, 662 insertions(+), 64 deletions(-) diff --git a/framework/encode/custom_vulkan_encoder_commands.h b/framework/encode/custom_vulkan_encoder_commands.h index 02275f8bf9..78dde9abf8 100644 --- a/framework/encode/custom_vulkan_encoder_commands.h +++ b/framework/encode/custom_vulkan_encoder_commands.h @@ -495,6 +495,26 @@ struct CustomEncoderPostCall } }; +template <> +struct CustomEncoderPostCall +{ + template + static void Dispatch(VulkanCaptureManager* manager, VkResult result, Args... args) + { + manager->PostProcess_vkCreateBuffer(result, args...); + } +}; + +template <> +struct CustomEncoderPostCall +{ + template + static void Dispatch(VulkanCaptureManager* manager, VkResult result, Args... args) + { + manager->PostProcess_vkCreateImage(result, args...); + } +}; + template <> struct CustomEncoderPostCall { diff --git a/framework/encode/vulkan_capture_manager.h b/framework/encode/vulkan_capture_manager.h index 284005e962..f2eb8ced1c 100644 --- a/framework/encode/vulkan_capture_manager.h +++ b/framework/encode/vulkan_capture_manager.h @@ -534,7 +534,7 @@ class VulkanCaptureManager : public ApiCaptureManager } void PostProcess_vkQueueBindSparse( - VkResult result, VkQueue, uint32_t bindInfoCount, const VkBindSparseInfo* pBindInfo, VkFence) + VkResult result, VkQueue queue, uint32_t bindInfoCount, const VkBindSparseInfo* pBindInfo, VkFence) { if (IsCaptureModeTrack() && (result == VK_SUCCESS)) { @@ -546,6 +546,110 @@ class VulkanCaptureManager : public ApiCaptureManager pBindInfo[i].signalSemaphoreCount, pBindInfo[i].pSignalSemaphores); } + + // In default mode, the capture manager uses a shared mutex to capture every API function. As a result, + // multiple threads may access the sparse resource maps concurrently. Therefore, we use a dedicated mutex + // for write access to these maps. + const std::lock_guard lock(sparse_resource_mutex); + for (uint32_t bind_info_index = 0; bind_info_index < bindInfoCount; bind_info_index++) + { + auto& bind_info = pBindInfo[bind_info_index]; + + // TODO: add device group support. In the following handling, we assume that the system only has one + // physical device or that resourceDeviceIndex and memoryDeviceIndex of VkDeviceGroupBindSparseInfo in + // the pnext chain are zero. + + if (bind_info.pBufferBinds != nullptr) + { + // The title binds sparse buffers to memory ranges, so we need to track the buffer binding + // information. The following updates will reflect the latest binding states for all buffers in this + // vkQueueBindSparse command, covering both fully-resident and partially-resident buffers. + for (uint32_t buffer_bind_index = 0; buffer_bind_index < bind_info.bufferBindCount; + buffer_bind_index++) + { + auto& buffer_bind = bind_info.pBufferBinds[buffer_bind_index]; + auto sparse_buffer = buffer_bind.buffer; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_buffer); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + for (uint32_t bind_memory_range_index = 0; bind_memory_range_index < buffer_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = buffer_bind.pBinds[bind_memory_range_index]; + graphics::UpdateSparseMemoryBindMap(wrapper->sparse_memory_bind_map, bind_memory_range); + } + } + } + } + + if (bind_info.pImageOpaqueBinds != nullptr) + { + // The title binds sparse images to opaque memory ranges, so we need to track the image binding + // information. The following handling will update the latest binding states for all images in this + // vkQueueBindSparse command, which utilizes opaque memory binding. There are two cases covered by + // the tracking. In the first case, the sparse image exclusively uses opaque memory binding. For + // this case, the target title treats the binding memory ranges as a linear unified region. This + // should represent a fully-resident binding because this linear region is entirely opaque, meaning + // there is no application-visible mapping between texel locations and memory offsets. In another + // case, the image utilizes subresource sparse memory binding, just binding only its mip tail region + // to an opaque memory range. For this situation, we use the sparse_opaque_memory_bind_map and + // sparse_subresource_memory_bind_map of the image wrapper to track the subresource bindings and + // opaque bindings separately. + for (uint32_t image_opaque_bind_index = 0; image_opaque_bind_index < bind_info.imageOpaqueBindCount; + image_opaque_bind_index++) + { + auto& image_opaque_bind = bind_info.pImageOpaqueBinds[image_opaque_bind_index]; + auto sparse_image = image_opaque_bind.image; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_image); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + + for (uint32_t bind_memory_range_index = 0; + bind_memory_range_index < image_opaque_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = image_opaque_bind.pBinds[bind_memory_range_index]; + graphics::UpdateSparseMemoryBindMap(wrapper->sparse_opaque_memory_bind_map, + bind_memory_range); + } + } + } + } + + if (bind_info.pImageBinds != nullptr) + { + // The title binds subresources of a sparse image to memory ranges, which requires us to keep track + // of the sparse image subresource binding information. It's important to note that while the image + // mainly use subresource sparse memory binding, its mip tail region must be bound to an opaque + // memory range. Therefore, we use the sparse_opaque_memory_bind_map and + // sparse_subresource_memory_bind_map of the image wrapper to separately track both the + // subresource bindings and the opaque bindings. + for (uint32_t image_bind_index = 0; image_bind_index < bind_info.imageBindCount; image_bind_index++) + { + auto& image_bind = bind_info.pImageBinds[image_bind_index]; + auto sparse_image = image_bind.image; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_image); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + + for (uint32_t bind_memory_range_index = 0; bind_memory_range_index < image_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = image_bind.pBinds[bind_memory_range_index]; + // TODO: Implement handling for tracking binding information of sparse image + // subresources. + GFXRECON_LOG_ERROR_ONCE("Binding of sparse image blocks is not supported!"); + } + } + } + } + } } } @@ -816,6 +920,50 @@ class VulkanCaptureManager : public ApiCaptureManager } } + void PostProcess_vkCreateBuffer(VkResult result, + VkDevice device, + const VkBufferCreateInfo* pCreateInfo, + const VkAllocationCallbacks* pAllocator, + VkBuffer* pBuffer) + { + if (IsCaptureModeTrack() && (result == VK_SUCCESS) && (pCreateInfo != nullptr)) + { + assert(state_tracker_ != nullptr); + + auto buffer_wrapper = vulkan_wrappers::GetWrapper(*pBuffer); + + if (buffer_wrapper->is_sparse_buffer) + { + // We will need to set the bind_device for handling sparse buffers. There will be no subsequent + // vkBindBufferMemory, vkBindBufferMemory2 or vkBindBufferMemory2KHR calls for sparse buffer, so we + // assign bind_device to the device that created the buffer. + buffer_wrapper->bind_device = vulkan_wrappers::GetWrapper(device); + } + } + } + + void PostProcess_vkCreateImage(VkResult result, + VkDevice device, + const VkImageCreateInfo* pCreateInfo, + const VkAllocationCallbacks* pAllocator, + VkImage* pImage) + { + if (IsCaptureModeTrack() && (result == VK_SUCCESS) && (pCreateInfo != nullptr)) + { + assert(state_tracker_ != nullptr); + + auto image_wrapper = vulkan_wrappers::GetWrapper(*pImage); + + if (image_wrapper->is_sparse_image) + { + // We will need to set the bind_device for handling sparse images. There will be no subsequent + // vkBindImageMemory, vkBindImageMemory2, or vkBindImageMemory2KHR calls for sparse image, so we assign + // bind_device to the device that created the image. + image_wrapper->bind_device = vulkan_wrappers::GetWrapper(device); + } + } + } + void PostProcess_vkCmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo* pRenderPassBegin, VkSubpassContents) @@ -1380,6 +1528,7 @@ class VulkanCaptureManager : public ApiCaptureManager std::unique_ptr state_tracker_; HardwareBufferMap hardware_buffers_; std::mutex deferred_operation_mutex; + std::mutex sparse_resource_mutex; }; GFXRECON_END_NAMESPACE(encode) diff --git a/framework/encode/vulkan_handle_wrappers.h b/framework/encode/vulkan_handle_wrappers.h index e131d8026d..3d0b26d3f8 100644 --- a/framework/encode/vulkan_handle_wrappers.h +++ b/framework/encode/vulkan_handle_wrappers.h @@ -30,6 +30,7 @@ #include "format/format.h" #include "generated/generated_vulkan_dispatch_table.h" #include "graphics/vulkan_device_util.h" +#include "graphics/vulkan_resources_util.h" #include "util/defines.h" #include "util/memory_output_stream.h" #include "util/page_guard_manager.h" @@ -210,6 +211,10 @@ struct BufferWrapper : public HandleWrapper const void* bind_pnext{ nullptr }; std::unique_ptr bind_pnext_memory; + bool is_sparse_buffer{ false }; + std::map sparse_memory_bind_map; + VkQueue sparse_bind_queue; + format::HandleId bind_memory_id{ format::kNullHandleId }; VkDeviceSize bind_offset{ 0 }; uint32_t queue_family_index{ 0 }; @@ -227,6 +232,11 @@ struct ImageWrapper : public HandleWrapper const void* bind_pnext{ nullptr }; std::unique_ptr bind_pnext_memory; + bool is_sparse_image{ false }; + std::map sparse_opaque_memory_bind_map; + graphics::VulkanSubresourceSparseImageMemoryBindMap sparse_subresource_memory_bind_map; + VkQueue sparse_bind_queue; + format::HandleId bind_memory_id{ format::kNullHandleId }; VkDeviceSize bind_offset{ 0 }; uint32_t queue_family_index{ 0 }; diff --git a/framework/encode/vulkan_state_tracker_initializers.h b/framework/encode/vulkan_state_tracker_initializers.h index 9178376771..91e188e71c 100644 --- a/framework/encode/vulkan_state_tracker_initializers.h +++ b/framework/encode/vulkan_state_tracker_initializers.h @@ -605,6 +605,11 @@ inline void InitializeStatecreated_size = create_info->size; + if ((create_info->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) != 0) + { + wrapper->is_sparse_buffer = true; + } + // TODO: Do we need to track the queue family that the buffer is actually used with? if ((create_info->queueFamilyIndexCount > 0) && (create_info->pQueueFamilyIndices != nullptr)) { @@ -638,6 +643,11 @@ inline void InitializeStatesamples = create_info->samples; wrapper->tiling = create_info->tiling; + if ((create_info->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) != 0) + { + wrapper->is_sparse_image = true; + } + // TODO: Do we need to track the queue family that the image is actually used with? if ((create_info->queueFamilyIndexCount > 0) && (create_info->pQueueFamilyIndices != nullptr)) { diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index 06aba4d98e..fb39c9ce2f 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -1361,7 +1361,7 @@ void VulkanStateWriter::ProcessBufferMemory(const vulkan_wrappers::DeviceWrapper const uint8_t* bytes = nullptr; std::vector data; - assert((buffer_wrapper != nullptr) && (memory_wrapper != nullptr)); + assert(buffer_wrapper != nullptr); if (snapshot_entry.need_staging_copy) { @@ -1375,6 +1375,7 @@ void VulkanStateWriter::ProcessBufferMemory(const vulkan_wrappers::DeviceWrapper } else { + assert(memory_wrapper != nullptr); assert((memory_wrapper->mapped_data == nullptr) || (memory_wrapper->mapped_offset == 0)); VkResult result = VK_SUCCESS; @@ -1612,57 +1613,164 @@ void VulkanStateWriter::WriteBufferMemoryState(const VulkanStateTable& state_tab state_table.VisitWrappers([&](const vulkan_wrappers::BufferWrapper* wrapper) { assert(wrapper != nullptr); + if (!wrapper->is_sparse_buffer) + { + // Perform memory binding for non-sparse buffer. + const vulkan_wrappers::DeviceMemoryWrapper* memory_wrapper = + state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); - // Perform memory binding. - const vulkan_wrappers::DeviceMemoryWrapper* memory_wrapper = - state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); + if (memory_wrapper != nullptr) + { + const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; + const VulkanDeviceTable* device_table = &device_wrapper->layer_table; + + assert((device_wrapper != nullptr) && (device_table != nullptr)); + + // Write memory requirements query before bind command. + VkMemoryRequirements memory_requirements; + + device_table->GetBufferMemoryRequirements( + device_wrapper->handle, wrapper->handle, &memory_requirements); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + EncodeStructPtr(&encoder_, &memory_requirements); - if (memory_wrapper != nullptr) + WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); + parameter_stream_.Clear(); + + // Write memory bind command. + if (wrapper->bind_pnext == nullptr) + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); + encoder_.EncodeUInt64Value(wrapper->bind_offset); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindBufferMemory, ¶meter_stream_); + } + else + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeUInt32Value(1); + + VkBindBufferMemoryInfo info = {}; + info.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO; + info.pNext = wrapper->bind_pnext; + info.buffer = wrapper->handle; + info.memory = memory_wrapper->handle; + info.memoryOffset = wrapper->bind_offset; + EncodeStructArray(&encoder_, &info, 1); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindBufferMemory2, ¶meter_stream_); + } + parameter_stream_.Clear(); + + // Group buffers with memory bindings by device for memory snapshot. + ResourceSnapshotQueueFamilyTable& snapshot_table = (*resources)[device_wrapper]; + ResourceSnapshotInfo& snapshot_entry = snapshot_table[wrapper->queue_family_index]; + + BufferSnapshotInfo snapshot_info; + snapshot_info.buffer_wrapper = wrapper; + snapshot_info.memory_wrapper = memory_wrapper; + snapshot_info.memory_properties = GetMemoryProperties(device_wrapper, memory_wrapper); + snapshot_info.need_staging_copy = !IsBufferReadable(snapshot_info.memory_properties, memory_wrapper); + + if ((*max_resource_size) < wrapper->created_size) + { + (*max_resource_size) = wrapper->created_size; + } + + if (snapshot_info.need_staging_copy && ((*max_staging_copy_size) < wrapper->created_size)) + { + (*max_staging_copy_size) = wrapper->created_size; + } + + snapshot_entry.buffers.emplace_back(snapshot_info); + } + } + else { + // Perform memory binding for sparse buffer. const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; const VulkanDeviceTable* device_table = &device_wrapper->layer_table; - assert((device_wrapper != nullptr) && (device_table != nullptr)); - // Write memory requirements query before bind command. - VkMemoryRequirements memory_requirements; + // We do not need to use sparse_resource_mutex for the access to the following sparse resource maps, as the + // writing states operation is included in the trim start handling, which is protected by an exclusive lock. + // No other API capturing handling occurs concurrently. + if (wrapper->sparse_memory_bind_map.size() != 0) + { + std::vector sparse_memory_binds; + VkSparseBufferMemoryBindInfo buffer_memory_bind_info = {}; - device_table->GetBufferMemoryRequirements(device_wrapper->handle, wrapper->handle, &memory_requirements); + // Write memory requirements query before vkQueueBindSparse command. For sparse buffer, the alignment of + // VkMemoryRequirements is the sparse block size in bytes which represents both the memory alignment + // requirement and the binding granularity (in bytes) for sparse buffer. + VkMemoryRequirements memory_requirements; - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeHandleIdValue(wrapper->handle_id); - EncodeStructPtr(&encoder_, &memory_requirements); + device_table->GetBufferMemoryRequirements( + device_wrapper->handle, wrapper->handle, &memory_requirements); - WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); - parameter_stream_.Clear(); - - // Write memory bind command. - if (wrapper->bind_pnext == nullptr) - { encoder_.EncodeHandleIdValue(device_wrapper->handle_id); encoder_.EncodeHandleIdValue(wrapper->handle_id); - encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); - encoder_.EncodeUInt64Value(wrapper->bind_offset); - encoder_.EncodeEnumValue(VK_SUCCESS); + EncodeStructPtr(&encoder_, &memory_requirements); - WriteFunctionCall(format::ApiCall_vkBindBufferMemory, ¶meter_stream_); - } - else - { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeUInt32Value(1); - - VkBindBufferMemoryInfo info = {}; - info.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO; - info.pNext = wrapper->bind_pnext; - info.buffer = wrapper->handle; - info.memory = memory_wrapper->handle; - info.memoryOffset = wrapper->bind_offset; - EncodeStructArray(&encoder_, &info, 1); - encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); + parameter_stream_.Clear(); - WriteFunctionCall(format::ApiCall_vkBindBufferMemory2, ¶meter_stream_); + const vulkan_wrappers::QueueWrapper* sparse_bind_queue_wrapper = + vulkan_wrappers::GetWrapper(wrapper->sparse_bind_queue); + + if ((wrapper->sparse_bind_queue != VK_NULL_HANDLE) && (sparse_bind_queue_wrapper != nullptr)) + { + for (auto& item : wrapper->sparse_memory_bind_map) + { + sparse_memory_binds.push_back(item.second); + } + + buffer_memory_bind_info.buffer = wrapper->handle; + buffer_memory_bind_info.bindCount = sparse_memory_binds.size(); + buffer_memory_bind_info.pBinds = sparse_memory_binds.data(); + + VkBindSparseInfo bind_sparse_info{}; + bind_sparse_info.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + bind_sparse_info.pNext = nullptr; + bind_sparse_info.waitSemaphoreCount = 0; + bind_sparse_info.pWaitSemaphores = nullptr; + bind_sparse_info.bufferBindCount = 1; + bind_sparse_info.pBufferBinds = &buffer_memory_bind_info; + bind_sparse_info.imageOpaqueBindCount = 0; + bind_sparse_info.pImageOpaqueBinds = nullptr; + bind_sparse_info.imageBindCount = 0; + bind_sparse_info.pImageBinds = nullptr; + bind_sparse_info.signalSemaphoreCount = 0; + bind_sparse_info.pSignalSemaphores = nullptr; + + encoder_.EncodeVulkanHandleValue(wrapper->sparse_bind_queue); + encoder_.EncodeUInt32Value(1); + EncodeStructArray(&encoder_, &bind_sparse_info, 1); + encoder_.EncodeVulkanHandleValue(VK_NULL_HANDLE); + encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkQueueBindSparse, ¶meter_stream_); + + parameter_stream_.Clear(); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkDeviceWaitIdle, ¶meter_stream_); + } + else + { + GFXRECON_LOG_WARNING("Unable to generate vkQueueBindSparse for the sparse buffer (id = %d) due to " + "the related sparse bind queue or its wrapper is invalid.", + wrapper->handle_id); + } } + parameter_stream_.Clear(); // Group buffers with memory bindings by device for memory snapshot. @@ -1670,10 +1778,16 @@ void VulkanStateWriter::WriteBufferMemoryState(const VulkanStateTable& state_tab ResourceSnapshotInfo& snapshot_entry = snapshot_table[wrapper->queue_family_index]; BufferSnapshotInfo snapshot_info; - snapshot_info.buffer_wrapper = wrapper; - snapshot_info.memory_wrapper = memory_wrapper; - snapshot_info.memory_properties = GetMemoryProperties(device_wrapper, memory_wrapper); - snapshot_info.need_staging_copy = !IsBufferReadable(snapshot_info.memory_properties, memory_wrapper); + snapshot_info.buffer_wrapper = wrapper; + snapshot_info.memory_wrapper = nullptr; + + // We enforce the memory properties to be `VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT`, and we set + // `need_staging_copy` to true for sparse buffers. When dumping buffer data, there are two distinct code + // paths: one involves a staging copy, while the other requires mapping host-visible memory. This latter + // method requires the buffer to be bound to a single range of a single memory, which is not applicable for + // sparse buffers. Therefore, we set the two values to use staging copy for dumping sparse buffers. + snapshot_info.memory_properties = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + snapshot_info.need_staging_copy = true; // Staging copy is needed for sparse buffer. if ((*max_resource_size) < wrapper->created_size) { @@ -1705,7 +1819,8 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); if ((wrapper->is_swapchain_image && memory_wrapper == nullptr && wrapper->bind_device != nullptr) || - (!wrapper->is_swapchain_image && memory_wrapper != nullptr)) + (!wrapper->is_swapchain_image && memory_wrapper != nullptr) || + (!wrapper->is_swapchain_image && wrapper->is_sparse_image && wrapper->bind_device != nullptr)) { const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; const VulkanDeviceTable* device_table = &device_wrapper->layer_table; @@ -1725,31 +1840,112 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl parameter_stream_.Clear(); // Write memory bind command. - if (wrapper->bind_pnext == nullptr) + if (!wrapper->is_sparse_image) { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeHandleIdValue(wrapper->handle_id); - encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); - encoder_.EncodeUInt64Value(wrapper->bind_offset); - encoder_.EncodeEnumValue(VK_SUCCESS); + if (wrapper->bind_pnext == nullptr) + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); + encoder_.EncodeUInt64Value(wrapper->bind_offset); + encoder_.EncodeEnumValue(VK_SUCCESS); - WriteFunctionCall(format::ApiCall_vkBindImageMemory, ¶meter_stream_); + WriteFunctionCall(format::ApiCall_vkBindImageMemory, ¶meter_stream_); + } + else + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeUInt32Value(1); + + VkBindImageMemoryInfo info = {}; + info.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; + info.pNext = wrapper->bind_pnext; + info.image = wrapper->handle; + info.memory = memory_wrapper->handle; + info.memoryOffset = wrapper->bind_offset; + EncodeStructArray(&encoder_, &info, 1); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindImageMemory2, ¶meter_stream_); + } } else { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeUInt32Value(1); - - VkBindImageMemoryInfo info = {}; - info.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; - info.pNext = wrapper->bind_pnext; - info.image = wrapper->handle; - info.memory = memory_wrapper->handle; - info.memoryOffset = wrapper->bind_offset; - EncodeStructArray(&encoder_, &info, 1); - encoder_.EncodeEnumValue(VK_SUCCESS); + const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; + const VulkanDeviceTable* device_table = &device_wrapper->layer_table; + assert((device_wrapper != nullptr) && (device_table != nullptr)); + + const vulkan_wrappers::QueueWrapper* sparse_bind_queue_wrapper = + vulkan_wrappers::GetWrapper(wrapper->sparse_bind_queue); + + GFXRECON_ASSERT((wrapper->sparse_bind_queue != VK_NULL_HANDLE) && + (sparse_bind_queue_wrapper != nullptr)); + + if ((wrapper->sparse_opaque_memory_bind_map.size() != 0) || + (wrapper->sparse_subresource_memory_bind_map.size() != 0)) + { + std::vector sparse_memory_binds; + VkSparseImageOpaqueMemoryBindInfo image_opaque_memory_bind_info = {}; + + for (auto& item : wrapper->sparse_opaque_memory_bind_map) + { + sparse_memory_binds.push_back(item.second); + } + + image_opaque_memory_bind_info.image = wrapper->handle; + image_opaque_memory_bind_info.bindCount = sparse_memory_binds.size(); + image_opaque_memory_bind_info.pBinds = + (sparse_memory_binds.size() == 0) ? nullptr : sparse_memory_binds.data(); - WriteFunctionCall(format::ApiCall_vkBindImageMemory2, ¶meter_stream_); + std::vector sparse_image_memory_binds; + VkSparseImageMemoryBindInfo image_memory_bind_info = {}; + + for (auto& subresource_bind_map : wrapper->sparse_subresource_memory_bind_map) + { + auto& offset_3d_to_memory_range_map = subresource_bind_map.second; + + for (auto& item : offset_3d_to_memory_range_map) + { + sparse_image_memory_binds.push_back(item.second); + } + } + + image_memory_bind_info.image = wrapper->handle; + image_memory_bind_info.bindCount = sparse_image_memory_binds.size(); + image_memory_bind_info.pBinds = + (sparse_image_memory_binds.size() == 0) ? nullptr : sparse_image_memory_binds.data(); + + VkBindSparseInfo bind_sparse_info{}; + + bind_sparse_info.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + bind_sparse_info.pNext = nullptr; + bind_sparse_info.waitSemaphoreCount = 0; + bind_sparse_info.pWaitSemaphores = nullptr; + bind_sparse_info.bufferBindCount = 0; + bind_sparse_info.pBufferBinds = nullptr; + bind_sparse_info.imageOpaqueBindCount = (image_opaque_memory_bind_info.bindCount == 0) ? 0 : 1; + bind_sparse_info.pImageOpaqueBinds = + (image_opaque_memory_bind_info.bindCount == 0) ? nullptr : &image_opaque_memory_bind_info; + bind_sparse_info.imageBindCount = (image_memory_bind_info.bindCount == 0) ? 0 : 1; + bind_sparse_info.pImageBinds = + (image_memory_bind_info.bindCount == 0) ? nullptr : &image_memory_bind_info; + bind_sparse_info.signalSemaphoreCount = 0; + bind_sparse_info.pSignalSemaphores = nullptr; + + encoder_.EncodeVulkanHandleValue(wrapper->sparse_bind_queue); + encoder_.EncodeUInt32Value(1); + EncodeStructArray(&encoder_, &bind_sparse_info, 1); + encoder_.EncodeVulkanHandleValue(VK_NULL_HANDLE); + encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkQueueBindSparse, ¶meter_stream_); + + parameter_stream_.Clear(); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkDeviceWaitIdle, ¶meter_stream_); + } } parameter_stream_.Clear(); @@ -1765,6 +1961,13 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl (wrapper->tiling == VK_IMAGE_TILING_LINEAR) && ((memory_properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + if (wrapper->is_sparse_image) + { + // The process for dumping host-visible image data requires binding the entire image to a single memory + // range. Since this is not applicable to sparse images, we set is_writable to false. + is_writable = false; + } + // If an image is not host writable and has not been transitioned from the undefined or preinitialized // layouts, no data could have been loaded into it and its data will be omitted from the state snapshot. if (is_transitioned || is_writable) @@ -1778,7 +1981,11 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl *device_wrapper->physical_device->layer_table_ref, device_wrapper->physical_device->memory_properties); - bool need_staging_copy = !IsImageReadable(memory_properties, memory_wrapper, wrapper); + // Sparse images require staging copy for the following process because dumping image data with mapping + // memory needs binding the entire image to a single memory range. Sparse image opaque binding allows + // binding to multiple memory objects and various memory ranges. + bool need_staging_copy = + !IsImageReadable(memory_properties, memory_wrapper, wrapper) || wrapper->is_sparse_image; std::vector aspects; bool combined_depth_stencil; diff --git a/framework/graphics/vulkan_resources_util.cpp b/framework/graphics/vulkan_resources_util.cpp index 0269588b20..7254f578e2 100644 --- a/framework/graphics/vulkan_resources_util.cpp +++ b/framework/graphics/vulkan_resources_util.cpp @@ -2067,6 +2067,137 @@ VkResult VulkanResourcesUtil::WriteToImageResourceStaging(VkImage return result; } +bool GetIntersectForSparseMemoryBind(uint32_t new_bind_resource_offset, + uint32_t new_bind_resource_size, + uint32_t existing_bind_resource_offset, + uint32_t existing_bind_resource_size, + uint32_t& intersection_resource_offset, + uint32_t& intersection_resource_size, + std::vector& remaining_resource_offsets, + std::vector& remaining_resource_sizes, + bool& new_bind_range_include_existing_bind_tange, + bool& existing_bind_range_include_new_bind_tange) +{ + bool intersection_exist = false; + uint32_t intersection_start = std::max(new_bind_resource_offset, existing_bind_resource_offset); + uint32_t intersection_end = std::min(new_bind_resource_offset + new_bind_resource_size, + existing_bind_resource_offset + existing_bind_resource_size); + + existing_bind_range_include_new_bind_tange = false; + new_bind_range_include_existing_bind_tange = false; + + if (intersection_start < intersection_end) + { + intersection_exist = true; + intersection_resource_offset = intersection_start; + intersection_resource_size = intersection_end - intersection_start; + + if ((intersection_resource_offset == new_bind_resource_offset) && + (intersection_resource_size == new_bind_resource_size)) + { + existing_bind_range_include_new_bind_tange = true; + } + + if ((intersection_resource_offset == existing_bind_resource_offset) && + (intersection_resource_size == existing_bind_resource_size)) + { + new_bind_range_include_existing_bind_tange = true; + } + + if (intersection_resource_offset > existing_bind_resource_offset) + { + remaining_resource_offsets.push_back(existing_bind_resource_offset); + remaining_resource_sizes.push_back(intersection_resource_offset - existing_bind_resource_offset); + } + + if ((intersection_resource_offset + intersection_resource_size) < + (existing_bind_resource_offset + existing_bind_resource_size)) + { + remaining_resource_offsets.push_back(intersection_resource_offset + intersection_resource_size); + remaining_resource_sizes.push_back((existing_bind_resource_offset + existing_bind_resource_size) - + (intersection_resource_offset + intersection_resource_size)); + } + } + + return intersection_exist; +} + +void UpdateSparseMemoryBindMap(std::map& sparse_memory_bind_map, + const VkSparseMemoryBind& new_sparse_memory_bind) +{ + std::vector all_remaining_existing_bind_ranges{}; + std::vector delete_existing_bind_ranges{}; + + VkDeviceSize search_key = new_sparse_memory_bind.resourceOffset + new_sparse_memory_bind.size; + auto iterator = sparse_memory_bind_map.lower_bound(search_key); + bool is_intersected_with_any_existing_bind = false; + + bool ignored = false; + + if ((sparse_memory_bind_map.size() != 0) && (iterator != sparse_memory_bind_map.begin())) + { + for (auto item = sparse_memory_bind_map.begin(); item != iterator; item++) + { + uint32_t intersection_resource_offset, intersection_resource_size; + std::vector remaining_resource_offsets, remaining_resource_sizes; + bool new_bind_range_include_existing_bind_tange, existing_bind_range_include_new_bind_tange; + + bool is_intersected = GetIntersectForSparseMemoryBind(new_sparse_memory_bind.resourceOffset, + new_sparse_memory_bind.size, + item->second.resourceOffset, + item->second.size, + intersection_resource_offset, + intersection_resource_size, + remaining_resource_offsets, + remaining_resource_sizes, + new_bind_range_include_existing_bind_tange, + existing_bind_range_include_new_bind_tange); + + if (is_intersected) + { + is_intersected_with_any_existing_bind = false; + + VkSparseMemoryBind add_sparse_memory_bind = { 0, 0, item->second.memory, 0, item->second.flags }; + GFXRECON_ASSERT(item->second.flags == new_sparse_memory_bind.flags); + + uint32_t index = 0; + for (auto& bind_offset : remaining_resource_offsets) + { + add_sparse_memory_bind.resourceOffset = bind_offset; + add_sparse_memory_bind.size = remaining_resource_sizes[index]; + add_sparse_memory_bind.memoryOffset = + item->second.memoryOffset + bind_offset - item->second.resourceOffset; + all_remaining_existing_bind_ranges.push_back(add_sparse_memory_bind); + + index++; + } + + delete_existing_bind_ranges.push_back(item->second); + } + } + } + + if (is_intersected_with_any_existing_bind) + { + for (auto& delete_item : delete_existing_bind_ranges) + { + sparse_memory_bind_map.erase(delete_item.resourceOffset); + } + + size_t index = 0, remaining_range_base = 0; + + for (auto add_item : all_remaining_existing_bind_ranges) + { + sparse_memory_bind_map[add_item.resourceOffset] = add_item; + } + } + + if (new_sparse_memory_bind.memory != VK_NULL_HANDLE) + { + sparse_memory_bind_map[new_sparse_memory_bind.resourceOffset] = new_sparse_memory_bind; + } +} + bool VulkanResourcesUtil::IsBlitSupported(VkFormat src_format, VkImageTiling src_image_tiling, VkFormat dst_format, diff --git a/framework/graphics/vulkan_resources_util.h b/framework/graphics/vulkan_resources_util.h index 1aeec6bbca..3b7e613072 100644 --- a/framework/graphics/vulkan_resources_util.h +++ b/framework/graphics/vulkan_resources_util.h @@ -31,6 +31,7 @@ #include "vulkan/vulkan_core.h" #include +#include GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(graphics) @@ -280,6 +281,76 @@ bool FindMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properti uint32_t* found_index, VkMemoryPropertyFlags* found_flags); +struct VkOffset3DComparator +{ + bool operator()(const VkOffset3D& l, const VkOffset3D& r) const + { + bool result = (l.x < r.x); + + if (l.x == r.x) + { + result = (l.y < r.y); + + if (l.y == r.y) + { + result = (l.z < r.z); + } + } + + return result; + } +}; + +typedef std::map + VulkanOffset3DSparseImageMemoryBindMap; + +struct VkImageSubresourceComparator +{ + bool operator()(const VkImageSubresource& l, const VkImageSubresource& r) const + { + bool result = (l.arrayLayer < r.arrayLayer); + + if (l.arrayLayer == r.arrayLayer) + { + result = (l.mipLevel < r.mipLevel); + + if (l.mipLevel == r.mipLevel) + { + result = (l.aspectMask < r.aspectMask); + } + } + + return result; + } +}; + +typedef std:: + map + VulkanSubresourceSparseImageMemoryBindMap; + +// Get the intersection of the new bind range and the existing bind range for sparse buffer or sparse image (opaque +// bind). +// If the intersection range exists, further get the remaining ranges for the existing bind range after removing +// the intersection range. For instance, if the new/existing bind range (offset, size) are (196608, 327680) and (0, +// 655360) respectively, the old range (0, 655360) completely covers the new range (196608, 327680). The intersection +// range is (196608, 327680), and the remaining ranges for the existing bind are (0, 196608) and (524288, 131072). So +// for the return of the function, remaining_resource_offsets will be a std::vector of [0, 524288], and +// remaining_resource_sizes will be [196608, 131072]. +// +bool GetIntersectForSparseMemoryBind(uint32_t new_bind_resource_offset, + uint32_t new_bind_resource_size, + uint32_t existing_bind_resource_offset, + uint32_t existing_bind_resource_size, + uint32_t& intersection_resource_offset, + uint32_t& intersection_resource_size, + std::vector& remaining_resource_offsets, + std::vector& remaining_resource_sizes, + bool& new_bind_range_include_existing_bind_tange, + bool& existing_bind_range_include_new_bind_tange); + +void UpdateSparseMemoryBindMap(std::map& sparse_memory_bind_map, + const VkSparseMemoryBind& new_sparse_memory_bind); + bool GetImageTexelSize(VkFormat format, VkDeviceSize* texel_size, bool* is_texel_block_size, @@ -323,7 +394,7 @@ bool NextRowTexelCoordinates(VkImageType imageType, uint32_t& z, uint32_t& layer); +GFXRECON_END_NAMESPACE(graphics) GFXRECON_END_NAMESPACE(gfxrecon) -GFXRECON_END_NAMESPACE(encode) #endif /* GFXRECON_GRAPHICS_VULKAN_RESOURCES_UTIL_H */