diff --git a/framework/encode/custom_vulkan_encoder_commands.h b/framework/encode/custom_vulkan_encoder_commands.h index 02275f8bf9..78dde9abf8 100644 --- a/framework/encode/custom_vulkan_encoder_commands.h +++ b/framework/encode/custom_vulkan_encoder_commands.h @@ -495,6 +495,26 @@ struct CustomEncoderPostCall } }; +template <> +struct CustomEncoderPostCall +{ + template + static void Dispatch(VulkanCaptureManager* manager, VkResult result, Args... args) + { + manager->PostProcess_vkCreateBuffer(result, args...); + } +}; + +template <> +struct CustomEncoderPostCall +{ + template + static void Dispatch(VulkanCaptureManager* manager, VkResult result, Args... args) + { + manager->PostProcess_vkCreateImage(result, args...); + } +}; + template <> struct CustomEncoderPostCall { diff --git a/framework/encode/vulkan_capture_manager.h b/framework/encode/vulkan_capture_manager.h index 284005e962..f2eb8ced1c 100644 --- a/framework/encode/vulkan_capture_manager.h +++ b/framework/encode/vulkan_capture_manager.h @@ -534,7 +534,7 @@ class VulkanCaptureManager : public ApiCaptureManager } void PostProcess_vkQueueBindSparse( - VkResult result, VkQueue, uint32_t bindInfoCount, const VkBindSparseInfo* pBindInfo, VkFence) + VkResult result, VkQueue queue, uint32_t bindInfoCount, const VkBindSparseInfo* pBindInfo, VkFence) { if (IsCaptureModeTrack() && (result == VK_SUCCESS)) { @@ -546,6 +546,110 @@ class VulkanCaptureManager : public ApiCaptureManager pBindInfo[i].signalSemaphoreCount, pBindInfo[i].pSignalSemaphores); } + + // In default mode, the capture manager uses a shared mutex to capture every API function. As a result, + // multiple threads may access the sparse resource maps concurrently. Therefore, we use a dedicated mutex + // for write access to these maps. + const std::lock_guard lock(sparse_resource_mutex); + for (uint32_t bind_info_index = 0; bind_info_index < bindInfoCount; bind_info_index++) + { + auto& bind_info = pBindInfo[bind_info_index]; + + // TODO: add device group support. In the following handling, we assume that the system only has one + // physical device or that resourceDeviceIndex and memoryDeviceIndex of VkDeviceGroupBindSparseInfo in + // the pnext chain are zero. + + if (bind_info.pBufferBinds != nullptr) + { + // The title binds sparse buffers to memory ranges, so we need to track the buffer binding + // information. The following updates will reflect the latest binding states for all buffers in this + // vkQueueBindSparse command, covering both fully-resident and partially-resident buffers. + for (uint32_t buffer_bind_index = 0; buffer_bind_index < bind_info.bufferBindCount; + buffer_bind_index++) + { + auto& buffer_bind = bind_info.pBufferBinds[buffer_bind_index]; + auto sparse_buffer = buffer_bind.buffer; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_buffer); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + for (uint32_t bind_memory_range_index = 0; bind_memory_range_index < buffer_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = buffer_bind.pBinds[bind_memory_range_index]; + graphics::UpdateSparseMemoryBindMap(wrapper->sparse_memory_bind_map, bind_memory_range); + } + } + } + } + + if (bind_info.pImageOpaqueBinds != nullptr) + { + // The title binds sparse images to opaque memory ranges, so we need to track the image binding + // information. The following handling will update the latest binding states for all images in this + // vkQueueBindSparse command, which utilizes opaque memory binding. There are two cases covered by + // the tracking. In the first case, the sparse image exclusively uses opaque memory binding. For + // this case, the target title treats the binding memory ranges as a linear unified region. This + // should represent a fully-resident binding because this linear region is entirely opaque, meaning + // there is no application-visible mapping between texel locations and memory offsets. In another + // case, the image utilizes subresource sparse memory binding, just binding only its mip tail region + // to an opaque memory range. For this situation, we use the sparse_opaque_memory_bind_map and + // sparse_subresource_memory_bind_map of the image wrapper to track the subresource bindings and + // opaque bindings separately. + for (uint32_t image_opaque_bind_index = 0; image_opaque_bind_index < bind_info.imageOpaqueBindCount; + image_opaque_bind_index++) + { + auto& image_opaque_bind = bind_info.pImageOpaqueBinds[image_opaque_bind_index]; + auto sparse_image = image_opaque_bind.image; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_image); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + + for (uint32_t bind_memory_range_index = 0; + bind_memory_range_index < image_opaque_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = image_opaque_bind.pBinds[bind_memory_range_index]; + graphics::UpdateSparseMemoryBindMap(wrapper->sparse_opaque_memory_bind_map, + bind_memory_range); + } + } + } + } + + if (bind_info.pImageBinds != nullptr) + { + // The title binds subresources of a sparse image to memory ranges, which requires us to keep track + // of the sparse image subresource binding information. It's important to note that while the image + // mainly use subresource sparse memory binding, its mip tail region must be bound to an opaque + // memory range. Therefore, we use the sparse_opaque_memory_bind_map and + // sparse_subresource_memory_bind_map of the image wrapper to separately track both the + // subresource bindings and the opaque bindings. + for (uint32_t image_bind_index = 0; image_bind_index < bind_info.imageBindCount; image_bind_index++) + { + auto& image_bind = bind_info.pImageBinds[image_bind_index]; + auto sparse_image = image_bind.image; + auto wrapper = vulkan_wrappers::GetWrapper(sparse_image); + + if (wrapper != nullptr) + { + wrapper->sparse_bind_queue = queue; + + for (uint32_t bind_memory_range_index = 0; bind_memory_range_index < image_bind.bindCount; + bind_memory_range_index++) + { + auto& bind_memory_range = image_bind.pBinds[bind_memory_range_index]; + // TODO: Implement handling for tracking binding information of sparse image + // subresources. + GFXRECON_LOG_ERROR_ONCE("Binding of sparse image blocks is not supported!"); + } + } + } + } + } } } @@ -816,6 +920,50 @@ class VulkanCaptureManager : public ApiCaptureManager } } + void PostProcess_vkCreateBuffer(VkResult result, + VkDevice device, + const VkBufferCreateInfo* pCreateInfo, + const VkAllocationCallbacks* pAllocator, + VkBuffer* pBuffer) + { + if (IsCaptureModeTrack() && (result == VK_SUCCESS) && (pCreateInfo != nullptr)) + { + assert(state_tracker_ != nullptr); + + auto buffer_wrapper = vulkan_wrappers::GetWrapper(*pBuffer); + + if (buffer_wrapper->is_sparse_buffer) + { + // We will need to set the bind_device for handling sparse buffers. There will be no subsequent + // vkBindBufferMemory, vkBindBufferMemory2 or vkBindBufferMemory2KHR calls for sparse buffer, so we + // assign bind_device to the device that created the buffer. + buffer_wrapper->bind_device = vulkan_wrappers::GetWrapper(device); + } + } + } + + void PostProcess_vkCreateImage(VkResult result, + VkDevice device, + const VkImageCreateInfo* pCreateInfo, + const VkAllocationCallbacks* pAllocator, + VkImage* pImage) + { + if (IsCaptureModeTrack() && (result == VK_SUCCESS) && (pCreateInfo != nullptr)) + { + assert(state_tracker_ != nullptr); + + auto image_wrapper = vulkan_wrappers::GetWrapper(*pImage); + + if (image_wrapper->is_sparse_image) + { + // We will need to set the bind_device for handling sparse images. There will be no subsequent + // vkBindImageMemory, vkBindImageMemory2, or vkBindImageMemory2KHR calls for sparse image, so we assign + // bind_device to the device that created the image. + image_wrapper->bind_device = vulkan_wrappers::GetWrapper(device); + } + } + } + void PostProcess_vkCmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo* pRenderPassBegin, VkSubpassContents) @@ -1380,6 +1528,7 @@ class VulkanCaptureManager : public ApiCaptureManager std::unique_ptr state_tracker_; HardwareBufferMap hardware_buffers_; std::mutex deferred_operation_mutex; + std::mutex sparse_resource_mutex; }; GFXRECON_END_NAMESPACE(encode) diff --git a/framework/encode/vulkan_handle_wrappers.h b/framework/encode/vulkan_handle_wrappers.h index e131d8026d..3d0b26d3f8 100644 --- a/framework/encode/vulkan_handle_wrappers.h +++ b/framework/encode/vulkan_handle_wrappers.h @@ -30,6 +30,7 @@ #include "format/format.h" #include "generated/generated_vulkan_dispatch_table.h" #include "graphics/vulkan_device_util.h" +#include "graphics/vulkan_resources_util.h" #include "util/defines.h" #include "util/memory_output_stream.h" #include "util/page_guard_manager.h" @@ -210,6 +211,10 @@ struct BufferWrapper : public HandleWrapper const void* bind_pnext{ nullptr }; std::unique_ptr bind_pnext_memory; + bool is_sparse_buffer{ false }; + std::map sparse_memory_bind_map; + VkQueue sparse_bind_queue; + format::HandleId bind_memory_id{ format::kNullHandleId }; VkDeviceSize bind_offset{ 0 }; uint32_t queue_family_index{ 0 }; @@ -227,6 +232,11 @@ struct ImageWrapper : public HandleWrapper const void* bind_pnext{ nullptr }; std::unique_ptr bind_pnext_memory; + bool is_sparse_image{ false }; + std::map sparse_opaque_memory_bind_map; + graphics::VulkanSubresourceSparseImageMemoryBindMap sparse_subresource_memory_bind_map; + VkQueue sparse_bind_queue; + format::HandleId bind_memory_id{ format::kNullHandleId }; VkDeviceSize bind_offset{ 0 }; uint32_t queue_family_index{ 0 }; diff --git a/framework/encode/vulkan_state_tracker_initializers.h b/framework/encode/vulkan_state_tracker_initializers.h index 9178376771..91e188e71c 100644 --- a/framework/encode/vulkan_state_tracker_initializers.h +++ b/framework/encode/vulkan_state_tracker_initializers.h @@ -605,6 +605,11 @@ inline void InitializeStatecreated_size = create_info->size; + if ((create_info->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) != 0) + { + wrapper->is_sparse_buffer = true; + } + // TODO: Do we need to track the queue family that the buffer is actually used with? if ((create_info->queueFamilyIndexCount > 0) && (create_info->pQueueFamilyIndices != nullptr)) { @@ -638,6 +643,11 @@ inline void InitializeStatesamples = create_info->samples; wrapper->tiling = create_info->tiling; + if ((create_info->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) != 0) + { + wrapper->is_sparse_image = true; + } + // TODO: Do we need to track the queue family that the image is actually used with? if ((create_info->queueFamilyIndexCount > 0) && (create_info->pQueueFamilyIndices != nullptr)) { diff --git a/framework/encode/vulkan_state_writer.cpp b/framework/encode/vulkan_state_writer.cpp index 06aba4d98e..fb39c9ce2f 100644 --- a/framework/encode/vulkan_state_writer.cpp +++ b/framework/encode/vulkan_state_writer.cpp @@ -1361,7 +1361,7 @@ void VulkanStateWriter::ProcessBufferMemory(const vulkan_wrappers::DeviceWrapper const uint8_t* bytes = nullptr; std::vector data; - assert((buffer_wrapper != nullptr) && (memory_wrapper != nullptr)); + assert(buffer_wrapper != nullptr); if (snapshot_entry.need_staging_copy) { @@ -1375,6 +1375,7 @@ void VulkanStateWriter::ProcessBufferMemory(const vulkan_wrappers::DeviceWrapper } else { + assert(memory_wrapper != nullptr); assert((memory_wrapper->mapped_data == nullptr) || (memory_wrapper->mapped_offset == 0)); VkResult result = VK_SUCCESS; @@ -1612,57 +1613,164 @@ void VulkanStateWriter::WriteBufferMemoryState(const VulkanStateTable& state_tab state_table.VisitWrappers([&](const vulkan_wrappers::BufferWrapper* wrapper) { assert(wrapper != nullptr); + if (!wrapper->is_sparse_buffer) + { + // Perform memory binding for non-sparse buffer. + const vulkan_wrappers::DeviceMemoryWrapper* memory_wrapper = + state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); - // Perform memory binding. - const vulkan_wrappers::DeviceMemoryWrapper* memory_wrapper = - state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); + if (memory_wrapper != nullptr) + { + const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; + const VulkanDeviceTable* device_table = &device_wrapper->layer_table; + + assert((device_wrapper != nullptr) && (device_table != nullptr)); + + // Write memory requirements query before bind command. + VkMemoryRequirements memory_requirements; + + device_table->GetBufferMemoryRequirements( + device_wrapper->handle, wrapper->handle, &memory_requirements); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + EncodeStructPtr(&encoder_, &memory_requirements); - if (memory_wrapper != nullptr) + WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); + parameter_stream_.Clear(); + + // Write memory bind command. + if (wrapper->bind_pnext == nullptr) + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); + encoder_.EncodeUInt64Value(wrapper->bind_offset); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindBufferMemory, ¶meter_stream_); + } + else + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeUInt32Value(1); + + VkBindBufferMemoryInfo info = {}; + info.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO; + info.pNext = wrapper->bind_pnext; + info.buffer = wrapper->handle; + info.memory = memory_wrapper->handle; + info.memoryOffset = wrapper->bind_offset; + EncodeStructArray(&encoder_, &info, 1); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindBufferMemory2, ¶meter_stream_); + } + parameter_stream_.Clear(); + + // Group buffers with memory bindings by device for memory snapshot. + ResourceSnapshotQueueFamilyTable& snapshot_table = (*resources)[device_wrapper]; + ResourceSnapshotInfo& snapshot_entry = snapshot_table[wrapper->queue_family_index]; + + BufferSnapshotInfo snapshot_info; + snapshot_info.buffer_wrapper = wrapper; + snapshot_info.memory_wrapper = memory_wrapper; + snapshot_info.memory_properties = GetMemoryProperties(device_wrapper, memory_wrapper); + snapshot_info.need_staging_copy = !IsBufferReadable(snapshot_info.memory_properties, memory_wrapper); + + if ((*max_resource_size) < wrapper->created_size) + { + (*max_resource_size) = wrapper->created_size; + } + + if (snapshot_info.need_staging_copy && ((*max_staging_copy_size) < wrapper->created_size)) + { + (*max_staging_copy_size) = wrapper->created_size; + } + + snapshot_entry.buffers.emplace_back(snapshot_info); + } + } + else { + // Perform memory binding for sparse buffer. const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; const VulkanDeviceTable* device_table = &device_wrapper->layer_table; - assert((device_wrapper != nullptr) && (device_table != nullptr)); - // Write memory requirements query before bind command. - VkMemoryRequirements memory_requirements; + // We do not need to use sparse_resource_mutex for the access to the following sparse resource maps, as the + // writing states operation is included in the trim start handling, which is protected by an exclusive lock. + // No other API capturing handling occurs concurrently. + if (wrapper->sparse_memory_bind_map.size() != 0) + { + std::vector sparse_memory_binds; + VkSparseBufferMemoryBindInfo buffer_memory_bind_info = {}; - device_table->GetBufferMemoryRequirements(device_wrapper->handle, wrapper->handle, &memory_requirements); + // Write memory requirements query before vkQueueBindSparse command. For sparse buffer, the alignment of + // VkMemoryRequirements is the sparse block size in bytes which represents both the memory alignment + // requirement and the binding granularity (in bytes) for sparse buffer. + VkMemoryRequirements memory_requirements; - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeHandleIdValue(wrapper->handle_id); - EncodeStructPtr(&encoder_, &memory_requirements); + device_table->GetBufferMemoryRequirements( + device_wrapper->handle, wrapper->handle, &memory_requirements); - WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); - parameter_stream_.Clear(); - - // Write memory bind command. - if (wrapper->bind_pnext == nullptr) - { encoder_.EncodeHandleIdValue(device_wrapper->handle_id); encoder_.EncodeHandleIdValue(wrapper->handle_id); - encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); - encoder_.EncodeUInt64Value(wrapper->bind_offset); - encoder_.EncodeEnumValue(VK_SUCCESS); + EncodeStructPtr(&encoder_, &memory_requirements); - WriteFunctionCall(format::ApiCall_vkBindBufferMemory, ¶meter_stream_); - } - else - { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeUInt32Value(1); - - VkBindBufferMemoryInfo info = {}; - info.sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO; - info.pNext = wrapper->bind_pnext; - info.buffer = wrapper->handle; - info.memory = memory_wrapper->handle; - info.memoryOffset = wrapper->bind_offset; - EncodeStructArray(&encoder_, &info, 1); - encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkGetBufferMemoryRequirements, ¶meter_stream_); + parameter_stream_.Clear(); - WriteFunctionCall(format::ApiCall_vkBindBufferMemory2, ¶meter_stream_); + const vulkan_wrappers::QueueWrapper* sparse_bind_queue_wrapper = + vulkan_wrappers::GetWrapper(wrapper->sparse_bind_queue); + + if ((wrapper->sparse_bind_queue != VK_NULL_HANDLE) && (sparse_bind_queue_wrapper != nullptr)) + { + for (auto& item : wrapper->sparse_memory_bind_map) + { + sparse_memory_binds.push_back(item.second); + } + + buffer_memory_bind_info.buffer = wrapper->handle; + buffer_memory_bind_info.bindCount = sparse_memory_binds.size(); + buffer_memory_bind_info.pBinds = sparse_memory_binds.data(); + + VkBindSparseInfo bind_sparse_info{}; + bind_sparse_info.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + bind_sparse_info.pNext = nullptr; + bind_sparse_info.waitSemaphoreCount = 0; + bind_sparse_info.pWaitSemaphores = nullptr; + bind_sparse_info.bufferBindCount = 1; + bind_sparse_info.pBufferBinds = &buffer_memory_bind_info; + bind_sparse_info.imageOpaqueBindCount = 0; + bind_sparse_info.pImageOpaqueBinds = nullptr; + bind_sparse_info.imageBindCount = 0; + bind_sparse_info.pImageBinds = nullptr; + bind_sparse_info.signalSemaphoreCount = 0; + bind_sparse_info.pSignalSemaphores = nullptr; + + encoder_.EncodeVulkanHandleValue(wrapper->sparse_bind_queue); + encoder_.EncodeUInt32Value(1); + EncodeStructArray(&encoder_, &bind_sparse_info, 1); + encoder_.EncodeVulkanHandleValue(VK_NULL_HANDLE); + encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkQueueBindSparse, ¶meter_stream_); + + parameter_stream_.Clear(); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkDeviceWaitIdle, ¶meter_stream_); + } + else + { + GFXRECON_LOG_WARNING("Unable to generate vkQueueBindSparse for the sparse buffer (id = %d) due to " + "the related sparse bind queue or its wrapper is invalid.", + wrapper->handle_id); + } } + parameter_stream_.Clear(); // Group buffers with memory bindings by device for memory snapshot. @@ -1670,10 +1778,16 @@ void VulkanStateWriter::WriteBufferMemoryState(const VulkanStateTable& state_tab ResourceSnapshotInfo& snapshot_entry = snapshot_table[wrapper->queue_family_index]; BufferSnapshotInfo snapshot_info; - snapshot_info.buffer_wrapper = wrapper; - snapshot_info.memory_wrapper = memory_wrapper; - snapshot_info.memory_properties = GetMemoryProperties(device_wrapper, memory_wrapper); - snapshot_info.need_staging_copy = !IsBufferReadable(snapshot_info.memory_properties, memory_wrapper); + snapshot_info.buffer_wrapper = wrapper; + snapshot_info.memory_wrapper = nullptr; + + // We enforce the memory properties to be `VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT`, and we set + // `need_staging_copy` to true for sparse buffers. When dumping buffer data, there are two distinct code + // paths: one involves a staging copy, while the other requires mapping host-visible memory. This latter + // method requires the buffer to be bound to a single range of a single memory, which is not applicable for + // sparse buffers. Therefore, we set the two values to use staging copy for dumping sparse buffers. + snapshot_info.memory_properties = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + snapshot_info.need_staging_copy = true; // Staging copy is needed for sparse buffer. if ((*max_resource_size) < wrapper->created_size) { @@ -1705,7 +1819,8 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl state_table.GetDeviceMemoryWrapper(wrapper->bind_memory_id); if ((wrapper->is_swapchain_image && memory_wrapper == nullptr && wrapper->bind_device != nullptr) || - (!wrapper->is_swapchain_image && memory_wrapper != nullptr)) + (!wrapper->is_swapchain_image && memory_wrapper != nullptr) || + (!wrapper->is_swapchain_image && wrapper->is_sparse_image && wrapper->bind_device != nullptr)) { const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; const VulkanDeviceTable* device_table = &device_wrapper->layer_table; @@ -1725,31 +1840,112 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl parameter_stream_.Clear(); // Write memory bind command. - if (wrapper->bind_pnext == nullptr) + if (!wrapper->is_sparse_image) { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeHandleIdValue(wrapper->handle_id); - encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); - encoder_.EncodeUInt64Value(wrapper->bind_offset); - encoder_.EncodeEnumValue(VK_SUCCESS); + if (wrapper->bind_pnext == nullptr) + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeHandleIdValue(wrapper->handle_id); + encoder_.EncodeHandleIdValue(memory_wrapper->handle_id); + encoder_.EncodeUInt64Value(wrapper->bind_offset); + encoder_.EncodeEnumValue(VK_SUCCESS); - WriteFunctionCall(format::ApiCall_vkBindImageMemory, ¶meter_stream_); + WriteFunctionCall(format::ApiCall_vkBindImageMemory, ¶meter_stream_); + } + else + { + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeUInt32Value(1); + + VkBindImageMemoryInfo info = {}; + info.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; + info.pNext = wrapper->bind_pnext; + info.image = wrapper->handle; + info.memory = memory_wrapper->handle; + info.memoryOffset = wrapper->bind_offset; + EncodeStructArray(&encoder_, &info, 1); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkBindImageMemory2, ¶meter_stream_); + } } else { - encoder_.EncodeHandleIdValue(device_wrapper->handle_id); - encoder_.EncodeUInt32Value(1); - - VkBindImageMemoryInfo info = {}; - info.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; - info.pNext = wrapper->bind_pnext; - info.image = wrapper->handle; - info.memory = memory_wrapper->handle; - info.memoryOffset = wrapper->bind_offset; - EncodeStructArray(&encoder_, &info, 1); - encoder_.EncodeEnumValue(VK_SUCCESS); + const vulkan_wrappers::DeviceWrapper* device_wrapper = wrapper->bind_device; + const VulkanDeviceTable* device_table = &device_wrapper->layer_table; + assert((device_wrapper != nullptr) && (device_table != nullptr)); + + const vulkan_wrappers::QueueWrapper* sparse_bind_queue_wrapper = + vulkan_wrappers::GetWrapper(wrapper->sparse_bind_queue); + + GFXRECON_ASSERT((wrapper->sparse_bind_queue != VK_NULL_HANDLE) && + (sparse_bind_queue_wrapper != nullptr)); + + if ((wrapper->sparse_opaque_memory_bind_map.size() != 0) || + (wrapper->sparse_subresource_memory_bind_map.size() != 0)) + { + std::vector sparse_memory_binds; + VkSparseImageOpaqueMemoryBindInfo image_opaque_memory_bind_info = {}; + + for (auto& item : wrapper->sparse_opaque_memory_bind_map) + { + sparse_memory_binds.push_back(item.second); + } + + image_opaque_memory_bind_info.image = wrapper->handle; + image_opaque_memory_bind_info.bindCount = sparse_memory_binds.size(); + image_opaque_memory_bind_info.pBinds = + (sparse_memory_binds.size() == 0) ? nullptr : sparse_memory_binds.data(); - WriteFunctionCall(format::ApiCall_vkBindImageMemory2, ¶meter_stream_); + std::vector sparse_image_memory_binds; + VkSparseImageMemoryBindInfo image_memory_bind_info = {}; + + for (auto& subresource_bind_map : wrapper->sparse_subresource_memory_bind_map) + { + auto& offset_3d_to_memory_range_map = subresource_bind_map.second; + + for (auto& item : offset_3d_to_memory_range_map) + { + sparse_image_memory_binds.push_back(item.second); + } + } + + image_memory_bind_info.image = wrapper->handle; + image_memory_bind_info.bindCount = sparse_image_memory_binds.size(); + image_memory_bind_info.pBinds = + (sparse_image_memory_binds.size() == 0) ? nullptr : sparse_image_memory_binds.data(); + + VkBindSparseInfo bind_sparse_info{}; + + bind_sparse_info.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + bind_sparse_info.pNext = nullptr; + bind_sparse_info.waitSemaphoreCount = 0; + bind_sparse_info.pWaitSemaphores = nullptr; + bind_sparse_info.bufferBindCount = 0; + bind_sparse_info.pBufferBinds = nullptr; + bind_sparse_info.imageOpaqueBindCount = (image_opaque_memory_bind_info.bindCount == 0) ? 0 : 1; + bind_sparse_info.pImageOpaqueBinds = + (image_opaque_memory_bind_info.bindCount == 0) ? nullptr : &image_opaque_memory_bind_info; + bind_sparse_info.imageBindCount = (image_memory_bind_info.bindCount == 0) ? 0 : 1; + bind_sparse_info.pImageBinds = + (image_memory_bind_info.bindCount == 0) ? nullptr : &image_memory_bind_info; + bind_sparse_info.signalSemaphoreCount = 0; + bind_sparse_info.pSignalSemaphores = nullptr; + + encoder_.EncodeVulkanHandleValue(wrapper->sparse_bind_queue); + encoder_.EncodeUInt32Value(1); + EncodeStructArray(&encoder_, &bind_sparse_info, 1); + encoder_.EncodeVulkanHandleValue(VK_NULL_HANDLE); + encoder_.EncodeEnumValue(VK_SUCCESS); + WriteFunctionCall(format::ApiCall_vkQueueBindSparse, ¶meter_stream_); + + parameter_stream_.Clear(); + + encoder_.EncodeHandleIdValue(device_wrapper->handle_id); + encoder_.EncodeEnumValue(VK_SUCCESS); + + WriteFunctionCall(format::ApiCall_vkDeviceWaitIdle, ¶meter_stream_); + } } parameter_stream_.Clear(); @@ -1765,6 +1961,13 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl (wrapper->tiling == VK_IMAGE_TILING_LINEAR) && ((memory_properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + if (wrapper->is_sparse_image) + { + // The process for dumping host-visible image data requires binding the entire image to a single memory + // range. Since this is not applicable to sparse images, we set is_writable to false. + is_writable = false; + } + // If an image is not host writable and has not been transitioned from the undefined or preinitialized // layouts, no data could have been loaded into it and its data will be omitted from the state snapshot. if (is_transitioned || is_writable) @@ -1778,7 +1981,11 @@ void VulkanStateWriter::WriteImageMemoryState(const VulkanStateTable& state_tabl *device_wrapper->physical_device->layer_table_ref, device_wrapper->physical_device->memory_properties); - bool need_staging_copy = !IsImageReadable(memory_properties, memory_wrapper, wrapper); + // Sparse images require staging copy for the following process because dumping image data with mapping + // memory needs binding the entire image to a single memory range. Sparse image opaque binding allows + // binding to multiple memory objects and various memory ranges. + bool need_staging_copy = + !IsImageReadable(memory_properties, memory_wrapper, wrapper) || wrapper->is_sparse_image; std::vector aspects; bool combined_depth_stencil; diff --git a/framework/graphics/vulkan_resources_util.cpp b/framework/graphics/vulkan_resources_util.cpp index 0269588b20..7254f578e2 100644 --- a/framework/graphics/vulkan_resources_util.cpp +++ b/framework/graphics/vulkan_resources_util.cpp @@ -2067,6 +2067,137 @@ VkResult VulkanResourcesUtil::WriteToImageResourceStaging(VkImage return result; } +bool GetIntersectForSparseMemoryBind(uint32_t new_bind_resource_offset, + uint32_t new_bind_resource_size, + uint32_t existing_bind_resource_offset, + uint32_t existing_bind_resource_size, + uint32_t& intersection_resource_offset, + uint32_t& intersection_resource_size, + std::vector& remaining_resource_offsets, + std::vector& remaining_resource_sizes, + bool& new_bind_range_include_existing_bind_tange, + bool& existing_bind_range_include_new_bind_tange) +{ + bool intersection_exist = false; + uint32_t intersection_start = std::max(new_bind_resource_offset, existing_bind_resource_offset); + uint32_t intersection_end = std::min(new_bind_resource_offset + new_bind_resource_size, + existing_bind_resource_offset + existing_bind_resource_size); + + existing_bind_range_include_new_bind_tange = false; + new_bind_range_include_existing_bind_tange = false; + + if (intersection_start < intersection_end) + { + intersection_exist = true; + intersection_resource_offset = intersection_start; + intersection_resource_size = intersection_end - intersection_start; + + if ((intersection_resource_offset == new_bind_resource_offset) && + (intersection_resource_size == new_bind_resource_size)) + { + existing_bind_range_include_new_bind_tange = true; + } + + if ((intersection_resource_offset == existing_bind_resource_offset) && + (intersection_resource_size == existing_bind_resource_size)) + { + new_bind_range_include_existing_bind_tange = true; + } + + if (intersection_resource_offset > existing_bind_resource_offset) + { + remaining_resource_offsets.push_back(existing_bind_resource_offset); + remaining_resource_sizes.push_back(intersection_resource_offset - existing_bind_resource_offset); + } + + if ((intersection_resource_offset + intersection_resource_size) < + (existing_bind_resource_offset + existing_bind_resource_size)) + { + remaining_resource_offsets.push_back(intersection_resource_offset + intersection_resource_size); + remaining_resource_sizes.push_back((existing_bind_resource_offset + existing_bind_resource_size) - + (intersection_resource_offset + intersection_resource_size)); + } + } + + return intersection_exist; +} + +void UpdateSparseMemoryBindMap(std::map& sparse_memory_bind_map, + const VkSparseMemoryBind& new_sparse_memory_bind) +{ + std::vector all_remaining_existing_bind_ranges{}; + std::vector delete_existing_bind_ranges{}; + + VkDeviceSize search_key = new_sparse_memory_bind.resourceOffset + new_sparse_memory_bind.size; + auto iterator = sparse_memory_bind_map.lower_bound(search_key); + bool is_intersected_with_any_existing_bind = false; + + bool ignored = false; + + if ((sparse_memory_bind_map.size() != 0) && (iterator != sparse_memory_bind_map.begin())) + { + for (auto item = sparse_memory_bind_map.begin(); item != iterator; item++) + { + uint32_t intersection_resource_offset, intersection_resource_size; + std::vector remaining_resource_offsets, remaining_resource_sizes; + bool new_bind_range_include_existing_bind_tange, existing_bind_range_include_new_bind_tange; + + bool is_intersected = GetIntersectForSparseMemoryBind(new_sparse_memory_bind.resourceOffset, + new_sparse_memory_bind.size, + item->second.resourceOffset, + item->second.size, + intersection_resource_offset, + intersection_resource_size, + remaining_resource_offsets, + remaining_resource_sizes, + new_bind_range_include_existing_bind_tange, + existing_bind_range_include_new_bind_tange); + + if (is_intersected) + { + is_intersected_with_any_existing_bind = false; + + VkSparseMemoryBind add_sparse_memory_bind = { 0, 0, item->second.memory, 0, item->second.flags }; + GFXRECON_ASSERT(item->second.flags == new_sparse_memory_bind.flags); + + uint32_t index = 0; + for (auto& bind_offset : remaining_resource_offsets) + { + add_sparse_memory_bind.resourceOffset = bind_offset; + add_sparse_memory_bind.size = remaining_resource_sizes[index]; + add_sparse_memory_bind.memoryOffset = + item->second.memoryOffset + bind_offset - item->second.resourceOffset; + all_remaining_existing_bind_ranges.push_back(add_sparse_memory_bind); + + index++; + } + + delete_existing_bind_ranges.push_back(item->second); + } + } + } + + if (is_intersected_with_any_existing_bind) + { + for (auto& delete_item : delete_existing_bind_ranges) + { + sparse_memory_bind_map.erase(delete_item.resourceOffset); + } + + size_t index = 0, remaining_range_base = 0; + + for (auto add_item : all_remaining_existing_bind_ranges) + { + sparse_memory_bind_map[add_item.resourceOffset] = add_item; + } + } + + if (new_sparse_memory_bind.memory != VK_NULL_HANDLE) + { + sparse_memory_bind_map[new_sparse_memory_bind.resourceOffset] = new_sparse_memory_bind; + } +} + bool VulkanResourcesUtil::IsBlitSupported(VkFormat src_format, VkImageTiling src_image_tiling, VkFormat dst_format, diff --git a/framework/graphics/vulkan_resources_util.h b/framework/graphics/vulkan_resources_util.h index 1aeec6bbca..3b7e613072 100644 --- a/framework/graphics/vulkan_resources_util.h +++ b/framework/graphics/vulkan_resources_util.h @@ -31,6 +31,7 @@ #include "vulkan/vulkan_core.h" #include +#include GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(graphics) @@ -280,6 +281,76 @@ bool FindMemoryTypeIndex(const VkPhysicalDeviceMemoryProperties& memory_properti uint32_t* found_index, VkMemoryPropertyFlags* found_flags); +struct VkOffset3DComparator +{ + bool operator()(const VkOffset3D& l, const VkOffset3D& r) const + { + bool result = (l.x < r.x); + + if (l.x == r.x) + { + result = (l.y < r.y); + + if (l.y == r.y) + { + result = (l.z < r.z); + } + } + + return result; + } +}; + +typedef std::map + VulkanOffset3DSparseImageMemoryBindMap; + +struct VkImageSubresourceComparator +{ + bool operator()(const VkImageSubresource& l, const VkImageSubresource& r) const + { + bool result = (l.arrayLayer < r.arrayLayer); + + if (l.arrayLayer == r.arrayLayer) + { + result = (l.mipLevel < r.mipLevel); + + if (l.mipLevel == r.mipLevel) + { + result = (l.aspectMask < r.aspectMask); + } + } + + return result; + } +}; + +typedef std:: + map + VulkanSubresourceSparseImageMemoryBindMap; + +// Get the intersection of the new bind range and the existing bind range for sparse buffer or sparse image (opaque +// bind). +// If the intersection range exists, further get the remaining ranges for the existing bind range after removing +// the intersection range. For instance, if the new/existing bind range (offset, size) are (196608, 327680) and (0, +// 655360) respectively, the old range (0, 655360) completely covers the new range (196608, 327680). The intersection +// range is (196608, 327680), and the remaining ranges for the existing bind are (0, 196608) and (524288, 131072). So +// for the return of the function, remaining_resource_offsets will be a std::vector of [0, 524288], and +// remaining_resource_sizes will be [196608, 131072]. +// +bool GetIntersectForSparseMemoryBind(uint32_t new_bind_resource_offset, + uint32_t new_bind_resource_size, + uint32_t existing_bind_resource_offset, + uint32_t existing_bind_resource_size, + uint32_t& intersection_resource_offset, + uint32_t& intersection_resource_size, + std::vector& remaining_resource_offsets, + std::vector& remaining_resource_sizes, + bool& new_bind_range_include_existing_bind_tange, + bool& existing_bind_range_include_new_bind_tange); + +void UpdateSparseMemoryBindMap(std::map& sparse_memory_bind_map, + const VkSparseMemoryBind& new_sparse_memory_bind); + bool GetImageTexelSize(VkFormat format, VkDeviceSize* texel_size, bool* is_texel_block_size, @@ -323,7 +394,7 @@ bool NextRowTexelCoordinates(VkImageType imageType, uint32_t& z, uint32_t& layer); +GFXRECON_END_NAMESPACE(graphics) GFXRECON_END_NAMESPACE(gfxrecon) -GFXRECON_END_NAMESPACE(encode) #endif /* GFXRECON_GRAPHICS_VULKAN_RESOURCES_UTIL_H */