Merge pull request #55954 from RandomShaper/vulkan_rd_goodies

This commit is contained in:
Rémi Verschelde 2022-06-28 11:04:10 +02:00 committed by GitHub
commit 5ab59ee7df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 86 additions and 84 deletions

View file

@ -38,7 +38,9 @@
#include <initializer_list>
template <class T, class U = uint32_t, bool force_trivial = false>
// If tight, it grows strictly as much as needed.
// Otherwise, it grows exponentially (the default and what you want in most cases).
template <class T, class U = uint32_t, bool force_trivial = false, bool tight = false>
class LocalVector {
private:
U count = 0;
@ -121,7 +123,7 @@ public:
_FORCE_INLINE_ bool is_empty() const { return count == 0; }
_FORCE_INLINE_ U get_capacity() const { return capacity; }
_FORCE_INLINE_ void reserve(U p_size) {
p_size = nearest_power_of_2_templated(p_size);
p_size = tight ? p_size : nearest_power_of_2_templated(p_size);
if (p_size > capacity) {
capacity = p_size;
data = (T *)memrealloc(data, capacity * sizeof(T));
@ -262,4 +264,7 @@ public:
}
};
template <class T, class U = uint32_t, bool force_trivial = false>
using TightLocalVector = LocalVector<T, U, force_trivial, true>;
#endif // LOCAL_VECTOR_H

View file

@ -1400,7 +1400,7 @@ Error RenderingDeviceVulkan::_insert_staging_block() {
return OK;
}
Error RenderingDeviceVulkan::_staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment, bool p_on_draw_command_buffer) {
Error RenderingDeviceVulkan::_staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment) {
//determine a block to use
r_alloc_size = p_amount;
@ -1542,7 +1542,7 @@ Error RenderingDeviceVulkan::_buffer_update(Buffer *p_buffer, size_t p_offset, c
uint32_t block_write_offset;
uint32_t block_write_amount;
Error err = _staging_buffer_allocate(MIN(to_submit, staging_buffer_block_size), p_required_align, block_write_offset, block_write_amount, p_use_draw_command_buffer);
Error err = _staging_buffer_allocate(MIN(to_submit, staging_buffer_block_size), p_required_align, block_write_offset, block_write_amount);
if (err) {
return err;
}
@ -2376,6 +2376,22 @@ Error RenderingDeviceVulkan::texture_update(RID p_texture, uint32_t p_layer, con
return _texture_update(p_texture, p_layer, p_data, p_post_barrier, false);
}
static _ALWAYS_INLINE_ void _copy_region(uint8_t const *__restrict p_src, uint8_t *__restrict p_dst, uint32_t p_src_x, uint32_t p_src_y, uint32_t p_src_w, uint32_t p_src_h, uint32_t p_src_full_w, uint32_t p_unit_size) {
uint32_t src_offset = (p_src_y * p_src_full_w + p_src_x) * p_unit_size;
uint32_t dst_offset = 0;
for (uint32_t y = p_src_h; y > 0; y--) {
uint8_t const *__restrict src = p_src + src_offset;
uint8_t *__restrict dst = p_dst + dst_offset;
for (uint32_t x = p_src_w * p_unit_size; x > 0; x--) {
*dst = *src;
src++;
dst++;
}
src_offset += p_src_full_w * p_unit_size;
dst_offset += p_src_w * p_unit_size;
}
}
Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, const Vector<uint8_t> &p_data, uint32_t p_post_barrier, bool p_use_setup_queue) {
_THREAD_SAFE_METHOD_
@ -2461,8 +2477,8 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
const uint8_t *read_ptr = read_ptr_mipmap + image_size * z / depth;
for (uint32_t x = 0; x < width; x += region_size) {
for (uint32_t y = 0; y < height; y += region_size) {
for (uint32_t y = 0; y < height; y += region_size) {
for (uint32_t x = 0; x < width; x += region_size) {
uint32_t region_w = MIN(region_size, width - x);
uint32_t region_h = MIN(region_size, height - y);
@ -2474,7 +2490,7 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
to_allocate >>= get_compressed_image_format_pixel_rshift(texture->format);
uint32_t alloc_offset, alloc_size;
Error err = _staging_buffer_allocate(to_allocate, required_align, alloc_offset, alloc_size, false, !p_use_setup_queue);
Error err = _staging_buffer_allocate(to_allocate, required_align, alloc_offset, alloc_size, false);
ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
uint8_t *write_ptr;
@ -2505,31 +2521,11 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
//uint32_t hb = height / block_h;
uint32_t region_wb = region_w / block_w;
uint32_t region_hb = region_h / block_h;
for (uint32_t xr = 0; xr < region_wb; xr++) {
for (uint32_t yr = 0; yr < region_hb; yr++) {
uint32_t src_offset = ((yr + yb) * wb + xr + xb) * block_size;
uint32_t dst_offset = (yr * region_wb + xr) * block_size;
//copy block
for (uint32_t i = 0; i < block_size; i++) {
write_ptr[dst_offset + i] = read_ptr[src_offset + i];
}
}
}
_copy_region(read_ptr, write_ptr, xb, yb, region_wb, region_hb, wb, block_size);
} else {
//regular image (pixels)
//must copy a pixel region
for (uint32_t xr = 0; xr < region_w; xr++) {
for (uint32_t yr = 0; yr < region_h; yr++) {
uint32_t src_offset = ((yr + y) * width + xr + x) * pixel_size;
uint32_t dst_offset = (yr * region_w + xr) * pixel_size;
//copy block
for (uint32_t i = 0; i < pixel_size; i++) {
write_ptr[dst_offset + i] = read_ptr[src_offset + i];
}
}
}
_copy_region(read_ptr, write_ptr, x, y, region_w, region_h, width, pixel_size);
}
{ //unmap
@ -2572,11 +2568,11 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
uint32_t access_flags = 0;
if (p_post_barrier & BARRIER_MASK_COMPUTE) {
barrier_flags |= VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
}
if (p_post_barrier & BARRIER_MASK_RASTER) {
barrier_flags |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
}
if (p_post_barrier & BARRIER_MASK_TRANSFER) {
barrier_flags |= VK_PIPELINE_STAGE_TRANSFER_BIT;
@ -2990,7 +2986,7 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture,
image_memory_barrier.subresourceRange.baseArrayLayer = p_src_layer;
image_memory_barrier.subresourceRange.layerCount = 1;
vkCmdPipelineBarrier(command_buffer, VK_ACCESS_TRANSFER_WRITE_BIT, barrier_flags, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier);
vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, barrier_flags, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier);
}
{ //make dst readable
@ -3016,6 +3012,13 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture,
}
}
if (dst_tex->used_in_frame != frames_drawn) {
dst_tex->used_in_raster = false;
dst_tex->used_in_compute = false;
dst_tex->used_in_frame = frames_drawn;
}
dst_tex->used_in_transfer = true;
return OK;
}
@ -4771,6 +4774,7 @@ Vector<uint8_t> RenderingDeviceVulkan::shader_compile_binary_from_spirv(const Ve
//just append stage mask and return
uniform_info.write[set].write[k].stages |= 1 << stage;
exists = true;
break;
}
}
@ -5437,7 +5441,6 @@ RID RenderingDeviceVulkan::storage_buffer_create(uint32_t p_size_bytes, const Ve
ERR_FAIL_COND_V(p_data.size() && (uint32_t)p_data.size() != p_size_bytes, RID());
Buffer buffer;
buffer.usage = p_usage;
uint32_t flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (p_usage & STORAGE_BUFFER_USAGE_DISPATCH_INDIRECT) {
flags |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
@ -7202,6 +7205,9 @@ RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_begin(RID p_framebu
Error RenderingDeviceVulkan::draw_list_begin_split(RID p_framebuffer, uint32_t p_splits, DrawListID *r_split_ids, InitialAction p_initial_color_action, FinalAction p_final_color_action, InitialAction p_initial_depth_action, FinalAction p_final_depth_action, const Vector<Color> &p_clear_color_values, float p_clear_depth, uint32_t p_clear_stencil, const Rect2 &p_region, const Vector<RID> &p_storage_textures) {
_THREAD_SAFE_METHOD_
ERR_FAIL_COND_V_MSG(draw_list != nullptr, ERR_BUSY, "Only one draw list can be active at the same time.");
ERR_FAIL_COND_V_MSG(compute_list != nullptr && !compute_list->state.allow_draw_overlap, ERR_BUSY, "Only one draw/compute list can be active at the same time.");
ERR_FAIL_COND_V(p_splits < 1, ERR_INVALID_DECLARATION);
Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer);
@ -7926,10 +7932,6 @@ void RenderingDeviceVulkan::draw_list_end(uint32_t p_post_barrier) {
// * Some buffer is copied
// * Another render pass happens (since we may be done)
#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#else
VkMemoryBarrier mem_barrier;
mem_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
mem_barrier.pNext = nullptr;
@ -7940,6 +7942,8 @@ void RenderingDeviceVulkan::draw_list_end(uint32_t p_post_barrier) {
vkCmdPipelineBarrier(frames[frame].draw_command_buffer, src_stage, barrier_flags, 0, 1, &mem_barrier, 0, nullptr, image_barrier_count, image_barriers);
}
#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#endif
}
@ -8294,7 +8298,7 @@ void RenderingDeviceVulkan::compute_list_dispatch_indirect(ComputeListID p_list,
Buffer *buffer = storage_buffer_owner.get_or_null(p_buffer);
ERR_FAIL_COND(!buffer);
ERR_FAIL_COND_MSG(!(buffer->usage & STORAGE_BUFFER_USAGE_DISPATCH_INDIRECT), "Buffer provided was not created to do indirect dispatch.");
ERR_FAIL_COND_MSG(!(buffer->usage & VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT), "Buffer provided was not created to do indirect dispatch.");
ERR_FAIL_COND_MSG(p_offset + 12 > buffer->size, "Offset provided (+12) is past the end of buffer.");
@ -8410,9 +8414,6 @@ void RenderingDeviceVulkan::compute_list_end(uint32_t p_post_barrier) {
}
}
#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#else
VkMemoryBarrier mem_barrier;
mem_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
mem_barrier.pNext = nullptr;
@ -8423,6 +8424,8 @@ void RenderingDeviceVulkan::compute_list_end(uint32_t p_post_barrier) {
vkCmdPipelineBarrier(compute_list->command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, barrier_flags, 0, 1, &mem_barrier, 0, nullptr, image_barrier_count, image_barriers);
}
#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#endif
memdelete(compute_list);
@ -8750,7 +8753,7 @@ void RenderingDeviceVulkan::_begin_frame() {
}
if (frames[frame].timestamp_count) {
vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values.ptr(), sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
vkCmdResetQueryPool(frames[frame].setup_command_buffer, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count);
SWAP(frames[frame].timestamp_names, frames[frame].timestamp_result_names);
SWAP(frames[frame].timestamp_cpu_values, frames[frame].timestamp_cpu_result_values);
@ -9041,7 +9044,7 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de
vmaCreateAllocator(&allocatorInfo, &allocator);
}
frames = memnew_arr(Frame, frame_count);
frames.resize(frame_count);
frame = 0;
//create setup and frame buffers
for (int i = 0; i < frame_count; i++) {
@ -9087,12 +9090,12 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de
vkCreateQueryPool(device, &query_pool_create_info, nullptr, &frames[i].timestamp_pool);
frames[i].timestamp_names = memnew_arr(String, max_timestamp_query_elements);
frames[i].timestamp_cpu_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_names.resize(max_timestamp_query_elements);
frames[i].timestamp_cpu_values.resize(max_timestamp_query_elements);
frames[i].timestamp_count = 0;
frames[i].timestamp_result_names = memnew_arr(String, max_timestamp_query_elements);
frames[i].timestamp_cpu_result_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_result_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_result_names.resize(max_timestamp_query_elements);
frames[i].timestamp_cpu_result_values.resize(max_timestamp_query_elements);
frames[i].timestamp_result_values.resize(max_timestamp_query_elements);
frames[i].timestamp_result_count = 0;
}
}
@ -9493,18 +9496,13 @@ void RenderingDeviceVulkan::finalize() {
_free_pending_resources(f);
vkDestroyCommandPool(device, frames[i].command_pool, nullptr);
vkDestroyQueryPool(device, frames[i].timestamp_pool, nullptr);
memdelete_arr(frames[i].timestamp_names);
memdelete_arr(frames[i].timestamp_cpu_values);
memdelete_arr(frames[i].timestamp_result_names);
memdelete_arr(frames[i].timestamp_result_values);
memdelete_arr(frames[i].timestamp_cpu_result_values);
}
for (int i = 0; i < split_draw_list_allocators.size(); i++) {
vkDestroyCommandPool(device, split_draw_list_allocators[i].command_pool, nullptr);
}
memdelete_arr(frames);
frames.clear();
for (int i = 0; i < staging_buffer_blocks.size(); i++) {
vmaDestroyBuffer(allocator, staging_buffer_blocks[i].buffer, staging_buffer_blocks[i].allocation);

View file

@ -206,7 +206,7 @@ class RenderingDeviceVulkan : public RenderingDevice {
uint64_t staging_buffer_max_size = 0;
bool staging_buffer_used = false;
Error _staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment = true, bool p_on_draw_command_buffer = false);
Error _staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment = true);
Error _insert_staging_block();
struct Buffer {
@ -637,7 +637,6 @@ class RenderingDeviceVulkan : public RenderingDevice {
};
bool is_compute = false;
int max_output = 0;
Vector<Set> sets;
Vector<uint32_t> set_formats;
Vector<VkPipelineShaderStageCreateInfo> pipeline_stages;
@ -870,11 +869,9 @@ class RenderingDeviceVulkan : public RenderingDevice {
uint32_t pipeline_dynamic_state = 0;
VertexFormatID pipeline_vertex_format = INVALID_ID;
RID pipeline_shader;
uint32_t invalid_set_from = 0;
bool pipeline_uses_restart_indices = false;
uint32_t pipeline_primitive_divisor = 0;
uint32_t pipeline_primitive_minimum = 0;
Vector<uint32_t> pipeline_set_formats;
uint32_t pipeline_push_constant_size = 0;
bool pipeline_push_constant_supplied = false;
} validation;
@ -948,7 +945,6 @@ class RenderingDeviceVulkan : public RenderingDevice {
bool pipeline_active = false;
RID pipeline_shader;
uint32_t invalid_set_from = 0;
Vector<uint32_t> pipeline_set_formats;
uint32_t pipeline_push_constant_size = 0;
bool pipeline_push_constant_supplied = false;
} validation;
@ -998,19 +994,19 @@ class RenderingDeviceVulkan : public RenderingDevice {
VkQueryPool timestamp_pool;
String *timestamp_names = nullptr;
uint64_t *timestamp_cpu_values = nullptr;
TightLocalVector<String> timestamp_names;
TightLocalVector<uint64_t> timestamp_cpu_values;
uint32_t timestamp_count = 0;
String *timestamp_result_names = nullptr;
uint64_t *timestamp_cpu_result_values = nullptr;
uint64_t *timestamp_result_values = nullptr;
TightLocalVector<String> timestamp_result_names;
TightLocalVector<uint64_t> timestamp_cpu_result_values;
TightLocalVector<uint64_t> timestamp_result_values;
uint32_t timestamp_result_count = 0;
uint64_t index = 0;
};
uint32_t max_timestamp_query_elements = 0;
Frame *frames = nullptr; //frames available, for main device they are cycled (usually 3), for local devices only 1
TightLocalVector<Frame> frames; //frames available, for main device they are cycled (usually 3), for local devices only 1
int frame = 0; //current frame
int frame_count = 0; //total amount of frames
uint64_t frames_drawn = 0;

View file

@ -1860,16 +1860,16 @@ Error VulkanContext::initialize() {
return OK;
}
void VulkanContext::set_setup_buffer(const VkCommandBuffer &pCommandBuffer) {
command_buffer_queue.write[0] = pCommandBuffer;
void VulkanContext::set_setup_buffer(VkCommandBuffer p_command_buffer) {
command_buffer_queue.write[0] = p_command_buffer;
}
void VulkanContext::append_command_buffer(const VkCommandBuffer &pCommandBuffer) {
void VulkanContext::append_command_buffer(VkCommandBuffer p_command_buffer) {
if (command_buffer_queue.size() <= command_buffer_count) {
command_buffer_queue.resize(command_buffer_count + 1);
}
command_buffer_queue.write[command_buffer_count] = pCommandBuffer;
command_buffer_queue.write[command_buffer_count] = p_command_buffer;
command_buffer_count++;
}
@ -1879,7 +1879,10 @@ void VulkanContext::flush(bool p_flush_setup, bool p_flush_pending) {
//flush the pending setup buffer
if (p_flush_setup && command_buffer_queue[0]) {
bool setup_flushable = p_flush_setup && command_buffer_queue[0];
bool pending_flushable = p_flush_pending && command_buffer_count > 1;
if (setup_flushable) {
//use a fence to wait for everything done
VkSubmitInfo submit_info;
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
@ -1889,33 +1892,33 @@ void VulkanContext::flush(bool p_flush_setup, bool p_flush_pending) {
submit_info.pWaitSemaphores = nullptr;
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = command_buffer_queue.ptr();
submit_info.signalSemaphoreCount = 0;
submit_info.pSignalSemaphores = nullptr;
submit_info.signalSemaphoreCount = pending_flushable ? 1 : 0;
submit_info.pSignalSemaphores = pending_flushable ? &draw_complete_semaphores[frame_index] : nullptr;
VkResult err = vkQueueSubmit(graphics_queue, 1, &submit_info, VK_NULL_HANDLE);
command_buffer_queue.write[0] = nullptr;
ERR_FAIL_COND(err);
vkDeviceWaitIdle(device);
}
if (p_flush_pending && command_buffer_count > 1) {
if (pending_flushable) {
//use a fence to wait for everything done
VkSubmitInfo submit_info;
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info.pNext = nullptr;
submit_info.pWaitDstStageMask = nullptr;
submit_info.waitSemaphoreCount = 0;
submit_info.pWaitSemaphores = nullptr;
VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
submit_info.pWaitDstStageMask = setup_flushable ? &wait_stage_mask : nullptr;
submit_info.waitSemaphoreCount = setup_flushable ? 1 : 0;
submit_info.pWaitSemaphores = setup_flushable ? &draw_complete_semaphores[frame_index] : nullptr;
submit_info.commandBufferCount = command_buffer_count - 1;
submit_info.pCommandBuffers = command_buffer_queue.ptr() + 1;
submit_info.signalSemaphoreCount = 0;
submit_info.pSignalSemaphores = nullptr;
VkResult err = vkQueueSubmit(graphics_queue, 1, &submit_info, VK_NULL_HANDLE);
ERR_FAIL_COND(err);
vkDeviceWaitIdle(device);
command_buffer_count = 1;
ERR_FAIL_COND(err);
}
vkDeviceWaitIdle(device);
}
Error VulkanContext::prepare_buffers() {

View file

@ -117,8 +117,8 @@ private:
// Present queue.
bool queues_initialized = false;
uint32_t graphics_queue_family_index = 0;
uint32_t present_queue_family_index = 0;
uint32_t graphics_queue_family_index = UINT32_MAX;
uint32_t present_queue_family_index = UINT32_MAX;
bool separate_present_queue = false;
VkQueue graphics_queue = VK_NULL_HANDLE;
VkQueue present_queue = VK_NULL_HANDLE;
@ -289,8 +289,8 @@ public:
VkFormat get_screen_format() const;
VkPhysicalDeviceLimits get_device_limits() const;
void set_setup_buffer(const VkCommandBuffer &pCommandBuffer);
void append_command_buffer(const VkCommandBuffer &pCommandBuffer);
void set_setup_buffer(VkCommandBuffer p_command_buffer);
void append_command_buffer(VkCommandBuffer p_command_buffer);
void resize_notify();
void flush(bool p_flush_setup = false, bool p_flush_pending = false);
Error prepare_buffers();