Skip to content

Commit

Permalink
Merge pull request #55954 from RandomShaper/vulkan_rd_goodies
Browse files Browse the repository at this point in the history
  • Loading branch information
@akien-mga
akien-mga authored Jun 28, 2022
2 parents 16d9918 + a82352c commit 5ab59ee
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 84 deletions.
9 changes: 7 additions & 2 deletions core/templates/local_vector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@

#include <initializer_list>

template <class T, class U = uint32_t, bool force_trivial = false>
// If tight, it grows strictly as much as needed.
// Otherwise, it grows exponentially (the default and what you want in most cases).
template <class T, class U = uint32_t, bool force_trivial = false, bool tight = false>
class LocalVector {
private:
U count = 0;
Expand Down Expand Up @@ -121,7 +123,7 @@ class LocalVector {
_FORCE_INLINE_ bool is_empty() const { return count == 0; }
_FORCE_INLINE_ U get_capacity() const { return capacity; }
_FORCE_INLINE_ void reserve(U p_size) {
p_size = nearest_power_of_2_templated(p_size);
p_size = tight ? p_size : nearest_power_of_2_templated(p_size);
if (p_size > capacity) {
capacity = p_size;
data = (T *)memrealloc(data, capacity * sizeof(T));
Expand Down Expand Up @@ -262,4 +264,7 @@ class LocalVector {
}
};

template <class T, class U = uint32_t, bool force_trivial = false>
using TightLocalVector = LocalVector<T, U, force_trivial, true>;

#endif // LOCAL_VECTOR_H
102 changes: 50 additions & 52 deletions drivers/vulkan/rendering_device_vulkan.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1400,7 +1400,7 @@ Error RenderingDeviceVulkan::_insert_staging_block() {
return OK;
}

Error RenderingDeviceVulkan::_staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment, bool p_on_draw_command_buffer) {
Error RenderingDeviceVulkan::_staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment) {
//determine a block to use

r_alloc_size = p_amount;
Expand Down Expand Up @@ -1542,7 +1542,7 @@ Error RenderingDeviceVulkan::_buffer_update(Buffer *p_buffer, size_t p_offset, c
uint32_t block_write_offset;
uint32_t block_write_amount;

Error err = _staging_buffer_allocate(MIN(to_submit, staging_buffer_block_size), p_required_align, block_write_offset, block_write_amount, p_use_draw_command_buffer);
Error err = _staging_buffer_allocate(MIN(to_submit, staging_buffer_block_size), p_required_align, block_write_offset, block_write_amount);
if (err) {
return err;
}
Expand Down Expand Up @@ -2376,6 +2376,22 @@ Error RenderingDeviceVulkan::texture_update(RID p_texture, uint32_t p_layer, con
return _texture_update(p_texture, p_layer, p_data, p_post_barrier, false);
}

static _ALWAYS_INLINE_ void _copy_region(uint8_t const *__restrict p_src, uint8_t *__restrict p_dst, uint32_t p_src_x, uint32_t p_src_y, uint32_t p_src_w, uint32_t p_src_h, uint32_t p_src_full_w, uint32_t p_unit_size) {
uint32_t src_offset = (p_src_y * p_src_full_w + p_src_x) * p_unit_size;
uint32_t dst_offset = 0;
for (uint32_t y = p_src_h; y > 0; y--) {
uint8_t const *__restrict src = p_src + src_offset;
uint8_t *__restrict dst = p_dst + dst_offset;
for (uint32_t x = p_src_w * p_unit_size; x > 0; x--) {
*dst = *src;
src++;
dst++;
}
src_offset += p_src_full_w * p_unit_size;
dst_offset += p_src_w * p_unit_size;
}
}

Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, const Vector<uint8_t> &p_data, uint32_t p_post_barrier, bool p_use_setup_queue) {
_THREAD_SAFE_METHOD_

Expand Down Expand Up @@ -2461,8 +2477,8 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co

const uint8_t *read_ptr = read_ptr_mipmap + image_size * z / depth;

for (uint32_t x = 0; x < width; x += region_size) {
for (uint32_t y = 0; y < height; y += region_size) {
for (uint32_t y = 0; y < height; y += region_size) {
for (uint32_t x = 0; x < width; x += region_size) {
uint32_t region_w = MIN(region_size, width - x);
uint32_t region_h = MIN(region_size, height - y);

Expand All @@ -2474,7 +2490,7 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
to_allocate >>= get_compressed_image_format_pixel_rshift(texture->format);

uint32_t alloc_offset, alloc_size;
Error err = _staging_buffer_allocate(to_allocate, required_align, alloc_offset, alloc_size, false, !p_use_setup_queue);
Error err = _staging_buffer_allocate(to_allocate, required_align, alloc_offset, alloc_size, false);
ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

uint8_t *write_ptr;
Expand Down Expand Up @@ -2505,31 +2521,11 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
//uint32_t hb = height / block_h;
uint32_t region_wb = region_w / block_w;
uint32_t region_hb = region_h / block_h;
for (uint32_t xr = 0; xr < region_wb; xr++) {
for (uint32_t yr = 0; yr < region_hb; yr++) {
uint32_t src_offset = ((yr + yb) * wb + xr + xb) * block_size;
uint32_t dst_offset = (yr * region_wb + xr) * block_size;
//copy block
for (uint32_t i = 0; i < block_size; i++) {
write_ptr[dst_offset + i] = read_ptr[src_offset + i];
}
}
}

_copy_region(read_ptr, write_ptr, xb, yb, region_wb, region_hb, wb, block_size);
} else {
//regular image (pixels)
//must copy a pixel region

for (uint32_t xr = 0; xr < region_w; xr++) {
for (uint32_t yr = 0; yr < region_h; yr++) {
uint32_t src_offset = ((yr + y) * width + xr + x) * pixel_size;
uint32_t dst_offset = (yr * region_w + xr) * pixel_size;
//copy block
for (uint32_t i = 0; i < pixel_size; i++) {
write_ptr[dst_offset + i] = read_ptr[src_offset + i];
}
}
}
_copy_region(read_ptr, write_ptr, x, y, region_w, region_h, width, pixel_size);
}

{ //unmap
Expand Down Expand Up @@ -2572,11 +2568,11 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co
uint32_t access_flags = 0;
if (p_post_barrier & BARRIER_MASK_COMPUTE) {
barrier_flags |= VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
}
if (p_post_barrier & BARRIER_MASK_RASTER) {
barrier_flags |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT;
access_flags |= VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
}
if (p_post_barrier & BARRIER_MASK_TRANSFER) {
barrier_flags |= VK_PIPELINE_STAGE_TRANSFER_BIT;
Expand Down Expand Up @@ -2990,7 +2986,7 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture,
image_memory_barrier.subresourceRange.baseArrayLayer = p_src_layer;
image_memory_barrier.subresourceRange.layerCount = 1;

vkCmdPipelineBarrier(command_buffer, VK_ACCESS_TRANSFER_WRITE_BIT, barrier_flags, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier);
vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, barrier_flags, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier);
}

{ //make dst readable
Expand All @@ -3016,6 +3012,13 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture,
}
}

if (dst_tex->used_in_frame != frames_drawn) {
dst_tex->used_in_raster = false;
dst_tex->used_in_compute = false;
dst_tex->used_in_frame = frames_drawn;
}
dst_tex->used_in_transfer = true;

return OK;
}

Expand Down Expand Up @@ -4771,6 +4774,7 @@ Vector<uint8_t> RenderingDeviceVulkan::shader_compile_binary_from_spirv(const Ve
//just append stage mask and return
uniform_info.write[set].write[k].stages |= 1 << stage;
exists = true;
break;
}
}

Expand Down Expand Up @@ -5437,7 +5441,6 @@ RID RenderingDeviceVulkan::storage_buffer_create(uint32_t p_size_bytes, const Ve
ERR_FAIL_COND_V(p_data.size() && (uint32_t)p_data.size() != p_size_bytes, RID());

Buffer buffer;
buffer.usage = p_usage;
uint32_t flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (p_usage & STORAGE_BUFFER_USAGE_DISPATCH_INDIRECT) {
flags |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
Expand Down Expand Up @@ -7202,6 +7205,9 @@ RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_begin(RID p_framebu
Error RenderingDeviceVulkan::draw_list_begin_split(RID p_framebuffer, uint32_t p_splits, DrawListID *r_split_ids, InitialAction p_initial_color_action, FinalAction p_final_color_action, InitialAction p_initial_depth_action, FinalAction p_final_depth_action, const Vector<Color> &p_clear_color_values, float p_clear_depth, uint32_t p_clear_stencil, const Rect2 &p_region, const Vector<RID> &p_storage_textures) {
_THREAD_SAFE_METHOD_

ERR_FAIL_COND_V_MSG(draw_list != nullptr, ERR_BUSY, "Only one draw list can be active at the same time.");
ERR_FAIL_COND_V_MSG(compute_list != nullptr && !compute_list->state.allow_draw_overlap, ERR_BUSY, "Only one draw/compute list can be active at the same time.");

ERR_FAIL_COND_V(p_splits < 1, ERR_INVALID_DECLARATION);

Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer);
Expand Down Expand Up @@ -7926,10 +7932,6 @@ void RenderingDeviceVulkan::draw_list_end(uint32_t p_post_barrier) {
// * Some buffer is copied
// * Another render pass happens (since we may be done)

#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#else

VkMemoryBarrier mem_barrier;
mem_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
mem_barrier.pNext = nullptr;
Expand All @@ -7940,6 +7942,8 @@ void RenderingDeviceVulkan::draw_list_end(uint32_t p_post_barrier) {
vkCmdPipelineBarrier(frames[frame].draw_command_buffer, src_stage, barrier_flags, 0, 1, &mem_barrier, 0, nullptr, image_barrier_count, image_barriers);
}

#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#endif
}

Expand Down Expand Up @@ -8294,7 +8298,7 @@ void RenderingDeviceVulkan::compute_list_dispatch_indirect(ComputeListID p_list,
Buffer *buffer = storage_buffer_owner.get_or_null(p_buffer);
ERR_FAIL_COND(!buffer);

ERR_FAIL_COND_MSG(!(buffer->usage & STORAGE_BUFFER_USAGE_DISPATCH_INDIRECT), "Buffer provided was not created to do indirect dispatch.");
ERR_FAIL_COND_MSG(!(buffer->usage & VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT), "Buffer provided was not created to do indirect dispatch.");

ERR_FAIL_COND_MSG(p_offset + 12 > buffer->size, "Offset provided (+12) is past the end of buffer.");

Expand Down Expand Up @@ -8410,9 +8414,6 @@ void RenderingDeviceVulkan::compute_list_end(uint32_t p_post_barrier) {
}
}

#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#else
VkMemoryBarrier mem_barrier;
mem_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
mem_barrier.pNext = nullptr;
Expand All @@ -8423,6 +8424,8 @@ void RenderingDeviceVulkan::compute_list_end(uint32_t p_post_barrier) {
vkCmdPipelineBarrier(compute_list->command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, barrier_flags, 0, 1, &mem_barrier, 0, nullptr, image_barrier_count, image_barriers);
}

#ifdef FORCE_FULL_BARRIER
_full_barrier(true);
#endif

memdelete(compute_list);
Expand Down Expand Up @@ -8750,7 +8753,7 @@ void RenderingDeviceVulkan::_begin_frame() {
}

if (frames[frame].timestamp_count) {
vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values.ptr(), sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
vkCmdResetQueryPool(frames[frame].setup_command_buffer, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count);
SWAP(frames[frame].timestamp_names, frames[frame].timestamp_result_names);
SWAP(frames[frame].timestamp_cpu_values, frames[frame].timestamp_cpu_result_values);
Expand Down Expand Up @@ -9041,7 +9044,7 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de
vmaCreateAllocator(&allocatorInfo, &allocator);
}

frames = memnew_arr(Frame, frame_count);
frames.resize(frame_count);
frame = 0;
//create setup and frame buffers
for (int i = 0; i < frame_count; i++) {
Expand Down Expand Up @@ -9087,12 +9090,12 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de

vkCreateQueryPool(device, &query_pool_create_info, nullptr, &frames[i].timestamp_pool);

frames[i].timestamp_names = memnew_arr(String, max_timestamp_query_elements);
frames[i].timestamp_cpu_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_names.resize(max_timestamp_query_elements);
frames[i].timestamp_cpu_values.resize(max_timestamp_query_elements);
frames[i].timestamp_count = 0;
frames[i].timestamp_result_names = memnew_arr(String, max_timestamp_query_elements);
frames[i].timestamp_cpu_result_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_result_values = memnew_arr(uint64_t, max_timestamp_query_elements);
frames[i].timestamp_result_names.resize(max_timestamp_query_elements);
frames[i].timestamp_cpu_result_values.resize(max_timestamp_query_elements);
frames[i].timestamp_result_values.resize(max_timestamp_query_elements);
frames[i].timestamp_result_count = 0;
}
}
Expand Down Expand Up @@ -9493,18 +9496,13 @@ void RenderingDeviceVulkan::finalize() {
_free_pending_resources(f);
vkDestroyCommandPool(device, frames[i].command_pool, nullptr);
vkDestroyQueryPool(device, frames[i].timestamp_pool, nullptr);
memdelete_arr(frames[i].timestamp_names);
memdelete_arr(frames[i].timestamp_cpu_values);
memdelete_arr(frames[i].timestamp_result_names);
memdelete_arr(frames[i].timestamp_result_values);
memdelete_arr(frames[i].timestamp_cpu_result_values);
}

for (int i = 0; i < split_draw_list_allocators.size(); i++) {
vkDestroyCommandPool(device, split_draw_list_allocators[i].command_pool, nullptr);
}

memdelete_arr(frames);
frames.clear();

for (int i = 0; i < staging_buffer_blocks.size(); i++) {
vmaDestroyBuffer(allocator, staging_buffer_blocks[i].buffer, staging_buffer_blocks[i].allocation);
Expand Down
18 changes: 7 additions & 11 deletions drivers/vulkan/rendering_device_vulkan.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ class RenderingDeviceVulkan : public RenderingDevice {
uint64_t staging_buffer_max_size = 0;
bool staging_buffer_used = false;

Error _staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment = true, bool p_on_draw_command_buffer = false);
Error _staging_buffer_allocate(uint32_t p_amount, uint32_t p_required_align, uint32_t &r_alloc_offset, uint32_t &r_alloc_size, bool p_can_segment = true);
Error _insert_staging_block();

struct Buffer {
Expand Down Expand Up @@ -637,7 +637,6 @@ class RenderingDeviceVulkan : public RenderingDevice {
};

bool is_compute = false;
int max_output = 0;
Vector<Set> sets;
Vector<uint32_t> set_formats;
Vector<VkPipelineShaderStageCreateInfo> pipeline_stages;
Expand Down Expand Up @@ -870,11 +869,9 @@ class RenderingDeviceVulkan : public RenderingDevice {
uint32_t pipeline_dynamic_state = 0;
VertexFormatID pipeline_vertex_format = INVALID_ID;
RID pipeline_shader;
uint32_t invalid_set_from = 0;
bool pipeline_uses_restart_indices = false;
uint32_t pipeline_primitive_divisor = 0;
uint32_t pipeline_primitive_minimum = 0;
Vector<uint32_t> pipeline_set_formats;
uint32_t pipeline_push_constant_size = 0;
bool pipeline_push_constant_supplied = false;
} validation;
Expand Down Expand Up @@ -948,7 +945,6 @@ class RenderingDeviceVulkan : public RenderingDevice {
bool pipeline_active = false;
RID pipeline_shader;
uint32_t invalid_set_from = 0;
Vector<uint32_t> pipeline_set_formats;
uint32_t pipeline_push_constant_size = 0;
bool pipeline_push_constant_supplied = false;
} validation;
Expand Down Expand Up @@ -998,19 +994,19 @@ class RenderingDeviceVulkan : public RenderingDevice {

VkQueryPool timestamp_pool;

String *timestamp_names = nullptr;
uint64_t *timestamp_cpu_values = nullptr;
TightLocalVector<String> timestamp_names;
TightLocalVector<uint64_t> timestamp_cpu_values;
uint32_t timestamp_count = 0;
String *timestamp_result_names = nullptr;
uint64_t *timestamp_cpu_result_values = nullptr;
uint64_t *timestamp_result_values = nullptr;
TightLocalVector<String> timestamp_result_names;
TightLocalVector<uint64_t> timestamp_cpu_result_values;
TightLocalVector<uint64_t> timestamp_result_values;
uint32_t timestamp_result_count = 0;
uint64_t index = 0;
};

uint32_t max_timestamp_query_elements = 0;

Frame *frames = nullptr; //frames available, for main device they are cycled (usually 3), for local devices only 1
TightLocalVector<Frame> frames; //frames available, for main device they are cycled (usually 3), for local devices only 1
int frame = 0; //current frame
int frame_count = 0; //total amount of frames
uint64_t frames_drawn = 0;
Expand Down
Loading

0 comments on commit 5ab59ee

Please sign in to comment.