Ability to create local RenderingDevice instances.

drivers/vulkan/rendering_device_vulkan.cpp

@@ -5336,17 +5336,19 @@ bool RenderingDeviceVulkan::compute_pipeline_is_valid(RID p_pipeline) {
 
 int RenderingDeviceVulkan::screen_get_width(DisplayServer::WindowID p_screen) const {
 	_THREAD_SAFE_METHOD_
-
+	ERR_FAIL_COND_V_MSG(local_device.is_valid(), -1, "Local devices have no screen");
 	return context->window_get_width(p_screen);
 }
 int RenderingDeviceVulkan::screen_get_height(DisplayServer::WindowID p_screen) const {
 	_THREAD_SAFE_METHOD_
+	ERR_FAIL_COND_V_MSG(local_device.is_valid(), -1, "Local devices have no screen");
 
 	return context->window_get_height(p_screen);
 }
 RenderingDevice::FramebufferFormatID RenderingDeviceVulkan::screen_get_framebuffer_format() const {
 
 	_THREAD_SAFE_METHOD_
+	ERR_FAIL_COND_V_MSG(local_device.is_valid(), INVALID_ID, "Local devices have no screen");
 
 	//very hacky, but not used often per frame so I guess ok
 	VkFormat vkformat = context->get_screen_format();
@@ -5376,6 +5378,7 @@ RenderingDevice::FramebufferFormatID RenderingDeviceVulkan::screen_get_framebuff
 RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_begin_for_screen(DisplayServer::WindowID p_screen, const Color &p_clear_color) {
 
 	_THREAD_SAFE_METHOD_
+	ERR_FAIL_COND_V_MSG(local_device.is_valid(), INVALID_ID, "Local devices have no screen");
 
 	ERR_FAIL_COND_V_MSG(draw_list != nullptr, INVALID_ID, "Only one draw list can be active at the same time.");
 	ERR_FAIL_COND_V_MSG(compute_list != nullptr, INVALID_ID, "Only one draw/compute list can be active at the same time.");
@@ -6695,75 +6698,104 @@ void RenderingDeviceVulkan::free(RID p_id) {
 	_free_dependencies(p_id); //recursively erase dependencies first, to avoid potential API problems
 	_free_internal(p_id);
 }
-void RenderingDeviceVulkan::swap_buffers() {
 
-	_THREAD_SAFE_METHOD_
+void RenderingDeviceVulkan::_finalize_command_bufers() {
 
-	{ //finalize frame
+	if (draw_list) {
-
+		ERR_PRINT("Found open draw list at the end of the frame, this should never happen (further drawing will likely not work).");
-		if (draw_list) {
-			ERR_PRINT("Found open draw list at the end of the frame, this should never happen (further drawing will likely not work).");
-		}
-
-		if (compute_list) {
-			ERR_PRINT("Found open compute list at the end of the frame, this should never happen (further compute will likely not work).");
-		}
-
-		{ //complete the setup buffer (that needs to be processed before anything else)
-			vkEndCommandBuffer(frames[frame].setup_command_buffer);
-			vkEndCommandBuffer(frames[frame].draw_command_buffer);
-		}
-		screen_prepared = false;
 	}
 
+	if (compute_list) {
+		ERR_PRINT("Found open compute list at the end of the frame, this should never happen (further compute will likely not work).");
+	}
+
+	{ //complete the setup buffer (that needs to be processed before anything else)
+		vkEndCommandBuffer(frames[frame].setup_command_buffer);
+		vkEndCommandBuffer(frames[frame].draw_command_buffer);
+	}
+}
+
+void RenderingDeviceVulkan::_begin_frame() {
+
+	//erase pending resources
+	_free_pending_resources(frame);
+
+	//create setup command buffer and set as the setup buffer
+
+	{
+		VkCommandBufferBeginInfo cmdbuf_begin;
+		cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+		cmdbuf_begin.pNext = nullptr;
+		cmdbuf_begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+		cmdbuf_begin.pInheritanceInfo = nullptr;
+
+		VkResult err = vkResetCommandBuffer(frames[frame].setup_command_buffer, 0);
+		ERR_FAIL_COND_MSG(err, "vkResetCommandBuffer failed with error " + itos(err) + ".");
+
+		err = vkBeginCommandBuffer(frames[frame].setup_command_buffer, &cmdbuf_begin);
+		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
+		err = vkBeginCommandBuffer(frames[frame].draw_command_buffer, &cmdbuf_begin);
+		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
+
+		if (local_device.is_null()) {
+			context->append_command_buffer(frames[frame].draw_command_buffer);
+			context->set_setup_buffer(frames[frame].setup_command_buffer); //append now so it's added before everything else
+		}
+	}
+
+	//advance current frame
+	frames_drawn++;
+	//advance staging buffer if used
+	if (staging_buffer_used) {
+		staging_buffer_current = (staging_buffer_current + 1) % staging_buffer_blocks.size();
+		staging_buffer_used = false;
+	}
+
+	if (frames[frame].timestamp_count) {
+		vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
+		SWAP(frames[frame].timestamp_names, frames[frame].timestamp_result_names);
+		SWAP(frames[frame].timestamp_cpu_values, frames[frame].timestamp_cpu_result_values);
+	}
+
+	frames[frame].timestamp_result_count = frames[frame].timestamp_count;
+	frames[frame].timestamp_count = 0;
+	frames[frame].index = Engine::get_singleton()->get_frames_drawn();
+}
+
+void RenderingDeviceVulkan::swap_buffers() {
+
+	ERR_FAIL_COND_MSG(local_device.is_valid(), "Local devices can't swap buffers.");
+	_THREAD_SAFE_METHOD_
+
+	_finalize_command_bufers();
+
+	screen_prepared = false;
 	//swap buffers
 	context->swap_buffers();
 
-	{ //advance frame
+	frame = (frame + 1) % frame_count;
 
-		frame = (frame + 1) % frame_count;
+	_begin_frame();
+}
 
-		//erase pending resources
+void RenderingDeviceVulkan::submit() {
-		_free_pending_resources(frame);
+	ERR_FAIL_COND_MSG(local_device.is_null(), "Only local devices can submit and sync.");
+	ERR_FAIL_COND_MSG(local_device_processing, "device already submitted, call sync to wait until done.");
 
-		//create setup command buffer and set as the setup buffer
+	_finalize_command_bufers();
 
-		{
+	VkCommandBuffer command_buffers[2] = { frames[frame].setup_command_buffer, frames[frame].draw_command_buffer };
-			VkCommandBufferBeginInfo cmdbuf_begin;
+	context->local_device_push_command_buffers(local_device, command_buffers, 2);
-			cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+	local_device_processing = true;
-			cmdbuf_begin.pNext = nullptr;
+}
-			cmdbuf_begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
-			cmdbuf_begin.pInheritanceInfo = nullptr;
 
-			VkResult err = vkResetCommandBuffer(frames[frame].setup_command_buffer, 0);
+void RenderingDeviceVulkan::sync() {
-			ERR_FAIL_COND_MSG(err, "vkResetCommandBuffer failed with error " + itos(err) + ".");
 
-			err = vkBeginCommandBuffer(frames[frame].setup_command_buffer, &cmdbuf_begin);
+	ERR_FAIL_COND_MSG(local_device.is_null(), "Only local devices can submit and sync.");
-			ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
+	ERR_FAIL_COND_MSG(!local_device_processing, "sync can only be called after a submit");
-			context->set_setup_buffer(frames[frame].setup_command_buffer); //append now so it's added before everything else
-			err = vkBeginCommandBuffer(frames[frame].draw_command_buffer, &cmdbuf_begin);
-			ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
-			context->append_command_buffer(frames[frame].draw_command_buffer);
-		}
 
-		//advance current frame
+	context->local_device_sync(local_device);
-		frames_drawn++;
+	_begin_frame();
-		//advance staging buffer if used
-		if (staging_buffer_used) {
-			staging_buffer_current = (staging_buffer_current + 1) % staging_buffer_blocks.size();
-			staging_buffer_used = false;
-		}
-
-		if (frames[frame].timestamp_count) {
-			vkGetQueryPoolResults(device, frames[frame].timestamp_pool, 0, frames[frame].timestamp_count, sizeof(uint64_t) * max_timestamp_query_elements, frames[frame].timestamp_result_values, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT);
-			SWAP(frames[frame].timestamp_names, frames[frame].timestamp_result_names);
-			SWAP(frames[frame].timestamp_cpu_values, frames[frame].timestamp_cpu_result_values);
-		}
-
-		frames[frame].timestamp_result_count = frames[frame].timestamp_count;
-		frames[frame].timestamp_count = 0;
-		frames[frame].index = Engine::get_singleton()->get_frames_drawn();
-	}
 }
 
 void RenderingDeviceVulkan::_free_pending_resources(int p_frame) {
@@ -6882,14 +6914,21 @@ uint32_t RenderingDeviceVulkan::get_frame_delay() const {
 
 void RenderingDeviceVulkan::_flush(bool p_current_frame) {
 
+	if (local_device.is_valid() && !p_current_frame) {
+		return; //flushign previous frames has no effect with local device
+	}
 	//not doing this crashes RADV (undefined behavior)
 	if (p_current_frame) {
 		vkEndCommandBuffer(frames[frame].setup_command_buffer);
 		vkEndCommandBuffer(frames[frame].draw_command_buffer);
 	}
-	context->flush(p_current_frame, p_current_frame);
+
-	//re-create the setup command
+	if (local_device.is_valid()) {
-	if (p_current_frame) {
+
+		VkCommandBuffer command_buffers[2] = { frames[frame].setup_command_buffer, frames[frame].draw_command_buffer };
+		context->local_device_push_command_buffers(local_device, command_buffers, 2);
+		context->local_device_sync(local_device);
+
 		VkCommandBufferBeginInfo cmdbuf_begin;
 		cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
 		cmdbuf_begin.pNext = nullptr;
@@ -6898,27 +6937,48 @@ void RenderingDeviceVulkan::_flush(bool p_current_frame) {
 
 		VkResult err = vkBeginCommandBuffer(frames[frame].setup_command_buffer, &cmdbuf_begin);
 		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
-		context->set_setup_buffer(frames[frame].setup_command_buffer); //append now so it's added before everything else
+		err = vkBeginCommandBuffer(frames[frame].draw_command_buffer, &cmdbuf_begin);
-	}
-
-	if (p_current_frame) {
-		VkCommandBufferBeginInfo cmdbuf_begin;
-		cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
-		cmdbuf_begin.pNext = nullptr;
-		cmdbuf_begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
-		cmdbuf_begin.pInheritanceInfo = nullptr;
-
-		VkResult err = vkBeginCommandBuffer(frames[frame].draw_command_buffer, &cmdbuf_begin);
 		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
-		context->append_command_buffer(frames[frame].draw_command_buffer);
+
+	} else {
+		context->flush(p_current_frame, p_current_frame);
+		//re-create the setup command
+		if (p_current_frame) {
+			VkCommandBufferBeginInfo cmdbuf_begin;
+			cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+			cmdbuf_begin.pNext = nullptr;
+			cmdbuf_begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+			cmdbuf_begin.pInheritanceInfo = nullptr;
+
+			VkResult err = vkBeginCommandBuffer(frames[frame].setup_command_buffer, &cmdbuf_begin);
+			ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
+			context->set_setup_buffer(frames[frame].setup_command_buffer); //append now so it's added before everything else
+		}
+
+		if (p_current_frame) {
+			VkCommandBufferBeginInfo cmdbuf_begin;
+			cmdbuf_begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+			cmdbuf_begin.pNext = nullptr;
+			cmdbuf_begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+			cmdbuf_begin.pInheritanceInfo = nullptr;
+
+			VkResult err = vkBeginCommandBuffer(frames[frame].draw_command_buffer, &cmdbuf_begin);
+			ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
+			context->append_command_buffer(frames[frame].draw_command_buffer);
+		}
 	}
 }
 
-void RenderingDeviceVulkan::initialize(VulkanContext *p_context) {
+void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_device) {
 
 	context = p_context;
 	device = p_context->get_device();
-	frame_count = p_context->get_swapchain_image_count() + 1; //always need one extra to ensure it's unused at any time, without having to use a fence for this.
+	if (p_local_device) {
+		frame_count = 1;
+		local_device = p_context->local_device_create();
+	} else {
+		frame_count = p_context->get_swapchain_image_count() + 1; //always need one extra to ensure it's unused at any time, without having to use a fence for this.
+	}
 	limits = p_context->get_device_limits();
 	max_timestamp_query_elements = 256;
 
@@ -6999,11 +7059,13 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context) {
 
 		VkResult err = vkBeginCommandBuffer(frames[0].setup_command_buffer, &cmdbuf_begin);
 		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
-		context->set_setup_buffer(frames[0].setup_command_buffer); //append now so it's added before everything else
 
 		err = vkBeginCommandBuffer(frames[0].draw_command_buffer, &cmdbuf_begin);
 		ERR_FAIL_COND_MSG(err, "vkBeginCommandBuffer failed with error " + itos(err) + ".");
-		context->append_command_buffer(frames[0].draw_command_buffer);
+		if (local_device.is_null()) {
+			context->set_setup_buffer(frames[0].setup_command_buffer); //append now so it's added before everything else
+			context->append_command_buffer(frames[0].draw_command_buffer);
+		}
 	}
 
 	staging_buffer_block_size = GLOBAL_DEF("rendering/vulkan/staging_buffer/block_size_kb", 256);
@@ -7285,6 +7347,19 @@ void RenderingDeviceVulkan::finalize() {
 	ERR_FAIL_COND(reverse_dependency_map.size());
 }
 
+RenderingDevice *RenderingDeviceVulkan::create_local_device() {
+	RenderingDeviceVulkan *rd = memnew(RenderingDeviceVulkan);
+	rd->initialize(context, true);
+	return rd;
+}
+
 RenderingDeviceVulkan::RenderingDeviceVulkan() {
 	screen_prepared = false;
 }
+
+RenderingDeviceVulkan::~RenderingDeviceVulkan() {
+	if (local_device.is_valid()) {
+		finalize();
+		context->local_device_free(local_device);
+	}
+}

drivers/vulkan/rendering_device_vulkan.h

@@ -952,10 +952,12 @@ class RenderingDeviceVulkan : public RenderingDevice {
 
 	uint32_t max_timestamp_query_elements;
 
-	Frame *frames; //frames available, they are cycled (usually 3)
+	Frame *frames; //frames available, for main device they are cycled (usually 3), for local devices only 1
 	int frame; //current frame
 	int frame_count; //total amount of frames
 	uint64_t frames_drawn;
+	RID local_device;
+	bool local_device_processing = false;
 
 	void _free_pending_resources(int p_frame);
 
@@ -971,6 +973,9 @@ class RenderingDeviceVulkan : public RenderingDevice {
 	template <class T>
 	void _free_rids(T &p_owner, const char *p_type);
 
+	void _finalize_command_bufers();
+	void _begin_frame();
+
 public:
 	virtual RID texture_create(const TextureFormat &p_format, const TextureView &p_view, const Vector<Vector<uint8_t>> &p_data = Vector<Vector<uint8_t>>());
 	virtual RID texture_create_shared(const TextureView &p_view, RID p_with_texture);
@@ -1121,14 +1126,20 @@ public:
 	virtual int limit_get(Limit p_limit);
 
 	virtual void prepare_screen_for_drawing();
-	void initialize(VulkanContext *p_context);
+	void initialize(VulkanContext *p_context, bool p_local_device = false);
 	void finalize();
 
-	virtual void swap_buffers();
+	virtual void swap_buffers(); //for main device
+
+	virtual void submit(); //for local device
+	virtual void sync(); //for local device
 
 	virtual uint32_t get_frame_delay() const;
 
+	virtual RenderingDevice *create_local_device();
+
 	RenderingDeviceVulkan();
+	~RenderingDeviceVulkan();
 };
 
 #endif // RENDERING_DEVICE_VULKAN_H

drivers/vulkan/vulkan_context.cpp

@@ -1499,6 +1499,87 @@ VulkanContext::VulkanContext() {
 	swapchainImageCount = 0;
 }
 
+RID VulkanContext::local_device_create() {
+	LocalDevice ld;
+
+	{ //create device
+		VkResult err;
+		float queue_priorities[1] = { 0.0 };
+		VkDeviceQueueCreateInfo queues[2];
+		queues[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
+		queues[0].pNext = nullptr;
+		queues[0].queueFamilyIndex = graphics_queue_family_index;
+		queues[0].queueCount = 1;
+		queues[0].pQueuePriorities = queue_priorities;
+		queues[0].flags = 0;
+
+		VkDeviceCreateInfo sdevice = {
+			/*sType =*/VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
+			/*pNext */ nullptr,
+			/*flags */ 0,
+			/*queueCreateInfoCount */ 1,
+			/*pQueueCreateInfos */ queues,
+			/*enabledLayerCount */ 0,
+			/*ppEnabledLayerNames */ nullptr,
+			/*enabledExtensionCount */ enabled_extension_count,
+			/*ppEnabledExtensionNames */ (const char *const *)extension_names,
+			/*pEnabledFeatures */ &physical_device_features, // If specific features are required, pass them in here
+		};
+		err = vkCreateDevice(gpu, &sdevice, nullptr, &ld.device);
+		ERR_FAIL_COND_V(err, RID());
+	}
+
+	{ //create graphics queue
+
+		vkGetDeviceQueue(ld.device, graphics_queue_family_index, 0, &ld.queue);
+	}
+
+	return local_device_owner.make_rid(ld);
+}
+
+VkDevice VulkanContext::local_device_get_vk_device(RID p_local_device) {
+	LocalDevice *ld = local_device_owner.getornull(p_local_device);
+	return ld->device;
+}
+
+void VulkanContext::local_device_push_command_buffers(RID p_local_device, const VkCommandBuffer *p_buffers, int p_count) {
+
+	LocalDevice *ld = local_device_owner.getornull(p_local_device);
+	ERR_FAIL_COND(ld->waiting);
+
+	VkSubmitInfo submit_info;
+	submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+	submit_info.pNext = nullptr;
+	submit_info.pWaitDstStageMask = nullptr;
+	submit_info.waitSemaphoreCount = 0;
+	submit_info.pWaitSemaphores = nullptr;
+	submit_info.commandBufferCount = p_count;
+	submit_info.pCommandBuffers = p_buffers;
+	submit_info.signalSemaphoreCount = 0;
+	submit_info.pSignalSemaphores = nullptr;
+
+	VkResult err = vkQueueSubmit(ld->queue, 1, &submit_info, VK_NULL_HANDLE);
+	ERR_FAIL_COND(err);
+
+	ld->waiting = true;
+}
+
+void VulkanContext::local_device_sync(RID p_local_device) {
+
+	LocalDevice *ld = local_device_owner.getornull(p_local_device);
+	ERR_FAIL_COND(!ld->waiting);
+
+	vkDeviceWaitIdle(ld->device);
+	ld->waiting = false;
+}
+
+void VulkanContext::local_device_free(RID p_local_device) {
+
+	LocalDevice *ld = local_device_owner.getornull(p_local_device);
+	vkDestroyDevice(ld->device, nullptr);
+	local_device_owner.free(p_local_device);
+}
+
 VulkanContext::~VulkanContext() {
 	if (queue_props) {
 		free(queue_props);

drivers/vulkan/vulkan_context.h

@@ -33,6 +33,8 @@
 
 #include "core/error_list.h"
 #include "core/map.h"
+#include "core/os/mutex.h"
+#include "core/rid_owner.h"
 #include "core/ustring.h"
 #include "servers/display_server.h"
 #include <vulkan/vulkan.h>
@@ -105,6 +107,14 @@ class VulkanContext {
 		}
 	};
 
+	struct LocalDevice {
+		bool waiting = false;
+		VkDevice device;
+		VkQueue queue;
+	};
+
+	RID_Owner<LocalDevice, true> local_device_owner;
+
 	Map<DisplayServer::WindowID, Window> windows;
 	uint32_t swapchainImageCount;
 
@@ -194,6 +204,12 @@ public:
 	VkFramebuffer window_get_framebuffer(DisplayServer::WindowID p_window = 0);
 	VkRenderPass window_get_render_pass(DisplayServer::WindowID p_window = 0);
 
+	RID local_device_create();
+	VkDevice local_device_get_vk_device(RID p_local_device);
+	void local_device_push_command_buffers(RID p_local_device, const VkCommandBuffer *p_buffers, int p_count);
+	void local_device_sync(RID p_local_device);
+	void local_device_free(RID p_local_device);
+
 	VkFormat get_screen_format() const;
 	VkPhysicalDeviceLimits get_device_limits() const;
 

editor/editor_node.cpp

@@ -64,6 +64,7 @@
 #include "servers/navigation_server_2d.h"
 #include "servers/navigation_server_3d.h"
 #include "servers/physics_server_2d.h"
+#include "servers/rendering/rendering_device.h"
 
 #include "editor/audio_stream_preview.h"
 #include "editor/debugger/editor_debugger_node.h"

servers/rendering/rendering_device.cpp

@@ -60,5 +60,7 @@ Vector<uint8_t> RenderingDevice::shader_compile_from_source(ShaderStage p_stage,
 }
 
 RenderingDevice::RenderingDevice() {
-	singleton = this;
+	if (singleton == nullptr) { // there may be more rendering devices later
+		singleton = this;
+	}
 }

servers/rendering/rendering_device.h

@@ -1024,6 +1024,11 @@ public:
 
 	virtual uint32_t get_frame_delay() const = 0;
 
+	virtual void submit() = 0;
+	virtual void sync() = 0;
+
+	virtual RenderingDevice *create_local_device() = 0;
+
 	static RenderingDevice *get_singleton();
 	RenderingDevice();
 };