virtualx-engine/drivers/vulkan/vulkan_context.cpp

#include "vulkan_context.h"
#include "core/print_string.h"
#include "core/project_settings.h"
#include "core/version.h"
#include "vk_enum_string_helper.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
#define VULKAN_DEBUG(m_text) print_line(m_text)
#define APP_SHORT_NAME "GodotEngine"

VKAPI_ATTR VkBool32 VKAPI_CALL VulkanContext::_debug_messenger_callback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity,
		VkDebugUtilsMessageTypeFlagsEXT messageType,
		const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData,
		void *pUserData) {
	char prefix[64] = "";
	char *message = (char *)malloc(strlen(pCallbackData->pMessage) + 5000);
	ERR_FAIL_COND_V(!message, false);

	if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT) {
		strcat(prefix, "VERBOSE : ");
	} else if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT) {
		strcat(prefix, "INFO : ");
	} else if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) {
		strcat(prefix, "WARNING : ");
	} else if (messageSeverity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) {
		strcat(prefix, "ERROR : ");
	}

	if (messageType & VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT) {
		strcat(prefix, "GENERAL");
	} else {
		if (messageType & VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT) {
			strcat(prefix, "VALIDATION");
			//validation_error = 1;
		}
		if (messageType & VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT) {
			if (messageType & VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT) {
				strcat(prefix, "|");
			}
			strcat(prefix, "PERFORMANCE");
		}
	}

	sprintf(message, "%s - Message Id Number: %d | Message Id Name: %s\n\t%s\n", prefix, pCallbackData->messageIdNumber,
			pCallbackData->pMessageIdName, pCallbackData->pMessage);

	if (pCallbackData->objectCount > 0) {
		char tmp_message[500];
		sprintf(tmp_message, "\n\tObjects - %d\n", pCallbackData->objectCount);
		strcat(message, tmp_message);
		for (uint32_t object = 0; object < pCallbackData->objectCount; ++object) {
			if (NULL != pCallbackData->pObjects[object].pObjectName && strlen(pCallbackData->pObjects[object].pObjectName) > 0) {
				sprintf(tmp_message, "\t\tObject[%d] - %s, Handle %p, Name \"%s\"\n", object,
						string_VkObjectType(pCallbackData->pObjects[object].objectType),
						(void *)(pCallbackData->pObjects[object].objectHandle), pCallbackData->pObjects[object].pObjectName);
			} else {
				sprintf(tmp_message, "\t\tObject[%d] - %s, Handle %p\n", object,
						string_VkObjectType(pCallbackData->pObjects[object].objectType),
						(void *)(pCallbackData->pObjects[object].objectHandle));
			}
			strcat(message, tmp_message);
		}
	}
	if (pCallbackData->cmdBufLabelCount > 0) {
		char tmp_message[500];
		sprintf(tmp_message, "\n\tCommand Buffer Labels - %d\n", pCallbackData->cmdBufLabelCount);
		strcat(message, tmp_message);
		for (uint32_t cmd_buf_label = 0; cmd_buf_label < pCallbackData->cmdBufLabelCount; ++cmd_buf_label) {
			sprintf(tmp_message, "\t\tLabel[%d] - %s { %f, %f, %f, %f}\n", cmd_buf_label,
					pCallbackData->pCmdBufLabels[cmd_buf_label].pLabelName, pCallbackData->pCmdBufLabels[cmd_buf_label].color[0],
					pCallbackData->pCmdBufLabels[cmd_buf_label].color[1], pCallbackData->pCmdBufLabels[cmd_buf_label].color[2],
					pCallbackData->pCmdBufLabels[cmd_buf_label].color[3]);
			strcat(message, tmp_message);
		}
	}

	ERR_PRINT(message);

	free(message);

	// Don't bail out, but keep going.
	return false;
}

VkBool32 VulkanContext::_check_layers(uint32_t check_count, const char **check_names, uint32_t layer_count, VkLayerProperties *layers) {
	for (uint32_t i = 0; i < check_count; i++) {
		VkBool32 found = 0;
		for (uint32_t j = 0; j < layer_count; j++) {
			if (!strcmp(check_names[i], layers[j].layerName)) {
				found = 1;
				break;
			}
		}
		if (!found) {
			ERR_PRINT("Cant find layer: " + String(check_names[i]));
			return 0;
		}
	}
	return 1;
}

Error VulkanContext::_create_validation_layers() {

	VkResult err;
	uint32_t instance_layer_count = 0;
	uint32_t validation_layer_count = 0;
	const char *instance_validation_layers_alt1[] = { "VK_LAYER_LUNARG_standard_validation" };
	const char *instance_validation_layers_alt2[] = { "VK_LAYER_GOOGLE_threading", "VK_LAYER_LUNARG_parameter_validation",
		"VK_LAYER_LUNARG_object_tracker", "VK_LAYER_LUNARG_core_validation",
		"VK_LAYER_GOOGLE_unique_objects" };
	VkBool32 validation_found = 0;
	err = vkEnumerateInstanceLayerProperties(&instance_layer_count, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	const char **instance_validation_layers = instance_validation_layers_alt1;
	if (instance_layer_count > 0) {
		VkLayerProperties *instance_layers = (VkLayerProperties *)malloc(sizeof(VkLayerProperties) * instance_layer_count);
		err = vkEnumerateInstanceLayerProperties(&instance_layer_count, instance_layers);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

		validation_found = _check_layers(ARRAY_SIZE(instance_validation_layers_alt1), instance_validation_layers,
				instance_layer_count, instance_layers);
		if (validation_found) {
			enabled_layer_count = ARRAY_SIZE(instance_validation_layers_alt1);
			enabled_layers[0] = "VK_LAYER_LUNARG_standard_validation";
			validation_layer_count = 1;
		} else {
			// use alternative set of validation layers
			instance_validation_layers = instance_validation_layers_alt2;
			enabled_layer_count = ARRAY_SIZE(instance_validation_layers_alt2);
			validation_found = _check_layers(ARRAY_SIZE(instance_validation_layers_alt2), instance_validation_layers,
					instance_layer_count, instance_layers);
			validation_layer_count = ARRAY_SIZE(instance_validation_layers_alt2);
			for (uint32_t i = 0; i < validation_layer_count; i++) {
				enabled_layers[i] = instance_validation_layers[i];
			}
		}
		free(instance_layers);
	}

	if (!validation_found) {
		return ERR_CANT_CREATE;
	}

	return OK;
}

Error VulkanContext::_initialize_extensions() {

	VkResult err;
	uint32_t instance_extension_count = 0;

	enabled_extension_count = 0;
	enabled_layer_count = 0;
	/* Look for instance extensions */
	VkBool32 surfaceExtFound = 0;
	VkBool32 platformSurfaceExtFound = 0;
	memset(extension_names, 0, sizeof(extension_names));

	err = vkEnumerateInstanceExtensionProperties(NULL, &instance_extension_count, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	if (instance_extension_count > 0) {
		VkExtensionProperties *instance_extensions = (VkExtensionProperties *)malloc(sizeof(VkExtensionProperties) * instance_extension_count);
		err = vkEnumerateInstanceExtensionProperties(NULL, &instance_extension_count, instance_extensions);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		for (uint32_t i = 0; i < instance_extension_count; i++) {
			if (!strcmp(VK_KHR_SURFACE_EXTENSION_NAME, instance_extensions[i].extensionName)) {
				surfaceExtFound = 1;
				extension_names[enabled_extension_count++] = VK_KHR_SURFACE_EXTENSION_NAME;
			}

			if (!strcmp(_get_platform_surface_extension(), instance_extensions[i].extensionName)) {
				platformSurfaceExtFound = 1;
				extension_names[enabled_extension_count++] = _get_platform_surface_extension();
			}
			if (!strcmp(VK_EXT_DEBUG_REPORT_EXTENSION_NAME, instance_extensions[i].extensionName)) {
				if (use_validation_layers) {
					extension_names[enabled_extension_count++] = VK_EXT_DEBUG_REPORT_EXTENSION_NAME;
				}
			}
			if (!strcmp(VK_EXT_DEBUG_UTILS_EXTENSION_NAME, instance_extensions[i].extensionName)) {
				if (use_validation_layers) {
					extension_names[enabled_extension_count++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
				}
			}
			ERR_FAIL_COND_V(enabled_extension_count >= MAX_EXTENSIONS, ERR_BUG); //??
		}

		free(instance_extensions);
	}

	ERR_FAIL_COND_V_MSG(!surfaceExtFound, ERR_CANT_CREATE, "No surface extension found, is a driver installed?");
	ERR_FAIL_COND_V_MSG(!platformSurfaceExtFound, ERR_CANT_CREATE, "No platform surface extension found, is a driver installed?");

	return OK;
}

Error VulkanContext::_create_physical_device() {

	/* Look for validation layers */
	if (use_validation_layers) {
		_create_validation_layers();
	}

	{
		Error err = _initialize_extensions();
		if (err != OK) {
			return err;
		}
	}

	CharString cs = ProjectSettings::get_singleton()->get("application/config/name").operator String().utf8();
	String name = "GodotEngine " + String(VERSION_FULL_NAME);
	CharString namecs = name.utf8();
	const VkApplicationInfo app = {
		.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
		.pNext = NULL,
		.pApplicationName = cs.get_data(),
		.applicationVersion = 0,
		.pEngineName = namecs.get_data(),
		.engineVersion = 0,
		.apiVersion = VK_API_VERSION_1_0,
	};
	VkInstanceCreateInfo inst_info = {
		.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
		.pNext = NULL,
		.pApplicationInfo = &app,
		.enabledLayerCount = enabled_layer_count,
		.ppEnabledLayerNames = (const char *const *)instance_validation_layers,
		.enabledExtensionCount = enabled_extension_count,
		.ppEnabledExtensionNames = (const char *const *)extension_names,
	};

	/*
	   * This is info for a temp callback to use during CreateInstance.
	   * After the instance is created, we use the instance-based
	   * function to register the final callback.
	   */
	VkDebugUtilsMessengerCreateInfoEXT dbg_messenger_create_info;
	if (use_validation_layers) {
		// VK_EXT_debug_utils style
		dbg_messenger_create_info.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT;
		dbg_messenger_create_info.pNext = NULL;
		dbg_messenger_create_info.flags = 0;
		dbg_messenger_create_info.messageSeverity =
				VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT;
		dbg_messenger_create_info.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
												VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
												VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT;
		dbg_messenger_create_info.pfnUserCallback = _debug_messenger_callback;
		dbg_messenger_create_info.pUserData = this;
		inst_info.pNext = &dbg_messenger_create_info;
	}

	uint32_t gpu_count;

	VkResult err = vkCreateInstance(&inst_info, NULL, &inst);
	ERR_FAIL_COND_V_MSG(err == VK_ERROR_INCOMPATIBLE_DRIVER, ERR_CANT_CREATE,
			"Cannot find a compatible Vulkan installable client driver (ICD).\n\n"
			"vkCreateInstance Failure");
	ERR_FAIL_COND_V_MSG(err == VK_ERROR_EXTENSION_NOT_PRESENT, ERR_CANT_CREATE,
			"Cannot find a specified extension library.\n"
			"Make sure your layers path is set appropriately.\n"
			"vkCreateInstance Failure");
	ERR_FAIL_COND_V_MSG(err, ERR_CANT_CREATE,
			"vkCreateInstance failed.\n\n"
			"Do you have a compatible Vulkan installable client driver (ICD) installed?\n"
			"Please look at the Getting Started guide for additional information.\n"
			"vkCreateInstance Failure");

	/* Make initial call to query gpu_count, then second call for gpu info*/
	err = vkEnumeratePhysicalDevices(inst, &gpu_count, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	ERR_FAIL_COND_V_MSG(gpu_count == 0, ERR_CANT_CREATE,
			"vkEnumeratePhysicalDevices reported zero accessible devices.\n\n"
			"Do you have a compatible Vulkan installable client driver (ICD) installed?\n"
			"vkEnumeratePhysicalDevices Failure");

	VkPhysicalDevice *physical_devices = (VkPhysicalDevice *)malloc(sizeof(VkPhysicalDevice) * gpu_count);
	err = vkEnumeratePhysicalDevices(inst, &gpu_count, physical_devices);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	/* for now, just grab the first physical device */
	gpu = physical_devices[0];
	free(physical_devices);

	/* Look for device extensions */
	uint32_t device_extension_count = 0;
	VkBool32 swapchainExtFound = 0;
	enabled_extension_count = 0;
	memset(extension_names, 0, sizeof(extension_names));

	err = vkEnumerateDeviceExtensionProperties(gpu, NULL, &device_extension_count, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	if (device_extension_count > 0) {
		VkExtensionProperties *device_extensions = (VkExtensionProperties *)malloc(sizeof(VkExtensionProperties) * device_extension_count);
		err = vkEnumerateDeviceExtensionProperties(gpu, NULL, &device_extension_count, device_extensions);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

		for (uint32_t i = 0; i < device_extension_count; i++) {
			if (!strcmp(VK_KHR_SWAPCHAIN_EXTENSION_NAME, device_extensions[i].extensionName)) {
				swapchainExtFound = 1;
				extension_names[enabled_extension_count++] = VK_KHR_SWAPCHAIN_EXTENSION_NAME;
			}
			ERR_FAIL_COND_V(enabled_extension_count >= MAX_EXTENSIONS, ERR_BUG);
		}

		if (VK_KHR_incremental_present_enabled) {
			// Even though the user "enabled" the extension via the command
			// line, we must make sure that it's enumerated for use with the
			// device.  Therefore, disable it here, and re-enable it again if
			// enumerated.
			VK_KHR_incremental_present_enabled = false;
			for (uint32_t i = 0; i < device_extension_count; i++) {
				if (!strcmp(VK_KHR_INCREMENTAL_PRESENT_EXTENSION_NAME, device_extensions[i].extensionName)) {
					extension_names[enabled_extension_count++] = VK_KHR_INCREMENTAL_PRESENT_EXTENSION_NAME;
					VK_KHR_incremental_present_enabled = true;
					VULKAN_DEBUG("VK_KHR_incremental_present extension enabled\n");
				}
				ERR_FAIL_COND_V(enabled_extension_count >= MAX_EXTENSIONS, ERR_BUG);
			}
			if (!VK_KHR_incremental_present_enabled) {
				VULKAN_DEBUG("VK_KHR_incremental_present extension NOT AVAILABLE\n");
			}
		}

		if (VK_GOOGLE_display_timing_enabled) {
			// Even though the user "enabled" the extension via the command
			// line, we must make sure that it's enumerated for use with the
			// device.  Therefore, disable it here, and re-enable it again if
			// enumerated.
			VK_GOOGLE_display_timing_enabled = false;
			for (uint32_t i = 0; i < device_extension_count; i++) {
				if (!strcmp(VK_GOOGLE_DISPLAY_TIMING_EXTENSION_NAME, device_extensions[i].extensionName)) {
					extension_names[enabled_extension_count++] = VK_GOOGLE_DISPLAY_TIMING_EXTENSION_NAME;
					VK_GOOGLE_display_timing_enabled = true;
					VULKAN_DEBUG("VK_GOOGLE_display_timing extension enabled\n");
				}
				ERR_FAIL_COND_V(enabled_extension_count >= MAX_EXTENSIONS, ERR_BUG);
			}
			if (!VK_GOOGLE_display_timing_enabled) {
				VULKAN_DEBUG("VK_GOOGLE_display_timing extension NOT AVAILABLE\n");
			}
		}

		free(device_extensions);
	}

	ERR_FAIL_COND_V_MSG(!swapchainExtFound, ERR_CANT_CREATE,
			"vkEnumerateDeviceExtensionProperties failed to find the " VK_KHR_SWAPCHAIN_EXTENSION_NAME
			" extension.\n\nDo you have a compatible Vulkan installable client driver (ICD) installed?\n"
			"vkCreateInstance Failure");

	if (use_validation_layers) {
		// Setup VK_EXT_debug_utils function pointers always (we use them for
		// debug labels and names).
		CreateDebugUtilsMessengerEXT =
				(PFN_vkCreateDebugUtilsMessengerEXT)vkGetInstanceProcAddr(inst, "vkCreateDebugUtilsMessengerEXT");
		DestroyDebugUtilsMessengerEXT =
				(PFN_vkDestroyDebugUtilsMessengerEXT)vkGetInstanceProcAddr(inst, "vkDestroyDebugUtilsMessengerEXT");
		SubmitDebugUtilsMessageEXT =
				(PFN_vkSubmitDebugUtilsMessageEXT)vkGetInstanceProcAddr(inst, "vkSubmitDebugUtilsMessageEXT");
		CmdBeginDebugUtilsLabelEXT =
				(PFN_vkCmdBeginDebugUtilsLabelEXT)vkGetInstanceProcAddr(inst, "vkCmdBeginDebugUtilsLabelEXT");
		CmdEndDebugUtilsLabelEXT =
				(PFN_vkCmdEndDebugUtilsLabelEXT)vkGetInstanceProcAddr(inst, "vkCmdEndDebugUtilsLabelEXT");
		CmdInsertDebugUtilsLabelEXT =
				(PFN_vkCmdInsertDebugUtilsLabelEXT)vkGetInstanceProcAddr(inst, "vkCmdInsertDebugUtilsLabelEXT");
		SetDebugUtilsObjectNameEXT =
				(PFN_vkSetDebugUtilsObjectNameEXT)vkGetInstanceProcAddr(inst, "vkSetDebugUtilsObjectNameEXT");
		if (NULL == CreateDebugUtilsMessengerEXT || NULL == DestroyDebugUtilsMessengerEXT ||
				NULL == SubmitDebugUtilsMessageEXT || NULL == CmdBeginDebugUtilsLabelEXT ||
				NULL == CmdEndDebugUtilsLabelEXT || NULL == CmdInsertDebugUtilsLabelEXT ||
				NULL == SetDebugUtilsObjectNameEXT) {
			ERR_FAIL_V_MSG(ERR_CANT_CREATE,
					"GetProcAddr: Failed to init VK_EXT_debug_utils\n"
					"GetProcAddr: Failure");
		}

		err = CreateDebugUtilsMessengerEXT(inst, &dbg_messenger_create_info, NULL, &dbg_messenger);
		switch (err) {
			case VK_SUCCESS:
				break;
			case VK_ERROR_OUT_OF_HOST_MEMORY:
				ERR_FAIL_V_MSG(ERR_CANT_CREATE,
						"CreateDebugUtilsMessengerEXT: out of host memory\n"
						"CreateDebugUtilsMessengerEXT Failure");
				break;
			default:
				ERR_FAIL_V_MSG(ERR_CANT_CREATE,
						"CreateDebugUtilsMessengerEXT: unknown failure\n"
						"CreateDebugUtilsMessengerEXT Failure");
				ERR_FAIL_V(ERR_CANT_CREATE);
				break;
		}
	}
	vkGetPhysicalDeviceProperties(gpu, &gpu_props);

	/* Call with NULL data to get count */
	vkGetPhysicalDeviceQueueFamilyProperties(gpu, &queue_family_count, NULL);
	ERR_FAIL_COND_V(queue_family_count == 0, ERR_CANT_CREATE);

	queue_props = (VkQueueFamilyProperties *)malloc(queue_family_count * sizeof(VkQueueFamilyProperties));
	vkGetPhysicalDeviceQueueFamilyProperties(gpu, &queue_family_count, queue_props);

	// Query fine-grained feature support for this device.
	//  If app has specific feature requirements it should check supported
	//  features based on this query
	VkPhysicalDeviceFeatures physDevFeatures;
	vkGetPhysicalDeviceFeatures(gpu, &physDevFeatures);

#define GET_INSTANCE_PROC_ADDR(inst, entrypoint)                                            \
	{                                                                                       \
		fp##entrypoint = (PFN_vk##entrypoint)vkGetInstanceProcAddr(inst, "vk" #entrypoint); \
		ERR_FAIL_COND_V_MSG(fp##entrypoint == NULL, ERR_CANT_CREATE,                        \
				"vkGetInstanceProcAddr failed to find vk" #entrypoint);                     \
	}

	GET_INSTANCE_PROC_ADDR(inst, GetPhysicalDeviceSurfaceSupportKHR);
	GET_INSTANCE_PROC_ADDR(inst, GetPhysicalDeviceSurfaceCapabilitiesKHR);
	GET_INSTANCE_PROC_ADDR(inst, GetPhysicalDeviceSurfaceFormatsKHR);
	GET_INSTANCE_PROC_ADDR(inst, GetPhysicalDeviceSurfacePresentModesKHR);
	GET_INSTANCE_PROC_ADDR(inst, GetSwapchainImagesKHR);

	return OK;
}

Error VulkanContext::_create_device() {

	VkResult err;
	float queue_priorities[1] = { 0.0 };
	VkDeviceQueueCreateInfo queues[2];
	queues[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
	queues[0].pNext = NULL;
	queues[0].queueFamilyIndex = graphics_queue_family_index;
	queues[0].queueCount = 1;
	queues[0].pQueuePriorities = queue_priorities;
	queues[0].flags = 0;

	VkDeviceCreateInfo sdevice = {
		.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
		.pNext = NULL,
		.queueCreateInfoCount = 1,
		.pQueueCreateInfos = queues,
		.enabledLayerCount = 0,
		.ppEnabledLayerNames = NULL,
		.enabledExtensionCount = enabled_extension_count,
		.ppEnabledExtensionNames = (const char *const *)extension_names,
		.pEnabledFeatures = NULL, // If specific features are required, pass them in here
	};
	if (separate_present_queue) {
		queues[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
		queues[1].pNext = NULL;
		queues[1].queueFamilyIndex = present_queue_family_index;
		queues[1].queueCount = 1;
		queues[1].pQueuePriorities = queue_priorities;
		queues[1].flags = 0;
		sdevice.queueCreateInfoCount = 2;
	}
	err = vkCreateDevice(gpu, &sdevice, NULL, &device);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	return OK;
}

Error VulkanContext::_create_swap_chain() {

	VkResult err = _create_surface(&surface, inst);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	// Iterate over each queue to learn whether it supports presenting:
	VkBool32 *supportsPresent = (VkBool32 *)malloc(queue_family_count * sizeof(VkBool32));
	for (uint32_t i = 0; i < queue_family_count; i++) {
		fpGetPhysicalDeviceSurfaceSupportKHR(gpu, i, surface, &supportsPresent[i]);
	}

	// Search for a graphics and a present queue in the array of queue
	// families, try to find one that supports both
	uint32_t graphicsQueueFamilyIndex = UINT32_MAX;
	uint32_t presentQueueFamilyIndex = UINT32_MAX;
	for (uint32_t i = 0; i < queue_family_count; i++) {
		if ((queue_props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) {
			if (graphicsQueueFamilyIndex == UINT32_MAX) {
				graphicsQueueFamilyIndex = i;
			}

			if (supportsPresent[i] == VK_TRUE) {
				graphicsQueueFamilyIndex = i;
				presentQueueFamilyIndex = i;
				break;
			}
		}
	}

	if (presentQueueFamilyIndex == UINT32_MAX) {
		// If didn't find a queue that supports both graphics and present, then
		// find a separate present queue.
		for (uint32_t i = 0; i < queue_family_count; ++i) {
			if (supportsPresent[i] == VK_TRUE) {
				presentQueueFamilyIndex = i;
				break;
			}
		}
	}

	// Generate error if could not find both a graphics and a present queue
	ERR_FAIL_COND_V_MSG(graphicsQueueFamilyIndex == UINT32_MAX || presentQueueFamilyIndex == UINT32_MAX, ERR_CANT_CREATE,
			"Could not find both graphics and present queues\n");

	graphics_queue_family_index = graphicsQueueFamilyIndex;
	present_queue_family_index = presentQueueFamilyIndex;
	separate_present_queue = (graphics_queue_family_index != present_queue_family_index);
	free(supportsPresent);

	_create_device();

	static PFN_vkGetDeviceProcAddr g_gdpa = NULL;
#define GET_DEVICE_PROC_ADDR(dev, entrypoint)                                                              \
	{                                                                                                      \
		if (!g_gdpa) g_gdpa = (PFN_vkGetDeviceProcAddr)vkGetInstanceProcAddr(inst, "vkGetDeviceProcAddr"); \
		fp##entrypoint = (PFN_vk##entrypoint)g_gdpa(dev, "vk" #entrypoint);                                \
		ERR_FAIL_COND_V_MSG(fp##entrypoint == NULL, ERR_CANT_CREATE,                                       \
				"vkGetDeviceProcAddr failed to find vk" #entrypoint);                                      \
	}

	GET_DEVICE_PROC_ADDR(device, CreateSwapchainKHR);
	GET_DEVICE_PROC_ADDR(device, DestroySwapchainKHR);
	GET_DEVICE_PROC_ADDR(device, GetSwapchainImagesKHR);
	GET_DEVICE_PROC_ADDR(device, AcquireNextImageKHR);
	GET_DEVICE_PROC_ADDR(device, QueuePresentKHR);
	if (VK_GOOGLE_display_timing_enabled) {
		GET_DEVICE_PROC_ADDR(device, GetRefreshCycleDurationGOOGLE);
		GET_DEVICE_PROC_ADDR(device, GetPastPresentationTimingGOOGLE);
	}

	vkGetDeviceQueue(device, graphics_queue_family_index, 0, &graphics_queue);

	if (!separate_present_queue) {
		present_queue = graphics_queue;
	} else {
		vkGetDeviceQueue(device, present_queue_family_index, 0, &present_queue);
	}

	// Get the list of VkFormat's that are supported:
	uint32_t formatCount;
	err = fpGetPhysicalDeviceSurfaceFormatsKHR(gpu, surface, &formatCount, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	VkSurfaceFormatKHR *surfFormats = (VkSurfaceFormatKHR *)malloc(formatCount * sizeof(VkSurfaceFormatKHR));
	err = fpGetPhysicalDeviceSurfaceFormatsKHR(gpu, surface, &formatCount, surfFormats);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	// If the format list includes just one entry of VK_FORMAT_UNDEFINED,
	// the surface has no preferred format.  Otherwise, at least one
	// supported format will be returned.
	if (true || (formatCount == 1 && surfFormats[0].format == VK_FORMAT_UNDEFINED)) {
		format = VK_FORMAT_B8G8R8A8_UNORM;
	} else {
		ERR_FAIL_COND_V(formatCount < 1, ERR_CANT_CREATE);
		format = surfFormats[0].format;
	}
	color_space = surfFormats[0].colorSpace;
	return OK;
}

Error VulkanContext::_create_semaphores() {
	VkResult err;

	// Create semaphores to synchronize acquiring presentable buffers before
	// rendering and waiting for drawing to be complete before presenting
	VkSemaphoreCreateInfo semaphoreCreateInfo = {
		.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
		.pNext = NULL,
		.flags = 0,
	};

	// Create fences that we can use to throttle if we get too far
	// ahead of the image presents
	VkFenceCreateInfo fence_ci = {
		.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, .pNext = NULL, .flags = VK_FENCE_CREATE_SIGNALED_BIT
	};
	for (uint32_t i = 0; i < FRAME_LAG; i++) {
		err = vkCreateFence(device, &fence_ci, NULL, &fences[i]);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

		err = vkCreateSemaphore(device, &semaphoreCreateInfo, NULL, &image_acquired_semaphores[i]);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

		err = vkCreateSemaphore(device, &semaphoreCreateInfo, NULL, &draw_complete_semaphores[i]);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

		if (separate_present_queue) {
			err = vkCreateSemaphore(device, &semaphoreCreateInfo, NULL, &image_ownership_semaphores[i]);
			ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		}
	}
	frame_index = 0;

	// Get Memory information and properties
	vkGetPhysicalDeviceMemoryProperties(gpu, &memory_properties);

	return OK;
}

Error VulkanContext::_prepare_buffers() {
	VkResult err;
	VkSwapchainKHR oldSwapchain = swapchain;

	// Check the surface capabilities and formats
	VkSurfaceCapabilitiesKHR surfCapabilities;
	err = fpGetPhysicalDeviceSurfaceCapabilitiesKHR(gpu, surface, &surfCapabilities);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	uint32_t presentModeCount;
	err = fpGetPhysicalDeviceSurfacePresentModesKHR(gpu, surface, &presentModeCount, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	VkPresentModeKHR *presentModes = (VkPresentModeKHR *)malloc(presentModeCount * sizeof(VkPresentModeKHR));
	ERR_FAIL_COND_V(!presentModes, ERR_CANT_CREATE);
	err = fpGetPhysicalDeviceSurfacePresentModesKHR(gpu, surface, &presentModeCount, presentModes);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	VkExtent2D swapchainExtent;
	// width and height are either both 0xFFFFFFFF, or both not 0xFFFFFFFF.
	if (surfCapabilities.currentExtent.width == 0xFFFFFFFF) {
		// If the surface size is undefined, the size is set to the size
		// of the images requested, which must fit within the minimum and
		// maximum values.
		swapchainExtent.width = width;
		swapchainExtent.height = height;

		if (swapchainExtent.width < surfCapabilities.minImageExtent.width) {
			swapchainExtent.width = surfCapabilities.minImageExtent.width;
		} else if (swapchainExtent.width > surfCapabilities.maxImageExtent.width) {
			swapchainExtent.width = surfCapabilities.maxImageExtent.width;
		}

		if (swapchainExtent.height < surfCapabilities.minImageExtent.height) {
			swapchainExtent.height = surfCapabilities.minImageExtent.height;
		} else if (swapchainExtent.height > surfCapabilities.maxImageExtent.height) {
			swapchainExtent.height = surfCapabilities.maxImageExtent.height;
		}
	} else {
		// If the surface size is defined, the swap chain size must match
		swapchainExtent = surfCapabilities.currentExtent;
		width = surfCapabilities.currentExtent.width;
		height = surfCapabilities.currentExtent.height;
	}

	if (width == 0 || height == 0) {
		is_minimized = true;
		return OK;
	} else {
		is_minimized = false;
	}

	// The FIFO present mode is guaranteed by the spec to be supported
	// and to have no tearing.  It's a great default present mode to use.
	VkPresentModeKHR swapchainPresentMode = VK_PRESENT_MODE_FIFO_KHR;

	//  There are times when you may wish to use another present mode.  The
	//  following code shows how to select them, and the comments provide some
	//  reasons you may wish to use them.
	//
	// It should be noted that Vulkan 1.0 doesn't provide a method for
	// synchronizing rendering with the presentation engine's display.  There
	// is a method provided for throttling rendering with the display, but
	// there are some presentation engines for which this method will not work.
	// If an application doesn't throttle its rendering, and if it renders much
	// faster than the refresh rate of the display, this can waste power on
	// mobile devices.  That is because power is being spent rendering images
	// that may never be seen.

	// VK_PRESENT_MODE_IMMEDIATE_KHR is for applications that don't care about
	// tearing, or have some way of synchronizing their rendering with the
	// display.
	// VK_PRESENT_MODE_MAILBOX_KHR may be useful for applications that
	// generally render a new presentable image every refresh cycle, but are
	// occasionally early.  In this case, the application wants the new image
	// to be displayed instead of the previously-queued-for-presentation image
	// that has not yet been displayed.
	// VK_PRESENT_MODE_FIFO_RELAXED_KHR is for applications that generally
	// render a new presentable image every refresh cycle, but are occasionally
	// late.  In this case (perhaps because of stuttering/latency concerns),
	// the application wants the late image to be immediately displayed, even
	// though that may mean some tearing.

	if (presentMode != swapchainPresentMode) {
		for (size_t i = 0; i < presentModeCount; ++i) {
			if (presentModes[i] == presentMode) {
				swapchainPresentMode = presentMode;
				break;
			}
		}
	}
	ERR_FAIL_COND_V_MSG(swapchainPresentMode != presentMode, ERR_CANT_CREATE, "Present mode specified is not supported\n");

	// Determine the number of VkImages to use in the swap chain.
	// Application desires to acquire 3 images at a time for triple
	// buffering
	uint32_t desiredNumOfSwapchainImages = 3;
	if (desiredNumOfSwapchainImages < surfCapabilities.minImageCount) {
		desiredNumOfSwapchainImages = surfCapabilities.minImageCount;
	}
	// If maxImageCount is 0, we can ask for as many images as we want;
	// otherwise we're limited to maxImageCount
	if ((surfCapabilities.maxImageCount > 0) && (desiredNumOfSwapchainImages > surfCapabilities.maxImageCount)) {
		// Application must settle for fewer images than desired:
		desiredNumOfSwapchainImages = surfCapabilities.maxImageCount;
	}

	VkSurfaceTransformFlagsKHR preTransform;
	if (surfCapabilities.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) {
		preTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR;
	} else {
		preTransform = surfCapabilities.currentTransform;
	}

	// Find a supported composite alpha mode - one of these is guaranteed to be set
	VkCompositeAlphaFlagBitsKHR compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR;
	VkCompositeAlphaFlagBitsKHR compositeAlphaFlags[4] = {
		VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
		VK_COMPOSITE_ALPHA_PRE_MULTIPLIED_BIT_KHR,
		VK_COMPOSITE_ALPHA_POST_MULTIPLIED_BIT_KHR,
		VK_COMPOSITE_ALPHA_INHERIT_BIT_KHR,
	};
	for (uint32_t i = 0; i < ARRAY_SIZE(compositeAlphaFlags); i++) {
		if (surfCapabilities.supportedCompositeAlpha & compositeAlphaFlags[i]) {
			compositeAlpha = compositeAlphaFlags[i];
			break;
		}
	}

	VkSwapchainCreateInfoKHR swapchain_ci = {
		.sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
		.pNext = NULL,
		.surface = surface,
		.minImageCount = desiredNumOfSwapchainImages,
		.imageFormat = format,
		.imageColorSpace = color_space,
		.imageExtent = {
				.width = swapchainExtent.width,
				.height = swapchainExtent.height,
		},
		.imageArrayLayers = 1,
		.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
		.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE,
		.queueFamilyIndexCount = 0,
		.pQueueFamilyIndices = NULL,
		.preTransform = (VkSurfaceTransformFlagBitsKHR)preTransform,
		.compositeAlpha = compositeAlpha,
		.presentMode = swapchainPresentMode,
		.clipped = true,
		.oldSwapchain = oldSwapchain,
	};
	uint32_t i;
	err = fpCreateSwapchainKHR(device, &swapchain_ci, NULL, &swapchain);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	// If we just re-created an existing swapchain, we should destroy the old
	// swapchain at this point.
	// Note: destroying the swapchain also cleans up all its associated
	// presentable images once the platform is done with them.
	if (oldSwapchain != VK_NULL_HANDLE) {
		fpDestroySwapchainKHR(device, oldSwapchain, NULL);
	}

	err = fpGetSwapchainImagesKHR(device, swapchain, &swapchainImageCount, NULL);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	VkImage *swapchainImages = (VkImage *)malloc(swapchainImageCount * sizeof(VkImage));
	ERR_FAIL_COND_V(!swapchainImages, ERR_CANT_CREATE);
	err = fpGetSwapchainImagesKHR(device, swapchain, &swapchainImageCount, swapchainImages);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	swapchain_image_resources =
			(SwapchainImageResources *)malloc(sizeof(SwapchainImageResources) * swapchainImageCount);
	ERR_FAIL_COND_V(!swapchain_image_resources, ERR_CANT_CREATE);

	for (i = 0; i < swapchainImageCount; i++) {
		VkImageViewCreateInfo color_image_view = {
			.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
			.pNext = NULL,
			.flags = 0,
			.viewType = VK_IMAGE_VIEW_TYPE_2D,
			.format = format,
			.components = {
					.r = VK_COMPONENT_SWIZZLE_R,
					.g = VK_COMPONENT_SWIZZLE_G,
					.b = VK_COMPONENT_SWIZZLE_B,
					.a = VK_COMPONENT_SWIZZLE_A,
			},
			.subresourceRange = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, .layerCount = 1 },
		};

		swapchain_image_resources[i].image = swapchainImages[i];

		color_image_view.image = swapchain_image_resources[i].image;

		err = vkCreateImageView(device, &color_image_view, NULL, &swapchain_image_resources[i].view);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	}

	if (VK_GOOGLE_display_timing_enabled) {
		VkRefreshCycleDurationGOOGLE rc_dur;
		err = fpGetRefreshCycleDurationGOOGLE(device, swapchain, &rc_dur);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		refresh_duration = rc_dur.refreshDuration;

		syncd_with_actual_presents = false;
		// Initially target 1X the refresh duration:
		target_IPD = refresh_duration;
		refresh_duration_multiplier = 1;
		prev_desired_present_time = 0;
		next_present_id = 1;
	}

	if (NULL != presentModes) {
		free(presentModes);
	}

	return OK;
}

Error VulkanContext::_prepare_framebuffers() {

	//for this, we only need color (no depth), since Godot does not render to the main
	//render buffer

	const VkAttachmentDescription attachment = {

		.flags = 0,
		.format = format,
		.samples = VK_SAMPLE_COUNT_1_BIT,
		.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR,
		.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
		.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
		.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE,
		.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
		.finalLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,

	};
	const VkAttachmentReference color_reference = {
		.attachment = 0,
		.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
	};

	const VkSubpassDescription subpass = {
		.flags = 0,
		.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
		.inputAttachmentCount = 0,
		.pInputAttachments = NULL,
		.colorAttachmentCount = 1,
		.pColorAttachments = &color_reference,
		.pResolveAttachments = NULL,
		.pDepthStencilAttachment = NULL,
		.preserveAttachmentCount = 0,
		.pPreserveAttachments = NULL,
	};
	const VkRenderPassCreateInfo rp_info = {
		.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
		.pNext = NULL,
		.flags = 0,
		.attachmentCount = 1,
		.pAttachments = &attachment,
		.subpassCount = 1,
		.pSubpasses = &subpass,
		.dependencyCount = 0,
		.pDependencies = NULL,
	};
	VkResult err;

	err = vkCreateRenderPass(device, &rp_info, NULL, &render_pass);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	for (uint32_t i = 0; i < swapchainImageCount; i++) {
		const VkFramebufferCreateInfo fb_info = {
			.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
			.pNext = NULL,
			.renderPass = render_pass,
			.attachmentCount = 1,
			.pAttachments = &swapchain_image_resources[i].view,
			.width = width,
			.height = height,
			.layers = 1,
		};

		err = vkCreateFramebuffer(device, &fb_info, NULL, &swapchain_image_resources[i].framebuffer);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	}

	return OK;
}

Error VulkanContext::_create_buffers() {

	Error error = _prepare_buffers();
	if (error != OK) {
		return error;
	}

	if (minimized) {
		prepared = false;
		return OK;
	}

	_prepare_framebuffers();

	if (separate_present_queue) {
		const VkCommandPoolCreateInfo present_cmd_pool_info = {
			.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
			.pNext = NULL,
			.flags = 0,
			.queueFamilyIndex = present_queue_family_index,
		};
		VkResult err = vkCreateCommandPool(device, &present_cmd_pool_info, NULL, &present_cmd_pool);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		const VkCommandBufferAllocateInfo present_cmd_info = {
			.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
			.pNext = NULL,
			.commandPool = present_cmd_pool,
			.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
			.commandBufferCount = 1,
		};
		for (uint32_t i = 0; i < swapchainImageCount; i++) {
			err = vkAllocateCommandBuffers(device, &present_cmd_info,
					&swapchain_image_resources[i].graphics_to_present_cmd);
			ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

			const VkCommandBufferBeginInfo cmd_buf_info = {
				.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
				.pNext = NULL,
				.flags = VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT,
				.pInheritanceInfo = NULL,
			};
			err = vkBeginCommandBuffer(swapchain_image_resources[i].graphics_to_present_cmd, &cmd_buf_info);
			ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

			VkImageMemoryBarrier image_ownership_barrier = { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
				.pNext = NULL,
				.srcAccessMask = 0,
				.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
				.oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
				.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
				.srcQueueFamilyIndex = graphics_queue_family_index,
				.dstQueueFamilyIndex = present_queue_family_index,
				.image = swapchain_image_resources[i].image,
				.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 } };

			vkCmdPipelineBarrier(swapchain_image_resources[i].graphics_to_present_cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
					VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, &image_ownership_barrier);
			err = vkEndCommandBuffer(swapchain_image_resources[i].graphics_to_present_cmd);
			ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		}
	}

	current_buffer = 0;
	prepared = true;

	return OK;
}

Error VulkanContext::initialize(int p_width, int p_height, bool p_minimized) {

	screen_width = p_width;
	screen_height = p_height;
	minimized = p_minimized;

	Error err = _create_physical_device();
	if (err) {
		return err;
	}

	err = _create_swap_chain();
	if (err) {
		return err;
	}

	err = _create_semaphores();
	if (err) {
		return err;
	}

	err = _create_buffers();
	if (err) {
		return err;
	}

	print_line("Vulkan context creation success o_O");
	return OK;
}

void VulkanContext::set_setup_buffer(const VkCommandBuffer &pCommandBuffer) {
	command_buffer_queue.write[0] = pCommandBuffer;
}

void VulkanContext::append_command_buffer(const VkCommandBuffer &pCommandBuffer) {

	if (command_buffer_queue.size() <= command_buffer_count) {
		command_buffer_queue.resize(command_buffer_count + 1);
	}

	command_buffer_queue.write[command_buffer_count] = pCommandBuffer;
	command_buffer_count++;
}

void VulkanContext::flush(bool p_flush_setup, bool p_flush_pending) {

	// ensure everything else pending is executed
	for (int i = 0; i < FRAME_LAG; i++) {
		int to_fence = (frame_index + i) % FRAME_LAG;
		vkWaitForFences(device, 1, &fences[to_fence], VK_TRUE, UINT64_MAX);
	}

	//flush the pending setup buffer

	if (p_flush_setup && command_buffer_queue[0]) {

		//use a fence to wait for everything done

		vkResetFences(device, 1, &fences[frame_index]);

		VkSubmitInfo submit_info;
		submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
		submit_info.pNext = NULL;
		submit_info.pWaitDstStageMask = NULL;
		submit_info.waitSemaphoreCount = 0;
		submit_info.pWaitSemaphores = NULL;
		submit_info.commandBufferCount = 1;
		submit_info.pCommandBuffers = command_buffer_queue.ptr();
		submit_info.signalSemaphoreCount = 0;
		submit_info.pSignalSemaphores = NULL;
		VkResult err = vkQueueSubmit(graphics_queue, 1, &submit_info, fences[frame_index]);
		command_buffer_queue.write[0] = NULL;
		ERR_FAIL_COND(err);
		vkWaitForFences(device, 1, &fences[frame_index], VK_TRUE, UINT64_MAX);
	}

	if (p_flush_pending && command_buffer_count > 1) {

		//use a fence to wait for everything done

		vkResetFences(device, 1, &fences[frame_index]);

		VkSubmitInfo submit_info;
		submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
		submit_info.pNext = NULL;
		submit_info.pWaitDstStageMask = NULL;
		submit_info.waitSemaphoreCount = 0;
		submit_info.pWaitSemaphores = NULL;
		submit_info.commandBufferCount = command_buffer_count - 1;
		submit_info.pCommandBuffers = command_buffer_queue.ptr() + 1;
		submit_info.signalSemaphoreCount = 0;
		submit_info.pSignalSemaphores = NULL;
		VkResult err = vkQueueSubmit(graphics_queue, 1, &submit_info, fences[frame_index]);
		command_buffer_queue.write[0] = NULL;
		ERR_FAIL_COND(err);
		vkWaitForFences(device, 1, &fences[frame_index], VK_TRUE, UINT64_MAX);

		command_buffer_count = 1;
	}
}

Error VulkanContext::swap_buffers() {

	//	print_line("swapbuffers?");
	VkResult err;

	// Ensure no more than FRAME_LAG renderings are outstanding
	vkWaitForFences(device, 1, &fences[frame_index], VK_TRUE, UINT64_MAX);
	vkResetFences(device, 1, &fences[frame_index]);

	do {
		// Get the index of the next available swapchain image:
		err =
				fpAcquireNextImageKHR(device, swapchain, UINT64_MAX,
						image_acquired_semaphores[frame_index], VK_NULL_HANDLE, &current_buffer);

		if (err == VK_ERROR_OUT_OF_DATE_KHR) {
			// swapchain is out of date (e.g. the window was resized) and
			// must be recreated:
			print_line("early out of data");
			resize_notify();
		} else if (err == VK_SUBOPTIMAL_KHR) {
			print_line("early suboptimal");
			// swapchain is not as optimal as it could be, but the platform's
			// presentation engine will still present the image correctly.
			break;
		} else {
			ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
		}
	} while (err != VK_SUCCESS);

#if 0
	if (VK_GOOGLE_display_timing_enabled) {
		// Look at what happened to previous presents, and make appropriate
		// adjustments in timing:
		DemoUpdateTargetIPD(demo);

		// Note: a real application would position its geometry to that it's in
		// the correct locatoin for when the next image is presented.  It might
		// also wait, so that there's less latency between any input and when
		// the next image is rendered/presented.  This demo program is so
		// simple that it doesn't do either of those.
	}
#endif
	// Wait for the image acquired semaphore to be signaled to ensure
	// that the image won't be rendered to until the presentation
	// engine has fully released ownership to the application, and it is
	// okay to render to the image.

	const VkCommandBuffer *commands_ptr = NULL;
	uint32_t commands_to_submit = 0;

	if (command_buffer_queue[0] == NULL) {
		//no setup command, but commands to submit, submit from the first and skip command
		if (command_buffer_count > 1) {
			commands_ptr = command_buffer_queue.ptr() + 1;
			commands_to_submit = command_buffer_count - 1;
		}
	} else {
		commands_ptr = command_buffer_queue.ptr();
		commands_to_submit = command_buffer_count;
	}

	VkPipelineStageFlags pipe_stage_flags;
	VkSubmitInfo submit_info;
	submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
	submit_info.pNext = NULL;
	submit_info.pWaitDstStageMask = &pipe_stage_flags;
	pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
	submit_info.waitSemaphoreCount = 1;
	submit_info.pWaitSemaphores = &image_acquired_semaphores[frame_index];
	submit_info.commandBufferCount = commands_to_submit;
	submit_info.pCommandBuffers = commands_ptr;
	submit_info.signalSemaphoreCount = 1;
	submit_info.pSignalSemaphores = &draw_complete_semaphores[frame_index];
	err = vkQueueSubmit(graphics_queue, 1, &submit_info, fences[frame_index]);
	ERR_FAIL_COND_V(err, ERR_CANT_CREATE);

	command_buffer_queue.write[0] = NULL;
	command_buffer_count = 1;

	if (separate_present_queue) {
		// If we are using separate queues, change image ownership to the
		// present queue before presenting, waiting for the draw complete
		// semaphore and signalling the ownership released semaphore when finished
		VkFence nullFence = VK_NULL_HANDLE;
		pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
		submit_info.waitSemaphoreCount = 1;
		submit_info.pWaitSemaphores = &draw_complete_semaphores[frame_index];
		submit_info.commandBufferCount = 1;
		submit_info.pCommandBuffers = &swapchain_image_resources[current_buffer].graphics_to_present_cmd;
		submit_info.signalSemaphoreCount = 1;
		submit_info.pSignalSemaphores = &image_ownership_semaphores[frame_index];
		err = vkQueueSubmit(present_queue, 1, &submit_info, nullFence);
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	}

	// If we are using separate queues we have to wait for image ownership,
	// otherwise wait for draw complete
	VkPresentInfoKHR present = {
		.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR,
		.pNext = NULL,
		.waitSemaphoreCount = 1,
		.pWaitSemaphores = (separate_present_queue) ? &image_ownership_semaphores[frame_index] : &draw_complete_semaphores[frame_index],
		.swapchainCount = 1,
		.pSwapchains = &swapchain,
		.pImageIndices = &current_buffer,
	};
#if 0
	if (VK_KHR_incremental_present_enabled) {
		// If using VK_KHR_incremental_present, we provide a hint of the region
		// that contains changed content relative to the previously-presented
		// image.  The implementation can use this hint in order to save
		// work/power (by only copying the region in the hint).  The
		// implementation is free to ignore the hint though, and so we must
		// ensure that the entire image has the correctly-drawn content.
		uint32_t eighthOfWidth = width / 8;
		uint32_t eighthOfHeight = height / 8;
		VkRectLayerKHR rect = {
			.offset.x = eighthOfWidth,
			.offset.y = eighthOfHeight,
			.extent.width = eighthOfWidth * 6,
			.extent.height = eighthOfHeight * 6,
			.layer = 0,
		};
		VkPresentRegionKHR region = {
			.rectangleCount = 1,
			.pRectangles = &rect,
		};
		VkPresentRegionsKHR regions = {
			.sType = VK_STRUCTURE_TYPE_PRESENT_REGIONS_KHR,
			.pNext = present.pNext,
			.swapchainCount = present.swapchainCount,
			.pRegions = &region,
		};
		present.pNext = &regions;
	}
#endif

#if 0
	if (VK_GOOGLE_display_timing_enabled) {
		VkPresentTimeGOOGLE ptime;
		if (prev_desired_present_time == 0) {
			// This must be the first present for this swapchain.
			//
			// We don't know where we are relative to the presentation engine's
			// display's refresh cycle.  We also don't know how long rendering
			// takes.  Let's make a grossly-simplified assumption that the
			// desiredPresentTime should be half way between now and
			// now+target_IPD.  We will adjust over time.
			uint64_t curtime = getTimeInNanoseconds();
			if (curtime == 0) {
				// Since we didn't find out the current time, don't give a
				// desiredPresentTime:
				ptime.desiredPresentTime = 0;
			} else {
				ptime.desiredPresentTime = curtime + (target_IPD >> 1);
			}
		} else {
			ptime.desiredPresentTime = (prev_desired_present_time + target_IPD);
		}
		ptime.presentID = next_present_id++;
		prev_desired_present_time = ptime.desiredPresentTime;

		VkPresentTimesInfoGOOGLE present_time = {
			.sType = VK_STRUCTURE_TYPE_PRESENT_TIMES_INFO_GOOGLE,
			.pNext = present.pNext,
			.swapchainCount = present.swapchainCount,
			.pTimes = &ptime,
		};
		if (VK_GOOGLE_display_timing_enabled) {
			present.pNext = &present_time;
		}
	}
#endif
	static int total_frames = 0;
	total_frames++;
	//	print_line("current buffer:  " + itos(current_buffer));
	err = fpQueuePresentKHR(present_queue, &present);

	frame_index += 1;
	frame_index %= FRAME_LAG;

	if (err == VK_ERROR_OUT_OF_DATE_KHR) {
		// swapchain is out of date (e.g. the window was resized) and
		// must be recreated:
		print_line("out of date");
		resize_notify();
	} else if (err == VK_SUBOPTIMAL_KHR) {
		// swapchain is not as optimal as it could be, but the platform's
		// presentation engine will still present the image correctly.
		print_line("suboptimal");
	} else {
		ERR_FAIL_COND_V(err, ERR_CANT_CREATE);
	}

	return OK;
}

void VulkanContext::resize_notify() {
}

VkDevice VulkanContext::get_device() {
	return device;
}

VkPhysicalDevice VulkanContext::get_physical_device() {
	return gpu;
}
int VulkanContext::get_frame_count() const {
	return swapchainImageCount;
}
uint32_t VulkanContext::get_graphics_queue() const {
	return graphics_queue_family_index;
}

int VulkanContext::get_screen_width(int p_screen) {
	return width;
}

int VulkanContext::get_screen_height(int p_screen) {
	return height;
}

VkFramebuffer VulkanContext::get_frame_framebuffer(int p_frame) {
	return swapchain_image_resources[p_frame].framebuffer;
}
VkFormat VulkanContext::get_screen_format() const {
	return format;
}

VkRenderPass VulkanContext::get_render_pass() {
	return render_pass;
}

VkPhysicalDeviceLimits VulkanContext::get_device_limits() const {
	return gpu_props.limits;
}

VulkanContext::VulkanContext() {
	presentMode = VK_PRESENT_MODE_FIFO_KHR;
	command_buffer_count = 0;
	instance_validation_layers = NULL;
	use_validation_layers = true;
	VK_KHR_incremental_present_enabled = true;
	VK_GOOGLE_display_timing_enabled = true;
	swapchain = NULL;
	prepared = false;

	command_buffer_queue.resize(1); //first one is the setup command always
	command_buffer_queue.write[0] = NULL;
	command_buffer_count = 1;
}