Implement shader caching

* Shader compilation is now cached. Subsequent loads take less than a millisecond.
* Improved game, editor and project manager startup time.
* Editor uses .godot/shader_cache to store shaders.
* Game uses user://shader_cache
* Project manager uses $config_dir/shader_cache
* Options to tweak shader caching in project settings.
* Editor path configuration moved from EditorSettings to new class, EditorPaths, so it can be available early on (before shaders are compiled).
* Reworked ShaderCompilerRD to ensure deterministic shader code creation (else shader may change and cache will be invalidated).
* Added shader compression with SMOLV: https://github.com/aras-p/smol-v

COPYRIGHT.txt

@@ -360,6 +360,12 @@ Comment: SMAZ
 Copyright: 2006-2009, Salvatore Sanfilippo
 License: BSD-3-clause
 
+Files: ./thirdparty/misc/smolv.cpp
+ ./thirdparty/misc/smolv.h
+Comment: SMOL-V
+Copyright: 2016-2020, Aras Pranckevicius
+License: public-domain or Unlicense or Expat
+
 Files: ./thirdparty/misc/stb_rect_pack.h
  ./thirdparty/misc/stb_vorbis.c
 Comment: stb libraries

core/SCsub

@@ -59,6 +59,7 @@ thirdparty_misc_sources = [
     "pcg.cpp",
     "polypartition.cpp",
     "clipper.cpp",
+    "smolv.cpp",
 ]
 thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources]
 env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources)

core/config/engine.cpp

@@ -31,6 +31,7 @@
 #include "engine.h"
 
 #include "core/authors.gen.h"
+#include "core/config/project_settings.h"
 #include "core/donors.gen.h"
 #include "core/license.gen.h"
 #include "core/version.h"
@@ -210,6 +211,13 @@ void Engine::get_singletons(List<Singleton> *p_singletons) {
 	}
 }
 
+void Engine::set_shader_cache_path(const String &p_path) {
+	shader_cache_path = p_path;
+}
+String Engine::get_shader_cache_path() const {
+	return shader_cache_path;
+}
+
 Engine *Engine::singleton = nullptr;
 
 Engine *Engine::get_singleton() {

core/config/engine.h

@@ -72,6 +72,8 @@ private:
 
 	static Engine *singleton;
 
+	String shader_cache_path;
+
 public:
 	static Engine *get_singleton();
 
@@ -121,6 +123,9 @@ public:
 	Dictionary get_license_info() const;
 	String get_license_text() const;
 
+	void set_shader_cache_path(const String &p_path);
+	String get_shader_cache_path() const;
+
 	bool is_abort_on_gpu_errors_enabled() const;
 	bool is_validation_layers_enabled() const;
 

doc/classes/EditorInterface.xml

@@ -57,6 +57,12 @@
 				[b]Note:[/b] This returns the main editor control containing the whole editor, not the 2D or 3D viewports specifically.
 			</description>
 		</method>
+		<method name="get_editor_paths">
+			<return type="EditorPaths">
+			</return>
+			<description>
+			</description>
+		</method>
 		<method name="get_editor_scale" qualifiers="const">
 			<return type="float">
 			</return>

doc/classes/EditorPaths.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="EditorPaths" inherits="Object" version="4.0">
+	<brief_description>
+	</brief_description>
+	<description>
+	</description>
+	<tutorials>
+	</tutorials>
+	<methods>
+		<method name="get_cache_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_config_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_data_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_self_contained_file" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_settings_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="is_self_contained" qualifiers="const">
+			<return type="bool">
+			</return>
+			<description>
+			</description>
+		</method>
+	</methods>
+	<constants>
+	</constants>
+</class>

doc/classes/EditorSettings.xml

@@ -124,15 +124,6 @@
 				Returns the value of the setting specified by [code]name[/code]. This is equivalent to using [method Object.get] on the EditorSettings instance.
 			</description>
 		</method>
-		<method name="get_settings_dir" qualifiers="const">
-			<return type="String">
-			</return>
-			<description>
-				Gets the global settings path for the engine. Inside this path, you can find some standard paths such as:
-				[code]settings/tmp[/code] - Used for temporary storage of files
-				[code]settings/templates[/code] - Where export templates are located
-			</description>
-		</method>
 		<method name="has_setting" qualifiers="const">
 			<return type="bool">
 			</return>

doc/classes/ProjectSettings.xml

@@ -1502,6 +1502,16 @@
 		<member name="rendering/reflections/sky_reflections/texture_array_reflections.mobile" type="bool" setter="" getter="" default="false">
 			Lower-end override for [member rendering/reflections/sky_reflections/texture_array_reflections] on mobile devices, due to performance concerns or driver support.
 		</member>
+		<member name="rendering/shader_compiler/shader_cache/compress" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/enabled" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/strip_debug" type="bool" setter="" getter="" default="false">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/strip_debug.release" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/use_zstd_compression" type="bool" setter="" getter="" default="true">
+		</member>
 		<member name="rendering/shading/overrides/force_blinn_over_ggx" type="bool" setter="" getter="" default="false">
 			If [code]true[/code], uses faster but lower-quality Blinn model to generate blurred reflections instead of the GGX model.
 		</member>

editor/editor_export.cpp

@@ -1065,7 +1065,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
 				}
 			} else {
 				// Use default text server data.
-				String icu_data_file = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp_icu_data");
+				String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp_icu_data");
 				TS->save_support_data(icu_data_file);
 				Vector<uint8_t> array = FileAccess::get_file_as_array(icu_data_file);
 				err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key);
@@ -1078,7 +1078,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
 	}
 
 	String config_file = "project.binary";
-	String engine_cfb = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp" + config_file);
+	String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp" + config_file);
 	ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list);
 	Vector<uint8_t> data = FileAccess::get_file_as_array(engine_cfb);
 	DirAccess::remove_file_or_error(engine_cfb);
@@ -1100,9 +1100,9 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 
 	// Create the temporary export directory if it doesn't exist.
 	DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
-	da->make_dir_recursive(EditorSettings::get_singleton()->get_cache_dir());
+	da->make_dir_recursive(EditorPaths::get_singleton()->get_cache_dir());
 
-	String tmppath = EditorSettings::get_singleton()->get_cache_dir().plus_file("packtmp");
+	String tmppath = EditorPaths::get_singleton()->get_cache_dir().plus_file("packtmp");
 	FileAccess *ftmp = FileAccess::open(tmppath, FileAccess::WRITE);
 	ERR_FAIL_COND_V_MSG(!ftmp, ERR_CANT_CREATE, "Cannot create file '" + tmppath + "'.");
 
@@ -1984,7 +1984,7 @@ void EditorExportTextSceneToBinaryPlugin::_export_file(const String &p_path, con
 	if (!convert) {
 		return;
 	}
-	String tmp_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpfile.res");
+	String tmp_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpfile.res");
 	Error err = ResourceFormatLoaderText::convert_file_to_binary(p_path, tmp_path);
 	if (err != OK) {
 		DirAccess::remove_file_or_error(tmp_path);

editor/editor_node.cpp

@@ -80,6 +80,7 @@
 #include "editor/editor_inspector.h"
 #include "editor/editor_layouts_dialog.h"
 #include "editor/editor_log.h"
+#include "editor/editor_paths.h"
 #include "editor/editor_plugin.h"
 #include "editor/editor_properties.h"
 #include "editor/editor_resource_picker.h"
@@ -1457,7 +1458,7 @@ void EditorNode::_save_scene_with_preview(String p_file, int p_idx) {
 			img->convert(Image::FORMAT_RGB8);
 
 			//save thumbnail directly, as thumbnailer may not update due to actual scene not changing md5
-			String temp_path = EditorSettings::get_singleton()->get_cache_dir();
+			String temp_path = EditorPaths::get_singleton()->get_cache_dir();
 			String cache_base = ProjectSettings::get_singleton()->globalize_path(p_file).md5_text();
 			cache_base = temp_path.plus_file("resthumb-" + cache_base);
 
@@ -2745,10 +2746,10 @@ void EditorNode::_menu_option_confirm(int p_option, bool p_confirmed) {
 			settings_config_dialog->popup_edit_settings();
 		} break;
 		case SETTINGS_EDITOR_DATA_FOLDER: {
-			OS::get_singleton()->shell_open(String("file://") + EditorSettings::get_singleton()->get_data_dir());
+			OS::get_singleton()->shell_open(String("file://") + EditorPaths::get_singleton()->get_data_dir());
 		} break;
 		case SETTINGS_EDITOR_CONFIG_FOLDER: {
-			OS::get_singleton()->shell_open(String("file://") + EditorSettings::get_singleton()->get_settings_dir());
+			OS::get_singleton()->shell_open(String("file://") + EditorPaths::get_singleton()->get_settings_dir());
 		} break;
 		case SETTINGS_MANAGE_EXPORT_TEMPLATES: {
 			export_template_manager->popup_manager();
@@ -3727,10 +3728,15 @@ bool EditorNode::is_scene_in_use(const String &p_path) {
 	return false;
 }
 
+void EditorNode::register_editor_paths(bool p_for_project_manager) {
+	EditorPaths::create(p_for_project_manager);
+}
+
 void EditorNode::register_editor_types() {
 	ResourceLoader::set_timestamp_on_load(true);
 	ResourceSaver::set_timestamp_on_save(true);
 
+	ClassDB::register_class<EditorPaths>();
 	ClassDB::register_class<EditorPlugin>();
 	ClassDB::register_class<EditorTranslationParserPlugin>();
 	ClassDB::register_class<EditorImportPlugin>();
@@ -3774,6 +3780,9 @@ void EditorNode::register_editor_types() {
 
 void EditorNode::unregister_editor_types() {
 	_init_callbacks.clear();
+	if (EditorPaths::get_singleton()) {
+		EditorPaths::free();
+	}
 }
 
 void EditorNode::stop_child_process(OS::ProcessID p_pid) {

editor/editor_node.h

@@ -798,6 +798,7 @@ public:
 
 	Error export_preset(const String &p_preset, const String &p_path, bool p_debug, bool p_pack_only);
 
+	static void register_editor_paths(bool p_for_project_manager);
 	static void register_editor_types();
 	static void unregister_editor_types();
 

editor/editor_paths.cpp

@@ -0,0 +1,156 @@
+/*************************************************************************/
+/*  editor_paths.cpp                                                     */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "editor_paths.h"
+#include "core/os/dir_access.h"
+#include "core/os/os.h"
+
+EditorPaths *EditorPaths::singleton = nullptr;
+
+bool EditorPaths::are_paths_valid() const {
+	return paths_valid;
+}
+
+String EditorPaths::get_settings_dir() const {
+	return settings_dir;
+}
+String EditorPaths::get_data_dir() const {
+	return data_dir;
+}
+String EditorPaths::get_config_dir() const {
+	return config_dir;
+}
+String EditorPaths::get_cache_dir() const {
+	return cache_dir;
+}
+bool EditorPaths::is_self_contained() const {
+	return self_contained;
+}
+String EditorPaths::get_self_contained_file() const {
+	return self_contained_file;
+}
+
+void EditorPaths::create(bool p_for_project_manager) {
+	ERR_FAIL_COND(singleton != nullptr);
+	memnew(EditorPaths(p_for_project_manager));
+}
+void EditorPaths::free() {
+	ERR_FAIL_COND(singleton == nullptr);
+	memdelete(singleton);
+}
+
+void EditorPaths::_bind_methods() {
+	ClassDB::bind_method(D_METHOD("get_settings_dir"), &EditorPaths::get_settings_dir);
+	ClassDB::bind_method(D_METHOD("get_data_dir"), &EditorPaths::get_data_dir);
+	ClassDB::bind_method(D_METHOD("get_config_dir"), &EditorPaths::get_config_dir);
+	ClassDB::bind_method(D_METHOD("get_cache_dir"), &EditorPaths::get_cache_dir);
+	ClassDB::bind_method(D_METHOD("is_self_contained"), &EditorPaths::is_self_contained);
+	ClassDB::bind_method(D_METHOD("get_self_contained_file"), &EditorPaths::get_self_contained_file);
+}
+
+EditorPaths::EditorPaths(bool p_for_project_mamanger) {
+	singleton = this;
+
+	String exe_path = OS::get_singleton()->get_executable_path().get_base_dir();
+	{
+		DirAccessRef d = DirAccess::create_for_path(exe_path);
+
+		if (d->file_exists(exe_path + "/._sc_")) {
+			self_contained = true;
+			self_contained_file = exe_path + "/._sc_";
+		} else if (d->file_exists(exe_path + "/_sc_")) {
+			self_contained = true;
+			self_contained_file = exe_path + "/_sc_";
+		}
+	}
+
+	String data_path;
+	String config_path;
+	String cache_path;
+
+	if (self_contained) {
+		// editor is self contained, all in same folder
+		data_path = exe_path;
+		data_dir = data_path.plus_file("editor_data");
+		config_path = exe_path;
+		config_dir = data_dir;
+		cache_path = exe_path;
+		cache_dir = data_dir.plus_file("cache");
+	} else {
+		// Typically XDG_DATA_HOME or %APPDATA%
+		data_path = OS::get_singleton()->get_data_path();
+		data_dir = data_path.plus_file(OS::get_singleton()->get_godot_dir_name());
+		// Can be different from data_path e.g. on Linux or macOS
+		config_path = OS::get_singleton()->get_config_path();
+		config_dir = config_path.plus_file(OS::get_singleton()->get_godot_dir_name());
+		// Can be different from above paths, otherwise a subfolder of data_dir
+		cache_path = OS::get_singleton()->get_cache_path();
+		if (cache_path == data_path) {
+			cache_dir = data_dir.plus_file("cache");
+		} else {
+			cache_dir = cache_path.plus_file(OS::get_singleton()->get_godot_dir_name());
+		}
+	}
+
+	paths_valid = (data_path != "" && config_path != "" && cache_path != "");
+
+	if (paths_valid) {
+		DirAccessRef dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
+		if (dir->change_dir(data_dir) != OK) {
+			dir->make_dir_recursive(data_dir);
+			if (dir->change_dir(data_dir) != OK) {
+				ERR_PRINT("Cannot create data directory!");
+				paths_valid = false;
+			}
+		}
+
+		// Validate/create cache dir
+
+		if (dir->change_dir(EditorPaths::get_singleton()->get_cache_dir()) != OK) {
+			dir->make_dir_recursive(cache_dir);
+			if (dir->change_dir(cache_dir) != OK) {
+				ERR_PRINT("Cannot create cache directory!");
+			}
+		}
+
+		if (p_for_project_mamanger) {
+			Engine::get_singleton()->set_shader_cache_path(get_data_dir());
+		} else {
+			DirAccessRef dir2 = DirAccess::open("res://");
+			if (dir2->change_dir(".godot") != OK) { //ensure the .godot subdir exists
+				if (dir2->make_dir(".godot") != OK) {
+					ERR_PRINT("Cannot create res://.godot directory!");
+				}
+			}
+
+			Engine::get_singleton()->set_shader_cache_path("res://.godot");
+		}
+	}
+}

editor/editor_paths.h

@@ -0,0 +1,72 @@
+/*************************************************************************/
+/*  editor_paths.h                                                       */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef EDITORPATHS_H
+#define EDITORPATHS_H
+
+#include "core/config/engine.h"
+
+class EditorPaths : public Object {
+	GDCLASS(EditorPaths, Object)
+
+	bool paths_valid = false;
+	String settings_dir;
+	String data_dir; //editor data dir
+	String config_dir; //editor config dir
+	String cache_dir; //editor cache dir
+	bool self_contained = false; //true if running self contained
+	String self_contained_file; //self contained file with configuration
+
+	static EditorPaths *singleton;
+
+protected:
+	static void _bind_methods();
+
+public:
+	bool are_paths_valid() const;
+
+	String get_settings_dir() const;
+	String get_data_dir() const;
+	String get_config_dir() const;
+	String get_cache_dir() const;
+	bool is_self_contained() const;
+	String get_self_contained_file() const;
+
+	static EditorPaths *get_singleton() {
+		return singleton;
+	}
+
+	static void create(bool p_for_project_manager);
+	static void free();
+
+	EditorPaths(bool p_for_project_mamanger = false);
+};
+
+#endif // EDITORPATHS_H

editor/editor_plugin.cpp

@@ -32,6 +32,7 @@
 
 #include "editor/editor_export.h"
 #include "editor/editor_node.h"
+#include "editor/editor_paths.h"
 #include "editor/editor_settings.h"
 #include "editor/filesystem_dock.h"
 #include "editor/project_settings_editor.h"
@@ -257,6 +258,9 @@ EditorSelection *EditorInterface::get_selection() {
 Ref<EditorSettings> EditorInterface::get_editor_settings() {
 	return EditorSettings::get_singleton();
 }
+EditorPaths *EditorInterface::get_editor_paths() {
+	return EditorPaths::get_singleton();
+}
 
 EditorResourcePreview *EditorInterface::get_resource_previewer() {
 	return EditorResourcePreview::get_singleton();
@@ -335,6 +339,7 @@ void EditorInterface::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("get_selected_path"), &EditorInterface::get_selected_path);
 	ClassDB::bind_method(D_METHOD("get_current_path"), &EditorInterface::get_current_path);
 	ClassDB::bind_method(D_METHOD("get_file_system_dock"), &EditorInterface::get_file_system_dock);
+	ClassDB::bind_method(D_METHOD("get_editor_paths"), &EditorInterface::get_editor_paths);
 
 	ClassDB::bind_method(D_METHOD("set_plugin_enabled", "plugin", "enabled"), &EditorInterface::set_plugin_enabled);
 	ClassDB::bind_method(D_METHOD("is_plugin_enabled", "plugin"), &EditorInterface::is_plugin_enabled);

editor/editor_plugin.h

@@ -54,6 +54,7 @@ class EditorNode3DGizmoPlugin;
 class EditorResourcePreview;
 class EditorFileSystem;
 class EditorToolAddons;
+class EditorPaths;
 class FileSystemDock;
 class ScriptEditor;
 
@@ -95,6 +96,7 @@ public:
 	EditorSelection *get_selection();
 	//EditorImportExport *get_import_export();
 	Ref<EditorSettings> get_editor_settings();
+	EditorPaths *get_editor_paths();
 	EditorResourcePreview *get_resource_previewer();
 	EditorFileSystem *get_resource_file_system();
 

editor/editor_resource_preview.cpp

@@ -241,7 +241,7 @@ void EditorResourcePreview::_thread() {
 					_preview_ready(item.path + ":" + itos(item.resource->hash_edited_version()), texture, small_texture, item.id, item.function, item.userdata);
 
 				} else {
-					String temp_path = EditorSettings::get_singleton()->get_cache_dir();
+					String temp_path = EditorPaths::get_singleton()->get_cache_dir();
 					String cache_base = ProjectSettings::get_singleton()->globalize_path(item.path).md5_text();
 					cache_base = temp_path.plus_file("resthumb-" + cache_base);
 

editor/editor_settings.cpp

@@ -902,67 +902,26 @@ void EditorSettings::create() {
 		return; //pointless
 	}
 
-	DirAccess *dir = nullptr;
-
-	String data_path;
-	String data_dir;
-	String config_path;
-	String config_dir;
-	String cache_path;
-	String cache_dir;
-
 	Ref<ConfigFile> extra_config = memnew(ConfigFile);
 
-	String exe_path = OS::get_singleton()->get_executable_path().get_base_dir();
+	if (EditorPaths::get_singleton()->is_self_contained()) {
-	DirAccess *d = DirAccess::create_for_path(exe_path);
+		Error err = extra_config->load(EditorPaths::get_singleton()->get_self_contained_file());
-	bool self_contained = false;
-
-	if (d->file_exists(exe_path + "/._sc_")) {
-		self_contained = true;
-		Error err = extra_config->load(exe_path + "/._sc_");
 		if (err != OK) {
-			ERR_PRINT("Can't load config from path '" + exe_path + "/._sc_'.");
+			ERR_PRINT("Can't load extra config from path :" + EditorPaths::get_singleton()->get_self_contained_file());
-		}
-	} else if (d->file_exists(exe_path + "/_sc_")) {
-		self_contained = true;
-		Error err = extra_config->load(exe_path + "/_sc_");
-		if (err != OK) {
-			ERR_PRINT("Can't load config from path '" + exe_path + "/_sc_'.");
 		}
 	}
-	memdelete(d);
 
-	if (self_contained) {
+	DirAccess *dir = nullptr;
-		// editor is self contained, all in same folder
-		data_path = exe_path;
-		data_dir = data_path.plus_file("editor_data");
-		config_path = exe_path;
-		config_dir = data_dir;
-		cache_path = exe_path;
-		cache_dir = data_dir.plus_file("cache");
-	} else {
-		// Typically XDG_DATA_HOME or %APPDATA%
-		data_path = OS::get_singleton()->get_data_path();
-		data_dir = data_path.plus_file(OS::get_singleton()->get_godot_dir_name());
-		// Can be different from data_path e.g. on Linux or macOS
-		config_path = OS::get_singleton()->get_config_path();
-		config_dir = config_path.plus_file(OS::get_singleton()->get_godot_dir_name());
-		// Can be different from above paths, otherwise a subfolder of data_dir
-		cache_path = OS::get_singleton()->get_cache_path();
-		if (cache_path == data_path) {
-			cache_dir = data_dir.plus_file("cache");
-		} else {
-			cache_dir = cache_path.plus_file(OS::get_singleton()->get_godot_dir_name());
-		}
-	}
 
 	ClassDB::register_class<EditorSettings>(); //otherwise it can't be unserialized
 
 	String config_file_path;
 
-	if (data_path != "" && config_path != "" && cache_path != "") {
+	if (EditorPaths::get_singleton()->are_paths_valid()) {
 		// Validate/create data dir and subdirectories
 
+		String data_dir = EditorPaths::get_singleton()->get_data_dir();
+
 		dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
 		if (dir->change_dir(data_dir) != OK) {
 			dir->make_dir_recursive(data_dir);
@@ -979,22 +938,11 @@ void EditorSettings::create() {
 			dir->change_dir("..");
 		}
 
-		// Validate/create cache dir
-
-		if (dir->change_dir(cache_dir) != OK) {
-			dir->make_dir_recursive(cache_dir);
-			if (dir->change_dir(cache_dir) != OK) {
-				ERR_PRINT("Cannot create cache directory!");
-				memdelete(dir);
-				goto fail;
-			}
-		}
-
 		// Validate/create config dir and subdirectories
 
-		if (dir->change_dir(config_dir) != OK) {
+		if (dir->change_dir(EditorPaths::get_singleton()->get_config_dir()) != OK) {
-			dir->make_dir_recursive(config_dir);
+			dir->make_dir_recursive(EditorPaths::get_singleton()->get_config_dir());
-			if (dir->change_dir(config_dir) != OK) {
+			if (dir->change_dir(EditorPaths::get_singleton()->get_config_dir()) != OK) {
 				ERR_PRINT("Cannot create config directory!");
 				memdelete(dir);
 				goto fail;
@@ -1035,7 +983,7 @@ void EditorSettings::create() {
 		// Validate editor config file
 
 		String config_file_name = "editor_settings-" + itos(VERSION_MAJOR) + ".tres";
-		config_file_path = config_dir.plus_file(config_file_name);
+		config_file_path = EditorPaths::get_singleton()->get_config_dir().plus_file(config_file_name);
 		if (!dir->file_exists(config_file_name)) {
 			memdelete(dir);
 			goto fail;
@@ -1052,9 +1000,6 @@ void EditorSettings::create() {
 
 		singleton->save_changed_setting = true;
 		singleton->config_file_path = config_file_path;
-		singleton->settings_dir = config_dir;
-		singleton->data_dir = data_dir;
-		singleton->cache_dir = cache_dir;
 
 		print_verbose("EditorSettings: Load OK!");
 
@@ -1069,6 +1014,8 @@ void EditorSettings::create() {
 fail:
 
 	// patch init projects
+	String exe_path = OS::get_singleton()->get_executable_path().get_base_dir();
+
 	if (extra_config->has_section("init_projects")) {
 		Vector<String> list = extra_config->get_value("init_projects", "list");
 		for (int i = 0; i < list.size(); i++) {
@@ -1080,9 +1027,6 @@ fail:
 	singleton = Ref<EditorSettings>(memnew(EditorSettings));
 	singleton->save_changed_setting = true;
 	singleton->config_file_path = config_file_path;
-	singleton->settings_dir = config_dir;
-	singleton->data_dir = data_dir;
-	singleton->cache_dir = cache_dir;
 	singleton->_load_defaults(extra_config);
 	singleton->setup_language();
 	singleton->setup_network();
@@ -1312,30 +1256,22 @@ void EditorSettings::add_property_hint(const PropertyInfo &p_hint) {
 
 // Data directories
 
-String EditorSettings::get_data_dir() const {
-	return data_dir;
-}
-
 String EditorSettings::get_templates_dir() const {
-	return get_data_dir().plus_file("templates");
+	return EditorPaths::get_singleton()->get_data_dir().plus_file("templates");
 }
 
 // Config directories
 
-String EditorSettings::get_settings_dir() const {
-	return settings_dir;
-}
-
 String EditorSettings::get_project_settings_dir() const {
 	return EditorSettings::PROJECT_EDITOR_SETTINGS_PATH;
 }
 
 String EditorSettings::get_text_editor_themes_dir() const {
-	return get_settings_dir().plus_file("text_editor_themes");
+	return EditorPaths::get_singleton()->get_settings_dir().plus_file("text_editor_themes");
 }
 
 String EditorSettings::get_script_templates_dir() const {
-	return get_settings_dir().plus_file("script_templates");
+	return EditorPaths::get_singleton()->get_settings_dir().plus_file("script_templates");
 }
 
 String EditorSettings::get_project_script_templates_dir() const {
@@ -1344,12 +1280,8 @@ String EditorSettings::get_project_script_templates_dir() const {
 
 // Cache directory
 
-String EditorSettings::get_cache_dir() const {
-	return cache_dir;
-}
-
 String EditorSettings::get_feature_profiles_dir() const {
-	return get_settings_dir().plus_file("feature_profiles");
+	return EditorPaths::get_singleton()->get_settings_dir().plus_file("feature_profiles");
 }
 
 // Metadata
@@ -1576,7 +1508,7 @@ Vector<String> EditorSettings::get_script_templates(const String &p_extension, c
 }
 
 String EditorSettings::get_editor_layouts_config() const {
-	return get_settings_dir().plus_file("editor_layouts.cfg");
+	return EditorPaths::get_singleton()->get_settings_dir().plus_file("editor_layouts.cfg");
 }
 
 // Shortcuts
@@ -1778,7 +1710,6 @@ void EditorSettings::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("property_get_revert", "name"), &EditorSettings::property_get_revert);
 	ClassDB::bind_method(D_METHOD("add_property_info", "info"), &EditorSettings::_add_property_info_bind);
 
-	ClassDB::bind_method(D_METHOD("get_settings_dir"), &EditorSettings::get_settings_dir);
 	ClassDB::bind_method(D_METHOD("get_project_settings_dir"), &EditorSettings::get_project_settings_dir);
 
 	ClassDB::bind_method(D_METHOD("set_project_metadata", "section", "key", "data"), &EditorSettings::set_project_metadata);

editor/editor_settings.h

@@ -36,6 +36,7 @@
 #include "core/object/class_db.h"
 #include "core/os/thread_safe.h"
 #include "core/string/translation.h"
+#include "editor/editor_paths.h"
 #include "scene/gui/shortcut.h"
 
 class EditorPlugin;
@@ -87,12 +88,7 @@ private:
 	mutable Map<String, Ref<Shortcut>> shortcuts;
 	Map<String, List<Ref<InputEvent>>> builtin_action_overrides;
 
-	String resource_path;
-	String settings_dir;
-	String data_dir;
-	String cache_dir;
 	String config_file_path;
-	String project_config_dir;
 
 	Vector<String> favorites;
 	Vector<String> recent_dirs;
@@ -153,12 +149,10 @@ public:
 
 	String get_data_dir() const;
 	String get_templates_dir() const;
-	String get_settings_dir() const;
 	String get_project_settings_dir() const;
 	String get_text_editor_themes_dir() const;
 	String get_script_templates_dir() const;
 	String get_project_script_templates_dir() const;
-	String get_cache_dir() const;
 	String get_feature_profiles_dir() const;
 
 	void set_project_metadata(const String &p_section, const String &p_key, Variant p_data);

editor/export_template_manager.cpp

@@ -444,7 +444,7 @@ void ExportTemplateManager::_begin_template_download(const String &p_url) {
 	}
 
 	download_data.clear();
-	download_templates->set_download_file(EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp_templates.tpz"));
+	download_templates->set_download_file(EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp_templates.tpz"));
 	download_templates->set_use_threads(true);
 
 	Error err = download_templates->request(p_url);

editor/plugins/asset_library_editor_plugin.cpp

@@ -464,7 +464,7 @@ void EditorAssetLibraryItemDownload::_make_request() {
 	retry->hide();
 
 	download->cancel_request();
-	download->set_download_file(EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp_asset_" + itos(asset_id)) + ".zip");
+	download->set_download_file(EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp_asset_" + itos(asset_id)) + ".zip");
 
 	Error err = download->request(host);
 	if (err != OK) {
@@ -702,7 +702,7 @@ void EditorAssetLibrary::_image_update(bool use_cache, bool final, const PackedB
 		PackedByteArray image_data = p_data;
 
 		if (use_cache) {
-			String cache_filename_base = EditorSettings::get_singleton()->get_cache_dir().plus_file("assetimage_" + image_queue[p_queue_id].image_url.md5_text());
+			String cache_filename_base = EditorPaths::get_singleton()->get_cache_dir().plus_file("assetimage_" + image_queue[p_queue_id].image_url.md5_text());
 
 			FileAccess *file = FileAccess::open(cache_filename_base + ".data", FileAccess::READ);
 
@@ -781,7 +781,7 @@ void EditorAssetLibrary::_image_request_completed(int p_status, int p_code, cons
 		if (p_code != HTTPClient::RESPONSE_NOT_MODIFIED) {
 			for (int i = 0; i < headers.size(); i++) {
 				if (headers[i].findn("ETag:") == 0) { // Save etag
-					String cache_filename_base = EditorSettings::get_singleton()->get_cache_dir().plus_file("assetimage_" + image_queue[p_queue_id].image_url.md5_text());
+					String cache_filename_base = EditorPaths::get_singleton()->get_cache_dir().plus_file("assetimage_" + image_queue[p_queue_id].image_url.md5_text());
 					String new_etag = headers[i].substr(headers[i].find(":") + 1, headers[i].length()).strip_edges();
 					FileAccess *file;
 
@@ -829,7 +829,7 @@ void EditorAssetLibrary::_update_image_queue() {
 	List<int> to_delete;
 	for (Map<int, ImageQueue>::Element *E = image_queue.front(); E; E = E->next()) {
 		if (!E->get().active && current_images < max_images) {
-			String cache_filename_base = EditorSettings::get_singleton()->get_cache_dir().plus_file("assetimage_" + E->get().image_url.md5_text());
+			String cache_filename_base = EditorPaths::get_singleton()->get_cache_dir().plus_file("assetimage_" + E->get().image_url.md5_text());
 			Vector<String> headers;
 
 			if (FileAccess::exists(cache_filename_base + ".etag") && FileAccess::exists(cache_filename_base + ".data")) {

editor/plugins/editor_preview_plugins.cpp

@@ -265,7 +265,7 @@ Ref<Texture2D> EditorPackedScenePreviewPlugin::generate(const RES &p_from, const
 }
 
 Ref<Texture2D> EditorPackedScenePreviewPlugin::generate_from_path(const String &p_path, const Size2 &p_size) const {
-	String temp_path = EditorSettings::get_singleton()->get_cache_dir();
+	String temp_path = EditorPaths::get_singleton()->get_cache_dir();
 	String cache_base = ProjectSettings::get_singleton()->globalize_path(p_path).md5_text();
 	cache_base = temp_path.plus_file("resthumb-" + cache_base);
 

main/main.cpp

@@ -1445,6 +1445,12 @@ Error Main::setup2(Thread::ID p_main_tid_override) {
 	}
 #endif
 
+#ifdef TOOLS_ENABLED
+	if (editor || project_manager) {
+		EditorNode::register_editor_paths(project_manager);
+	}
+#endif
+
 	/* Determine text driver */
 
 	if (text_driver == "") {

modules/glslang/register_types.cpp

@@ -179,11 +179,18 @@ static Vector<uint8_t> _compile_shader_glsl(RenderingDevice::ShaderStage p_stage
 	return ret;
 }
 
+static String _get_cache_key_function_glsl(const RenderingDevice::Capabilities *p_capabilities) {
+	String version;
+	version = "SpirVGen=" + itos(glslang::GetSpirvGeneratorVersion()) + ", major=" + itos(p_capabilities->version_major) + ", minor=" + itos(p_capabilities->version_minor) + " , subgroup_size=" + itos(p_capabilities->subgroup_operations) + " , subgroup_ops=" + itos(p_capabilities->subgroup_operations) + " , subgroup_in_shaders=" + itos(p_capabilities->subgroup_in_shaders);
+	return version;
+}
+
 void preregister_glslang_types() {
 	// initialize in case it's not initialized. This is done once per thread
 	// and it's safe to call multiple times
 	glslang::InitializeProcess();
 	RenderingDevice::shader_set_compile_function(_compile_shader_glsl);
+	RenderingDevice::shader_set_get_cache_key_function(_get_cache_key_function_glsl);
 }
 
 void register_glslang_types() {

modules/mono/godotsharp_dirs.cpp

@@ -63,8 +63,8 @@ String _get_expected_build_config() {
 
 String _get_mono_user_dir() {
 #ifdef TOOLS_ENABLED
-	if (EditorSettings::get_singleton()) {
+	if (EditorPaths::get_singleton()) {
-		return EditorSettings::get_singleton()->get_data_dir().plus_file("mono");
+		return EditorPaths::get_singleton()->get_data_dir().plus_file("mono");
 	} else {
 		String settings_path;
 

platform/android/export/export.cpp

@@ -1793,7 +1793,7 @@ public:
 			p_debug_flags |= DEBUG_FLAG_REMOTE_DEBUG_LOCALHOST;
 		}
 
-		String tmp_export_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpexport." + uitos(OS::get_singleton()->get_unix_time()) + ".apk");
+		String tmp_export_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpexport." + uitos(OS::get_singleton()->get_unix_time()) + ".apk");
 
 #define CLEANUP_AND_RETURN(m_err)                         \
 	{                                                     \
@@ -2651,7 +2651,7 @@ public:
 		FileAccess *dst_f = nullptr;
 		io2.opaque = &dst_f;
 
-		String tmp_unaligned_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpexport-unaligned." + uitos(OS::get_singleton()->get_unix_time()) + ".apk");
+		String tmp_unaligned_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpexport-unaligned." + uitos(OS::get_singleton()->get_unix_time()) + ".apk");
 
 #define CLEANUP_AND_RETURN(m_err)                            \
 	{                                                        \

platform/javascript/export/export.cpp

@@ -63,7 +63,7 @@ private:
 	}
 
 	void _set_internal_certs(Ref<Crypto> p_crypto) {
-		const String cache_path = EditorSettings::get_singleton()->get_cache_dir();
+		const String cache_path = EditorPaths::get_singleton()->get_cache_dir();
 		const String key_path = cache_path.plus_file("html5_server.key");
 		const String crt_path = cache_path.plus_file("html5_server.crt");
 		bool regen = !FileAccess::exists(key_path) || !FileAccess::exists(crt_path);
@@ -138,7 +138,7 @@ public:
 
 		const String req_file = req[1].get_file();
 		const String req_ext = req[1].get_extension();
-		const String cache_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("web");
+		const String cache_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("web");
 		const String filepath = cache_path.plus_file(req_file);
 
 		if (!mimes.has(req_ext) || !FileAccess::exists(filepath)) {
@@ -888,7 +888,7 @@ Error EditorExportPlatformJavaScript::run(const Ref<EditorExportPreset> &p_prese
 		return OK;
 	}
 
-	const String dest = EditorSettings::get_singleton()->get_cache_dir().plus_file("web");
+	const String dest = EditorPaths::get_singleton()->get_cache_dir().plus_file("web");
 	DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
 	if (!da->dir_exists(dest)) {
 		Error err = da->make_dir_recursive(dest);

platform/osx/export/export.cpp

@@ -301,7 +301,7 @@ void EditorExportPlatformOSX::_make_icon(const Ref<Image> &p_icon, Vector<uint8_
 		if (icon_infos[i].is_png) {
 			// Encode PNG icon.
 			it->create_from_image(copy);
-			String path = EditorSettings::get_singleton()->get_cache_dir().plus_file("icon.png");
+			String path = EditorPaths::get_singleton()->get_cache_dir().plus_file("icon.png");
 			ResourceSaver::save(path, it);
 
 			FileAccess *f = FileAccess::open(path, FileAccess::READ);
@@ -610,7 +610,7 @@ Error EditorExportPlatformOSX::export_project(const Ref<EditorExportPreset> &p_p
 
 	// Create our application bundle.
 	String tmp_app_dir_name = pkg_name + ".app";
-	String tmp_app_path_name = EditorSettings::get_singleton()->get_cache_dir().plus_file(tmp_app_dir_name);
+	String tmp_app_path_name = EditorPaths::get_singleton()->get_cache_dir().plus_file(tmp_app_dir_name);
 	print_line("Exporting to " + tmp_app_path_name);
 
 	Error err = OK;
@@ -774,7 +774,7 @@ Error EditorExportPlatformOSX::export_project(const Ref<EditorExportPreset> &p_p
 
 		String ent_path = p_preset->get("codesign/entitlements/custom_file");
 		if (sign_enabled && (ent_path == "")) {
-			ent_path = EditorSettings::get_singleton()->get_cache_dir().plus_file(pkg_name + ".entitlements");
+			ent_path = EditorPaths::get_singleton()->get_cache_dir().plus_file(pkg_name + ".entitlements");
 
 			FileAccess *ent_f = FileAccess::open(ent_path, FileAccess::WRITE);
 			if (ent_f) {
@@ -959,7 +959,7 @@ Error EditorExportPlatformOSX::export_project(const Ref<EditorExportPreset> &p_p
 				zlib_filefunc_def io_dst = zipio_create_io_from_file(&dst_f);
 				zipFile zip = zipOpen2(p_path.utf8().get_data(), APPEND_STATUS_CREATE, nullptr, &io_dst);
 
-				_zip_folder_recursive(zip, EditorSettings::get_singleton()->get_cache_dir(), pkg_name + ".app", pkg_name);
+				_zip_folder_recursive(zip, EditorPaths::get_singleton()->get_cache_dir(), pkg_name + ".app", pkg_name);
 
 				zipClose(zip, nullptr);
 			}

platform/uwp/export/export.cpp

@@ -567,7 +567,7 @@ void AppxPackager::finish() {
 	// Create and add block map file
 	EditorNode::progress_task_step("export", "Creating block map...", 4);
 
-	const String &tmp_blockmap_file_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpblockmap.xml");
+	const String &tmp_blockmap_file_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpblockmap.xml");
 	make_block_map(tmp_blockmap_file_path);
 
 	FileAccess *blockmap_file = FileAccess::open(tmp_blockmap_file_path, FileAccess::READ);
@@ -585,7 +585,7 @@ void AppxPackager::finish() {
 
 	EditorNode::progress_task_step("export", "Setting content types...", 5);
 
-	const String &tmp_content_types_file_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpcontenttypes.xml");
+	const String &tmp_content_types_file_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpcontenttypes.xml");
 	make_content_types(tmp_content_types_file_path);
 
 	FileAccess *types_file = FileAccess::open(tmp_content_types_file_path, FileAccess::READ);
@@ -879,7 +879,7 @@ class EditorExportPlatformUWP : public EditorExportPlatform {
 			return data;
 		}
 
-		String tmp_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("uwp_tmp_logo.png");
+		String tmp_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("uwp_tmp_logo.png");
 
 		Error err = texture->get_image()->save_png(tmp_path);
 

servers/rendering/renderer_rd/renderer_compositor_rd.cpp

@@ -31,6 +31,7 @@
 #include "renderer_compositor_rd.h"
 
 #include "core/config/project_settings.h"
+#include "core/os/dir_access.h"
 
 void RendererCompositorRD::prepare_for_blitting_render_targets() {
 	RD::get_singleton()->prepare_screen_for_drawing();
@@ -155,6 +156,43 @@ void RendererCompositorRD::finalize() {
 RendererCompositorRD *RendererCompositorRD::singleton = nullptr;
 
 RendererCompositorRD::RendererCompositorRD() {
+	{
+		String shader_cache_dir = Engine::get_singleton()->get_shader_cache_path();
+		if (shader_cache_dir == String()) {
+			shader_cache_dir = "user://";
+		}
+		DirAccessRef da = DirAccess::open(shader_cache_dir);
+		if (!da) {
+			ERR_PRINT("Can't create shader cache folder, no shader caching will happen: " + shader_cache_dir);
+		} else {
+			Error err = da->change_dir("shader_cache");
+			if (err != OK) {
+				err = da->make_dir("shader_cache");
+			}
+			if (err != OK) {
+				ERR_PRINT("Can't create shader cache folder, no shader caching will happen: " + shader_cache_dir);
+			} else {
+				shader_cache_dir = shader_cache_dir.plus_file("shader_cache");
+
+				bool shader_cache_enabled = GLOBAL_GET("rendering/shader_compiler/shader_cache/enabled");
+				if (!Engine::get_singleton()->is_editor_hint() && !shader_cache_enabled) {
+					shader_cache_dir = String(); //disable only if not editor
+				}
+
+				if (shader_cache_dir != String()) {
+					bool compress = GLOBAL_GET("rendering/shader_compiler/shader_cache/compress");
+					bool use_zstd = GLOBAL_GET("rendering/shader_compiler/shader_cache/use_zstd_compression");
+					bool strip_debug = GLOBAL_GET("rendering/shader_compiler/shader_cache/strip_debug");
+
+					ShaderRD::set_shader_cache_dir(shader_cache_dir);
+					ShaderRD::set_shader_cache_save_compressed(compress);
+					ShaderRD::set_shader_cache_save_compressed_zstd(use_zstd);
+					ShaderRD::set_shader_cache_save_debug(!strip_debug);
+				}
+			}
+		}
+	}
+
 	singleton = this;
 	time = 0;
 
@@ -171,3 +209,7 @@ RendererCompositorRD::RendererCompositorRD() {
 		scene = memnew(RendererSceneRenderImplementation::RenderForwardClustered(storage));
 	}
 }
+
+RendererCompositorRD::~RendererCompositorRD() {
+	ShaderRD::set_shader_cache_dir(String());
+}

servers/rendering/renderer_rd/renderer_compositor_rd.h

@@ -118,6 +118,6 @@ public:
 
 	static RendererCompositorRD *singleton;
 	RendererCompositorRD();
-	~RendererCompositorRD() {}
+	~RendererCompositorRD();
 };
 #endif // RASTERIZER_RD_H

servers/rendering/renderer_rd/shader_compiler_rd.cpp

@@ -369,17 +369,24 @@ void ShaderCompilerRD::_dump_function_deps(const SL::ShaderNode *p_node, const S
 
 	ERR_FAIL_COND(fidx == -1);
 
+	Vector<StringName> uses_functions;
+
 	for (Set<StringName>::Element *E = p_node->functions[fidx].uses_function.front(); E; E = E->next()) {
-		if (added.has(E->get())) {
+		uses_functions.push_back(E->get());
+	}
+	uses_functions.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+	for (int k = 0; k < uses_functions.size(); k++) {
+		if (added.has(uses_functions[k])) {
 			continue; //was added already
 		}
 
-		_dump_function_deps(p_node, E->get(), p_func_code, r_to_add, added);
+		_dump_function_deps(p_node, uses_functions[k], p_func_code, r_to_add, added);
 
 		SL::FunctionNode *fnode = nullptr;
 
 		for (int i = 0; i < p_node->functions.size(); i++) {
-			if (p_node->functions[i].name == E->get()) {
+			if (p_node->functions[i].name == uses_functions[k]) {
 				fnode = p_node->functions[i].function;
 				break;
 			}
@@ -427,9 +434,9 @@ void ShaderCompilerRD::_dump_function_deps(const SL::ShaderNode *p_node, const S
 
 		header += ")\n";
 		r_to_add += header;
-		r_to_add += p_func_code[E->get()];
+		r_to_add += p_func_code[uses_functions[k]];
 
-		added.insert(E->get());
+		added.insert(uses_functions[k]);
 	}
 }
 
@@ -581,63 +588,74 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge
 			uniform_defines.resize(max_uniforms);
 			bool uses_uniforms = false;
 
+			Vector<StringName> uniform_names;
+
 			for (Map<StringName, SL::ShaderNode::Uniform>::Element *E = pnode->uniforms.front(); E; E = E->next()) {
+				uniform_names.push_back(E->key());
+			}
+
+			uniform_names.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+			for (int k = 0; k < uniform_names.size(); k++) {
+				StringName uniform_name = uniform_names[k];
+				const SL::ShaderNode::Uniform &uniform = pnode->uniforms[uniform_name];
+
 				String ucode;
 
-				if (E->get().scope == SL::ShaderNode::Uniform::SCOPE_INSTANCE) {
+				if (uniform.scope == SL::ShaderNode::Uniform::SCOPE_INSTANCE) {
 					//insert, but don't generate any code.
-					p_actions.uniforms->insert(E->key(), E->get());
+					p_actions.uniforms->insert(uniform_name, uniform);
 					continue; //instances are indexed directly, dont need index uniforms
 				}
-				if (SL::is_sampler_type(E->get().type)) {
+				if (SL::is_sampler_type(uniform.type)) {
-					ucode = "layout(set = " + itos(actions.texture_layout_set) + ", binding = " + itos(actions.base_texture_binding_index + E->get().texture_order) + ") uniform ";
+					ucode = "layout(set = " + itos(actions.texture_layout_set) + ", binding = " + itos(actions.base_texture_binding_index + uniform.texture_order) + ") uniform ";
 				}
 
-				bool is_buffer_global = !SL::is_sampler_type(E->get().type) && E->get().scope == SL::ShaderNode::Uniform::SCOPE_GLOBAL;
+				bool is_buffer_global = !SL::is_sampler_type(uniform.type) && uniform.scope == SL::ShaderNode::Uniform::SCOPE_GLOBAL;
 
 				if (is_buffer_global) {
 					//this is an integer to index the global table
 					ucode += _typestr(ShaderLanguage::TYPE_UINT);
 				} else {
-					ucode += _prestr(E->get().precision);
+					ucode += _prestr(uniform.precision);
-					ucode += _typestr(E->get().type);
+					ucode += _typestr(uniform.type);
 				}
 
-				ucode += " " + _mkid(E->key());
+				ucode += " " + _mkid(uniform_name);
 				ucode += ";\n";
-				if (SL::is_sampler_type(E->get().type)) {
+				if (SL::is_sampler_type(uniform.type)) {
 					for (int j = 0; j < STAGE_MAX; j++) {
 						r_gen_code.stage_globals[j] += ucode;
 					}
 
 					GeneratedCode::Texture texture;
-					texture.name = E->key();
+					texture.name = uniform_name;
-					texture.hint = E->get().hint;
+					texture.hint = uniform.hint;
-					texture.type = E->get().type;
+					texture.type = uniform.type;
-					texture.filter = E->get().filter;
+					texture.filter = uniform.filter;
-					texture.repeat = E->get().repeat;
+					texture.repeat = uniform.repeat;
-					texture.global = E->get().scope == ShaderLanguage::ShaderNode::Uniform::SCOPE_GLOBAL;
+					texture.global = uniform.scope == ShaderLanguage::ShaderNode::Uniform::SCOPE_GLOBAL;
 					if (texture.global) {
 						r_gen_code.uses_global_textures = true;
 					}
 
-					r_gen_code.texture_uniforms.write[E->get().texture_order] = texture;
+					r_gen_code.texture_uniforms.write[uniform.texture_order] = texture;
 				} else {
 					if (!uses_uniforms) {
 						uses_uniforms = true;
 					}
-					uniform_defines.write[E->get().order] = ucode;
+					uniform_defines.write[uniform.order] = ucode;
 					if (is_buffer_global) {
 						//globals are indices into the global table
-						uniform_sizes.write[E->get().order] = _get_datatype_size(ShaderLanguage::TYPE_UINT);
+						uniform_sizes.write[uniform.order] = _get_datatype_size(ShaderLanguage::TYPE_UINT);
-						uniform_alignments.write[E->get().order] = _get_datatype_alignment(ShaderLanguage::TYPE_UINT);
+						uniform_alignments.write[uniform.order] = _get_datatype_alignment(ShaderLanguage::TYPE_UINT);
 					} else {
-						uniform_sizes.write[E->get().order] = _get_datatype_size(E->get().type);
+						uniform_sizes.write[uniform.order] = _get_datatype_size(uniform.type);
-						uniform_alignments.write[E->get().order] = _get_datatype_alignment(E->get().type);
+						uniform_alignments.write[uniform.order] = _get_datatype_alignment(uniform.type);
 					}
 				}
 
-				p_actions.uniforms->insert(E->key(), E->get());
+				p_actions.uniforms->insert(uniform_name, uniform);
 			}
 
 			for (int i = 0; i < max_uniforms; i++) {
@@ -704,21 +722,32 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge
 
 			List<Pair<StringName, SL::ShaderNode::Varying>> var_frag_to_light;
 
+			Vector<StringName> varying_names;
+
 			for (Map<StringName, SL::ShaderNode::Varying>::Element *E = pnode->varyings.front(); E; E = E->next()) {
-				if (E->get().stage == SL::ShaderNode::Varying::STAGE_FRAGMENT_TO_LIGHT || E->get().stage == SL::ShaderNode::Varying::STAGE_FRAGMENT) {
+				varying_names.push_back(E->key());
-					var_frag_to_light.push_back(Pair<StringName, SL::ShaderNode::Varying>(E->key(), E->get()));
+			}
-					fragment_varyings.insert(E->key());
+
+			varying_names.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+			for (int k = 0; k < varying_names.size(); k++) {
+				StringName varying_name = varying_names[k];
+				const SL::ShaderNode::Varying &varying = pnode->varyings[varying_name];
+
+				if (varying.stage == SL::ShaderNode::Varying::STAGE_FRAGMENT_TO_LIGHT || varying.stage == SL::ShaderNode::Varying::STAGE_FRAGMENT) {
+					var_frag_to_light.push_back(Pair<StringName, SL::ShaderNode::Varying>(varying_name, varying));
+					fragment_varyings.insert(varying_name);
 					continue;
 				}
 
 				String vcode;
-				String interp_mode = _interpstr(E->get().interpolation);
+				String interp_mode = _interpstr(varying.interpolation);
-				vcode += _prestr(E->get().precision);
+				vcode += _prestr(varying.precision);
-				vcode += _typestr(E->get().type);
+				vcode += _typestr(varying.type);
-				vcode += " " + _mkid(E->key());
+				vcode += " " + _mkid(varying_name);
-				if (E->get().array_size > 0) {
+				if (varying.array_size > 0) {
 					vcode += "[";
-					vcode += itos(E->get().array_size);
+					vcode += itos(varying.array_size);
 					vcode += "]";
 				}
 				vcode += ";\n";

servers/rendering/renderer_rd/shader_rd.cpp

@@ -30,8 +30,12 @@
 
 #include "shader_rd.h"
 
+#include "core/io/compression.h"
+#include "core/os/dir_access.h"
+#include "core/os/file_access.h"
 #include "renderer_compositor_rd.h"
 #include "servers/rendering/rendering_device.h"
+#include "thirdparty/misc/smolv.h"
 
 void ShaderRD::_add_stage(const char *p_code, StageType p_stage_type) {
 	Vector<String> lines = String(p_code).split("\n");
@@ -97,6 +101,7 @@ void ShaderRD::_add_stage(const char *p_code, StageType p_stage_type) {
 
 void ShaderRD::setup(const char *p_vertex_code, const char *p_fragment_code, const char *p_compute_code, const char *p_name) {
 	name = p_name;
+
 	if (p_compute_code) {
 		_add_stage(p_compute_code, STAGE_TYPE_COMPUTE);
 		is_compute = true;
@@ -109,6 +114,18 @@ void ShaderRD::setup(const char *p_vertex_code, const char *p_fragment_code, con
 			_add_stage(p_fragment_code, STAGE_TYPE_FRAGMENT);
 		}
 	}
+
+	StringBuilder tohash;
+	tohash.append("[VersionKey]");
+	tohash.append(RenderingDevice::get_singleton()->shader_get_cache_key());
+	tohash.append("[Vertex]");
+	tohash.append(p_vertex_code ? p_vertex_code : "");
+	tohash.append("[Fragment]");
+	tohash.append(p_fragment_code ? p_fragment_code : "");
+	tohash.append("[Compute]");
+	tohash.append(p_compute_code ? p_compute_code : "");
+
+	base_sha256 = tohash.as_string().sha256_text();
 }
 
 RID ShaderRD::version_create() {
@@ -131,6 +148,9 @@ void ShaderRD::_clear_version(Version *p_version) {
 		}
 
 		memdelete_arr(p_version->variants);
+		if (p_version->variant_stages) {
+			memdelete_arr(p_version->variant_stages);
+		}
 		p_version->variants = nullptr;
 	}
 }
@@ -183,7 +203,7 @@ void ShaderRD::_compile_variant(uint32_t p_variant, Version *p_version) {
 		return; //variant is disabled, return
 	}
 
-	Vector<RD::ShaderStageData> stages;
+	Vector<RD::ShaderStageData> &stages = p_version->variant_stages[p_variant];
 
 	String error;
 	String current_source;
@@ -313,6 +333,197 @@ RS::ShaderNativeSourceCode ShaderRD::version_get_native_source_code(RID p_versio
 	return source_code;
 }
 
+String ShaderRD::_version_get_sha1(Version *p_version) const {
+	StringBuilder hash_build;
+
+	hash_build.append("[uniforms]");
+	hash_build.append(p_version->uniforms.get_data());
+	hash_build.append("[vertex_globals]");
+	hash_build.append(p_version->vertex_globals.get_data());
+	hash_build.append("[fragment_globals]");
+	hash_build.append(p_version->fragment_globals.get_data());
+	hash_build.append("[compute_globals]");
+	hash_build.append(p_version->compute_globals.get_data());
+
+	Vector<StringName> code_sections;
+	for (Map<StringName, CharString>::Element *E = p_version->code_sections.front(); E; E = E->next()) {
+		code_sections.push_back(E->key());
+	}
+	code_sections.sort_custom<StringName::AlphCompare>();
+
+	for (int i = 0; i < code_sections.size(); i++) {
+		hash_build.append(String("[code:") + String(code_sections[i]) + "]");
+		hash_build.append(p_version->code_sections[code_sections[i]].get_data());
+	}
+	for (int i = 0; i < p_version->custom_defines.size(); i++) {
+		hash_build.append("[custom_defines:" + itos(i) + "]");
+		hash_build.append(p_version->custom_defines[i].get_data());
+	}
+
+	return hash_build.as_string().sha1_text();
+}
+
+static const char *shader_file_header = "GDSC";
+static const uint32_t cache_file_version = 1;
+
+bool ShaderRD::_load_from_cache(Version *p_version) {
+	String sha1 = _version_get_sha1(p_version);
+	String path = shader_cache_dir.plus_file(name).plus_file(base_sha256).plus_file(sha1) + ".cache";
+
+	uint64_t time_from = OS::get_singleton()->get_ticks_usec();
+
+	FileAccessRef f = FileAccess::open(path, FileAccess::READ);
+	if (!f) {
+		return false;
+	}
+
+	char header[5] = { 0, 0, 0, 0, 0 };
+	f->get_buffer((uint8_t *)header, 4);
+	ERR_FAIL_COND_V(header != String(shader_file_header), false);
+
+	uint32_t file_version = f->get_32();
+	if (file_version != cache_file_version) {
+		return false; // wrong version
+	}
+
+	uint32_t variant_count = f->get_32();
+
+	ERR_FAIL_COND_V(variant_count != (uint32_t)variant_defines.size(), false); //should not happen but check
+
+	bool success = true;
+	for (uint32_t i = 0; i < variant_count; i++) {
+		uint32_t stage_count = f->get_32();
+		p_version->variant_stages[i].resize(stage_count);
+		for (uint32_t j = 0; j < stage_count; j++) {
+			p_version->variant_stages[i].write[j].shader_stage = RD::ShaderStage(f->get_32());
+
+			int compression = f->get_32();
+			uint32_t length = f->get_32();
+
+			if (compression == 0) {
+				Vector<uint8_t> data;
+				data.resize(length);
+
+				f->get_buffer(data.ptrw(), length);
+
+				p_version->variant_stages[i].write[j].spir_v = data;
+			} else {
+				Vector<uint8_t> data;
+
+				if (compression == 2) {
+					//zstd
+					int smol_length = f->get_32();
+					Vector<uint8_t> zstd_data;
+
+					zstd_data.resize(smol_length);
+					f->get_buffer(zstd_data.ptrw(), smol_length);
+
+					data.resize(length);
+					Compression::decompress(data.ptrw(), data.size(), zstd_data.ptr(), zstd_data.size(), Compression::MODE_ZSTD);
+
+				} else {
+					data.resize(length);
+					f->get_buffer(data.ptrw(), length);
+				}
+
+				Vector<uint8_t> spirv;
+				uint32_t spirv_size = smolv::GetDecodedBufferSize(data.ptr(), data.size());
+				spirv.resize(spirv_size);
+				if (!smolv::Decode(data.ptr(), data.size(), spirv.ptrw(), spirv_size)) {
+					ERR_PRINT("Malformed smolv input uncompressing shader " + name + ", variant #" + itos(i) + " stage :" + itos(j));
+					success = false;
+					break;
+				}
+				p_version->variant_stages[i].write[j].spir_v = spirv;
+			}
+		}
+	}
+
+	if (!success) {
+		for (uint32_t i = 0; i < variant_count; i++) {
+			p_version->variant_stages[i].resize(0);
+		}
+		return false;
+	}
+
+	float time_ms = double(OS::get_singleton()->get_ticks_usec() - time_from) / 1000.0;
+
+	print_verbose("Shader cache load success '" + path + "' " + rtos(time_ms) + "ms.");
+
+	for (uint32_t i = 0; i < variant_count; i++) {
+		RID shader = RD::get_singleton()->shader_create(p_version->variant_stages[i]);
+		{
+			MutexLock lock(variant_set_mutex);
+			p_version->variants[i] = shader;
+		}
+	}
+
+	memdelete_arr(p_version->variant_stages); //clear stages
+	p_version->variant_stages = nullptr;
+	p_version->valid = true;
+	return true;
+}
+
+void ShaderRD::_save_to_cache(Version *p_version) {
+	String sha1 = _version_get_sha1(p_version);
+	String path = shader_cache_dir.plus_file(name).plus_file(base_sha256).plus_file(sha1) + ".cache";
+
+	FileAccessRef f = FileAccess::open(path, FileAccess::WRITE);
+	ERR_FAIL_COND(!f);
+	f->store_buffer((const uint8_t *)shader_file_header, 4);
+	f->store_32(cache_file_version); //file version
+	uint32_t variant_count = variant_defines.size();
+	f->store_32(variant_count); //variant count
+
+	for (uint32_t i = 0; i < variant_count; i++) {
+		f->store_32(p_version->variant_stages[i].size()); //stage count
+		for (int j = 0; j < p_version->variant_stages[i].size(); j++) {
+			f->store_32(p_version->variant_stages[i][j].shader_stage); //stage count
+			Vector<uint8_t> spirv = p_version->variant_stages[i][j].spir_v;
+
+			bool save_uncompressed = true;
+			if (shader_cache_save_compressed) {
+				smolv::ByteArray smolv;
+				bool strip_debug = !shader_cache_save_debug;
+				if (!smolv::Encode(spirv.ptr(), spirv.size(), smolv, strip_debug ? smolv::kEncodeFlagStripDebugInfo : 0)) {
+					ERR_PRINT("Error compressing shader " + name + ", variant #" + itos(i) + " stage :" + itos(i));
+				} else {
+					bool compress_zstd = shader_cache_save_compressed_zstd;
+
+					if (compress_zstd) {
+						Vector<uint8_t> zstd;
+						zstd.resize(Compression::get_max_compressed_buffer_size(smolv.size(), Compression::MODE_ZSTD));
+						int dst_size = Compression::compress(zstd.ptrw(), &smolv[0], smolv.size(), Compression::MODE_ZSTD);
+						if (dst_size >= 0 && (uint32_t)dst_size < smolv.size()) {
+							f->store_32(2); //compressed zstd
+							f->store_32(smolv.size()); //size of smolv buffer
+							f->store_32(dst_size); //size of smolv buffer
+							f->store_buffer(zstd.ptr(), dst_size); //smolv buffer
+						} else {
+							compress_zstd = false;
+						}
+					}
+
+					if (!compress_zstd) {
+						f->store_32(1); //compressed
+						f->store_32(smolv.size()); //size of smolv buffer
+						f->store_buffer(&smolv[0], smolv.size()); //smolv buffer
+					}
+					save_uncompressed = false;
+				}
+			}
+
+			if (save_uncompressed) {
+				f->store_32(0); //uncompressed
+				f->store_32(spirv.size()); //stage count
+				f->store_buffer(spirv.ptr(), spirv.size()); //stage count
+			}
+		}
+	}
+
+	f->close();
+}
+
 void ShaderRD::_compile_version(Version *p_version) {
 	_clear_version(p_version);
 
@@ -320,6 +531,15 @@ void ShaderRD::_compile_version(Version *p_version) {
 	p_version->dirty = false;
 
 	p_version->variants = memnew_arr(RID, variant_defines.size());
+	typedef Vector<RD::ShaderStageData> ShaderStageArray;
+	p_version->variant_stages = memnew_arr(ShaderStageArray, variant_defines.size());
+
+	if (shader_cache_dir_valid) {
+		if (_load_from_cache(p_version)) {
+			return;
+		}
+	}
+
 #if 1
 
 	RendererThreadPool::singleton->thread_work_pool.do_work(variant_defines.size(), this, &ShaderRD::_compile_variant, p_version);
@@ -351,9 +571,19 @@ void ShaderRD::_compile_version(Version *p_version) {
 			}
 		}
 		memdelete_arr(p_version->variants);
-		p_version->variants = nullptr;
+		if (p_version->variant_stages) {
-		return;
+			memdelete_arr(p_version->variant_stages);
 		}
+		p_version->variants = nullptr;
+		p_version->variant_stages = nullptr;
+		return;
+	} else if (shader_cache_dir_valid) {
+		//save shader cache
+		_save_to_cache(p_version);
+	}
+
+	memdelete_arr(p_version->variant_stages); //clear stages
+	p_version->variant_stages = nullptr;
 
 	p_version->valid = true;
 }
@@ -443,6 +673,8 @@ bool ShaderRD::is_variant_enabled(int p_variant) const {
 	return variants_enabled[p_variant];
 }
 
+bool ShaderRD::shader_cache_cleanup_on_start = false;
+
 ShaderRD::ShaderRD() {
 	// Do not feel forced to use this, in most cases it makes little to no difference.
 	bool use_32_threads = false;
@@ -469,8 +701,64 @@ void ShaderRD::initialize(const Vector<String> &p_variant_defines, const String
 		variant_defines.push_back(p_variant_defines[i].utf8());
 		variants_enabled.push_back(true);
 	}
+
+	if (shader_cache_dir != String()) {
+		StringBuilder hash_build;
+
+		hash_build.append("[base_hash]");
+		hash_build.append(base_sha256);
+		hash_build.append("[general_defines]");
+		hash_build.append(general_defines.get_data());
+		for (int i = 0; i < variant_defines.size(); i++) {
+			hash_build.append("[variant_defines:" + itos(i) + "]");
+			hash_build.append(variant_defines[i].get_data());
 		}
 
+		base_sha256 = hash_build.as_string().sha256_text();
+
+		DirAccessRef d = DirAccess::open(shader_cache_dir);
+		ERR_FAIL_COND(!d);
+		if (d->change_dir(name) != OK) {
+			Error err = d->make_dir(name);
+			ERR_FAIL_COND(err != OK);
+			d->change_dir(name);
+		}
+
+		//erase other versions?
+		if (shader_cache_cleanup_on_start) {
+		}
+		//
+		if (d->change_dir(base_sha256) != OK) {
+			Error err = d->make_dir(base_sha256);
+			ERR_FAIL_COND(err != OK);
+		}
+		shader_cache_dir_valid = true;
+
+		print_verbose("Shader '" + name + "' SHA256: " + base_sha256);
+	}
+}
+
+void ShaderRD::set_shader_cache_dir(const String &p_dir) {
+	shader_cache_dir = p_dir;
+}
+
+void ShaderRD::set_shader_cache_save_compressed(bool p_enable) {
+	shader_cache_save_compressed = p_enable;
+}
+
+void ShaderRD::set_shader_cache_save_compressed_zstd(bool p_enable) {
+	shader_cache_save_compressed_zstd = p_enable;
+}
+
+void ShaderRD::set_shader_cache_save_debug(bool p_enable) {
+	shader_cache_save_debug = p_enable;
+}
+
+String ShaderRD::shader_cache_dir;
+bool ShaderRD::shader_cache_save_compressed = true;
+bool ShaderRD::shader_cache_save_compressed_zstd = true;
+bool ShaderRD::shader_cache_save_debug = true;
+
 ShaderRD::~ShaderRD() {
 	List<RID> remaining;
 	version_owner.get_owned_list(&remaining);

servers/rendering/renderer_rd/shader_rd.h

@@ -59,7 +59,8 @@ class ShaderRD {
 		Map<StringName, CharString> code_sections;
 		Vector<CharString> custom_defines;
 
-		RID *variants; //same size as version defines
+		Vector<RD::ShaderStageData> *variant_stages = nullptr;
+		RID *variants = nullptr; //same size as version defines
 
 		bool valid;
 		bool dirty;
@@ -96,10 +97,19 @@ class ShaderRD {
 
 	bool is_compute = false;
 
-	const char *name;
+	String name;
 
 	CharString base_compute_defines;
 
+	String base_sha256;
+
+	static String shader_cache_dir;
+	static bool shader_cache_cleanup_on_start;
+	static bool shader_cache_save_compressed;
+	static bool shader_cache_save_compressed_zstd;
+	static bool shader_cache_save_debug;
+	bool shader_cache_dir_valid = false;
+
 	enum StageType {
 		STAGE_TYPE_VERTEX,
 		STAGE_TYPE_FRAGMENT,
@@ -113,6 +123,10 @@ class ShaderRD {
 
 	void _add_stage(const char *p_code, StageType p_stage_type);
 
+	String _version_get_sha1(Version *p_version) const;
+	bool _load_from_cache(Version *p_version);
+	void _save_to_cache(Version *p_version);
+
 protected:
 	ShaderRD();
 	void setup(const char *p_vertex_code, const char *p_fragment_code, const char *p_compute_code, const char *p_name);
@@ -148,6 +162,11 @@ public:
 	void set_variant_enabled(int p_variant, bool p_enabled);
 	bool is_variant_enabled(int p_variant) const;
 
+	static void set_shader_cache_dir(const String &p_dir);
+	static void set_shader_cache_save_compressed(bool p_enable);
+	static void set_shader_cache_save_compressed_zstd(bool p_enable);
+	static void set_shader_cache_save_debug(bool p_enable);
+
 	RS::ShaderNativeSourceCode version_get_native_source_code(RID p_version);
 
 	void initialize(const Vector<String> &p_variant_defines, const String &p_general_defines = "");

servers/rendering/rendering_device.cpp

@@ -40,6 +40,7 @@ RenderingDevice *RenderingDevice::get_singleton() {
 
 RenderingDevice::ShaderCompileFunction RenderingDevice::compile_function = nullptr;
 RenderingDevice::ShaderCacheFunction RenderingDevice::cache_function = nullptr;
+RenderingDevice::ShaderGetCacheKeyFunction RenderingDevice::get_cache_key_function = nullptr;
 
 void RenderingDevice::shader_set_compile_function(ShaderCompileFunction p_function) {
 	compile_function = p_function;
@@ -49,6 +50,10 @@ void RenderingDevice::shader_set_cache_function(ShaderCacheFunction p_function)
 	cache_function = p_function;
 }
 
+void RenderingDevice::shader_set_get_cache_key_function(ShaderGetCacheKeyFunction p_function) {
+	get_cache_key_function = p_function;
+}
+
 Vector<uint8_t> RenderingDevice::shader_compile_from_source(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language, String *r_error, bool p_allow_cache) {
 	if (p_allow_cache && cache_function) {
 		Vector<uint8_t> cache = cache_function(p_stage, p_source_code, p_language);
@@ -62,6 +67,13 @@ Vector<uint8_t> RenderingDevice::shader_compile_from_source(ShaderStage p_stage,
 	return compile_function(p_stage, p_source_code, p_language, r_error, &device_capabilities);
 }
 
+String RenderingDevice::shader_get_cache_key() const {
+	if (get_cache_key_function) {
+		return get_cache_key_function(&device_capabilities);
+	}
+	return String();
+}
+
 RID RenderingDevice::_texture_create(const Ref<RDTextureFormat> &p_format, const Ref<RDTextureView> &p_view, const TypedArray<PackedByteArray> &p_data) {
 	ERR_FAIL_COND_V(p_format.is_null(), RID());
 	ERR_FAIL_COND_V(p_view.is_null(), RID());

servers/rendering/rendering_device.h

@@ -103,12 +103,14 @@ public:
 		bool supports_multiview = false; // If true this device supports multiview options
 	};
 
+	typedef String (*ShaderGetCacheKeyFunction)(const Capabilities *p_capabilities);
 	typedef Vector<uint8_t> (*ShaderCompileFunction)(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language, String *r_error, const Capabilities *p_capabilities);
 	typedef Vector<uint8_t> (*ShaderCacheFunction)(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language);
 
 private:
 	static ShaderCompileFunction compile_function;
 	static ShaderCacheFunction cache_function;
+	static ShaderGetCacheKeyFunction get_cache_key_function;
 
 	static RenderingDevice *singleton;
 
@@ -635,9 +637,11 @@ public:
 	const Capabilities *get_device_capabilities() const { return &device_capabilities; };
 
 	virtual Vector<uint8_t> shader_compile_from_source(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language = SHADER_LANGUAGE_GLSL, String *r_error = nullptr, bool p_allow_cache = true);
+	virtual String shader_get_cache_key() const;
 
 	static void shader_set_compile_function(ShaderCompileFunction p_function);
 	static void shader_set_cache_function(ShaderCacheFunction p_function);
+	static void shader_set_get_cache_key_function(ShaderGetCacheKeyFunction p_function);
 
 	struct ShaderStageData {
 		ShaderStage shader_stage;

servers/rendering_server.cpp

@@ -2306,6 +2306,12 @@ RenderingServer::RenderingServer() {
 					"rendering/vulkan/rendering/back_end",
 					PROPERTY_HINT_ENUM, "ForwardClustered,ForwardMobile"));
 
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/enabled", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/compress", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/use_zstd_compression", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/strip_debug", false);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/strip_debug.release", true);
+
 	GLOBAL_DEF("rendering/reflections/sky_reflections/roughness_layers", 8);
 	GLOBAL_DEF("rendering/reflections/sky_reflections/texture_array_reflections", true);
 	GLOBAL_DEF("rendering/reflections/sky_reflections/texture_array_reflections.mobile", false);

thirdparty/README.md

@@ -469,6 +469,10 @@ Collection of single-file libraries used in Godot components.
   * Version: git (2f625846a775501fb69456567409a8b12f10ea25, 2012)
   * License: BSD-3-Clause
   * Modifications: use `const char*` instead of `char*` for input string
+- `smolv.h`
+  * Upstream: https://github.com/aras-p/smol-v
+  * Version: git (4b52c165c13763051a18e80ffbc2ee436314ceb2, 2020)
+  * License: Public Domain or MIT
 - `stb_rect_pack.h`
   * Upstream: https://github.com/nothings/stb
   * Version: 1.00 (2bb4a0accd4003c1db4c24533981e01b1adfd656, 2019)
@@ -731,3 +735,4 @@ Files extracted from upstream source:
 
 - lib/{common/,compress/,decompress/,zstd.h}
 - LICENSE
+

thirdparty/misc/smolv.h

@@ -0,0 +1,169 @@
+// smol-v - public domain - https://github.com/aras-p/smol-v
+// authored 2016-2020 by Aras Pranckevicius
+// no warranty implied; use at your own risk
+// See end of file for license information.
+//
+//
+// ### OVERVIEW:
+//
+// SMOL-V encodes Vulkan/Khronos SPIR-V format programs into a form that is smaller, and is more
+// compressible. Normally no changes to the programs are done; they decode
+// into exactly same program as what was encoded. Optionally, debug information
+// can be removed too.
+//
+// SPIR-V is a very verbose format, several times larger than same programs expressed in other
+// shader formats (e.g. DX11 bytecode, GLSL, DX9 bytecode etc.). The SSA-form with ever increasing
+// IDs is not very appreciated by regular data compressors either. SMOL-V does several things
+// to improve this:
+// - Many words, especially ones that most often have small values, are encoded using
+//   "varint" scheme (1-5 bytes per word, with just one byte for values in 0..127 range).
+//   See https://developers.google.com/protocol-buffers/docs/encoding
+// - Some IDs used in the program are delta-encoded, relative to previously seen IDs (e.g. Result
+//   IDs). Often instructions reference things that were computed just before, so this results in
+//   small deltas. These values are also encoded using "varint" scheme.
+// - Reordering instruction opcodes so that the most common ones are the smallest values, for smaller
+//  varint encoding.
+// - Encoding several instructions in a more compact form, e.g. the "typical <=4 component swizzle"
+//  shape of a VectorShuffle instruction, or sequences of MemberDecorate instructions.
+//
+// A somewhat similar utility is spirv-remap from glslang, see
+// https://github.com/KhronosGroup/glslang/blob/master/README-spirv-remap.txt
+//
+//
+// ### USAGE:
+//
+// Add source/smolv.h and source/smolv.cpp to your C++ project build.
+// Currently it might require C++11 or somesuch; I only tested with Visual Studio 2017/2019, Mac Xcode 11 and Gcc 5.4.
+//
+// smolv::Encode and smolv::Decode is the basic functionality.
+//
+// Other functions are for development/statistics purposes, to figure out frequencies and
+// distributions of the instructions.
+//
+// There's a test + compression benchmarking suite in testing/testmain.cpp; using that needs adding
+// other files under testing/external to the build too (3rd party code: glslang remapper, Zstd, LZ4).
+//
+//
+// ### LIMITATIONS / TODO:
+//
+// - SPIR-V where the words got stored in big-endian layout is not supported yet.
+// - The whole thing might not work on Big-Endian CPUs. It might, but I'm not 100% sure.
+// - Not much prevention is done against malformed/corrupted inputs, TODO.
+// - Out of memory cases are not handled. The code will either throw exception
+//   or crash, depending on your compilation flags.
+
+#pragma once
+
+#include <stdint.h>
+#include <vector>
+#include <cstddef>
+
+namespace smolv
+{
+	typedef std::vector<uint8_t> ByteArray;
+
+	enum EncodeFlags
+	{
+		kEncodeFlagNone = 0,
+		kEncodeFlagStripDebugInfo = (1<<0), // Strip all optional SPIR-V instructions (debug names etc.)
+	};
+	enum DecodeFlags
+	{
+		kDecodeFlagNone = 0,
+		kDecodeFlagUse20160831AsZeroVersion = (1 << 0), // For "version zero" of SMOL-V encoding, use 2016 08 31 code path (this is what happens to be used by Unity 2017-2020)
+	};
+
+	// Preserve *some* OpName debug names.
+	// Return true to preserve, false to strip.
+	// This is really only used to implement a workaround for problems with some Vulkan drivers.
+	typedef bool(*StripOpNameFilterFunc)(const char* name);
+
+	// -------------------------------------------------------------------
+	// Encoding / Decoding
+
+	// Encode SPIR-V into SMOL-V.
+	//
+	// Resulting data is appended to outSmolv array (the array is not cleared).
+	//
+	// flags is bitset of EncodeFlags values.
+	//
+	// Returns false on malformed SPIR-V input; if that happens the output array might get
+	// partial/broken SMOL-V program.
+	bool Encode(const void* spirvData, size_t spirvSize, ByteArray& outSmolv, uint32_t flags = kEncodeFlagNone, StripOpNameFilterFunc stripFilter = 0);
+
+
+	// Decode SMOL-V into SPIR-V.
+	//
+	// Resulting data is written into the passed buffer. Get required buffer space with
+	// GetDecodeBufferSize; this is the size of decoded SPIR-V program.
+	//
+	// flags is bitset of DecodeFlags values.
+
+	// Decoding does no memory allocations.
+	//
+	// Returns false on malformed input; if that happens the output buffer might be only partially
+	// written to.
+	bool Decode(const void* smolvData, size_t smolvSize, void* spirvOutputBuffer, size_t spirvOutputBufferSize, uint32_t flags = kDecodeFlagNone);
+
+
+	// Given a SMOL-V program, get size of the decoded SPIR-V program.
+	// This is the buffer size that Decode expects.
+	//
+	// Returns zero on malformed input (just checks the header, not the full input).
+	size_t GetDecodedBufferSize(const void* smolvData, size_t smolvSize);
+
+
+	// -------------------------------------------------------------------
+	// Computing instruction statistics on SPIR-V/SMOL-V programs
+
+	struct Stats;
+
+	Stats* StatsCreate();
+	void StatsDelete(Stats* s);
+
+	bool StatsCalculate(Stats* stats, const void* spirvData, size_t spirvSize);
+	bool StatsCalculateSmol(Stats* stats, const void* smolvData, size_t smolvSize);
+	void StatsPrint(const Stats* stats);
+
+} // namespace smolv
+
+
+// ------------------------------------------------------------------------------
+// This software is available under 2 licenses -- choose whichever you prefer.
+// ------------------------------------------------------------------------------
+// ALTERNATIVE A - MIT License
+// Copyright (c) 2016-2020 Aras Pranckevicius
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+// ------------------------------------------------------------------------------
+// ALTERNATIVE B - Public Domain (www.unlicense.org)
+// This is free and unencumbered software released into the public domain.
+// Anyone is free to copy, modify, publish, use, compile, sell, or distribute this
+// software, either in source code form or as a compiled binary, for any purpose,
+// commercial or non-commercial, and by any means.
+// In jurisdictions that recognize copyright laws, the author or authors of this
+// software dedicate any and all copyright interest in the software to the public
+// domain. We make this dedication for the benefit of the public at large and to
+// the detriment of our heirs and successors. We intend this dedication to be an
+// overt act of relinquishment in perpetuity of all present and future rights to
+// this software under copyright law.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+// ------------------------------------------------------------------------------