/**************************************************************************/ /* editor_file_system.cpp */ /**************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ /* https://godotengine.org */ /**************************************************************************/ /* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ /* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ /* "Software"), to deal in the Software without restriction, including */ /* without limitation the rights to use, copy, modify, merge, publish, */ /* distribute, sublicense, and/or sell copies of the Software, and to */ /* permit persons to whom the Software is furnished to do so, subject to */ /* the following conditions: */ /* */ /* The above copyright notice and this permission notice shall be */ /* included in all copies or substantial portions of the Software. */ /* */ /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ /* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ /* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ /* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ /* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /**************************************************************************/ #include "editor_file_system.h" #include "core/config/project_settings.h" #include "core/extension/gdextension_manager.h" #include "core/io/file_access.h" #include "core/io/resource_importer.h" #include "core/io/resource_loader.h" #include "core/io/resource_saver.h" #include "core/object/worker_thread_pool.h" #include "core/os/os.h" #include "core/variant/variant_parser.h" #include "editor/editor_help.h" #include "editor/editor_node.h" #include "editor/editor_paths.h" #include "editor/editor_resource_preview.h" #include "editor/editor_settings.h" #include "scene/resources/packed_scene.h" EditorFileSystem *EditorFileSystem::singleton = nullptr; //the name is the version, to keep compatibility with different versions of Godot #define CACHE_FILE_NAME "filesystem_cache8" int EditorFileSystemDirectory::find_file_index(const String &p_file) const { for (int i = 0; i < files.size(); i++) { if (files[i]->file == p_file) { return i; } } return -1; } int EditorFileSystemDirectory::find_dir_index(const String &p_dir) const { for (int i = 0; i < subdirs.size(); i++) { if (subdirs[i]->name == p_dir) { return i; } } return -1; } void EditorFileSystemDirectory::force_update() { // We set modified_time to 0 to force `EditorFileSystem::_scan_fs_changes` to search changes in the directory modified_time = 0; } int EditorFileSystemDirectory::get_subdir_count() const { return subdirs.size(); } EditorFileSystemDirectory *EditorFileSystemDirectory::get_subdir(int p_idx) { ERR_FAIL_INDEX_V(p_idx, subdirs.size(), nullptr); return subdirs[p_idx]; } int EditorFileSystemDirectory::get_file_count() const { return files.size(); } String EditorFileSystemDirectory::get_file(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->file; } String EditorFileSystemDirectory::get_path() const { String p; const EditorFileSystemDirectory *d = this; while (d->parent) { p = d->name.path_join(p); d = d->parent; } return "res://" + p; } String EditorFileSystemDirectory::get_file_path(int p_idx) const { String file = get_file(p_idx); const EditorFileSystemDirectory *d = this; while (d->parent) { file = d->name.path_join(file); d = d->parent; } return "res://" + file; } Vector EditorFileSystemDirectory::get_file_deps(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), Vector()); Vector deps; for (int i = 0; i < files[p_idx]->deps.size(); i++) { String dep = files[p_idx]->deps[i]; int sep_idx = dep.find("::"); //may contain type information, unwanted if (sep_idx != -1) { dep = dep.substr(0, sep_idx); } ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(dep); if (uid != ResourceUID::INVALID_ID) { //return proper dependency resource from uid if (ResourceUID::get_singleton()->has_id(uid)) { dep = ResourceUID::get_singleton()->get_id_path(uid); } else { continue; } } deps.push_back(dep); } return deps; } bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), false); return files[p_idx]->import_valid; } uint64_t EditorFileSystemDirectory::get_file_modified_time(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), 0); return files[p_idx]->modified_time; } String EditorFileSystemDirectory::get_file_script_class_name(int p_idx) const { return files[p_idx]->script_class_name; } String EditorFileSystemDirectory::get_file_script_class_extends(int p_idx) const { return files[p_idx]->script_class_extends; } String EditorFileSystemDirectory::get_file_script_class_icon_path(int p_idx) const { return files[p_idx]->script_class_icon_path; } StringName EditorFileSystemDirectory::get_file_type(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->type; } StringName EditorFileSystemDirectory::get_file_resource_script_class(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->resource_script_class; } String EditorFileSystemDirectory::get_name() { return name; } EditorFileSystemDirectory *EditorFileSystemDirectory::get_parent() { return parent; } void EditorFileSystemDirectory::_bind_methods() { ClassDB::bind_method(D_METHOD("get_subdir_count"), &EditorFileSystemDirectory::get_subdir_count); ClassDB::bind_method(D_METHOD("get_subdir", "idx"), &EditorFileSystemDirectory::get_subdir); ClassDB::bind_method(D_METHOD("get_file_count"), &EditorFileSystemDirectory::get_file_count); ClassDB::bind_method(D_METHOD("get_file", "idx"), &EditorFileSystemDirectory::get_file); ClassDB::bind_method(D_METHOD("get_file_path", "idx"), &EditorFileSystemDirectory::get_file_path); ClassDB::bind_method(D_METHOD("get_file_type", "idx"), &EditorFileSystemDirectory::get_file_type); ClassDB::bind_method(D_METHOD("get_file_script_class_name", "idx"), &EditorFileSystemDirectory::get_file_script_class_name); ClassDB::bind_method(D_METHOD("get_file_script_class_extends", "idx"), &EditorFileSystemDirectory::get_file_script_class_extends); ClassDB::bind_method(D_METHOD("get_file_import_is_valid", "idx"), &EditorFileSystemDirectory::get_file_import_is_valid); ClassDB::bind_method(D_METHOD("get_name"), &EditorFileSystemDirectory::get_name); ClassDB::bind_method(D_METHOD("get_path"), &EditorFileSystemDirectory::get_path); ClassDB::bind_method(D_METHOD("get_parent"), &EditorFileSystemDirectory::get_parent); ClassDB::bind_method(D_METHOD("find_file_index", "name"), &EditorFileSystemDirectory::find_file_index); ClassDB::bind_method(D_METHOD("find_dir_index", "name"), &EditorFileSystemDirectory::find_dir_index); } EditorFileSystemDirectory::EditorFileSystemDirectory() { modified_time = 0; parent = nullptr; } EditorFileSystemDirectory::~EditorFileSystemDirectory() { for (int i = 0; i < files.size(); i++) { memdelete(files[i]); } for (int i = 0; i < subdirs.size(); i++) { memdelete(subdirs[i]); } } void EditorFileSystem::_scan_filesystem() { ERR_FAIL_COND(!scanning || new_filesystem); //read .fscache String cpath; sources_changed.clear(); file_cache.clear(); String project = ProjectSettings::get_singleton()->get_resource_path(); String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join(CACHE_FILE_NAME); { Ref f = FileAccess::open(fscache, FileAccess::READ); bool first = true; if (f.is_valid()) { //read the disk cache while (!f->eof_reached()) { String l = f->get_line().strip_edges(); if (first) { if (first_scan) { // only use this on first scan, afterwards it gets ignored // this is so on first reimport we synchronize versions, then // we don't care until editor restart. This is for usability mainly so // your workflow is not killed after changing a setting by forceful reimporting // everything there is. filesystem_settings_version_for_import = l.strip_edges(); if (filesystem_settings_version_for_import != ResourceFormatImporter::get_singleton()->get_import_settings_hash()) { revalidate_import_files = true; } } first = false; continue; } if (l.is_empty()) { continue; } if (l.begins_with("::")) { Vector split = l.split("::"); ERR_CONTINUE(split.size() != 3); const String &name = split[1]; cpath = name; } else { // The last section (deps) may contain the same splitter, so limit the maxsplit to 8 to get the complete deps. Vector split = l.split("::", true, 8); ERR_CONTINUE(split.size() < 9); String name = split[0]; String file; file = name; name = cpath.path_join(name); FileCache fc; fc.type = split[1]; if (fc.type.contains("/")) { fc.type = fc.type.get_slice("/", 0); fc.resource_script_class = fc.type.get_slice("/", 1); } fc.uid = split[2].to_int(); fc.modification_time = split[3].to_int(); fc.import_modification_time = split[4].to_int(); fc.import_valid = split[5].to_int() != 0; fc.import_group_file = split[6].strip_edges(); fc.script_class_name = split[7].get_slice("<>", 0); fc.script_class_extends = split[7].get_slice("<>", 1); fc.script_class_icon_path = split[7].get_slice("<>", 2); String deps = split[8].strip_edges(); if (deps.length()) { Vector dp = deps.split("<>"); for (int i = 0; i < dp.size(); i++) { const String &path = dp[i]; fc.deps.push_back(path); } } file_cache[name] = fc; } } } } const String update_cache = EditorPaths::get_singleton()->get_project_settings_dir().path_join("filesystem_update4"); if (first_scan && FileAccess::exists(update_cache)) { { Ref f2 = FileAccess::open(update_cache, FileAccess::READ); String l = f2->get_line().strip_edges(); while (!l.is_empty()) { dep_update_list.insert(l); file_cache.erase(l); // Erase cache for this, so it gets updated. l = f2->get_line().strip_edges(); } } Ref d = DirAccess::create(DirAccess::ACCESS_RESOURCES); d->remove(update_cache); // Bye bye update cache. } EditorProgressBG scan_progress("efs", "ScanFS", 1000); ScanProgress sp; sp.progress = &scan_progress; new_filesystem = memnew(EditorFileSystemDirectory); new_filesystem->parent = nullptr; Ref d = DirAccess::create(DirAccess::ACCESS_RESOURCES); d->change_dir("res://"); _scan_new_dir(new_filesystem, d, &sp, 1); dep_update_list.clear(); file_cache.clear(); //clear caches, no longer needed if (!first_scan) { //on the first scan this is done from the main thread after re-importing _save_filesystem_cache(); } scanning = false; } void EditorFileSystem::_save_filesystem_cache() { group_file_cache.clear(); String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join(CACHE_FILE_NAME); Ref f = FileAccess::open(fscache, FileAccess::WRITE); ERR_FAIL_COND_MSG(f.is_null(), "Cannot create file '" + fscache + "'. Check user write permissions."); f->store_line(filesystem_settings_version_for_import); _save_filesystem_cache(filesystem, f); } void EditorFileSystem::_thread_func(void *_userdata) { EditorFileSystem *sd = (EditorFileSystem *)_userdata; sd->_scan_filesystem(); } bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_imported_files) { if (!reimport_on_missing_imported_files && p_only_imported_files) { return false; } if (!FileAccess::exists(p_path + ".import")) { return true; } if (!ResourceFormatImporter::get_singleton()->are_import_settings_valid(p_path)) { //reimport settings are not valid, reimport return true; } Error err; Ref f = FileAccess::open(p_path + ".import", FileAccess::READ, &err); if (f.is_null()) { //no import file, do reimport return true; } VariantParser::StreamFile stream; stream.f = f; String assign; Variant value; VariantParser::Tag next_tag; int lines = 0; String error_text; List to_check; String importer_name; String source_file = ""; String source_md5 = ""; Vector dest_files; String dest_md5 = ""; int version = 0; bool found_uid = false; while (true) { assign = Variant(); next_tag.fields.clear(); next_tag.name = String(); err = VariantParser::parse_tag_assign_eof(&stream, lines, error_text, next_tag, assign, value, nullptr, true); if (err == ERR_FILE_EOF) { break; } else if (err != OK) { ERR_PRINT("ResourceFormatImporter::load - '" + p_path + ".import:" + itos(lines) + "' error '" + error_text + "'."); return false; //parse error, try reimport manually (Avoid reimport loop on broken file) } if (!assign.is_empty()) { if (assign.begins_with("path")) { to_check.push_back(value); } else if (assign == "files") { Array fa = value; for (int i = 0; i < fa.size(); i++) { to_check.push_back(fa[i]); } } else if (assign == "importer_version") { version = value; } else if (assign == "importer") { importer_name = value; } else if (assign == "uid") { found_uid = true; } else if (!p_only_imported_files) { if (assign == "source_file") { source_file = value; } else if (assign == "dest_files") { dest_files = value; } } } else if (next_tag.name != "remap" && next_tag.name != "deps") { break; } } if (importer_name == "keep" || importer_name == "skip") { return false; //keep mode, do not reimport } if (!found_uid) { return true; //UID not found, old format, reimport. } Ref importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); if (importer.is_null()) { return true; // the importer has possibly changed, try to reimport. } if (importer->get_format_version() > version) { return true; // version changed, reimport } // Read the md5's from a separate file (so the import parameters aren't dependent on the file version String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_path); Ref md5s = FileAccess::open(base_path + ".md5", FileAccess::READ, &err); if (md5s.is_null()) { // No md5's stored for this resource return true; } VariantParser::StreamFile md5_stream; md5_stream.f = md5s; while (true) { assign = Variant(); next_tag.fields.clear(); next_tag.name = String(); err = VariantParser::parse_tag_assign_eof(&md5_stream, lines, error_text, next_tag, assign, value, nullptr, true); if (err == ERR_FILE_EOF) { break; } else if (err != OK) { ERR_PRINT("ResourceFormatImporter::load - '" + p_path + ".import.md5:" + itos(lines) + "' error '" + error_text + "'."); return false; // parse error } if (!assign.is_empty()) { if (!p_only_imported_files) { if (assign == "source_md5") { source_md5 = value; } else if (assign == "dest_md5") { dest_md5 = value; } } } } //imported files are gone, reimport for (const String &E : to_check) { if (!FileAccess::exists(E)) { return true; } } //check source md5 matching if (!p_only_imported_files) { if (!source_file.is_empty() && source_file != p_path) { return true; //file was moved, reimport } if (source_md5.is_empty()) { return true; //lacks md5, so just reimport } String md5 = FileAccess::get_md5(p_path); if (md5 != source_md5) { return true; } if (dest_files.size() && !dest_md5.is_empty()) { md5 = FileAccess::get_multiple_md5(dest_files); if (md5 != dest_md5) { return true; } } } return false; //nothing changed } bool EditorFileSystem::_scan_import_support(const Vector &reimports) { if (import_support_queries.size() == 0) { return false; } HashMap import_support_test; Vector import_support_tested; import_support_tested.resize(import_support_queries.size()); for (int i = 0; i < import_support_queries.size(); i++) { import_support_tested.write[i] = false; if (import_support_queries[i]->is_active()) { Vector extensions = import_support_queries[i]->get_file_extensions(); for (int j = 0; j < extensions.size(); j++) { import_support_test.insert(extensions[j], i); } } } if (import_support_test.size() == 0) { return false; //well nothing to do } for (int i = 0; i < reimports.size(); i++) { HashMap::Iterator E = import_support_test.find(reimports[i].get_extension().to_lower()); if (E) { import_support_tested.write[E->value] = true; } } for (int i = 0; i < import_support_tested.size(); i++) { if (import_support_tested[i]) { if (import_support_queries.write[i]->query()) { return true; } } } return false; } bool EditorFileSystem::_update_scan_actions(bool p_show_progress) { sources_changed.clear(); bool fs_changed = false; Vector reimports; Vector reloads; HashMap import_group_files; EditorProgress *editor_progress = nullptr; if (p_show_progress && scan_actions.size() > 0) { editor_progress = memnew(EditorProgress("_update_scan_actions", TTR("Scanning actions..."), scan_actions.size())); editor_progress->step("", 0, true); } int idx_action = 0; for (const ItemAction &ia : scan_actions) { if (editor_progress) { editor_progress->step(ia.file, idx_action, false); } idx_action++; switch (ia.action) { case ItemAction::ACTION_NONE: { } break; case ItemAction::ACTION_DIR_ADD: { int idx = 0; for (int i = 0; i < ia.dir->subdirs.size(); i++) { if (ia.new_dir->name.filenocasecmp_to(ia.dir->subdirs[i]->name) < 0) { break; } idx++; } if (idx == ia.dir->subdirs.size()) { ia.dir->subdirs.push_back(ia.new_dir); } else { ia.dir->subdirs.insert(idx, ia.new_dir); } fs_changed = true; } break; case ItemAction::ACTION_DIR_REMOVE: { ERR_CONTINUE(!ia.dir->parent); ia.dir->parent->subdirs.erase(ia.dir); memdelete(ia.dir); fs_changed = true; } break; case ItemAction::ACTION_FILE_ADD: { int idx = 0; for (int i = 0; i < ia.dir->files.size(); i++) { if (ia.new_file->file.filenocasecmp_to(ia.dir->files[i]->file) < 0) { break; } idx++; } if (idx == ia.dir->files.size()) { ia.dir->files.push_back(ia.new_file); } else { ia.dir->files.insert(idx, ia.new_file); } fs_changed = true; if (ClassDB::is_parent_class(ia.new_file->type, SNAME("Script"))) { _queue_update_script_class(ia.dir->get_file_path(idx)); } if (ia.new_file->type == SNAME("PackedScene")) { _queue_update_scene_groups(ia.dir->get_file_path(idx)); } } break; case ItemAction::ACTION_FILE_REMOVE: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); if (ClassDB::is_parent_class(ia.dir->files[idx]->type, SNAME("Script"))) { _queue_update_script_class(ia.dir->get_file_path(idx)); } if (ia.dir->files[idx]->type == SNAME("PackedScene")) { _queue_update_scene_groups(ia.dir->get_file_path(idx)); } _delete_internal_files(ia.dir->files[idx]->file); memdelete(ia.dir->files[idx]); ia.dir->files.remove_at(idx); fs_changed = true; } break; case ItemAction::ACTION_FILE_TEST_REIMPORT: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); String full_path = ia.dir->get_file_path(idx); if (_test_for_reimport(full_path, false)) { //must reimport reimports.push_back(full_path); if (!ia.import_group_file.is_empty()) { import_group_files[full_path] = ia.import_group_file; } Vector dependencies = _get_dependencies(full_path); for (const String &dependency_path : dependencies) { if (import_extensions.has(dependency_path.get_extension())) { reimports.push_back(dependency_path); String import_groupe_file = ResourceFormatImporter::get_singleton()->get_import_group_file(dependency_path); if (!ia.import_group_file.is_empty()) { import_group_files[dependency_path] = import_groupe_file; } } } } else { //must not reimport, all was good //update modified times, to avoid reimport ia.dir->files[idx]->modified_time = FileAccess::get_modified_time(full_path); ia.dir->files[idx]->import_modified_time = FileAccess::get_modified_time(full_path + ".import"); } fs_changed = true; } break; case ItemAction::ACTION_FILE_RELOAD: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); String full_path = ia.dir->get_file_path(idx); if (ClassDB::is_parent_class(ia.dir->files[idx]->type, SNAME("Script"))) { _queue_update_script_class(full_path); } if (ia.dir->files[idx]->type == SNAME("PackedScene")) { _queue_update_scene_groups(full_path); } reloads.push_back(full_path); } break; } } memdelete_notnull(editor_progress); if (_scan_extensions()) { //needs editor restart //extensions also may provide filetypes to be imported, so they must run before importing if (EditorNode::immediate_confirmation_dialog(TTR("Some extensions need the editor to restart to take effect."), first_scan ? TTR("Restart") : TTR("Save & Restart"), TTR("Continue"))) { if (!first_scan) { EditorNode::get_singleton()->save_all_scenes(); } EditorNode::get_singleton()->restart_editor(); //do not import return true; } } if (reimports.size()) { if (_scan_import_support(reimports)) { return true; } _reimport_files_internal(reimports, &import_group_files); } else { //reimport files will update the uid cache file so if nothing was reimported, update it manually ResourceUID::get_singleton()->update_cache(); } if (first_scan) { //only on first scan this is valid and updated, then settings changed. revalidate_import_files = false; filesystem_settings_version_for_import = ResourceFormatImporter::get_singleton()->get_import_settings_hash(); _save_filesystem_cache(); } if (reloads.size()) { emit_signal(SNAME("resources_reload"), reloads); } scan_actions.clear(); return fs_changed; } void EditorFileSystem::scan() { if (false /*&& bool(Globals::get_singleton()->get("debug/disable_scan"))*/) { return; } if (scanning || scanning_changes || thread.is_started()) { return; } _update_extensions(); if (!use_threads) { scanning = true; scan_total = 0; emit_signal(SNAME("scan_started"), true); _scan_filesystem(); if (filesystem) { memdelete(filesystem); } //file_type_cache.clear(); filesystem = new_filesystem; new_filesystem = nullptr; _update_scan_actions(true); scanning = false; _update_pending_script_classes(); _update_pending_scene_groups(); emit_signal(SNAME("filesystem_changed")); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); first_scan = false; } else { ERR_FAIL_COND(thread.is_started()); set_process(true); Thread::Settings s; scanning = true; scan_total = 0; emit_signal(SNAME("scan_started"), true); s.priority = Thread::PRIORITY_LOW; thread.start(_thread_func, this, s); } } void EditorFileSystem::ScanProgress::increment() { real++; nb_file_to_scan--; current = CLAMP(real / estimated, current, 1); progress->step(current * 1000); EditorFileSystem::singleton->scan_total = current; } void EditorFileSystem::ScanProgress::update_estimated(int p_depth, int p_nb_sub_dirs, int p_nb_files) { // The algorithm calculates an estimated value for the progress of scanning directories and files. // It adjusts the estimate based on the number of directories and files discovered so far, // and ensures the estimated value does not decrease drastically between updates. float average = 10; directories_processed++; if (nb_dir_to_scan > 0) { nb_dir_to_scan--; } nb_dir_to_scan += p_nb_sub_dirs; nb_file_to_scan += p_nb_files; if (directories_processed > 2) { average = (real + nb_file_to_scan) / directories_processed; if (average < 10) { average = 10; } } float old_estimated = estimated; estimated = real + nb_file_to_scan + (nb_dir_to_scan * average * p_depth); if (estimated < old_estimated * 0.95) { estimated = old_estimated * 0.95; } if (estimated <= real + average) { estimated = real + average; } } void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, Ref &da, ScanProgress *p_progress, int p_depth) { List dirs; List files; String cd = da->get_current_dir(); p_dir->modified_time = FileAccess::get_modified_time(cd); da->list_dir_begin(); while (true) { String f = da->get_next(); if (f.is_empty()) { break; } if (da->current_is_hidden()) { continue; } if (da->current_is_dir()) { if (f.begins_with(".")) { // Ignore special and . / .. continue; } if (_should_skip_directory(cd.path_join(f))) { continue; } dirs.push_back(f); } else { files.push_back(f); } } da->list_dir_end(); dirs.sort_custom(); files.sort_custom(); p_progress->update_estimated(p_depth, dirs.size(), files.size()); int idx = 0; for (List::Element *E = dirs.front(); E; E = E->next(), idx++) { String name = E->get(); if (da->change_dir(name) == OK) { String d = da->get_current_dir(); if (d == cd || !d.begins_with(cd)) { da->change_dir(cd); //avoid recursion } else { EditorFileSystemDirectory *efd = memnew(EditorFileSystemDirectory); efd->parent = p_dir; efd->name = E->get(); _scan_new_dir(efd, da, p_progress, p_depth + 1); // Folders are already sorted with FileNoCaseComparator. p_dir->subdirs.push_back(efd); da->change_dir(".."); } } else { ERR_PRINT("Cannot go into subdir '" + E->get() + "'."); } } for (List::Element *E = files.front(); E; E = E->next(), idx++) { String name = E->get(); String ext = name.get_extension().to_lower(); if (!valid_extensions.has(ext)) { continue; //invalid } EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); fi->file = name; String path = cd.path_join(fi->file); FileCache *fc = file_cache.getptr(path); uint64_t mt = FileAccess::get_modified_time(path); if (import_extensions.has(ext)) { //is imported uint64_t import_mt = 0; if (FileAccess::exists(path + ".import")) { import_mt = FileAccess::get_modified_time(path + ".import"); } if (fc && fc->modification_time == mt && fc->import_modification_time == import_mt && !_test_for_reimport(path, true)) { fi->type = fc->type; fi->resource_script_class = fc->resource_script_class; fi->uid = fc->uid; fi->deps = fc->deps; fi->modified_time = fc->modification_time; fi->import_modified_time = fc->import_modification_time; fi->import_valid = fc->import_valid; fi->script_class_name = fc->script_class_name; fi->import_group_file = fc->import_group_file; fi->script_class_extends = fc->script_class_extends; fi->script_class_icon_path = fc->script_class_icon_path; if (fc->type.is_empty()) { fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); fi->import_group_file = ResourceLoader::get_import_group_file(path); //there is also the chance that file type changed due to reimport, must probably check this somehow here (or kind of note it for next time in another file?) //note: I think this should not happen any longer.. } if (revalidate_import_files && !ResourceFormatImporter::get_singleton()->are_import_settings_valid(path)) { ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = E->get(); ia.import_group_file = fi->import_group_file; scan_actions.push_back(ia); } if (fc->uid == ResourceUID::INVALID_ID) { // imported files should always have a UID, so attempt to fetch it. fi->uid = ResourceLoader::get_resource_uid(path); } } else { // Using get_resource_import_info() to prevent calling 3 times ResourceFormatImporter::_get_path_and_type. ResourceFormatImporter::get_singleton()->get_resource_import_info(path, fi->type, fi->uid, fi->import_group_file); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->modified_time = 0; fi->import_modified_time = 0; fi->import_valid = fi->type == "TextFile" ? true : ResourceLoader::is_import_valid(path); ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = E->get(); ia.import_group_file = fi->import_group_file; scan_actions.push_back(ia); } } else { if (fc && fc->modification_time == mt) { //not imported, so just update type if changed fi->type = fc->type; fi->resource_script_class = fc->resource_script_class; fi->uid = fc->uid; fi->modified_time = fc->modification_time; fi->deps = fc->deps; fi->import_modified_time = 0; fi->import_valid = true; fi->script_class_name = fc->script_class_name; fi->script_class_extends = fc->script_class_extends; fi->script_class_icon_path = fc->script_class_icon_path; } else { //new or modified time fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); if (fi->type == "" && textfile_extensions.has(ext)) { fi->type = "TextFile"; } fi->uid = ResourceLoader::get_resource_uid(path); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->deps = _get_dependencies(path); fi->modified_time = mt; fi->import_modified_time = 0; fi->import_valid = true; // Files in dep_update_list are forced for rescan to update dependencies. They don't need other updates. if (!dep_update_list.has(path)) { if (ClassDB::is_parent_class(fi->type, SNAME("Script"))) { _queue_update_script_class(path); } if (fi->type == SNAME("PackedScene")) { _queue_update_scene_groups(path); } } } } if (fi->uid != ResourceUID::INVALID_ID) { if (ResourceUID::get_singleton()->has_id(fi->uid)) { ResourceUID::get_singleton()->set_id(fi->uid, path); } else { ResourceUID::get_singleton()->add_id(fi->uid, path); } } p_dir->files.push_back(fi); p_progress->increment(); } } void EditorFileSystem::_scan_fs_changes(EditorFileSystemDirectory *p_dir, ScanProgress *p_progress) { uint64_t current_mtime = FileAccess::get_modified_time(p_dir->get_path()); bool updated_dir = false; String cd = p_dir->get_path(); if (current_mtime != p_dir->modified_time || using_fat32_or_exfat) { updated_dir = true; p_dir->modified_time = current_mtime; //ooooops, dir changed, see what's going on //first mark everything as verified for (int i = 0; i < p_dir->files.size(); i++) { p_dir->files[i]->verified = false; } for (int i = 0; i < p_dir->subdirs.size(); i++) { p_dir->get_subdir(i)->verified = false; } //then scan files and directories and check what's different Ref da = DirAccess::create(DirAccess::ACCESS_RESOURCES); Error ret = da->change_dir(cd); ERR_FAIL_COND_MSG(ret != OK, "Cannot change to '" + cd + "' folder."); da->list_dir_begin(); while (true) { String f = da->get_next(); if (f.is_empty()) { break; } if (da->current_is_hidden()) { continue; } if (da->current_is_dir()) { if (f.begins_with(".")) { // Ignore special and . / .. continue; } int idx = p_dir->find_dir_index(f); if (idx == -1) { if (_should_skip_directory(cd.path_join(f))) { continue; } EditorFileSystemDirectory *efd = memnew(EditorFileSystemDirectory); efd->parent = p_dir; efd->name = f; Ref d = DirAccess::create(DirAccess::ACCESS_RESOURCES); d->change_dir(cd.path_join(f)); _scan_new_dir(efd, d, p_progress, 1); ItemAction ia; ia.action = ItemAction::ACTION_DIR_ADD; ia.dir = p_dir; ia.file = f; ia.new_dir = efd; scan_actions.push_back(ia); } else { p_dir->subdirs[idx]->verified = true; } } else { String ext = f.get_extension().to_lower(); if (!valid_extensions.has(ext)) { continue; //invalid } int idx = p_dir->find_file_index(f); if (idx == -1) { //never seen this file, add actition to add it EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); fi->file = f; String path = cd.path_join(fi->file); fi->modified_time = FileAccess::get_modified_time(path); fi->import_modified_time = 0; fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); if (fi->type == "" && textfile_extensions.has(ext)) { fi->type = "TextFile"; } fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->import_valid = fi->type == "TextFile" ? true : ResourceLoader::is_import_valid(path); fi->import_group_file = ResourceLoader::get_import_group_file(path); { ItemAction ia; ia.action = ItemAction::ACTION_FILE_ADD; ia.dir = p_dir; ia.file = f; ia.new_file = fi; scan_actions.push_back(ia); } if (import_extensions.has(ext)) { //if it can be imported, and it was added, it needs to be reimported ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = f; ia.import_group_file = fi->import_group_file; scan_actions.push_back(ia); } } else { p_dir->files[idx]->verified = true; } } } da->list_dir_end(); } for (int i = 0; i < p_dir->files.size(); i++) { if (updated_dir && !p_dir->files[i]->verified) { //this file was removed, add action to remove it ItemAction ia; ia.action = ItemAction::ACTION_FILE_REMOVE; ia.dir = p_dir; ia.file = p_dir->files[i]->file; scan_actions.push_back(ia); continue; } String path = cd.path_join(p_dir->files[i]->file); if (import_extensions.has(p_dir->files[i]->file.get_extension().to_lower())) { //check here if file must be imported or not uint64_t mt = FileAccess::get_modified_time(path); bool reimport = false; if (mt != p_dir->files[i]->modified_time) { reimport = true; //it was modified, must be reimported. } else if (!FileAccess::exists(path + ".import")) { reimport = true; //no .import file, obviously reimport } else { uint64_t import_mt = FileAccess::get_modified_time(path + ".import"); if (import_mt != p_dir->files[i]->import_modified_time) { reimport = true; } else if (_test_for_reimport(path, true)) { reimport = true; } } if (reimport) { ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = p_dir->files[i]->file; ia.import_group_file = p_dir->files[i]->import_group_file; scan_actions.push_back(ia); } } else if (ResourceCache::has(path)) { //test for potential reload uint64_t mt = FileAccess::get_modified_time(path); if (mt != p_dir->files[i]->modified_time) { p_dir->files[i]->modified_time = mt; //save new time, but test for reload ItemAction ia; ia.action = ItemAction::ACTION_FILE_RELOAD; ia.dir = p_dir; ia.file = p_dir->files[i]->file; scan_actions.push_back(ia); } } } for (int i = 0; i < p_dir->subdirs.size(); i++) { if ((updated_dir && !p_dir->subdirs[i]->verified) || _should_skip_directory(p_dir->subdirs[i]->get_path())) { //this directory was removed or ignored, add action to remove it ItemAction ia; ia.action = ItemAction::ACTION_DIR_REMOVE; ia.dir = p_dir->subdirs[i]; scan_actions.push_back(ia); continue; } _scan_fs_changes(p_dir->get_subdir(i), p_progress); } } void EditorFileSystem::_delete_internal_files(const String &p_file) { if (FileAccess::exists(p_file + ".import")) { List paths; ResourceFormatImporter::get_singleton()->get_internal_resource_path_list(p_file, &paths); Ref da = DirAccess::create(DirAccess::ACCESS_RESOURCES); for (const String &E : paths) { da->remove(E); } da->remove(p_file + ".import"); } } void EditorFileSystem::_thread_func_sources(void *_userdata) { EditorFileSystem *efs = (EditorFileSystem *)_userdata; if (efs->filesystem) { EditorProgressBG pr("sources", TTR("ScanSources"), 1000); ScanProgress sp; sp.progress = ≺ efs->_scan_fs_changes(efs->filesystem, &sp); } efs->scanning_changes_done.set(); } void EditorFileSystem::scan_changes() { if (first_scan || // Prevent a premature changes scan from inhibiting the first full scan scanning || scanning_changes || thread.is_started()) { scan_changes_pending = true; set_process(true); return; } _update_extensions(); sources_changed.clear(); scanning_changes = true; scanning_changes_done.clear(); if (!use_threads) { if (filesystem) { EditorProgressBG pr("sources", TTR("ScanSources"), 1000); ScanProgress sp; sp.progress = ≺ scan_total = 0; emit_signal(SNAME("scan_started"), false); _scan_fs_changes(filesystem, &sp); bool changed = _update_scan_actions(); _update_pending_script_classes(); _update_pending_scene_groups(); if (changed) { emit_signal(SNAME("filesystem_changed")); } } scanning_changes = false; scanning_changes_done.set(); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); } else { ERR_FAIL_COND(thread_sources.is_started()); set_process(true); scan_total = 0; emit_signal(SNAME("scan_started"), false); Thread::Settings s; s.priority = Thread::PRIORITY_LOW; thread_sources.start(_thread_func_sources, this, s); } } void EditorFileSystem::_notification(int p_what) { switch (p_what) { case NOTIFICATION_EXIT_TREE: { Thread &active_thread = thread.is_started() ? thread : thread_sources; if (use_threads && active_thread.is_started()) { while (scanning) { OS::get_singleton()->delay_usec(1000); } active_thread.wait_to_finish(); WARN_PRINT("Scan thread aborted..."); set_process(false); } if (filesystem) { memdelete(filesystem); } if (new_filesystem) { memdelete(new_filesystem); } filesystem = nullptr; new_filesystem = nullptr; } break; case NOTIFICATION_PROCESS: { if (use_threads) { /** This hack exists because of the EditorProgress nature * of processing events recursively. This needs to be rewritten * at some point entirely, but in the meantime the following * hack prevents deadlock on import. */ static bool prevent_recursive_process_hack = false; if (prevent_recursive_process_hack) { break; } prevent_recursive_process_hack = true; bool done_importing = false; if (scanning_changes) { if (scanning_changes_done.is_set()) { set_process(false); if (thread_sources.is_started()) { thread_sources.wait_to_finish(); } emit_signal(SNAME("scan_stopped"), false); bool changed = _update_scan_actions(); _update_pending_script_classes(); _update_pending_scene_groups(); if (changed) { emit_signal(SNAME("filesystem_changed")); } emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); first_scan = false; scanning_changes = false; // Changed to false here to prevent recursive triggering of scan thread. done_importing = true; } } else if (!scanning && thread.is_started()) { set_process(false); if (filesystem) { memdelete(filesystem); } filesystem = new_filesystem; new_filesystem = nullptr; thread.wait_to_finish(); emit_signal(SNAME("scan_stopped"), true); _update_scan_actions(true); _update_pending_script_classes(); _update_pending_scene_groups(); emit_signal(SNAME("filesystem_changed")); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); first_scan = false; } if (done_importing && scan_changes_pending) { scan_changes_pending = false; scan_changes(); } prevent_recursive_process_hack = false; } } break; } } bool EditorFileSystem::is_scanning() const { return scanning || scanning_changes; } float EditorFileSystem::get_scanning_progress() const { return scan_total; } EditorFileSystemDirectory *EditorFileSystem::get_filesystem() { return filesystem; } void EditorFileSystem::_save_filesystem_cache(EditorFileSystemDirectory *p_dir, Ref p_file) { if (!p_dir) { return; //none } p_file->store_line("::" + p_dir->get_path() + "::" + String::num(p_dir->modified_time)); for (int i = 0; i < p_dir->files.size(); i++) { const EditorFileSystemDirectory::FileInfo *file_info = p_dir->files[i]; if (!file_info->import_group_file.is_empty()) { group_file_cache.insert(file_info->import_group_file); } String type = file_info->type; if (file_info->resource_script_class) { type += "/" + String(file_info->resource_script_class); } PackedStringArray cache_string; cache_string.append(file_info->file); cache_string.append(type); cache_string.append(itos(file_info->uid)); cache_string.append(itos(file_info->modified_time)); cache_string.append(itos(file_info->import_modified_time)); cache_string.append(itos(file_info->import_valid)); cache_string.append(file_info->import_group_file); cache_string.append(String("<>").join({ file_info->script_class_name, file_info->script_class_extends, file_info->script_class_icon_path })); cache_string.append(String("<>").join(file_info->deps)); p_file->store_line(String("::").join(cache_string)); } for (int i = 0; i < p_dir->subdirs.size(); i++) { _save_filesystem_cache(p_dir->subdirs[i], p_file); } } bool EditorFileSystem::_find_file(const String &p_file, EditorFileSystemDirectory **r_d, int &r_file_pos) const { //todo make faster if (!filesystem || scanning) { return false; } String f = ProjectSettings::get_singleton()->localize_path(p_file); // Note: Only checks if base directory is case sensitive. Ref dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); bool fs_case_sensitive = dir->is_case_sensitive("res://"); if (!f.begins_with("res://")) { return false; } f = f.substr(6, f.length()); f = f.replace("\\", "/"); Vector path = f.split("/"); if (path.size() == 0) { return false; } String file = path[path.size() - 1]; path.resize(path.size() - 1); EditorFileSystemDirectory *fs = filesystem; for (int i = 0; i < path.size(); i++) { if (path[i].begins_with(".")) { return false; } int idx = -1; for (int j = 0; j < fs->get_subdir_count(); j++) { if (fs_case_sensitive) { if (fs->get_subdir(j)->get_name() == path[i]) { idx = j; break; } } else { if (fs->get_subdir(j)->get_name().to_lower() == path[i].to_lower()) { idx = j; break; } } } if (idx == -1) { //does not exist, create i guess? EditorFileSystemDirectory *efsd = memnew(EditorFileSystemDirectory); efsd->name = path[i]; efsd->parent = fs; int idx2 = 0; for (int j = 0; j < fs->get_subdir_count(); j++) { if (efsd->name.filenocasecmp_to(fs->get_subdir(j)->get_name()) < 0) { break; } idx2++; } if (idx2 == fs->get_subdir_count()) { fs->subdirs.push_back(efsd); } else { fs->subdirs.insert(idx2, efsd); } fs = efsd; } else { fs = fs->get_subdir(idx); } } int cpos = -1; for (int i = 0; i < fs->files.size(); i++) { if (fs->files[i]->file == file) { cpos = i; break; } } r_file_pos = cpos; *r_d = fs; return cpos != -1; } String EditorFileSystem::get_file_type(const String &p_file) const { EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!_find_file(p_file, &fs, cpos)) { return ""; } return fs->files[cpos]->type; } EditorFileSystemDirectory *EditorFileSystem::find_file(const String &p_file, int *r_index) const { if (!filesystem || scanning) { return nullptr; } EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!_find_file(p_file, &fs, cpos)) { return nullptr; } if (r_index) { *r_index = cpos; } return fs; } EditorFileSystemDirectory *EditorFileSystem::get_filesystem_path(const String &p_path) { if (!filesystem || scanning) { return nullptr; } String f = ProjectSettings::get_singleton()->localize_path(p_path); if (!f.begins_with("res://")) { return nullptr; } f = f.substr(6, f.length()); f = f.replace("\\", "/"); if (f.is_empty()) { return filesystem; } if (f.ends_with("/")) { f = f.substr(0, f.length() - 1); } Vector path = f.split("/"); if (path.size() == 0) { return nullptr; } EditorFileSystemDirectory *fs = filesystem; for (int i = 0; i < path.size(); i++) { int idx = -1; for (int j = 0; j < fs->get_subdir_count(); j++) { if (fs->get_subdir(j)->get_name() == path[i]) { idx = j; break; } } if (idx == -1) { return nullptr; } else { fs = fs->get_subdir(idx); } } return fs; } void EditorFileSystem::_save_late_updated_files() { //files that already existed, and were modified, need re-scanning for dependencies upon project restart. This is done via saving this special file String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join("filesystem_update4"); Ref f = FileAccess::open(fscache, FileAccess::WRITE); ERR_FAIL_COND_MSG(f.is_null(), "Cannot create file '" + fscache + "'. Check user write permissions."); for (const String &E : late_update_files) { f->store_line(E); } } Vector EditorFileSystem::_get_dependencies(const String &p_path) { // Avoid error spam on first opening of a not yet imported project by treating the following situation // as a benign one, not letting the file open error happen: the resource is of an importable type but // it has not been imported yet. if (ResourceFormatImporter::get_singleton()->recognize_path(p_path)) { const String &internal_path = ResourceFormatImporter::get_singleton()->get_internal_resource_path(p_path); if (!internal_path.is_empty() && !FileAccess::exists(internal_path)) { // If path is empty (error), keep the code flow to the error. return Vector(); } } List deps; ResourceLoader::get_dependencies(p_path, &deps); Vector ret; for (const String &E : deps) { ret.push_back(E); } return ret; } String EditorFileSystem::_get_global_script_class(const String &p_type, const String &p_path, String *r_extends, String *r_icon_path) const { for (int i = 0; i < ScriptServer::get_language_count(); i++) { if (ScriptServer::get_language(i)->handles_global_class_type(p_type)) { String global_name; String extends; String icon_path; global_name = ScriptServer::get_language(i)->get_global_class_name(p_path, &extends, &icon_path); *r_extends = extends; *r_icon_path = icon_path; return global_name; } } *r_extends = String(); *r_icon_path = String(); return String(); } void EditorFileSystem::_update_script_classes(bool p_show_progress) { update_script_mutex.lock(); EditorProgress *ep = nullptr; if (p_show_progress && update_script_paths.size() > 0) { ep = memnew(EditorProgress("update_scripts_classes", TTR("(Re)importing Scripts"), update_script_paths.size() * 2)); ep->step("", 0, true); } int step_count = 0; for (const String &path : update_script_paths) { if (ep) { ep->step(TTR("Registering global classes..."), step_count++, false); } EditorFileSystem::get_singleton()->register_global_class_script(path, path); } // Parse documentation second, as it requires the class names to be correct and registered for (const String &path : update_script_paths) { int index = -1; EditorFileSystemDirectory *efd = find_file(path, &index); if (ep) { ep->step(TTR("Updating documentation..."), step_count++, false); } if (!efd || index < 0) { // The file was removed continue; } for (int i = 0; i < ScriptServer::get_language_count(); i++) { ScriptLanguage *lang = ScriptServer::get_language(i); if (lang->supports_documentation() && efd->files[index]->type == lang->get_type()) { Ref