From bea7c9be5d0e05e2ccf3fd0578533fa2cb2f62da Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Thu, 28 Apr 2022 10:24:29 +0300 Subject: [PATCH] [3.x] Improve embedded PCK loading and exporting. --- core/io/file_access_pack.cpp | 71 +++++++++++++++++++------ core/os/os.cpp | 5 ++ core/os/os.h | 2 + editor/editor_export.cpp | 72 +++++++++++++++---------- editor/editor_export.h | 3 ++ platform/windows/export/export.cpp | 68 ++++++++++++++++-------- platform/windows/godot_windows.cpp | 18 ++++--- platform/windows/os_windows.cpp | 52 ++++++++++++++++++ platform/windows/os_windows.h | 2 + platform/x11/os_x11.cpp | 85 ++++++++++++++++++++++++++++++ platform/x11/os_x11.h | 2 + 11 files changed, 307 insertions(+), 73 deletions(-) diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index db2720c36d7..6ba8a9bcad4 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -30,6 +30,7 @@ #include "file_access_pack.h" +#include "core/os/os.h" #include "core/version.h" #include @@ -130,40 +131,76 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, return false; } + bool pck_header_found = false; + + // Search for the header at the start offset - standalone PCK file. f->seek(p_offset); uint32_t magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { + pck_header_found = true; + } - if (magic != PACK_HEADER_MAGIC) { - // loading with offset feature not supported for self contained exe files + // Search for the header in the executable "pck" section - self contained executable. + if (!pck_header_found) { + // Loading with offset feature not supported for self contained exe files. + if (p_offset != 0) { + f->close(); + memdelete(f); + ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported."); + } + + int64_t pck_off = OS::get_singleton()->get_embedded_pck_offset(); + if (pck_off != 0) { + // Search for the header, in case PCK start and section have different alignment. + for (int i = 0; i < 8; i++) { + f->seek(pck_off); + magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { +#ifdef DEBUG_ENABLED + print_verbose("PCK header found in executable pck section, loading from offset 0x" + String::num_int64(pck_off - 4, 16)); +#endif + pck_header_found = true; + break; + } + pck_off++; + } + } + } + + // Search for the header at the end of file - self contained executable. + if (!pck_header_found) { + // Loading with offset feature not supported for self contained exe files. if (p_offset != 0) { f->close(); memdelete(f); ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported."); } - //maybe at the end.... self contained exe f->seek_end(); f->seek(f->get_position() - 4); - magic = f->get_32(); - if (magic != PACK_HEADER_MAGIC) { - f->close(); - memdelete(f); - return false; - } - f->seek(f->get_position() - 12); - - uint64_t ds = f->get_64(); - f->seek(f->get_position() - ds - 8); magic = f->get_32(); - if (magic != PACK_HEADER_MAGIC) { - f->close(); - memdelete(f); - return false; + if (magic == PACK_HEADER_MAGIC) { + f->seek(f->get_position() - 12); + uint64_t ds = f->get_64(); + f->seek(f->get_position() - ds - 8); + magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { +#ifdef DEBUG_ENABLED + print_verbose("PCK header found at the end of executable, loading from offset 0x" + String::num_int64(f->get_position() - 4, 16)); +#endif + pck_header_found = true; + } } } + if (!pck_header_found) { + f->close(); + memdelete(f); + return false; + } + uint32_t version = f->get_32(); uint32_t ver_major = f->get_32(); uint32_t ver_minor = f->get_32(); diff --git a/core/os/os.cpp b/core/os/os.cpp index 580bf09995b..c5b828150ee 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -305,6 +305,11 @@ String OS::get_locale_language() const { return get_locale().left(3).replace("_", ""); } +// Embedded PCK offset. +uint64_t OS::get_embedded_pck_offset() const { + return 0; +} + // Helper function to ensure that a dir name/path will be valid on the OS String OS::get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator) const { Vector invalid_chars = String(": * ? \" < > |").split(" "); diff --git a/core/os/os.h b/core/os/os.h index 5809f486a3a..4c3d1f680d5 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -463,6 +463,8 @@ public: virtual String get_locale() const; String get_locale_language() const; + virtual uint64_t get_embedded_pck_offset() const; + String get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator = false) const; virtual String get_godot_dir_name() const; diff --git a/editor/editor_export.cpp b/editor/editor_export.cpp index b3e24dac99d..02b37b51367 100644 --- a/editor/editor_export.cpp +++ b/editor/editor_export.cpp @@ -1625,6 +1625,19 @@ List EditorExportPlatformPC::get_binary_extensions(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags); + Error err = prepare_template(p_preset, p_debug, p_path, p_flags); + if (err == OK) { + err = export_project_data(p_preset, p_debug, p_path, p_flags); + } + + return err; +} + +Error EditorExportPlatformPC::prepare_template(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { + if (!DirAccess::exists(p_path.get_base_dir())) { + return ERR_FILE_BAD_PATH; + } + String custom_debug = p_preset->get("custom_template/debug"); String custom_release = p_preset->get("custom_template/release"); @@ -1657,38 +1670,41 @@ Error EditorExportPlatformPC::export_project(const Ref &p_pr da->make_dir_recursive(p_path.get_base_dir()); Error err = da->copy(template_path, p_path, get_chmod_flags()); - if (err == OK) { - String pck_path; - if (p_preset->get("binary_format/embed_pck")) { - pck_path = p_path; - } else { - pck_path = p_path.get_basename() + ".pck"; + return err; +} + +Error EditorExportPlatformPC::export_project_data(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { + String pck_path; + if (p_preset->get("binary_format/embed_pck")) { + pck_path = p_path; + } else { + pck_path = p_path.get_basename() + ".pck"; + } + + Vector so_files; + + int64_t embedded_pos; + int64_t embedded_size; + Error err = save_pack(p_preset, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); + if (err == OK && p_preset->get("binary_format/embed_pck")) { + if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) { + EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); + return ERR_INVALID_PARAMETER; } - Vector so_files; - - int64_t embedded_pos; - int64_t embedded_size; - err = save_pack(p_preset, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); - if (err == OK && p_preset->get("binary_format/embed_pck")) { - if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) { - EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); - return ERR_INVALID_PARAMETER; - } - - FixUpEmbeddedPckFunc fixup_func = get_fixup_embedded_pck_func(); - if (fixup_func) { - err = fixup_func(p_path, embedded_pos, embedded_size); - } + FixUpEmbeddedPckFunc fixup_func = get_fixup_embedded_pck_func(); + if (fixup_func) { + err = fixup_func(p_path, embedded_pos, embedded_size); } + } - if (err == OK && !so_files.empty()) { - // If shared object files, copy them. - for (int i = 0; i < so_files.size() && err == OK; i++) { - err = da->copy(so_files[i].path, p_path.get_base_dir().plus_file(so_files[i].path.get_file())); - if (err == OK) { - err = sign_shared_object(p_preset, p_debug, p_path.get_base_dir().plus_file(so_files[i].path.get_file())); - } + if (err == OK && !so_files.empty()) { + // If shared object files, copy them. + DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + for (int i = 0; i < so_files.size() && err == OK; i++) { + err = da->copy(so_files[i].path, p_path.get_base_dir().plus_file(so_files[i].path.get_file())); + if (err == OK) { + err = sign_shared_object(p_preset, p_debug, p_path.get_base_dir().plus_file(so_files[i].path.get_file())); } } } diff --git a/editor/editor_export.h b/editor/editor_export.h index 80926c1e7f6..3c7bae326e7 100644 --- a/editor/editor_export.h +++ b/editor/editor_export.h @@ -434,6 +434,9 @@ public: virtual Error export_project(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags = 0); virtual Error sign_shared_object(const Ref &p_preset, bool p_debug, const String &p_path); + Error prepare_template(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags); + Error export_project_data(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags); + void set_extension(const String &p_extension, const String &p_feature_key = "default"); void set_name(const String &p_name); void set_os_name(const String &p_name); diff --git a/platform/windows/export/export.cpp b/platform/windows/export/export.cpp index 52b7e410756..9592c3ea895 100644 --- a/platform/windows/export/export.cpp +++ b/platform/windows/export/export.cpp @@ -40,7 +40,7 @@ static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size); class EditorExportPlatformWindows : public EditorExportPlatformPC { - void _rcedit_add_data(const Ref &p_preset, const String &p_path); + Error _rcedit_add_data(const Ref &p_preset, const String &p_path); Error _code_sign(const Ref &p_preset, const String &p_path); public: @@ -60,16 +60,27 @@ Error EditorExportPlatformWindows::sign_shared_object(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { - Error err = EditorExportPlatformPC::export_project(p_preset, p_debug, p_path, p_flags); - - if (err != OK) { - return err; + String pck_path = p_path; + if (p_preset->get("binary_format/embed_pck")) { + pck_path = p_path.get_basename() + ".tmp"; } - _rcedit_add_data(p_preset, p_path); + Error err = EditorExportPlatformPC::prepare_template(p_preset, p_debug, pck_path, p_flags); + if (p_preset->get("application/modify_resources") && err == OK) { + err = _rcedit_add_data(p_preset, pck_path); + } + + if (err == OK) { + err = EditorExportPlatformPC::export_project_data(p_preset, p_debug, pck_path, p_flags); + } if (p_preset->get("codesign/enable") && err == OK) { - err = _code_sign(p_preset, p_path); + err = _code_sign(p_preset, pck_path); + } + + if (p_preset->get("binary_format/embed_pck") && err == OK) { + DirAccessRef tmp_dir = DirAccess::create_for_path(p_path.get_base_dir()); + err = tmp_dir->rename(pck_path, p_path); } return err; @@ -96,6 +107,7 @@ void EditorExportPlatformWindows::get_export_options(List *r_optio r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "codesign/description"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::POOL_STRING_ARRAY, "codesign/custom_options"), PoolStringArray())); + r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "application/modify_resources"), true)); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/icon", PROPERTY_HINT_FILE, "*.ico"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/file_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/product_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), "")); @@ -106,17 +118,16 @@ void EditorExportPlatformWindows::get_export_options(List *r_optio r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/trademarks"), "")); } -void EditorExportPlatformWindows::_rcedit_add_data(const Ref &p_preset, const String &p_path) { +Error EditorExportPlatformWindows::_rcedit_add_data(const Ref &p_preset, const String &p_path) { String rcedit_path = EditorSettings::get_singleton()->get("export/windows/rcedit"); - if (rcedit_path.empty()) { - WARN_PRINT("The rcedit tool is not configured in the Editor Settings (Export > Windows > Rcedit). No custom icon or app information data will be embedded in the exported executable."); - return; + if (rcedit_path != String() && !FileAccess::exists(rcedit_path)) { + ERR_PRINT("Could not find rcedit executable at " + rcedit_path + ", aborting."); + return ERR_FILE_NOT_FOUND; } - if (!FileAccess::exists(rcedit_path)) { - ERR_PRINT("Could not find rcedit executable at " + rcedit_path + ", no icon or app information data will be included."); - return; + if (rcedit_path == String()) { + rcedit_path = "rcedit"; // try to run rcedit from PATH } #ifndef WINDOWS_ENABLED @@ -124,8 +135,8 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref String wine_path = EditorSettings::get_singleton()->get("export/windows/wine"); if (wine_path != String() && !FileAccess::exists(wine_path)) { - ERR_PRINT("Could not find wine executable at " + wine_path + ", no icon or app information data will be included."); - return; + ERR_PRINT("Could not find wine executable at " + wine_path + ", aborting."); + return ERR_FILE_NOT_FOUND; } if (wine_path == String()) { @@ -183,13 +194,22 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref args.push_back(trademarks); } -#ifdef WINDOWS_ENABLED - OS::get_singleton()->execute(rcedit_path, args, true); -#else +#ifndef WINDOWS_ENABLED // On non-Windows we need WINE to run rcedit args.push_front(rcedit_path); - OS::get_singleton()->execute(wine_path, args, true); + rcedit_path = wine_path; #endif + + String str; + Error err = OS::get_singleton()->execute(rcedit_path, args, true, nullptr, &str, nullptr, true); + ERR_FAIL_COND_V_MSG(err != OK, err, "Could not start rcedit executable, configure rcedit path in the Editor Settings (Export > Windows > Rcedit)."); + print_line("rcedit (" + p_path + "): " + str); + + if (str.find("Fatal error") != -1) { + return FAILED; + } + + return OK; } Error EditorExportPlatformWindows::_code_sign(const Ref &p_preset, const String &p_path) { @@ -326,7 +346,7 @@ Error EditorExportPlatformWindows::_code_sign(const Ref &p_p String str; Error err = OS::get_singleton()->execute(signtool_path, args, true, nullptr, &str, nullptr, true); - ERR_FAIL_COND_V(err != OK, err); + ERR_FAIL_COND_V_MSG(err != OK, err, "Could not start signtool executable, configure signtool path in the Editor Settings (Export > Windows > Signtool)."); print_line("codesign (" + p_path + "): " + str); #ifndef WINDOWS_ENABLED @@ -355,7 +375,7 @@ bool EditorExportPlatformWindows::can_export(const Ref &p_pr bool valid = EditorExportPlatformPC::can_export(p_preset, err, r_missing_templates); String rcedit_path = EditorSettings::get_singleton()->get("export/windows/rcedit"); - if (rcedit_path.empty()) { + if (p_preset->get("application/modify_resources") && rcedit_path.empty()) { err += TTR("The rcedit tool must be configured in the Editor Settings (Export > Windows > Rcedit) to change the icon or app information data.") + "\n"; } @@ -430,6 +450,10 @@ void register_windows_exporter() { static Error fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size) { // Patch the header of the "pck" section in the PE file so that it corresponds to the embedded data + if (p_embedded_size + p_embedded_start >= 0x100000000) { // Check for total executable size + ERR_FAIL_V_MSG(ERR_INVALID_DATA, "Windows executables cannot be >= 4 GiB."); + } + FileAccess *f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (!f) { return ERR_CANT_OPEN; diff --git a/platform/windows/godot_windows.cpp b/platform/windows/godot_windows.cpp index d7d9e4eace0..2a6748201ef 100644 --- a/platform/windows/godot_windows.cpp +++ b/platform/windows/godot_windows.cpp @@ -39,7 +39,18 @@ #ifndef TOOLS_ENABLED #if defined _MSC_VER #pragma section("pck", read) -__declspec(allocate("pck")) static const char dummy[8] = { 0 }; +__declspec(allocate("pck")) static char dummy[8] = { 0 }; + +// Dummy function to prevent LTO from discarding "pck" section. +extern "C" char *__cdecl pck_section_dummy_call() { + return &dummy[0]; +}; +#if defined _AMD64_ +#pragma comment(linker, "/include:pck_section_dummy_call") +#elif defined _X86_ +#pragma comment(linker, "/include:_pck_section_dummy_call") +#endif + #elif defined __GNUC__ static const char dummy[8] __attribute__((section("pck"), used)) = { 0 }; #endif @@ -140,11 +151,6 @@ __declspec(dllexport) int widechar_main(int argc, wchar_t **argv) { setlocale(LC_CTYPE, ""); -#ifndef TOOLS_ENABLED - // Workaround to prevent LTCG (MSVC LTO) from removing "pck" section - const char *dummy_guard = dummy; -#endif - char **argv_utf8 = new char *[argc]; for (int i = 0; i < argc; ++i) { diff --git a/platform/windows/os_windows.cpp b/platform/windows/os_windows.cpp index 74612be99d6..69983c6cac8 100644 --- a/platform/windows/os_windows.cpp +++ b/platform/windows/os_windows.cpp @@ -3439,6 +3439,58 @@ MainLoop *OS_Windows::get_main_loop() const { return main_loop; } +uint64_t OS_Windows::get_embedded_pck_offset() const { + FileAccessRef f = FileAccess::open(get_executable_path(), FileAccess::READ); + if (!f) { + return 0; + } + + // Process header. + { + f->seek(0x3c); + uint32_t pe_pos = f->get_32(); + + f->seek(pe_pos); + uint32_t magic = f->get_32(); + if (magic != 0x00004550) { + return 0; + } + } + + int num_sections; + { + int64_t header_pos = f->get_position(); + + f->seek(header_pos + 2); + num_sections = f->get_16(); + f->seek(header_pos + 16); + uint16_t opt_header_size = f->get_16(); + + // Skip rest of header + optional header to go to the section headers. + f->seek(f->get_position() + 2 + opt_header_size); + } + int64_t section_table_pos = f->get_position(); + + // Search for the "pck" section. + int64_t off = 0; + for (int i = 0; i < num_sections; ++i) { + int64_t section_header_pos = section_table_pos + i * 40; + f->seek(section_header_pos); + + uint8_t section_name[9]; + f->get_buffer(section_name, 8); + section_name[8] = '\0'; + + if (strcmp((char *)section_name, "pck") == 0) { + f->seek(section_header_pos + 20); + off = f->get_32(); + break; + } + } + + return off; +} + String OS_Windows::get_config_path() const { // The XDG Base Directory specification technically only applies on Linux/*BSD, but it doesn't hurt to support it on Windows as well. if (has_environment("XDG_CONFIG_HOME")) { diff --git a/platform/windows/os_windows.h b/platform/windows/os_windows.h index 56d5f8b3d2d..789837bf802 100644 --- a/platform/windows/os_windows.h +++ b/platform/windows/os_windows.h @@ -482,6 +482,8 @@ public: virtual MainLoop *get_main_loop() const; + virtual uint64_t get_embedded_pck_offset() const; + virtual String get_name() const; virtual Date get_date(bool utc) const; diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp index 3971762a2ce..4a52ae15cec 100644 --- a/platform/x11/os_x11.cpp +++ b/platform/x11/os_x11.cpp @@ -3078,6 +3078,91 @@ MainLoop *OS_X11::get_main_loop() const { return main_loop; } +uint64_t OS_X11::get_embedded_pck_offset() const { + FileAccessRef f = FileAccess::open(get_executable_path(), FileAccess::READ); + if (!f) { + return 0; + } + + // Read and check ELF magic number. + { + uint32_t magic = f->get_32(); + if (magic != 0x464c457f) { // 0x7F + "ELF" + return 0; + } + } + + // Read program architecture bits from class field. + int bits = f->get_8() * 32; + + // Get info about the section header table. + int64_t section_table_pos; + int64_t section_header_size; + if (bits == 32) { + section_header_size = 40; + f->seek(0x20); + section_table_pos = f->get_32(); + f->seek(0x30); + } else { // 64 + section_header_size = 64; + f->seek(0x28); + section_table_pos = f->get_64(); + f->seek(0x3c); + } + int num_sections = f->get_16(); + int string_section_idx = f->get_16(); + + // Load the strings table. + uint8_t *strings; + { + // Jump to the strings section header. + f->seek(section_table_pos + string_section_idx * section_header_size); + + // Read strings data size and offset. + int64_t string_data_pos; + int64_t string_data_size; + if (bits == 32) { + f->seek(f->get_position() + 0x10); + string_data_pos = f->get_32(); + string_data_size = f->get_32(); + } else { // 64 + f->seek(f->get_position() + 0x18); + string_data_pos = f->get_64(); + string_data_size = f->get_64(); + } + + // Read strings data. + f->seek(string_data_pos); + strings = (uint8_t *)memalloc(string_data_size); + if (!strings) { + return 0; + } + f->get_buffer(strings, string_data_size); + } + + // Search for the "pck" section. + int64_t off = 0; + for (int i = 0; i < num_sections; ++i) { + int64_t section_header_pos = section_table_pos + i * section_header_size; + f->seek(section_header_pos); + + uint32_t name_offset = f->get_32(); + if (strcmp((char *)strings + name_offset, "pck") == 0) { + if (bits == 32) { + f->seek(section_header_pos + 0x10); + off = f->get_32(); + } else { // 64 + f->seek(section_header_pos + 0x18); + off = f->get_64(); + } + break; + } + } + memfree(strings); + + return off; +} + void OS_X11::delete_main_loop() { // Send owned clipboard data to clipboard manager before exit. // This has to be done here because the clipboard data is cleared before finalize(). diff --git a/platform/x11/os_x11.h b/platform/x11/os_x11.h index cbf68b17603..35507e6d2ec 100644 --- a/platform/x11/os_x11.h +++ b/platform/x11/os_x11.h @@ -269,6 +269,8 @@ public: virtual MainLoop *get_main_loop() const; + virtual uint64_t get_embedded_pck_offset() const; + virtual bool can_draw() const; virtual void set_clipboard(const String &p_text);