From 570e6c173004b50c56596332ed1457ca78960611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 21 Jul 2022 15:15:54 +0200 Subject: [PATCH] SCons: Refactor LTO options with `lto=` Adds support for LTO on macOS and Android. Disable LTO by default on iOS even if `production=yes` is set. Also add `linker` option to `server` platform missed in #63283. Refactor code handling old arguments to make it simpler (breaks compat, but is explicit enough about it and scripts are easy to fix). --- SConstruct | 69 ++++++++++++++++++++--------------- platform/android/detect.py | 12 ++++++ platform/iphone/detect.py | 16 ++++++-- platform/javascript/detect.py | 14 +++---- platform/osx/detect.py | 15 +++++++- platform/server/detect.py | 39 ++++++++++++++++++-- platform/windows/detect.py | 24 +++++++----- platform/x11/detect.py | 26 ++++--------- 8 files changed, 142 insertions(+), 73 deletions(-) diff --git a/SConstruct b/SConstruct index 2a5c8d5464b..d14fb09bac1 100644 --- a/SConstruct +++ b/SConstruct @@ -123,7 +123,7 @@ opts.Add("arch", "Platform-dependent architecture (arm/arm64/x86/x64/mips/...)", opts.Add(EnumVariable("bits", "Target platform bits", "default", ("default", "32", "64"))) opts.Add(EnumVariable("optimize", "Optimization type", "speed", ("speed", "size", "none"))) opts.Add(BoolVariable("production", "Set defaults to build Godot for use in production", False)) -opts.Add(BoolVariable("use_lto", "Use link-time optimization", False)) +opts.Add(EnumVariable("lto", "Link-time optimization (for production builds)", "none", ("none", "thin", "full"))) # Components opts.Add(BoolVariable("deprecated", "Enable deprecated features", True)) @@ -399,34 +399,6 @@ if selected_platform in platform_list: env.Tool("compilation_db") env.Alias("compiledb", env.CompilationDatabase()) - # 'dev' and 'production' are aliases to set default options if they haven't been set - # manually by the user. - if env["dev"]: - env["verbose"] = methods.get_cmdline_bool("verbose", True) - env["warnings"] = ARGUMENTS.get("warnings", "extra") - env["werror"] = methods.get_cmdline_bool("werror", True) - if env["production"]: - env["use_static_cpp"] = methods.get_cmdline_bool("use_static_cpp", True) - env["use_lto"] = methods.get_cmdline_bool("use_lto", True) - print("use_lto is: " + str(env["use_lto"])) - env["debug_symbols"] = methods.get_cmdline_bool("debug_symbols", False) - if not env["tools"] and env["target"] == "debug": - print( - "WARNING: Requested `production` build with `tools=no target=debug`, " - "this will give you a full debug template (use `target=release_debug` " - "for an optimized template with debug features)." - ) - if env.msvc: - print( - "WARNING: For `production` Windows builds, you should use MinGW with GCC " - "or Clang instead of Visual Studio, as they can better optimize the " - "GDScript VM in a very significant way. MSVC LTO also doesn't work " - "reliably for our use case." - "If you want to use MSVC nevertheless for production builds, set " - "`debug_symbols=no use_lto=no` instead of the `production=yes` option." - ) - Exit(255) - env.extra_suffix = "" if env["extra_suffix"] != "": @@ -472,6 +444,45 @@ if selected_platform in platform_list: # We apply it to CCFLAGS (both C and C++ code) in case it impacts C features. env.Prepend(CCFLAGS=["/std:c++14"]) + # 'dev' and 'production' are aliases to set default options if they haven't been set + # manually by the user. + if env["dev"]: + env["verbose"] = methods.get_cmdline_bool("verbose", True) + env["warnings"] = ARGUMENTS.get("warnings", "extra") + env["werror"] = methods.get_cmdline_bool("werror", True) + if env["production"]: + env["use_static_cpp"] = methods.get_cmdline_bool("use_static_cpp", True) + env["lto"] = ARGUMENTS.get("lto", "full") + env["debug_symbols"] = methods.get_cmdline_bool("debug_symbols", False) + if not env["tools"] and env["target"] == "debug": + print( + "WARNING: Requested `production` build with `tools=no target=debug`, " + "this will give you a full debug template (use `target=release_debug` " + "for an optimized template with debug features)." + ) + if env.msvc: + print( + "WARNING: For `production` Windows builds, you should use MinGW with GCC " + "or Clang instead of Visual Studio, as they can better optimize the " + "GDScript VM in a very significant way. MSVC LTO also doesn't work " + "reliably for our use case." + "If you want to use MSVC nevertheless for production builds, set " + "`debug_symbols=no lto=none` instead of the `production=yes` option." + ) + Exit(255) + if env["lto"] != "none": + print("Using LTO: " + env["lto"]) + + # Handle renamed options. + if "use_lto" in ARGUMENTS or "use_thinlto" in ARGUMENTS: + print("Error: The `use_lto` and `use_thinlto` boolean options have been unified to `lto=`.") + print(" Please adjust your scripts accordingly.") + Exit(255) + if "use_lld" in ARGUMENTS: + print("Error: The `use_lld` boolean option has been replaced by `linker=`.") + print(" Please adjust your scripts accordingly.") + Exit(255) + # Configure compiler warnings if env.msvc: # MSVC # Truncations, narrowing conversions, signed/unsigned comparisons... diff --git a/platform/android/detect.py b/platform/android/detect.py index adfe15b42b6..c3206330f94 100644 --- a/platform/android/detect.py +++ b/platform/android/detect.py @@ -48,6 +48,9 @@ def get_ndk_version(): def get_flags(): return [ ("tools", False), + # Benefits of LTO for Android (size, performance) haven't been clearly established yet. + # So for now we override the default value which may be set when using `production=yes`. + ("lto", "none"), ] @@ -137,6 +140,15 @@ def configure(env): env.Append(CPPDEFINES=["_DEBUG"]) env.Append(CPPFLAGS=["-UNDEBUG"]) + # LTO + if env["lto"] != "none": + if env["lto"] == "thin": + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + else: + env.Append(CCFLAGS=["-flto"]) + env.Append(LINKFLAGS=["-flto"]) + # Compiler configuration env["SHLIBSUFFIX"] = ".so" diff --git a/platform/iphone/detect.py b/platform/iphone/detect.py index c3077fe066c..4c962a03dc2 100644 --- a/platform/iphone/detect.py +++ b/platform/iphone/detect.py @@ -42,6 +42,9 @@ def get_opts(): def get_flags(): return [ ("tools", False), + # Disable by default even if production is set, as it makes linking in Xcode + # on exports very slow and that's not what most users expect. + ("lto", "none"), ] @@ -64,11 +67,16 @@ def configure(env): env.Append(CCFLAGS=["-gdwarf-2", "-O0"]) env.Append(CPPDEFINES=["_DEBUG", ("DEBUG", 1)]) - if env["use_lto"]: - env.Append(CCFLAGS=["-flto"]) - env.Append(LINKFLAGS=["-flto"]) + # LTO + if env["lto"] != "none": + if env["lto"] == "thin": + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + else: + env.Append(CCFLAGS=["-flto"]) + env.Append(LINKFLAGS=["-flto"]) - ## Architecture + # Architecture if env["arch"] == "x86": # i386 env["bits"] = "32" elif env["arch"] == "x86_64": diff --git a/platform/javascript/detect.py b/platform/javascript/detect.py index e04e9a9cab8..1441c39cec8 100644 --- a/platform/javascript/detect.py +++ b/platform/javascript/detect.py @@ -31,7 +31,6 @@ def get_opts(): return [ ("initial_memory", "Initial WASM memory (in MiB)", 32), BoolVariable("use_assertions", "Use Emscripten runtime assertions", False), - BoolVariable("use_thinlto", "Use ThinLTO", False), BoolVariable("use_ubsan", "Use Emscripten undefined behavior sanitizer (UBSAN)", False), BoolVariable("use_asan", "Use Emscripten address sanitizer (ASAN)", False), BoolVariable("use_lsan", "Use Emscripten leak sanitizer (LSAN)", False), @@ -101,12 +100,13 @@ def configure(env): env["ENV"] = os.environ # LTO - if env["use_thinlto"]: - env.Append(CCFLAGS=["-flto=thin"]) - env.Append(LINKFLAGS=["-flto=thin"]) - elif env["use_lto"]: - env.Append(CCFLAGS=["-flto=full"]) - env.Append(LINKFLAGS=["-flto=full"]) + if env["lto"] != "none": + if env["lto"] == "thin": + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + else: + env.Append(CCFLAGS=["-flto"]) + env.Append(LINKFLAGS=["-flto"]) # Sanitizers if env["use_ubsan"]: diff --git a/platform/osx/detect.py b/platform/osx/detect.py index 4e54c54eb27..d18c250620c 100644 --- a/platform/osx/detect.py +++ b/platform/osx/detect.py @@ -35,7 +35,11 @@ def get_opts(): def get_flags(): - return [] + return [ + # Benefits of LTO for macOS (size, performance) haven't been clearly established yet. + # So for now we override the default value which may be set when using `production=yes`. + ("lto", "none"), + ] def configure(env): @@ -127,6 +131,15 @@ def configure(env): env["AS"] = basecmd + "as" env.Append(CPPDEFINES=["__MACPORTS__"]) # hack to fix libvpx MM256_BROADCASTSI128_SI256 define + # LTO + if env["lto"] != "none": + if env["lto"] == "thin": + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + else: + env.Append(CCFLAGS=["-flto"]) + env.Append(LINKFLAGS=["-flto"]) + if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"]: env.extra_suffix += "s" diff --git a/platform/server/detect.py b/platform/server/detect.py index 98ecb771ae0..e0e574e81d2 100644 --- a/platform/server/detect.py +++ b/platform/server/detect.py @@ -1,6 +1,7 @@ import os import platform import sys +from methods import get_compiler_version, using_gcc # This file is mostly based on platform/x11/detect.py. # If editing this file, make sure to apply relevant changes here too. @@ -31,6 +32,7 @@ def get_opts(): from SCons.Variables import BoolVariable, EnumVariable return [ + EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")), BoolVariable("use_llvm", "Use the LLVM compiler", False), BoolVariable("use_static_cpp", "Link libgcc and libstdc++ statically for better portability", True), BoolVariable("use_ubsan", "Use LLVM/GCC compiler undefined behavior sanitizer (UBSAN)", False), @@ -100,6 +102,27 @@ def configure(env): env.extra_suffix = ".llvm" + env.extra_suffix env.Append(LIBS=["atomic"]) + # Linker + if env["linker"] != "default": + print("Using linker program: " + env["linker"]) + if env["linker"] == "mold" and using_gcc(env): # GCC < 12.1 doesn't support -fuse-ld=mold. + cc_semver = tuple(get_compiler_version(env)) + if cc_semver < (12, 1): + found_wrapper = False + for path in ["/usr/libexec", "/usr/local/libexec", "/usr/lib", "/usr/local/lib"]: + if os.path.isfile(path + "/mold/ld"): + env.Append(LINKFLAGS=["-B" + path + "/mold"]) + found_wrapper = True + break + if not found_wrapper: + print("Couldn't locate mold installation path. Make sure it's installed in /usr or /usr/local.") + sys.exit(255) + else: + env.Append(LINKFLAGS=["-fuse-ld=mold"]) + else: + env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]]) + + # Sanitizers if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"] or env["use_msan"]: env.extra_suffix += "s" @@ -123,18 +146,26 @@ def configure(env): env.Append(CCFLAGS=["-fsanitize=memory"]) env.Append(LINKFLAGS=["-fsanitize=memory"]) - if env["use_lto"]: - env.Append(CCFLAGS=["-flto"]) - if not env["use_llvm"] and env.GetOption("num_jobs") > 1: + # LTO + if env["lto"] != "none": + if env["lto"] == "thin": + if not env["use_llvm"]: + print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.") + sys.exit(255) + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + elif not env["use_llvm"] and env.GetOption("num_jobs") > 1: + env.Append(CCFLAGS=["-flto"]) env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))]) else: + env.Append(CCFLAGS=["-flto"]) env.Append(LINKFLAGS=["-flto"]) + if not env["use_llvm"]: env["RANLIB"] = "gcc-ranlib" env["AR"] = "gcc-ar" env.Append(CCFLAGS=["-pipe"]) - env.Append(LINKFLAGS=["-pipe"]) ## Dependencies diff --git a/platform/windows/detect.py b/platform/windows/detect.py index 466e40ce9ab..d7f507ba87c 100644 --- a/platform/windows/detect.py +++ b/platform/windows/detect.py @@ -70,7 +70,6 @@ def get_opts(): ("msvc_version", "MSVC version to use. Ignored if VCINSTALLDIR is set in shell env.", None), BoolVariable("use_mingw", "Use the Mingw compiler, even if MSVC is installed.", False), BoolVariable("use_llvm", "Use the LLVM compiler", False), - BoolVariable("use_thinlto", "Use ThinLTO", False), BoolVariable("use_static_cpp", "Link MinGW/MSVC C++ runtime libraries statically", True), BoolVariable("use_asan", "Use address sanitizer (ASAN)", False), ] @@ -282,7 +281,10 @@ def configure_msvc(env, manual_msvc_config): ## LTO - if env["use_lto"]: + if env["lto"] != "none": + if env["lto"] == "thin": + print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.") + sys.exit(255) env.AppendUnique(CCFLAGS=["/GL"]) env.AppendUnique(ARFLAGS=["/LTCG"]) if env["progress"]: @@ -393,17 +395,19 @@ def configure_mingw(env): env["x86_libtheora_opt_gcc"] = True - if env["use_lto"]: - if not env["use_llvm"] and env.GetOption("num_jobs") > 1: + if env["lto"] != "none": + if env["lto"] == "thin": + if not env["use_llvm"]: + print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.") + sys.exit(255) + env.Append(CCFLAGS=["-flto=thin"]) + env.Append(LINKFLAGS=["-flto=thin"]) + elif not env["use_llvm"] and env.GetOption("num_jobs") > 1: env.Append(CCFLAGS=["-flto"]) env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))]) else: - if env["use_thinlto"]: - env.Append(CCFLAGS=["-flto=thin"]) - env.Append(LINKFLAGS=["-flto=thin"]) - else: - env.Append(CCFLAGS=["-flto"]) - env.Append(LINKFLAGS=["-flto"]) + env.Append(CCFLAGS=["-flto"]) + env.Append(LINKFLAGS=["-flto"]) env.Append(LINKFLAGS=["-Wl,--stack," + str(STACK_SIZE)]) diff --git a/platform/x11/detect.py b/platform/x11/detect.py index 592c8bbb5f4..f7ce0b1d9b9 100644 --- a/platform/x11/detect.py +++ b/platform/x11/detect.py @@ -66,8 +66,6 @@ def get_opts(): return [ EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")), BoolVariable("use_llvm", "Use the LLVM compiler", False), - BoolVariable("use_lld", "Use the LLD linker (deprecated, use `linker=lld` instead).", False), - BoolVariable("use_thinlto", "Use ThinLTO (LLVM only, requires linker=lld, implies use_lto=yes)", False), BoolVariable("use_static_cpp", "Link libgcc and libstdc++ statically for better portability", True), BoolVariable("use_ubsan", "Use LLVM/GCC compiler undefined behavior sanitizer (UBSAN)", False), BoolVariable("use_asan", "Use LLVM/GCC compiler address sanitizer (ASAN))", False), @@ -147,12 +145,7 @@ def configure(env): env["CXX"] = "clang++" env.extra_suffix = ".llvm" + env.extra_suffix - if env["use_lld"]: - if env["linker"] != "default": - print("Can't specify both `use_lld=yes` and a non-default `linker`. Remove `use_lld=yes`.") - sys.exit(255) - print("The `use_lld=yes` option is deprecated, use `linker=lld` instead.") - env["linker"] = "lld" + # Linker if env["linker"] != "default": print("Using linker program: " + env["linker"]) @@ -173,13 +166,7 @@ def configure(env): else: env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]]) - if env["use_thinlto"]: - if not env["use_llvm"] or env["linker"] != "lld": - print("ThinLTO is only compatible with LLVM and the LLD linker, use `use_llvm=yes linker=lld`.") - sys.exit(255) - else: - env["use_lto"] = True # ThinLTO implies LTO - + # Sanitizers if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"] or env["use_msan"]: env.extra_suffix += "s" @@ -216,8 +203,12 @@ def configure(env): env.Append(CCFLAGS=["-fsanitize=memory"]) env.Append(LINKFLAGS=["-fsanitize=memory"]) - if env["use_lto"]: - if env["use_thinlto"]: + # LTO + if env["lto"] != "none": + if env["lto"] == "thin": + if not env["use_llvm"]: + print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.") + sys.exit(255) env.Append(CCFLAGS=["-flto=thin"]) env.Append(LINKFLAGS=["-flto=thin"]) elif not env["use_llvm"] and env.GetOption("num_jobs") > 1: @@ -232,7 +223,6 @@ def configure(env): env["AR"] = "gcc-ar" env.Append(CCFLAGS=["-pipe"]) - env.Append(LINKFLAGS=["-pipe"]) # Check for gcc version >= 6 before adding -no-pie version = get_compiler_version(env) or [-1, -1]