Merge pull request #63309 from akien-mga/3.x-scons-refactor-lto

SConstruct

@@ -123,7 +123,7 @@ opts.Add("arch", "Platform-dependent architecture (arm/arm64/x86/x64/mips/...)",
 opts.Add(EnumVariable("bits", "Target platform bits", "default", ("default", "32", "64")))
 opts.Add(EnumVariable("optimize", "Optimization type", "speed", ("speed", "size", "none")))
 opts.Add(BoolVariable("production", "Set defaults to build Godot for use in production", False))
-opts.Add(BoolVariable("use_lto", "Use link-time optimization", False))
+opts.Add(EnumVariable("lto", "Link-time optimization (for production builds)", "none", ("none", "thin", "full")))
 
 # Components
 opts.Add(BoolVariable("deprecated", "Enable deprecated features", True))
@@ -402,34 +402,6 @@ if selected_platform in platform_list:
         env.Tool("compilation_db")
         env.Alias("compiledb", env.CompilationDatabase())
 
-    # 'dev' and 'production' are aliases to set default options if they haven't been set
-    # manually by the user.
-    if env["dev"]:
-        env["verbose"] = methods.get_cmdline_bool("verbose", True)
-        env["warnings"] = ARGUMENTS.get("warnings", "extra")
-        env["werror"] = methods.get_cmdline_bool("werror", True)
-    if env["production"]:
-        env["use_static_cpp"] = methods.get_cmdline_bool("use_static_cpp", True)
-        env["use_lto"] = methods.get_cmdline_bool("use_lto", True)
-        print("use_lto is: " + str(env["use_lto"]))
-        env["debug_symbols"] = methods.get_cmdline_bool("debug_symbols", False)
-        if not env["tools"] and env["target"] == "debug":
-            print(
-                "WARNING: Requested `production` build with `tools=no target=debug`, "
-                "this will give you a full debug template (use `target=release_debug` "
-                "for an optimized template with debug features)."
-            )
-        if env.msvc:
-            print(
-                "WARNING: For `production` Windows builds, you should use MinGW with GCC "
-                "or Clang instead of Visual Studio, as they can better optimize the "
-                "GDScript VM in a very significant way. MSVC LTO also doesn't work "
-                "reliably for our use case."
-                "If you want to use MSVC nevertheless for production builds, set "
-                "`debug_symbols=no use_lto=no` instead of the `production=yes` option."
-            )
-            Exit(255)
-
     env.extra_suffix = ""
 
     if env["extra_suffix"] != "":
@@ -475,6 +447,45 @@ if selected_platform in platform_list:
         # We apply it to CCFLAGS (both C and C++ code) in case it impacts C features.
         env.Prepend(CCFLAGS=["/std:c++14"])
 
+    # 'dev' and 'production' are aliases to set default options if they haven't been set
+    # manually by the user.
+    if env["dev"]:
+        env["verbose"] = methods.get_cmdline_bool("verbose", True)
+        env["warnings"] = ARGUMENTS.get("warnings", "extra")
+        env["werror"] = methods.get_cmdline_bool("werror", True)
+    if env["production"]:
+        env["use_static_cpp"] = methods.get_cmdline_bool("use_static_cpp", True)
+        env["lto"] = ARGUMENTS.get("lto", "full")
+        env["debug_symbols"] = methods.get_cmdline_bool("debug_symbols", False)
+        if not env["tools"] and env["target"] == "debug":
+            print(
+                "WARNING: Requested `production` build with `tools=no target=debug`, "
+                "this will give you a full debug template (use `target=release_debug` "
+                "for an optimized template with debug features)."
+            )
+        if env.msvc:
+            print(
+                "WARNING: For `production` Windows builds, you should use MinGW with GCC "
+                "or Clang instead of Visual Studio, as they can better optimize the "
+                "GDScript VM in a very significant way. MSVC LTO also doesn't work "
+                "reliably for our use case."
+                "If you want to use MSVC nevertheless for production builds, set "
+                "`debug_symbols=no lto=none` instead of the `production=yes` option."
+            )
+            Exit(255)
+    if env["lto"] != "none":
+        print("Using LTO: " + env["lto"])
+
+    # Handle renamed options.
+    if "use_lto" in ARGUMENTS or "use_thinlto" in ARGUMENTS:
+        print("Error: The `use_lto` and `use_thinlto` boolean options have been unified to `lto=<none|thin|full>`.")
+        print("       Please adjust your scripts accordingly.")
+        Exit(255)
+    if "use_lld" in ARGUMENTS:
+        print("Error: The `use_lld` boolean option has been replaced by `linker=<default|bfd|gold|lld|mold>`.")
+        print("       Please adjust your scripts accordingly.")
+        Exit(255)
+
     # Configure compiler warnings
     if env.msvc:  # MSVC
         # Truncations, narrowing conversions, signed/unsigned comparisons...

platform/android/detect.py

@@ -48,6 +48,9 @@ def get_ndk_version():
 def get_flags():
     return [
         ("tools", False),
+        # Benefits of LTO for Android (size, performance) haven't been clearly established yet.
+        # So for now we override the default value which may be set when using `production=yes`.
+        ("lto", "none"),
     ]
 
 
@@ -137,6 +140,15 @@ def configure(env):
         env.Append(CPPDEFINES=["_DEBUG"])
         env.Append(CPPFLAGS=["-UNDEBUG"])
 
+    # LTO
+    if env["lto"] != "none":
+        if env["lto"] == "thin":
+            env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto=thin"])
+        else:
+            env.Append(CCFLAGS=["-flto"])
+            env.Append(LINKFLAGS=["-flto"])
+
     # Compiler configuration
 
     env["SHLIBSUFFIX"] = ".so"

platform/iphone/detect.py

@@ -42,6 +42,9 @@ def get_opts():
 def get_flags():
     return [
         ("tools", False),
+        # Disable by default even if production is set, as it makes linking in Xcode
+        # on exports very slow and that's not what most users expect.
+        ("lto", "none"),
     ]
 
 
@@ -64,11 +67,16 @@ def configure(env):
         env.Append(CCFLAGS=["-gdwarf-2", "-O0"])
         env.Append(CPPDEFINES=["_DEBUG", ("DEBUG", 1)])
 
-    if env["use_lto"]:
+    # LTO
-        env.Append(CCFLAGS=["-flto"])
+    if env["lto"] != "none":
-        env.Append(LINKFLAGS=["-flto"])
+        if env["lto"] == "thin":
+            env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto=thin"])
+        else:
+            env.Append(CCFLAGS=["-flto"])
+            env.Append(LINKFLAGS=["-flto"])
 
-    ## Architecture
+    # Architecture
     if env["arch"] == "x86":  # i386
         env["bits"] = "32"
     elif env["arch"] == "x86_64":

platform/javascript/detect.py

@@ -31,7 +31,6 @@ def get_opts():
     return [
         ("initial_memory", "Initial WASM memory (in MiB)", 32),
         BoolVariable("use_assertions", "Use Emscripten runtime assertions", False),
-        BoolVariable("use_thinlto", "Use ThinLTO", False),
         BoolVariable("use_ubsan", "Use Emscripten undefined behavior sanitizer (UBSAN)", False),
         BoolVariable("use_asan", "Use Emscripten address sanitizer (ASAN)", False),
         BoolVariable("use_lsan", "Use Emscripten leak sanitizer (LSAN)", False),
@@ -101,12 +100,13 @@ def configure(env):
     env["ENV"] = os.environ
 
     # LTO
-    if env["use_thinlto"]:
+    if env["lto"] != "none":
-        env.Append(CCFLAGS=["-flto=thin"])
+        if env["lto"] == "thin":
-        env.Append(LINKFLAGS=["-flto=thin"])
+            env.Append(CCFLAGS=["-flto=thin"])
-    elif env["use_lto"]:
+            env.Append(LINKFLAGS=["-flto=thin"])
-        env.Append(CCFLAGS=["-flto=full"])
+        else:
-        env.Append(LINKFLAGS=["-flto=full"])
+            env.Append(CCFLAGS=["-flto"])
+            env.Append(LINKFLAGS=["-flto"])
 
     # Sanitizers
     if env["use_ubsan"]:

platform/osx/detect.py

@@ -35,7 +35,11 @@ def get_opts():
 
 
 def get_flags():
-    return []
+    return [
+        # Benefits of LTO for macOS (size, performance) haven't been clearly established yet.
+        # So for now we override the default value which may be set when using `production=yes`.
+        ("lto", "none"),
+    ]
 
 
 def configure(env):
@@ -127,6 +131,15 @@ def configure(env):
         env["AS"] = basecmd + "as"
         env.Append(CPPDEFINES=["__MACPORTS__"])  # hack to fix libvpx MM256_BROADCASTSI128_SI256 define
 
+    # LTO
+    if env["lto"] != "none":
+        if env["lto"] == "thin":
+            env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto=thin"])
+        else:
+            env.Append(CCFLAGS=["-flto"])
+            env.Append(LINKFLAGS=["-flto"])
+
     if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"]:
         env.extra_suffix += "s"
 

platform/server/detect.py

@@ -1,6 +1,7 @@
 import os
 import platform
 import sys
+from methods import get_compiler_version, using_gcc
 
 # This file is mostly based on platform/x11/detect.py.
 # If editing this file, make sure to apply relevant changes here too.
@@ -31,6 +32,7 @@ def get_opts():
     from SCons.Variables import BoolVariable, EnumVariable
 
     return [
+        EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")),
         BoolVariable("use_llvm", "Use the LLVM compiler", False),
         BoolVariable("use_static_cpp", "Link libgcc and libstdc++ statically for better portability", True),
         BoolVariable("use_ubsan", "Use LLVM/GCC compiler undefined behavior sanitizer (UBSAN)", False),
@@ -100,6 +102,27 @@ def configure(env):
         env.extra_suffix = ".llvm" + env.extra_suffix
         env.Append(LIBS=["atomic"])
 
+    # Linker
+    if env["linker"] != "default":
+        print("Using linker program: " + env["linker"])
+        if env["linker"] == "mold" and using_gcc(env):  # GCC < 12.1 doesn't support -fuse-ld=mold.
+            cc_semver = tuple(get_compiler_version(env))
+            if cc_semver < (12, 1):
+                found_wrapper = False
+                for path in ["/usr/libexec", "/usr/local/libexec", "/usr/lib", "/usr/local/lib"]:
+                    if os.path.isfile(path + "/mold/ld"):
+                        env.Append(LINKFLAGS=["-B" + path + "/mold"])
+                        found_wrapper = True
+                        break
+                if not found_wrapper:
+                    print("Couldn't locate mold installation path. Make sure it's installed in /usr or /usr/local.")
+                    sys.exit(255)
+            else:
+                env.Append(LINKFLAGS=["-fuse-ld=mold"])
+        else:
+            env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]])
+
+    # Sanitizers
     if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"] or env["use_msan"]:
         env.extra_suffix += "s"
 
@@ -123,18 +146,26 @@ def configure(env):
             env.Append(CCFLAGS=["-fsanitize=memory"])
             env.Append(LINKFLAGS=["-fsanitize=memory"])
 
-    if env["use_lto"]:
+    # LTO
-        env.Append(CCFLAGS=["-flto"])
+    if env["lto"] != "none":
-        if not env["use_llvm"] and env.GetOption("num_jobs") > 1:
+        if env["lto"] == "thin":
+            if not env["use_llvm"]:
+                print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
+                sys.exit(255)
+            env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto=thin"])
+        elif not env["use_llvm"] and env.GetOption("num_jobs") > 1:
+            env.Append(CCFLAGS=["-flto"])
             env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))])
         else:
+            env.Append(CCFLAGS=["-flto"])
             env.Append(LINKFLAGS=["-flto"])
+
         if not env["use_llvm"]:
             env["RANLIB"] = "gcc-ranlib"
             env["AR"] = "gcc-ar"
 
     env.Append(CCFLAGS=["-pipe"])
-    env.Append(LINKFLAGS=["-pipe"])
 
     ## Dependencies
 

platform/windows/detect.py

@@ -70,7 +70,6 @@ def get_opts():
         ("msvc_version", "MSVC version to use. Ignored if VCINSTALLDIR is set in shell env.", None),
         BoolVariable("use_mingw", "Use the Mingw compiler, even if MSVC is installed.", False),
         BoolVariable("use_llvm", "Use the LLVM compiler", False),
-        BoolVariable("use_thinlto", "Use ThinLTO", False),
         BoolVariable("use_static_cpp", "Link MinGW/MSVC C++ runtime libraries statically", True),
         BoolVariable("use_asan", "Use address sanitizer (ASAN)", False),
     ]
@@ -282,7 +281,10 @@ def configure_msvc(env, manual_msvc_config):
 
     ## LTO
 
-    if env["use_lto"]:
+    if env["lto"] != "none":
+        if env["lto"] == "thin":
+            print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
+            sys.exit(255)
         env.AppendUnique(CCFLAGS=["/GL"])
         env.AppendUnique(ARFLAGS=["/LTCG"])
         if env["progress"]:
@@ -393,17 +395,19 @@ def configure_mingw(env):
 
     env["x86_libtheora_opt_gcc"] = True
 
-    if env["use_lto"]:
+    if env["lto"] != "none":
-        if not env["use_llvm"] and env.GetOption("num_jobs") > 1:
+        if env["lto"] == "thin":
+            if not env["use_llvm"]:
+                print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
+                sys.exit(255)
+            env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto=thin"])
+        elif not env["use_llvm"] and env.GetOption("num_jobs") > 1:
             env.Append(CCFLAGS=["-flto"])
             env.Append(LINKFLAGS=["-flto=" + str(env.GetOption("num_jobs"))])
         else:
-            if env["use_thinlto"]:
+            env.Append(CCFLAGS=["-flto"])
-                env.Append(CCFLAGS=["-flto=thin"])
+            env.Append(LINKFLAGS=["-flto"])
-                env.Append(LINKFLAGS=["-flto=thin"])
-            else:
-                env.Append(CCFLAGS=["-flto"])
-                env.Append(LINKFLAGS=["-flto"])
 
     env.Append(LINKFLAGS=["-Wl,--stack," + str(STACK_SIZE)])
 

platform/x11/detect.py

@@ -66,8 +66,6 @@ def get_opts():
     return [
         EnumVariable("linker", "Linker program", "default", ("default", "bfd", "gold", "lld", "mold")),
         BoolVariable("use_llvm", "Use the LLVM compiler", False),
-        BoolVariable("use_lld", "Use the LLD linker (deprecated, use `linker=lld` instead).", False),
-        BoolVariable("use_thinlto", "Use ThinLTO (LLVM only, requires linker=lld, implies use_lto=yes)", False),
         BoolVariable("use_static_cpp", "Link libgcc and libstdc++ statically for better portability", True),
         BoolVariable("use_ubsan", "Use LLVM/GCC compiler undefined behavior sanitizer (UBSAN)", False),
         BoolVariable("use_asan", "Use LLVM/GCC compiler address sanitizer (ASAN))", False),
@@ -147,12 +145,7 @@ def configure(env):
             env["CXX"] = "clang++"
         env.extra_suffix = ".llvm" + env.extra_suffix
 
-    if env["use_lld"]:
+    # Linker
-        if env["linker"] != "default":
-            print("Can't specify both `use_lld=yes` and a non-default `linker`. Remove `use_lld=yes`.")
-            sys.exit(255)
-        print("The `use_lld=yes` option is deprecated, use `linker=lld` instead.")
-        env["linker"] = "lld"
 
     if env["linker"] != "default":
         print("Using linker program: " + env["linker"])
@@ -173,13 +166,7 @@ def configure(env):
         else:
             env.Append(LINKFLAGS=["-fuse-ld=%s" % env["linker"]])
 
-    if env["use_thinlto"]:
+    # Sanitizers
-        if not env["use_llvm"] or env["linker"] != "lld":
-            print("ThinLTO is only compatible with LLVM and the LLD linker, use `use_llvm=yes linker=lld`.")
-            sys.exit(255)
-        else:
-            env["use_lto"] = True  # ThinLTO implies LTO
-
     if env["use_ubsan"] or env["use_asan"] or env["use_lsan"] or env["use_tsan"] or env["use_msan"]:
         env.extra_suffix += "s"
 
@@ -216,8 +203,12 @@ def configure(env):
             env.Append(CCFLAGS=["-fsanitize=memory"])
             env.Append(LINKFLAGS=["-fsanitize=memory"])
 
-    if env["use_lto"]:
+    # LTO
-        if env["use_thinlto"]:
+    if env["lto"] != "none":
+        if env["lto"] == "thin":
+            if not env["use_llvm"]:
+                print("ThinLTO is only compatible with LLVM, use `use_llvm=yes` or `lto=full`.")
+                sys.exit(255)
             env.Append(CCFLAGS=["-flto=thin"])
             env.Append(LINKFLAGS=["-flto=thin"])
         elif not env["use_llvm"] and env.GetOption("num_jobs") > 1:
@@ -232,7 +223,6 @@ def configure(env):
             env["AR"] = "gcc-ar"
 
     env.Append(CCFLAGS=["-pipe"])
-    env.Append(LINKFLAGS=["-pipe"])
 
     # Check for gcc version >= 6 before adding -no-pie
     version = get_compiler_version(env) or [-1, -1]