Add custom_modules build option to compile external user modules

This patch adds ability to include external, user-defined C++ modules
to be compiled as part of Godot via `custom_modules` build option
which can be passed to `scons`.

```
scons platform=x11 tools=yes custom_modules="../project/modules"
```

Features:

- detects all available modules under `custom_modules` directory the
same way as it does for built-in modules (not recursive);
- works with both relative and absolute paths on the filesystem;
- multiple search paths can be specified as a comma-separated list.

Module custom documentation and editor icons collection and generation
process is adapted to work with absolute paths needed by such modules.
Also fixed doctool bug mixing absolute and relative paths respectively.

Implementation details:

- `env.module_list` is a dictionary now, which holds both module name as
  key and either a relative or absolute path to a module as a value.
- `methods.detect_modules` is run twice: once for built-in modules, and
  second for external modules, all combined later.
- `methods.detect_modules` was not doing what it says on the tin. It is
  split into `detect_modules` which collects a list of available modules
  and `write_modules` which generates `register_types` sources for each.
- whether a module is built-in or external is distinguished by relative
  or absolute paths respectively. `custom_modules` scons converter
  ensures that the path is absolute even if relative path is supplied,
  including expanding user paths and symbolic links.
- treats the parent directory as if it was Godot's base directory, so
  that there's no need to change include paths in cases where custom
  modules are included as dependencies in other modules.

(cherry picked from commit a96f0e98d7)
This commit is contained in:
Andrii Doroshenko (Xrayez) 2020-04-04 14:46:15 +03:00
parent 11d6c0f20d
commit 133997654c
6 changed files with 125 additions and 66 deletions

View file

@ -48,8 +48,6 @@ for x in sorted(glob.glob("platform/*")):
sys.path.remove(tmppath)
sys.modules.pop('detect')
module_list = methods.detect_modules()
methods.save_active_platforms(active_platforms, active_platform_ids)
custom_tools = ['default']
@ -121,6 +119,7 @@ opts.Add(BoolVariable('deprecated', "Enable deprecated features", True))
opts.Add(BoolVariable('gdscript', "Enable GDScript support", True))
opts.Add(BoolVariable('minizip', "Enable ZIP archive support using minizip", True))
opts.Add(BoolVariable('xaudio2', "Enable the XAudio2 audio driver", False))
opts.Add("custom_modules", "A list of comma-separated directory paths containing custom modules to build.", "")
# Advanced options
opts.Add(BoolVariable('verbose', "Enable verbose output for the compilation", False))
@ -177,17 +176,41 @@ for k in platform_opts.keys():
for o in opt_list:
opts.Add(o)
for x in module_list:
module_enabled = True
tmppath = "./modules/" + x
sys.path.insert(0, tmppath)
# Detect modules.
modules_detected = {}
module_search_paths = ["modules"] # Built-in path.
if ARGUMENTS.get("custom_modules"):
paths = ARGUMENTS.get("custom_modules").split(",")
for p in paths:
try:
module_search_paths.append(methods.convert_custom_modules_path(p))
except ValueError as e:
print(e)
sys.exit(255)
for path in module_search_paths:
# Note: custom modules can override built-in ones.
modules_detected.update(methods.detect_modules(path))
include_path = os.path.dirname(path)
if include_path:
env_base.Prepend(CPPPATH=[include_path])
# Add module options
for name, path in modules_detected.items():
enabled = True
sys.path.insert(0, path)
import config
enabled_attr = getattr(config, "is_enabled", None)
if (callable(enabled_attr) and not config.is_enabled()):
module_enabled = False
sys.path.remove(tmppath)
sys.modules.pop('config')
opts.Add(BoolVariable('module_' + x + '_enabled', "Enable module '%s'" % (x, ), module_enabled))
try:
enabled = config.is_enabled()
except AttributeError:
pass
sys.path.remove(path)
sys.modules.pop("config")
opts.Add(BoolVariable("module_" + name + "_enabled", "Enable module '%s'" % (name,), enabled))
methods.write_modules(modules_detected)
opts.Update(env_base) # update environment
Help(opts.GenerateHelpText(env_base)) # generate help
@ -427,16 +450,15 @@ if selected_platform in platform_list:
sys.path.remove(tmppath)
sys.modules.pop('detect')
env.module_list = []
modules_enabled = {}
env.module_icons_paths = []
env.doc_class_path = {}
for x in module_list:
if not env['module_' + x + '_enabled']:
for name, path in sorted(modules_detected.items()):
if not env["module_" + name + "_enabled"]:
continue
tmppath = "./modules/" + x
sys.path.insert(0, tmppath)
env.current_module = x
sys.path.insert(0, path)
env.current_module = name
import config
# can_build changed number of arguments between 3.0 (1) and 3.1 (2),
# so try both to preserve compatibility for 3.0 modules
@ -450,26 +472,27 @@ if selected_platform in platform_list:
can_build = config.can_build(selected_platform)
if (can_build):
config.configure(env)
env.module_list.append(x)
# Get doc classes paths (if present)
try:
doc_classes = config.get_doc_classes()
doc_path = config.get_doc_path()
for c in doc_classes:
env.doc_class_path[c] = "modules/" + x + "/" + doc_path
env.doc_class_path[c] = path + "/" + doc_path
except:
pass
# Get icon paths (if present)
try:
icons_path = config.get_icons_path()
env.module_icons_paths.append("modules/" + x + "/" + icons_path)
env.module_icons_paths.append(path + "/" + icons_path)
except:
# Default path for module icons
env.module_icons_paths.append("modules/" + x + "/" + "icons")
env.module_icons_paths.append(path + "/" + "icons")
modules_enabled[name] = path
sys.path.remove(tmppath)
sys.modules.pop('config')
sys.path.remove(path)
sys.modules.pop("config")
env.module_list = modules_enabled
methods.update_version(env.module_version_string)

View file

@ -6,6 +6,7 @@ env.editor_sources = []
import os
import os.path
import glob
from platform_methods import run_in_subprocess
from compat import open_utf8
import editor_builders
@ -41,20 +42,21 @@ if env['tools']:
f.write(reg_exporters_inc)
f.write(reg_exporters)
# API documentation
# Core API documentation.
docs = []
doc_dirs = ["doc/classes"]
docs += Glob("#doc/classes/*.xml")
for p in env.doc_class_path.values():
if p not in doc_dirs:
doc_dirs.append(p)
# Module API documentation.
module_dirs = []
for d in env.doc_class_path.values():
if d not in module_dirs:
module_dirs.append(d)
for d in doc_dirs:
try:
for f in os.listdir(os.path.join(env.Dir('#').abspath, d)):
docs.append("#" + os.path.join(d, f))
except OSError:
pass
for d in module_dirs:
if not os.path.isabs(d):
docs += Glob("#" + d + "/*.xml") # Built-in.
else:
docs += Glob(d + "/*.xml") # Custom.
_make_doc_data_class_path(os.path.join(env.Dir('#').abspath, "editor/doc"))
@ -62,9 +64,7 @@ if env['tools']:
env.Depends("#editor/doc_data_compressed.gen.h", docs)
env.CommandNoCache("#editor/doc_data_compressed.gen.h", docs, run_in_subprocess(editor_builders.make_doc_header))
import glob
path = env.Dir('.').abspath
path = env.Dir(".").abspath
# Translations
tlist = glob.glob(path + "/translations/*.po")

View file

@ -2,6 +2,8 @@
Import('env')
import os
from platform_methods import run_in_subprocess
import editor_icons_builders
@ -15,7 +17,10 @@ env['BUILDERS']['MakeEditorIconsBuilder'] = make_editor_icons_builder
icon_sources = Glob("*.svg")
# Module icons
for module_icons in env.module_icons_paths:
icon_sources += Glob('#' + module_icons + "/*.svg")
for path in env.module_icons_paths:
if not os.path.isabs(path):
icon_sources += Glob("#" + path + "/*.svg") # Built-in.
else:
icon_sources += Glob(path + "/*.svg") # Custom.
env.Alias('editor_icons', [env.MakeEditorIconsBuilder('#editor/editor_icons.gen.h', icon_sources)])

View file

@ -1554,7 +1554,11 @@ bool Main::start() {
print_line("Loading docs...");
for (int i = 0; i < _doc_data_class_path_count; i++) {
String path = doc_tool.plus_file(_doc_data_class_paths[i].path);
// Custom modules are always located by absolute path.
String path = _doc_data_class_paths[i].path;
if (path.is_rel_path()) {
path = doc_tool.plus_file(path);
}
String name = _doc_data_class_paths[i].name;
doc_data_classes[name] = path;
if (!checked_paths.has(path)) {

View file

@ -130,31 +130,41 @@ def parse_cg_file(fname, uniforms, sizes, conditionals):
fs.close()
def detect_modules():
def detect_modules(at_path):
module_list = {} # name : path
module_list = []
modules_glob = os.path.join(at_path, "*")
files = glob.glob(modules_glob)
files.sort() # so register_module_types does not change that often, and also plugins are registered in alphabetic order
for x in files:
if not is_module(x):
continue
name = os.path.basename(x)
path = x.replace("\\", "/") # win32
module_list[name] = path
return module_list
def is_module(path):
return os.path.isdir(path) and os.path.exists(path + "/config.py")
def write_modules(module_list):
includes_cpp = ""
register_cpp = ""
unregister_cpp = ""
files = glob.glob("modules/*")
files.sort() # so register_module_types does not change that often, and also plugins are registered in alphabetic order
for x in files:
if not os.path.isdir(x):
continue
if not os.path.exists(x + "/config.py"):
continue
x = x.replace("modules/", "") # rest of world
x = x.replace("modules\\", "") # win32
module_list.append(x)
for name, path in module_list.items():
try:
with open("modules/" + x + "/register_types.h"):
includes_cpp += '#include "modules/' + x + '/register_types.h"\n'
register_cpp += '#ifdef MODULE_' + x.upper() + '_ENABLED\n'
register_cpp += '\tregister_' + x + '_types();\n'
with open(os.path.join(path, "register_types.h")):
includes_cpp += '#include "' + path + '/register_types.h"\n'
register_cpp += '#ifdef MODULE_' + name.upper() + '_ENABLED\n'
register_cpp += '\tregister_' + name + '_types();\n'
register_cpp += '#endif\n'
unregister_cpp += '#ifdef MODULE_' + x.upper() + '_ENABLED\n'
unregister_cpp += '\tunregister_' + x + '_types();\n'
unregister_cpp += '#ifdef MODULE_' + name.upper() + '_ENABLED\n'
unregister_cpp += '\tunregister_' + name + '_types();\n'
unregister_cpp += '#endif\n'
except IOError:
pass
@ -178,7 +188,18 @@ void unregister_module_types() {
with open("modules/register_module_types.gen.cpp", "w") as f:
f.write(modules_cpp)
return module_list
def convert_custom_modules_path(path):
if not path:
return path
err_msg = "Build option 'custom_modules' must %s"
if not os.path.isdir(path):
raise ValueError(err_msg % "point to an existing directory.")
if os.path.realpath(path) == os.path.realpath("modules"):
raise ValueError(err_msg % "be a directory other than built-in `modules` directory.")
if is_module(path):
raise ValueError(err_msg % "point to a directory with modules, not a single module.")
return os.path.realpath(os.path.expanduser(path))
def disable_module(self):

View file

@ -1,6 +1,8 @@
#!/usr/bin/env python
Import('env')
Import("env")
import os
env_modules = env.Clone()
@ -10,11 +12,15 @@ env.modules_sources = []
env_modules.add_source_files(env.modules_sources, "register_module_types.gen.cpp")
for x in env.module_list:
if (x in env.disabled_modules):
for name, path in env.module_list.items():
if (name in env.disabled_modules):
continue
env_modules.Append(CPPDEFINES=["MODULE_" + x.upper() + "_ENABLED"])
SConscript(x + "/SCsub")
env_modules.Append(CPPDEFINES=["MODULE_" + name.upper() + "_ENABLED"])
if not os.path.isabs(path):
SConscript(name + "/SCsub") # Built-in.
else:
SConscript(path + "/SCsub") # Custom.
if env['split_libmodules']:
env.split_lib("modules", env_lib = env_modules)