virtualx-engine/editor/SCsub
Viktor Ferenczi c5bd0c37ce Running builder (content generator) functions in subprocesses on Windows
- Refactored all builder (make_*) functions into separate Python modules along to the build tree
- Introduced utility function to wrap all invocations on Windows, but does not change it elsewhere
- Introduced stub to use the builders module as a stand alone script and invoke a selected function

There is a problem with file handles related to writing generated content (*.gen.h and *.gen.cpp)
on Windows, which randomly causes a SHARING VIOLATION error to the compiler resulting in flaky
builds. Running all such content generators in a new subprocess instead of directly inside the
build script works around the issue.

Yes, I tried the multiprocessing module. It did not work due to conflict with SCons on cPickle.
Suggested workaround did not fully work either.

Using the run_in_subprocess wrapper on osx and x11 platforms as well for consistency. In case of
running a cross-compilation on Windows they would still be used, but likely it will not happen
in practice. What counts is that the build itself is running on which platform, not the target
platform.

Some generated files are written directly in an SConstruct or SCsub file, before the parallel build starts. They don't need to be written in a subprocess, apparently, so I left them as is.
2018-07-27 21:37:55 +02:00

108 lines
4 KiB
Python

#!/usr/bin/env python
Import('env')
env.editor_sources = []
import os
import os.path
from platform_methods import run_in_subprocess
from compat import open_utf8
import editor_builders
def _make_doc_data_class_path(to_path):
# NOTE: It is safe to generate this file here, since this is still executed serially
g = open_utf8(os.path.join(to_path, "doc_data_class_path.gen.h"), "w")
g.write("static const int _doc_data_class_path_count = " + str(len(env.doc_class_path)) + ";\n")
g.write("struct _DocDataClassPath { const char* name; const char* path; };\n")
g.write("static const _DocDataClassPath _doc_data_class_paths[" + str(len(env.doc_class_path) + 1) + "] = {\n");
for c in sorted(env.doc_class_path):
g.write("\t{\"" + c + "\", \"" + env.doc_class_path[c] + "\"},\n")
g.write("\t{NULL, NULL}\n")
g.write("};\n")
g.close()
if env['tools']:
# Register exporters
reg_exporters_inc = '#include "register_exporters.h"\n'
reg_exporters = 'void register_exporters() {\n'
for e in env.platform_exporters:
env.editor_sources.append("#platform/" + e + "/export/export.cpp")
reg_exporters += '\tregister_' + e + '_exporter();\n'
reg_exporters_inc += '#include "platform/' + e + '/export/export.h"\n'
reg_exporters += '}\n'
# NOTE: It is safe to generate this file here, since this is still executed serially
with open_utf8("register_exporters.gen.cpp", "w") as f:
f.write(reg_exporters_inc)
f.write(reg_exporters)
# API documentation
docs = []
doc_dirs = ["doc/classes"]
for p in env.doc_class_path.values():
if p not in doc_dirs:
doc_dirs.append(p)
for d in doc_dirs:
try:
for f in os.listdir(os.path.join(env.Dir('#').abspath, d)):
docs.append("#" + os.path.join(d, f))
except OSError:
pass
_make_doc_data_class_path(os.path.join(env.Dir('#').abspath, "editor/doc"))
docs = sorted(docs)
env.Depends("#editor/doc_data_compressed.gen.h", docs)
env.CommandNoCache("#editor/doc_data_compressed.gen.h", docs, run_in_subprocess(editor_builders.make_doc_header))
# Certificates
env.Depends("#editor/certs_compressed.gen.h", "#thirdparty/certs/ca-certificates.crt")
env.CommandNoCache("#editor/certs_compressed.gen.h", "#thirdparty/certs/ca-certificates.crt", run_in_subprocess(editor_builders.make_certs_header))
import glob
path = env.Dir('.').abspath
# Translations
tlist = glob.glob(path + "/translations/*.po")
env.Depends('#editor/translations.gen.h', tlist)
env.CommandNoCache('#editor/translations.gen.h', tlist, run_in_subprocess(editor_builders.make_translations_header))
# Fonts
flist = glob.glob(path + "/../thirdparty/fonts/*.ttf")
flist.append(glob.glob(path + "/../thirdparty/fonts/*.otf"))
env.Depends('#editor/builtin_fonts.gen.h', flist)
env.CommandNoCache('#editor/builtin_fonts.gen.h', flist, run_in_subprocess(editor_builders.make_fonts_header))
# Authors
env.Depends('#editor/authors.gen.h', "../AUTHORS.md")
env.CommandNoCache('#editor/authors.gen.h', "../AUTHORS.md", run_in_subprocess(editor_builders.make_authors_header))
# Donors
env.Depends('#editor/donors.gen.h', "../DONORS.md")
env.CommandNoCache('#editor/donors.gen.h', "../DONORS.md", run_in_subprocess(editor_builders.make_donors_header))
# License
env.Depends('#editor/license.gen.h', ["../COPYRIGHT.txt", "../LICENSE.txt"])
env.CommandNoCache('#editor/license.gen.h', ["../COPYRIGHT.txt", "../LICENSE.txt"], run_in_subprocess(editor_builders.make_license_header))
env.add_source_files(env.editor_sources, "*.cpp")
env.add_source_files(env.editor_sources, ["#thirdparty/misc/clipper.cpp"])
SConscript('collada/SCsub')
SConscript('doc/SCsub')
SConscript('fileserver/SCsub')
SConscript('icons/SCsub')
SConscript('import/SCsub')
SConscript('plugins/SCsub')
lib = env.add_library("editor", env.editor_sources)
env.Prepend(LIBS=[lib])
Export('env')