Merge pull request #13359 from rraallvv/cache_pruning

Fixes for SCons shared cache for Travis-CI and AppVeyor-CI (master)

.appveyor.yml

@@ -3,8 +3,8 @@ os: Visual Studio 2015
 environment:
   HOME: "%HOMEDRIVE%%HOMEPATH%"
   PYTHON: C:\Python27
-  SCONS_CACHE: "%HOME%\\scons_cache"
-  SCONS_CACHE_LIMIT: 128
+  SCONS_CACHE_ROOT: "%HOME%\\scons_cache"
+  SCONS_CACHE_LIMIT: 512
   matrix:
     - VS: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat
       GD_PLATFORM: windows
@@ -13,7 +13,7 @@ environment:
       ARCH: amd64
 
 cache:
-  - "%SCONS_CACHE%"
+  - "%SCONS_CACHE_ROOT%"
 
 install:
   - SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -25,6 +25,7 @@ before_build:
   - python --version
   - scons --version
   - cl.exe
+  - SET "SCONS_CACHE=%SCONS_CACHE_ROOT%\master"
 
 build_script:
-- scons platform=%GD_PLATFORM% target=%TARGET% tools=%TOOLS% progress=no
+- scons platform=%GD_PLATFORM% target=%TARGET% tools=%TOOLS% verbose=yes progress=no

.travis.yml

@@ -7,7 +7,7 @@ sudo: false
 env:
   global:
     - SCONS_CACHE=$HOME/.scons_cache
-    - SCONS_CACHE_LIMIT=128
+    - SCONS_CACHE_LIMIT=1024
 
 cache:
   directories:
@@ -18,22 +18,22 @@ matrix:
     - env: STATIC_CHECKS=yes
       os: linux
       compiler: clang
-    - env: GODOT_TARGET=x11 TOOLS=yes
+    - env: GODOT_TARGET=x11 TOOLS=yes CACHE_NAME=${GODOT_TARGET}-gcc-tools
       os: linux
       compiler: gcc
-    - env: GODOT_TARGET=x11 TOOLS=no
+    - env: GODOT_TARGET=x11 TOOLS=no CACHE_NAME=${GODOT_TARGET}-clang
       os: linux
       compiler: clang
-    #- env: GODOT_TARGET=windows TOOLS=yes
+    #- env: GODOT_TARGET=windows TOOLS=yes CACHE_NAME=${GODOT_TARGET}-gcc-tools
     #  os: linux
     #  compiler: gcc
-    - env: GODOT_TARGET=android TOOLS=no
+    - env: GODOT_TARGET=android TOOLS=no CACHE_NAME=${GODOT_TARGET}-gcc
       os: linux
       compiler: gcc
-    - env: GODOT_TARGET=osx TOOLS=yes
+    - env: GODOT_TARGET=osx TOOLS=yes CACHE_NAME=${GODOT_TARGET}-clang-tools
       os: osx
       compiler: clang
-    #- env: GODOT_TARGET=iphone TOOLS=no
+    #- env: GODOT_TARGET=iphone TOOLS=no CACHE_NAME=${GODOT_TARGET}-clang
     #  os: osx
     #  compiler: clang
 

SConstruct

@@ -487,6 +487,7 @@ node_count_interval = 1
 node_pruning = 8 # Number of nodes to process before prunning the cache
 if ('env' in locals()):
     node_count_fname = str(env.Dir('#')) + '/.scons_node_count'
+show_progress = env['progress']
 
 import time, math
 
@@ -497,23 +498,26 @@ class cache_progress:
         self.path = path
         self.limit = limit
         self.exponent_scale = math.log(2) / half_life
+        if env['verbose'] and path != None:
+            screen.write('Current cache limit is ' + self.convert_size(limit) + ' (used: ' + self.convert_size(self.get_size(path)) + ')\n')
         self.pruning = node_pruning
         self.delete(self.file_list())
 
     def __call__(self, node, *args, **kw):
-        global node_count, node_count_max, node_count_interval, node_count_fname, node_pruning
-        # Print the progress percentage
-        node_count += node_count_interval
-        if (node_count_max > 0 and node_count <= node_count_max):
-            screen.write('\r[%3d%%] ' % (node_count * 100 / node_count_max))
-            screen.flush()
-        elif (node_count_max > 0 and node_count > node_count_max):
-            screen.write('\r[100%] ')
-            screen.flush()
-        else:
-            screen.write('\r[Initial build] ')
-            screen.flush()
-        # Prune if the number of nodes proccessed is 'node_pruning' or bigger
+        global node_count, node_count_max, node_count_interval, node_count_fname, node_pruning, show_progress
+        if show_progress:
+            # Print the progress percentage
+            node_count += node_count_interval
+            if (node_count_max > 0 and node_count <= node_count_max):
+                screen.write('\r[%3d%%] ' % (node_count * 100 / node_count_max))
+                screen.flush()
+            elif (node_count_max > 0 and node_count > node_count_max):
+                screen.write('\r[100%] ')
+                screen.flush()
+            else:
+                screen.write('\r[Initial build] ')
+                screen.flush()
+        # Prune if the number of nodes processed is 'node_pruning' or bigger
         self.pruning -= node_count_interval
         if self.pruning <= 0:
             self.pruning = node_pruning
@@ -522,8 +526,9 @@ class cache_progress:
     def delete(self, files):
         if len(files) == 0:
             return
-        # Utter something
-        screen.write('\rPurging %d %s from cache...\n' % (len(files), len(files) > 1 and 'files' or 'file'))
+        if env['verbose']:
+            # Utter something
+            screen.write('\rPurging %d %s from cache...\n' % (len(files), len(files) > 1 and 'files' or 'file'))
         map(os.remove, files)
 
     def file_list(self):
@@ -557,22 +562,40 @@ class cache_progress:
         else:
             return [x[0] for x in file_stat[mark:]]
 
+    def convert_size(self, size_bytes):
+       if size_bytes == 0:
+           return "0 bytes"
+       size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+       i = int(math.floor(math.log(size_bytes, 1024)))
+       p = math.pow(1024, i)
+       s = round(size_bytes / p, 2)
+       return "%s %s" % (int(s) if i == 0 else s, size_name[i])
+
+    def get_size(self, start_path = '.'):
+        total_size = 0
+        for dirpath, dirnames, filenames in os.walk(start_path):
+            for f in filenames:
+                fp = os.path.join(dirpath, f)
+                total_size += os.path.getsize(fp)
+        return total_size
+
 def progress_finish(target, source, env):
-    global node_count
+    global node_count, progressor
     with open(node_count_fname, 'w') as f:
         f.write('%d\n' % node_count)
+    progressor.delete(progressor.file_list())
 
-if 'env' in locals() and env['progress']:
-    try:
-        with open(node_count_fname) as f:
-            node_count_max = int(f.readline())
-    except:
-        pass
-    cache_directory = os.environ.get("SCONS_CACHE")
-    # Simple cache pruning, attached to SCons' progress callback. Trim the
-    # cache directory to a size not larger than cache_limit.
-    cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024
-    progress = cache_progress(cache_directory, cache_limit)
-    Progress(progress, interval = node_count_interval)
-    progress_finish_command = Command('progress_finish', [], progress_finish)
-    AlwaysBuild(progress_finish_command)
+try:
+    with open(node_count_fname) as f:
+        node_count_max = int(f.readline())
+except:
+    pass
+cache_directory = os.environ.get("SCONS_CACHE")
+# Simple cache pruning, attached to SCons' progress callback. Trim the
+# cache directory to a size not larger than cache_limit.
+cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024
+progressor = cache_progress(cache_directory, cache_limit)
+Progress(progressor, interval = node_count_interval)
+
+progress_finish_command = Command('progress_finish', [], progress_finish)
+AlwaysBuild(progress_finish_command)

core/SCsub

@@ -106,5 +106,6 @@ SConscript('helper/SCsub')
 
 # Build it all as a library
 lib = env.Library("core", env.core_sources)
+env.NoCache(lib)
 env.Prepend(LIBS=[lib])
 Export('env')

drivers/SCsub

@@ -46,4 +46,5 @@ if env.split_drivers:
 else:
     env.add_source_files(env.drivers_sources, "*.cpp")
     lib = env.Library("drivers", env.drivers_sources)
+    env.NoCache(lib)
     env.Prepend(LIBS=[lib])

editor/SCsub

@@ -475,6 +475,7 @@ if env['tools']:
     SConscript('plugins/SCsub')
 
     lib = env.Library("editor", env.editor_sources)
+    env.NoCache(lib)
     env.Prepend(LIBS=[lib])
 
     Export('env')

main/SCsub

@@ -57,5 +57,5 @@ env.Command("#main/app_icon.gen.h", "#main/app_icon.png", make_app_icon)
 SConscript('tests/SCsub')
 
 lib = env.Library("main", env.main_sources)
-
+env.NoCache(lib)
 env.Prepend(LIBS=[lib])

main/tests/SCsub

@@ -10,5 +10,5 @@ Export('env')
 # SConscript('math/SCsub');
 
 lib = env.Library("tests", env.tests_sources)
-
+env.NoCache(lib)
 env.Prepend(LIBS=[lib])

methods.py

@@ -1496,6 +1496,7 @@ def split_lib(self, libname):
         if base != cur_base and len(list) > max_src:
             if num > 0:
                 lib = env.Library(libname + str(num), list)
+                env.NoCache(lib)
                 lib_list.append(lib)
                 list = []
             num = num + 1
@@ -1503,6 +1504,7 @@ def split_lib(self, libname):
         list.append(f)
 
     lib = env.Library(libname + str(num), list)
+    env.NoCache(lib)
     lib_list.append(lib)
 
     if len(lib_list) > 0:
@@ -1510,11 +1512,14 @@ def split_lib(self, libname):
         if os.name == 'posix' and sys.platform == 'msys':
             env.Replace(ARFLAGS=['rcsT'])
             lib = env.Library(libname + "_collated", lib_list)
+            env.NoCache(lib)
             lib_list = [lib]
 
     lib_base = []
     env.add_source_files(lib_base, "*.cpp")
-    lib_list.insert(0, env.Library(libname, lib_base))
+    lib = env.Library(libname, lib_base)
+    env.NoCache(lib)
+    lib_list.insert(0, lib)
 
     env.Prepend(LIBS=lib_list)
 

modules/SCsub

@@ -18,5 +18,6 @@ for x in env.module_list:
     SConscript(x + "/SCsub")
 
 lib = env_modules.Library("modules", env.modules_sources)
+env_modules.NoCache(lib)
 
 env.Prepend(LIBS=[lib])

modules/freetype/SCsub

@@ -69,6 +69,7 @@ if env['builtin_freetype']:
         env.Append(CPPPATH=["#thirdparty/libpng"])
 
     lib = env.Library("freetype_builtin", thirdparty_sources)
+    env.NoCache(lib)
     # Needs to be appended to arrive after libscene in the linker call,
     # but we don't want it to arrive *after* system libs, so manual hack
     # LIBS contains first SCons Library objects ("SCons.Node.FS.File object")

modules/gdnative/SCsub

@@ -248,4 +248,5 @@ if ARGUMENTS.get('gdnative_wrapper', False):
     if not env.msvc:
         gd_wrapper_env.Append(CCFLAGS=['-fPIC'])
 
-    gd_wrapper_env.Library("#bin/gdnative_wrapper_code", [gensource])
+    lib = gd_wrapper_env.Library("#bin/gdnative_wrapper_code", [gensource])
+    gd_wrapper_env.NoCache(lib)

modules/recast/SCsub

@@ -25,6 +25,7 @@ if env['builtin_recast']:
     env.Append(CPPPATH=[thirdparty_dir, thirdparty_dir + "/Include"])
 
     lib = env.Library("recast_builtin", thirdparty_sources)
+    env.NoCache(lib)
     env.Append(LIBS=[lib])
 
 # Godot source files

modules/svg/SCsub

@@ -13,6 +13,8 @@ thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources]
 # env.add_source_files(env.modules_sources, thirdparty_sources)
 
 lib = env.Library("svg_builtin", thirdparty_sources)
+env.NoCache(lib)
+
 # Needs to be appended to arrive after libscene in the linker call,
 # but we don't want it to arrive *after* system libs, so manual hack
 # LIBS contains first SCons Library objects ("SCons.Node.FS.File object")

platform/SCsub

@@ -25,6 +25,8 @@ f.write(unreg_apis)
 f.close()
 platform_sources.append('register_platform_apis.gen.cpp')
 
-env.Prepend(LIBS=env.Library('platform', platform_sources))
+lib = env.Library('platform', platform_sources)
+env.NoCache(lib)
+env.Prepend(LIBS=lib)
 
 Export('env')

platform/android/SCsub

@@ -144,8 +144,8 @@ manifest = manifest.replace("$$ADD_APPATTRIBUTE_CHUNKS$$", env.android_appattrib
 pp_baseout.write(manifest)
 
 
-env_android.SharedLibrary("#bin/libgodot", [android_objects], SHLIBSUFFIX=env["SHLIBSUFFIX"])
-
+lib = env_android.SharedLibrary("#bin/libgodot", [android_objects], SHLIBSUFFIX=env["SHLIBSUFFIX"])
+env_android.NoCache(lib)
 
 lib_arch_dir = ''
 if env['android_arch'] == 'armv6':

platform/iphone/SCsub

@@ -18,6 +18,7 @@ iphone_lib = [
 
 env_ios = env.Clone()
 ios_lib = env_ios.Library('iphone', iphone_lib)
+env_ios.NoCache(ios_lib)
 
 def combine_libs(target=None, source=None, env=None):
     lib_path = target[0].srcnode().abspath

platform/osx/SCsub

@@ -16,7 +16,9 @@ files = [
     'power_osx.cpp',
 ]
 
-binary = env.Program('#bin/godot', files)
-if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
-    env.AddPostAction(binary, make_debug)
+prog = env.Program('#bin/godot', files)
+env.NoCache(prog)
+
+if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
+    env.AddPostAction(prog, make_debug)
 

platform/server/SCsub

@@ -7,4 +7,5 @@ common_server = [\
     "os_server.cpp",\
 ]
 
-env.Program('#bin/godot_server', ['godot_server.cpp'] + common_server)
+prog = env.Program('#bin/godot_server', ['godot_server.cpp'] + common_server)
+env.NoCache(prog)

platform/uwp/SCsub

@@ -20,6 +20,7 @@ if "build_angle" in env and env["build_angle"]:
     cmd = env.AlwaysBuild(env.ANGLE('libANGLE.lib', None))
 
 prog = env.Program('#bin/godot', files)
+env.NoCache(prog)
 
 if "build_angle" in env and env["build_angle"]:
     env.Depends(prog, [cmd])

platform/windows/SCsub

@@ -28,7 +28,8 @@ obj = env.RES(restarget, 'godot_res.rc')
 
 common_win.append(obj)
 
-binary = env.Program('#bin/godot', ['godot_win.cpp'] + common_win, PROGSUFFIX=env["PROGSUFFIX"])
+prog = env.Program('#bin/godot', ['godot_win.cpp'] + common_win, PROGSUFFIX=env["PROGSUFFIX"])
+env.NoCache(prog)
 
 # Microsoft Visual Studio Project Generation
 if env['vsproj']:
@@ -38,4 +39,4 @@ if env['vsproj']:
 
 if not os.getenv("VCINSTALLDIR"):
     if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
-        env.AddPostAction(binary, make_debug_mingw)
+        env.AddPostAction(prog, make_debug_mingw)

platform/x11/SCsub

@@ -17,6 +17,8 @@ common_x11 = [
     "power_x11.cpp",
 ]
 
-binary = env.Program('#bin/godot', ['godot_x11.cpp'] + common_x11)
+prog = env.Program('#bin/godot', ['godot_x11.cpp'] + common_x11)
+env.NoCache(prog)
+
 if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
-    env.AddPostAction(binary, make_debug)
+    env.AddPostAction(prog, make_debug)

scene/SCsub

@@ -31,6 +31,7 @@ SConscript('resources/SCsub')
 
 # Build it all as a library
 lib = env.Library("scene", env.scene_sources)
+env.NoCache(lib)
 env.Prepend(LIBS=[lib])
 
 Export('env')

servers/SCsub

@@ -14,5 +14,6 @@ SConscript('visual/SCsub')
 SConscript('audio/SCsub')
 
 lib = env.Library("servers", env.servers_sources)
+env.NoCache(lib)
 
 env.Prepend(LIBS=[lib])