JS synchronous start, better persistent FS sync.

The engine now expects to emscripten FS to be setup and sync-ed before
main is called. This is exposed via `Module["initFS"]` which also allows
to setup multiple persistence paths (internal use only for now).

Additionally, FS syncing is done **once** for every loop if at least one
file in a persistent path was open for writing and closed, and if the FS
is not syncing already.

This should potentially fix issues reported by users where "autosave"
would not work on the web (never calling `syncfs` because of too many
writes).

platform/javascript/engine/engine.js

@@ -33,6 +33,7 @@ Function('return this')()['Engine'] = (function() {
 		this.resizeCanvasOnStart = false;
 		this.onExecute = null;
 		this.onExit = null;
+		this.persistentPaths = [];
 	};
 
 	Engine.prototype.init = /** @param {string=} basePath */ function(basePath) {
@@ -56,12 +57,14 @@ Function('return this')()['Engine'] = (function() {
 			config['locateFile'] = Utils.createLocateRewrite(loadPath);
 			config['instantiateWasm'] = Utils.createInstantiatePromise(loadPromise);
 			Godot(config).then(function(module) {
-				me.rtenv = module;
-				if (unloadAfterInit) {
-					unload();
-				}
-				resolve();
-				config = null;
+				module['initFS'](me.persistentPaths).then(function(fs_err) {
+					me.rtenv = module;
+					if (unloadAfterInit) {
+						unload();
+					}
+					resolve();
+					config = null;
+				});
 			});
 		});
 		return initPromise;
@@ -220,6 +223,10 @@ Function('return this')()['Engine'] = (function() {
 		this.rtenv['copyToFS'](path, buffer);
 	}
 
+	Engine.prototype.setPersistentPaths = function(persistentPaths) {
+		this.persistentPaths = persistentPaths;
+	};
+
 	// Closure compiler exported engine methods.
 	/** @export */
 	Engine['isWebGLAvailable'] = Utils.isWebGLAvailable;
@@ -241,5 +248,6 @@ Function('return this')()['Engine'] = (function() {
 	Engine.prototype['setOnExecute'] = Engine.prototype.setOnExecute;
 	Engine.prototype['setOnExit'] = Engine.prototype.setOnExit;
 	Engine.prototype['copyToFS'] = Engine.prototype.copyToFS;
+	Engine.prototype['setPersistentPaths'] = Engine.prototype.setPersistentPaths;
 	return Engine;
 })();

platform/javascript/javascript_main.cpp

@@ -94,12 +94,27 @@ void main_loop_callback() {
 		/* clang-format on */
 		os->get_main_loop()->finish();
 		os->finalize_async(); // Will add all the async finish functions.
+		/* clang-format off */
 		EM_ASM({
 			Promise.all(Module.async_finish).then(function() {
 				Module.async_finish = [];
+				return new Promise(function(accept, reject) {
+					if (!Module.idbfs) {
+						accept();
+						return;
+					}
+					FS.syncfs(function(error) {
+						if (error) {
+							err('Failed to save IDB file system: ' + error.message);
+						}
+						accept();
+					});
+				});
+			}).then(function() {
 				ccall("cleanup_after_sync", null, []);
 			});
 		});
+		/* clang-format on */
 	}
 }
 
@@ -107,13 +122,8 @@ extern "C" EMSCRIPTEN_KEEPALIVE void cleanup_after_sync() {
 	emscripten_set_main_loop(exit_callback, -1, false);
 }
 
-extern "C" EMSCRIPTEN_KEEPALIVE void main_after_fs_sync(char *p_idbfs_err) {
-	// Set IDBFS status
-	String idbfs_err = String::utf8(p_idbfs_err);
-	if (!idbfs_err.empty()) {
-		print_line("IndexedDB not available: " + idbfs_err);
-	}
-	os->set_idb_available(idbfs_err.empty());
+int main(int argc, char *argv[]) {
+	os = new OS_JavaScript(argc, argv);
 
 	// Set canvas ID
 	char canvas_ptr[256];
@@ -133,7 +143,10 @@ extern "C" EMSCRIPTEN_KEEPALIVE void main_after_fs_sync(char *p_idbfs_err) {
 	/* clang-format on */
 	setenv("LANG", locale_ptr, true);
 
-	Main::setup2();
+	// Set IDBFS status
+	os->set_idb_available((bool)EM_ASM_INT({ return Module.idbfs }));
+
+	Main::setup(argv[0], argc - 1, &argv[1]);
 	// Ease up compatibility.
 	ResourceLoader::set_abort_on_missing_resources(false);
 	Main::start();
@@ -144,35 +157,8 @@ extern "C" EMSCRIPTEN_KEEPALIVE void main_after_fs_sync(char *p_idbfs_err) {
 			ccall("_request_quit_callback", null, []);
 		};
 	});
+	emscripten_set_main_loop(main_loop_callback, -1, false);
 	// Immediately run the first iteration.
 	// We are inside an animation frame, we want to immediately draw on the newly setup canvas.
 	main_loop_callback();
-	emscripten_resume_main_loop();
-}
-
-int main(int argc, char *argv[]) {
-	// Create and mount userfs immediately.
-	EM_ASM({
-		FS.mkdir('/userfs');
-		FS.mount(IDBFS, {}, '/userfs');
-	});
-	os = new OS_JavaScript(argc, argv);
-	Main::setup(argv[0], argc - 1, &argv[1], false);
-	emscripten_set_main_loop(main_loop_callback, -1, false);
-	emscripten_pause_main_loop(); // Will need to wait for FS sync.
-
-	// Sync from persistent state into memory and then
-	// run the 'main_after_fs_sync' function.
-	/* clang-format off */
-	EM_ASM({
-		FS.syncfs(true, function(err) {
-			requestAnimationFrame(function() {
-				ccall('main_after_fs_sync', null, ['string'], [err ? err.message : ""]);
-			});
-		});
-	});
-	/* clang-format on */
-
-	return 0;
-	// Continued async in main_after_fs_sync() from the syncfs() callback.
 }

platform/javascript/native/utils.js

@@ -28,6 +28,38 @@
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
 /*************************************************************************/
 
+Module['initFS'] = function(persistentPaths) {
+	FS.mkdir('/userfs');
+	FS.mount(IDBFS, {}, '/userfs');
+
+	function createRecursive(dir) {
+		try {
+			FS.stat(dir);
+		} catch (e) {
+			if (e.errno !== ERRNO_CODES.ENOENT) {
+				throw e;
+			}
+			FS.mkdirTree(dir);
+		}
+	}
+
+	persistentPaths.forEach(function(path) {
+		createRecursive(path);
+		FS.mount(IDBFS, {}, path);
+	});
+	return new Promise(function(resolve, reject) {
+		FS.syncfs(true, function(err) {
+			if (err) {
+				Module.idbfs = false;
+				console.log("IndexedDB not available: " + err.message);
+			} else {
+				Module.idbfs = true;
+			}
+			resolve(err);
+		});
+	});
+};
+
 Module['copyToFS'] = function(path, buffer) {
 	var p = path.lastIndexOf("/");
 	var dir = "/";
@@ -37,7 +69,7 @@ Module['copyToFS'] = function(path, buffer) {
 	try {
 		FS.stat(dir);
 	} catch (e) {
-		if (e.errno !== ERRNO_CODES.ENOENT) { // 'ENOENT', see https://github.com/emscripten-core/emscripten/blob/master/system/lib/libc/musl/arch/emscripten/bits/errno.h
+		if (e.errno !== ERRNO_CODES.ENOENT) {
 			throw e;
 		}
 		FS.mkdirTree(dir);

platform/javascript/os_javascript.cpp

@@ -1119,24 +1119,25 @@ void OS_JavaScript::resume_audio() {
 	}
 }
 
+extern "C" EMSCRIPTEN_KEEPALIVE void _idb_synced() {
+	OS_JavaScript::get_singleton()->idb_is_syncing = false;
+}
+
 bool OS_JavaScript::main_loop_iterate() {
 
-	if (is_userfs_persistent() && sync_wait_time >= 0) {
-		int64_t current_time = get_ticks_msec();
-		int64_t elapsed_time = current_time - last_sync_check_time;
-		last_sync_check_time = current_time;
-
-		sync_wait_time -= elapsed_time;
-
-		if (sync_wait_time < 0) {
-			/* clang-format off */
-			EM_ASM(
-				FS.syncfs(function(error) {
-					if (error) { err('Failed to save IDB file system: ' + error.message); }
-				});
-			);
-			/* clang-format on */
-		}
+	if (is_userfs_persistent() && idb_needs_sync && !idb_is_syncing) {
+		idb_is_syncing = true;
+		idb_needs_sync = false;
+		/* clang-format off */
+		EM_ASM(
+			FS.syncfs(function(error) {
+				if (error) {
+					err('Failed to save IDB file system: ' + error.message);
+				}
+				ccall("_idb_synced", 'void', [], []);
+			});
+		);
+		/* clang-format on */
 	}
 
 	if (emscripten_sample_gamepad_data() == EMSCRIPTEN_RESULT_SUCCESS)
@@ -1174,9 +1175,11 @@ void OS_JavaScript::delete_main_loop() {
 }
 
 void OS_JavaScript::finalize_async() {
+	/* clang-format off */
 	EM_ASM({
 		Module.listeners.clear();
 	});
+	/* clang-format on */
 	if (audio_driver_javascript) {
 		audio_driver_javascript->finish_async();
 	}
@@ -1387,10 +1390,17 @@ int OS_JavaScript::get_power_percent_left() {
 void OS_JavaScript::file_access_close_callback(const String &p_file, int p_flags) {
 
 	OS_JavaScript *os = get_singleton();
-	if (os->is_userfs_persistent() && p_file.begins_with("/userfs") && p_flags & FileAccess::WRITE) {
-		os->last_sync_check_time = OS::get_singleton()->get_ticks_msec();
-		// Wait five seconds in case more files are about to be closed.
-		os->sync_wait_time = 5000;
+
+	if (!(os->is_userfs_persistent() && p_flags & FileAccess::WRITE)) {
+		return; // FS persistence is not working or we are not writing.
+	}
+	bool is_file_persistent = p_file.begins_with("/userfs");
+#ifdef TOOLS_ENABLED
+	// Hack for editor persistence (can we track).
+	is_file_persistent = is_file_persistent || p_file.begins_with("/home/web_user/");
+#endif
+	if (is_file_persistent) {
+		os->idb_needs_sync = true;
 	}
 }
 
@@ -1435,7 +1445,8 @@ OS_JavaScript::OS_JavaScript(int p_argc, char *p_argv[]) {
 
 	swap_ok_cancel = false;
 	idb_available = false;
-	sync_wait_time = -1;
+	idb_needs_sync = false;
+	idb_is_syncing = false;
 
 	if (AudioDriverJavaScript::is_available()) {
 		audio_driver_javascript = memnew(AudioDriverJavaScript);

platform/javascript/os_javascript.h

@@ -71,8 +71,7 @@ class OS_JavaScript : public OS_Unix {
 
 	bool swap_ok_cancel;
 	bool idb_available;
-	int64_t sync_wait_time;
-	int64_t last_sync_check_time;
+	bool idb_needs_sync;
 
 	static EM_BOOL fullscreen_change_callback(int p_event_type, const EmscriptenFullscreenChangeEvent *p_event, void *p_user_data);
 
@@ -110,6 +109,7 @@ protected:
 
 public:
 	String canvas_id;
+	bool idb_is_syncing;
 	void finalize_async();
 	bool check_size_force_redraw();