Merge pull request #47701 from vnen/gdscript-test-runner

core/config/project_settings.cpp

@@ -467,16 +467,17 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 	d->change_dir(p_path);
 
 	String current_dir = d->get_current_dir();
-	String candidate = current_dir;
 	bool found = false;
 	Error err;
 
 	while (true) {
+		// Set the resource path early so things can be resolved when loading.
+		resource_path = current_dir;
+		resource_path = resource_path.replace("\\", "/"); // Windows path to Unix path just in case.
 		err = _load_settings_text_or_binary(current_dir.plus_file("project.godot"), current_dir.plus_file("project.binary"));
 		if (err == OK) {
 			// Optional, we don't mind if it fails.
 			_load_settings_text(current_dir.plus_file("override.cfg"));
-			candidate = current_dir;
 			found = true;
 			break;
 		}
@@ -493,8 +494,6 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
 		}
 	}
 
-	resource_path = candidate;
-	resource_path = resource_path.replace("\\", "/"); // Windows path to Unix path just in case.
 	memdelete(d);
 
 	if (!found) {

main/main.cpp

@@ -390,6 +390,8 @@ Error Main::test_setup() {
 	register_core_types();
 	register_core_driver_types();
 
+	packed_data = memnew(PackedData);
+
 	globals = memnew(ProjectSettings);
 
 	GLOBAL_DEF("debug/settings/crash_handler/message",
@@ -459,6 +461,9 @@ void Main::test_cleanup() {
 	if (globals) {
 		memdelete(globals);
 	}
+	if (packed_data) {
+		memdelete(packed_data);
+	}
 	if (engine) {
 		memdelete(engine);
 	}

modules/gdscript/gdscript.cpp

@@ -45,6 +45,10 @@
 #include "gdscript_parser.h"
 #include "gdscript_warning.h"
 
+#ifdef TESTS_ENABLED
+#include "tests/gdscript_test_runner.h"
+#endif
+
 ///////////////////////////
 
 GDScriptNativeClass::GDScriptNativeClass(const StringName &p_name) {
@@ -1766,6 +1770,10 @@ void GDScriptLanguage::init() {
 	for (List<Engine::Singleton>::Element *E = singletons.front(); E; E = E->next()) {
 		_add_global(E->get().name, E->get().ptr);
 	}
+
+#ifdef TESTS_ENABLED
+	GDScriptTests::GDScriptTestRunner::handle_cmdline();
+#endif
 }
 
 String GDScriptLanguage::get_type() const {

modules/gdscript/register_types.cpp

@@ -163,19 +163,19 @@ void unregister_gdscript_types() {
 
 #ifdef TESTS_ENABLED
 void test_tokenizer() {
-	TestGDScript::test(TestGDScript::TestType::TEST_TOKENIZER);
+	GDScriptTests::test(GDScriptTests::TestType::TEST_TOKENIZER);
 }
 
 void test_parser() {
-	TestGDScript::test(TestGDScript::TestType::TEST_PARSER);
+	GDScriptTests::test(GDScriptTests::TestType::TEST_PARSER);
 }
 
 void test_compiler() {
-	TestGDScript::test(TestGDScript::TestType::TEST_COMPILER);
+	GDScriptTests::test(GDScriptTests::TestType::TEST_COMPILER);
 }
 
 void test_bytecode() {
-	TestGDScript::test(TestGDScript::TestType::TEST_BYTECODE);
+	GDScriptTests::test(GDScriptTests::TestType::TEST_BYTECODE);
 }
 
 REGISTER_TEST_COMMAND("gdscript-tokenizer", &test_tokenizer);

modules/gdscript/tests/gdscript_test_runner.cpp

@@ -0,0 +1,584 @@
+/*************************************************************************/
+/*  gdscript_test_runner.cpp                                             */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "gdscript_test_runner.h"
+
+#include "../gdscript.h"
+#include "../gdscript_analyzer.h"
+#include "../gdscript_compiler.h"
+#include "../gdscript_parser.h"
+
+#include "core/config/project_settings.h"
+#include "core/core_string_names.h"
+#include "core/io/file_access_pack.h"
+#include "core/os/dir_access.h"
+#include "core/os/os.h"
+#include "core/string/string_builder.h"
+#include "scene/resources/packed_scene.h"
+
+#include "tests/test_macros.h"
+
+namespace GDScriptTests {
+
+void init_autoloads() {
+	Map<StringName, ProjectSettings::AutoloadInfo> autoloads = ProjectSettings::get_singleton()->get_autoload_list();
+
+	// First pass, add the constants so they exist before any script is loaded.
+	for (Map<StringName, ProjectSettings::AutoloadInfo>::Element *E = autoloads.front(); E; E = E->next()) {
+		const ProjectSettings::AutoloadInfo &info = E->get();
+
+		if (info.is_singleton) {
+			for (int i = 0; i < ScriptServer::get_language_count(); i++) {
+				ScriptServer::get_language(i)->add_global_constant(info.name, Variant());
+			}
+		}
+	}
+
+	// Second pass, load into global constants.
+	for (Map<StringName, ProjectSettings::AutoloadInfo>::Element *E = autoloads.front(); E; E = E->next()) {
+		const ProjectSettings::AutoloadInfo &info = E->get();
+
+		if (!info.is_singleton) {
+			// Skip non-singletons since we don't have a scene tree here anyway.
+			continue;
+		}
+
+		RES res = ResourceLoader::load(info.path);
+		ERR_CONTINUE_MSG(res.is_null(), "Can't autoload: " + info.path);
+		Node *n = nullptr;
+		if (res->is_class("PackedScene")) {
+			Ref<PackedScene> ps = res;
+			n = ps->instance();
+		} else if (res->is_class("Script")) {
+			Ref<Script> script_res = res;
+			StringName ibt = script_res->get_instance_base_type();
+			bool valid_type = ClassDB::is_parent_class(ibt, "Node");
+			ERR_CONTINUE_MSG(!valid_type, "Script does not inherit a Node: " + info.path);
+
+			Object *obj = ClassDB::instance(ibt);
+
+			ERR_CONTINUE_MSG(obj == nullptr,
+					"Cannot instance script for autoload, expected 'Node' inheritance, got: " +
+							String(ibt));
+
+			n = Object::cast_to<Node>(obj);
+			n->set_script(script_res);
+		}
+
+		ERR_CONTINUE_MSG(!n, "Path in autoload not a node or script: " + info.path);
+		n->set_name(info.name);
+
+		for (int i = 0; i < ScriptServer::get_language_count(); i++) {
+			ScriptServer::get_language(i)->add_global_constant(info.name, n);
+		}
+	}
+}
+
+void init_language(const String &p_base_path) {
+	// Setup project settings since it's needed by the languages to get the global scripts.
+	// This also sets up the base resource path.
+	Error err = ProjectSettings::get_singleton()->setup(p_base_path, String(), true);
+	if (err) {
+		print_line("Could not load project settings.");
+		// Keep going since some scripts still work without this.
+	}
+
+	// Initialize the language for the test routine.
+	GDScriptLanguage::get_singleton()->init();
+	init_autoloads();
+}
+
+void finish_language() {
+	GDScriptLanguage::get_singleton()->finish();
+	ScriptServer::global_classes_clear();
+}
+
+StringName GDScriptTestRunner::test_function_name;
+
+GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language) {
+	test_function_name = StaticCString::create("test");
+	do_init_languages = p_init_language;
+
+	source_dir = p_source_dir;
+	if (!source_dir.ends_with("/")) {
+		source_dir += "/";
+	}
+
+	if (do_init_languages) {
+		init_language(p_source_dir);
+
+		// Enable all warnings for GDScript, so we can test them.
+		ProjectSettings::get_singleton()->set_setting("debug/gdscript/warnings/enable", true);
+		for (int i = 0; i < (int)GDScriptWarning::WARNING_MAX; i++) {
+			String warning = GDScriptWarning::get_name_from_code((GDScriptWarning::Code)i).to_lower();
+			ProjectSettings::get_singleton()->set_setting("debug/gdscript/warnings/" + warning, true);
+		}
+	}
+
+	// Enable printing to show results
+	_print_line_enabled = true;
+	_print_error_enabled = true;
+}
+
+GDScriptTestRunner::~GDScriptTestRunner() {
+	test_function_name = StringName();
+	if (do_init_languages) {
+		finish_language();
+	}
+}
+
+int GDScriptTestRunner::run_tests() {
+	if (!make_tests()) {
+		FAIL("An error occurred while making the tests.");
+		return -1;
+	}
+
+	if (!generate_class_index()) {
+		FAIL("An error occurred while generating class index.");
+		return -1;
+	}
+
+	int failed = 0;
+	for (int i = 0; i < tests.size(); i++) {
+		GDScriptTest test = tests[i];
+		GDScriptTest::TestResult result = test.run_test();
+
+		String expected = FileAccess::get_file_as_string(test.get_output_file());
+		INFO(test.get_source_file());
+		if (!result.passed) {
+			INFO(expected);
+			failed++;
+		}
+
+		CHECK_MESSAGE(result.passed, (result.passed ? String() : result.output));
+	}
+
+	return failed;
+}
+
+bool GDScriptTestRunner::generate_outputs() {
+	is_generating = true;
+
+	if (!make_tests()) {
+		print_line("Failed to generate a test output.");
+		return false;
+	}
+
+	if (!generate_class_index()) {
+		return false;
+	}
+
+	for (int i = 0; i < tests.size(); i++) {
+		OS::get_singleton()->print(".");
+		GDScriptTest test = tests[i];
+		bool result = test.generate_output();
+
+		if (!result) {
+			print_line("\nCould not generate output for " + test.get_source_file());
+			return false;
+		}
+	}
+	print_line("\nGenerated output files for " + itos(tests.size()) + " tests successfully.");
+
+	return true;
+}
+
+bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
+	Error err = OK;
+	DirAccessRef dir(DirAccess::open(p_dir, &err));
+
+	if (err != OK) {
+		return false;
+	}
+
+	String current_dir = dir->get_current_dir();
+
+	dir->list_dir_begin();
+	String next = dir->get_next();
+
+	while (!next.is_empty()) {
+		if (dir->current_is_dir()) {
+			if (next == "." || next == "..") {
+				next = dir->get_next();
+				continue;
+			}
+			if (!make_tests_for_dir(current_dir.plus_file(next))) {
+				return false;
+			}
+		} else {
+			if (next.get_extension().to_lower() == "gd") {
+				String out_file = next.get_basename() + ".out";
+				if (!is_generating && !dir->file_exists(out_file)) {
+					ERR_FAIL_V_MSG(false, "Could not find output file for " + next);
+				}
+				GDScriptTest test(current_dir.plus_file(next), current_dir.plus_file(out_file), source_dir);
+				tests.push_back(test);
+			}
+		}
+
+		next = dir->get_next();
+	}
+
+	dir->list_dir_end();
+
+	return true;
+}
+
+bool GDScriptTestRunner::make_tests() {
+	Error err = OK;
+	DirAccessRef dir(DirAccess::open(source_dir, &err));
+
+	ERR_FAIL_COND_V_MSG(err != OK, false, "Could not open specified test directory.");
+
+	return make_tests_for_dir(dir->get_current_dir());
+}
+
+bool GDScriptTestRunner::generate_class_index() {
+	StringName gdscript_name = GDScriptLanguage::get_singleton()->get_name();
+	for (int i = 0; i < tests.size(); i++) {
+		GDScriptTest test = tests[i];
+		String base_type;
+
+		String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(test.get_source_file(), &base_type);
+		if (class_name == String()) {
+			continue;
+		}
+		ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false,
+				"Class name '" + class_name + "' from " + test.get_source_file() + " is already used in " + ScriptServer::get_global_class_path(class_name));
+
+		ScriptServer::add_global_class(class_name, base_type, gdscript_name, test.get_source_file());
+	}
+	return true;
+}
+
+GDScriptTest::GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir) {
+	source_file = p_source_path;
+	output_file = p_output_path;
+	base_dir = p_base_dir;
+	_print_handler.printfunc = print_handler;
+	_error_handler.errfunc = error_handler;
+}
+
+void GDScriptTestRunner::handle_cmdline() {
+	List<String> cmdline_args = OS::get_singleton()->get_cmdline_args();
+	// TODO: this could likely be ported to use test commands:
+	// https://github.com/godotengine/godot/pull/41355
+	// Currently requires to startup the whole engine, which is slow.
+	String test_cmd = "--gdscript-test";
+	String gen_cmd = "--gdscript-generate-tests";
+
+	for (List<String>::Element *E = cmdline_args.front(); E != nullptr; E = E->next()) {
+		String &cmd = E->get();
+		if (cmd == test_cmd || cmd == gen_cmd) {
+			if (E->next() == nullptr) {
+				ERR_PRINT("Needed a path for the test files.");
+				exit(-1);
+			}
+
+			const String &path = E->next()->get();
+
+			GDScriptTestRunner runner(path, false);
+			int failed = 0;
+			if (cmd == test_cmd) {
+				failed = runner.run_tests();
+			} else {
+				bool completed = runner.generate_outputs();
+				failed = completed ? 0 : -1;
+			}
+			exit(failed);
+		}
+	}
+}
+
+void GDScriptTest::enable_stdout() {
+	// TODO: this could likely be handled by doctest or `tests/test_macros.h`.
+	OS::get_singleton()->set_stdout_enabled(true);
+	OS::get_singleton()->set_stderr_enabled(true);
+}
+
+void GDScriptTest::disable_stdout() {
+	// TODO: this could likely be handled by doctest or `tests/test_macros.h`.
+	OS::get_singleton()->set_stdout_enabled(false);
+	OS::get_singleton()->set_stderr_enabled(false);
+}
+
+void GDScriptTest::print_handler(void *p_this, const String &p_message, bool p_error) {
+	TestResult *result = (TestResult *)p_this;
+	result->output += p_message + "\n";
+}
+
+void GDScriptTest::error_handler(void *p_this, const char *p_function, const char *p_file, int p_line, const char *p_error, const char *p_explanation, ErrorHandlerType p_type) {
+	ErrorHandlerData *data = (ErrorHandlerData *)p_this;
+	GDScriptTest *self = data->self;
+	TestResult *result = data->result;
+
+	result->status = GDTEST_RUNTIME_ERROR;
+
+	StringBuilder builder;
+	builder.append(">> ");
+	switch (p_type) {
+		case ERR_HANDLER_ERROR:
+			builder.append("ERROR");
+			break;
+		case ERR_HANDLER_WARNING:
+			builder.append("WARNING");
+			break;
+		case ERR_HANDLER_SCRIPT:
+			builder.append("SCRIPT ERROR");
+			break;
+		case ERR_HANDLER_SHADER:
+			builder.append("SHADER ERROR");
+			break;
+		default:
+			builder.append("Unknown error type");
+			break;
+	}
+
+	builder.append("\n>> ");
+	builder.append(p_function);
+	builder.append("\n>> ");
+	builder.append(p_function);
+	builder.append("\n>> ");
+	builder.append(String(p_file).trim_prefix(self->base_dir));
+	builder.append("\n>> ");
+	builder.append(itos(p_line));
+	builder.append("\n>> ");
+	builder.append(p_error);
+	if (strlen(p_explanation) > 0) {
+		builder.append("\n>> ");
+		builder.append(p_explanation);
+	}
+	builder.append("\n");
+
+	result->output = builder.as_string();
+}
+
+bool GDScriptTest::check_output(const String &p_output) const {
+	Error err = OK;
+	String expected = FileAccess::get_file_as_string(output_file, &err);
+
+	ERR_FAIL_COND_V_MSG(err != OK, false, "Error when opening the output file.");
+
+	String got = p_output.strip_edges(); // TODO: may be hacky.
+	got += "\n"; // Make sure to insert newline for CI static checks.
+
+	return got == expected;
+}
+
+String GDScriptTest::get_text_for_status(GDScriptTest::TestStatus p_status) const {
+	switch (p_status) {
+		case GDTEST_OK:
+			return "GDTEST_OK";
+		case GDTEST_LOAD_ERROR:
+			return "GDTEST_LOAD_ERROR";
+		case GDTEST_PARSER_ERROR:
+			return "GDTEST_PARSER_ERROR";
+		case GDTEST_ANALYZER_ERROR:
+			return "GDTEST_ANALYZER_ERROR";
+		case GDTEST_COMPILER_ERROR:
+			return "GDTEST_COMPILER_ERROR";
+		case GDTEST_RUNTIME_ERROR:
+			return "GDTEST_RUNTIME_ERROR";
+	}
+	return "";
+}
+
+GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
+	disable_stdout();
+
+	TestResult result;
+	result.status = GDTEST_OK;
+	result.output = String();
+
+	Error err = OK;
+
+	// Create script.
+	Ref<GDScript> script;
+	script.instance();
+	script->set_path(source_file);
+	script->set_script_path(source_file);
+	err = script->load_source_code(source_file);
+	if (err != OK) {
+		enable_stdout();
+		result.status = GDTEST_LOAD_ERROR;
+		result.passed = false;
+		ERR_FAIL_V_MSG(result, "\nCould not load source code for: '" + source_file + "'");
+	}
+
+	// Test parsing.
+	GDScriptParser parser;
+	err = parser.parse(script->get_source_code(), source_file, false);
+	if (err != OK) {
+		enable_stdout();
+		result.status = GDTEST_PARSER_ERROR;
+		result.output = get_text_for_status(result.status) + "\n";
+
+		const List<GDScriptParser::ParserError> &errors = parser.get_errors();
+		for (auto *E = errors.front(); E; E = E->next()) {
+			result.output += E->get().message + "\n"; // TODO: line, column?
+			break; // Only the first error since the following might be cascading.
+		}
+		if (!p_is_generating) {
+			result.passed = check_output(result.output);
+		}
+		return result;
+	}
+
+	// Test type-checking.
+	GDScriptAnalyzer analyzer(&parser);
+	err = analyzer.analyze();
+	if (err != OK) {
+		enable_stdout();
+		result.status = GDTEST_ANALYZER_ERROR;
+		result.output = get_text_for_status(result.status) + "\n";
+
+		const List<GDScriptParser::ParserError> &errors = parser.get_errors();
+		for (auto *E = errors.front(); E; E = E->next()) {
+			result.output += E->get().message + "\n"; // TODO: line, column?
+			break; // Only the first error since the following might be cascading.
+		}
+		if (!p_is_generating) {
+			result.passed = check_output(result.output);
+		}
+		return result;
+	}
+
+	StringBuilder warning_string;
+	for (const List<GDScriptWarning>::Element *E = parser.get_warnings().front(); E != nullptr; E = E->next()) {
+		const GDScriptWarning warning = E->get();
+		warning_string.append(">> WARNING");
+		warning_string.append("\n>> Line: ");
+		warning_string.append(itos(warning.start_line));
+		warning_string.append("\n>> ");
+		warning_string.append(warning.get_name());
+		warning_string.append("\n>> ");
+		warning_string.append(warning.get_message());
+		warning_string.append("\n");
+	}
+	result.output += warning_string.as_string();
+
+	// Test compiling.
+	GDScriptCompiler compiler;
+	err = compiler.compile(&parser, script.ptr(), false);
+	if (err != OK) {
+		enable_stdout();
+		result.status = GDTEST_COMPILER_ERROR;
+		result.output = get_text_for_status(result.status) + "\n";
+		result.output = compiler.get_error();
+		if (!p_is_generating) {
+			result.passed = check_output(result.output);
+		}
+		return result;
+	}
+
+	// Test running.
+	const Map<StringName, GDScriptFunction *>::Element *test_function_element = script->get_member_functions().find(GDScriptTestRunner::test_function_name);
+	if (test_function_element == nullptr) {
+		enable_stdout();
+		result.status = GDTEST_LOAD_ERROR;
+		result.output = "";
+		result.passed = false;
+		ERR_FAIL_V_MSG(result, "\nCould not find test function on: '" + source_file + "'");
+	}
+
+	script->reload();
+
+	// Create object instance for test.
+	Object *obj = ClassDB::instance(script->get_native()->get_name());
+	Ref<Reference> obj_ref;
+	if (obj->is_reference()) {
+		obj_ref = Ref<Reference>(Object::cast_to<Reference>(obj));
+	}
+	obj->set_script(script);
+	GDScriptInstance *instance = static_cast<GDScriptInstance *>(obj->get_script_instance());
+
+	// Setup output handlers.
+	ErrorHandlerData error_data(&result, this);
+
+	_print_handler.userdata = &result;
+	_error_handler.userdata = &error_data;
+	add_print_handler(&_print_handler);
+	add_error_handler(&_error_handler);
+
+	// Call test function.
+	Callable::CallError call_err;
+	instance->call(GDScriptTestRunner::test_function_name, nullptr, 0, call_err);
+
+	// Tear down output handlers.
+	remove_print_handler(&_print_handler);
+	remove_error_handler(&_error_handler);
+
+	// Check results.
+	if (call_err.error != Callable::CallError::CALL_OK) {
+		enable_stdout();
+		result.status = GDTEST_LOAD_ERROR;
+		result.passed = false;
+		ERR_FAIL_V_MSG(result, "\nCould not call test function on: '" + source_file + "'");
+	}
+
+	result.output = get_text_for_status(result.status) + "\n" + result.output;
+	if (!p_is_generating) {
+		result.passed = check_output(result.output);
+	}
+
+	if (obj_ref.is_null()) {
+		memdelete(obj);
+	}
+
+	enable_stdout();
+	return result;
+}
+
+GDScriptTest::TestResult GDScriptTest::run_test() {
+	return execute_test_code(false);
+}
+
+bool GDScriptTest::generate_output() {
+	TestResult result = execute_test_code(true);
+	if (result.status == GDTEST_LOAD_ERROR) {
+		return false;
+	}
+
+	Error err = OK;
+	FileAccessRef out_file = FileAccess::open(output_file, FileAccess::WRITE, &err);
+	if (err != OK) {
+		return false;
+	}
+
+	String output = result.output.strip_edges(); // TODO: may be hacky.
+	output += "\n"; // Make sure to insert newline for CI static checks.
+
+	out_file->store_string(output);
+	out_file->close();
+
+	return true;
+}
+
+} // namespace GDScriptTests

modules/gdscript/tests/gdscript_test_runner.h

@@ -0,0 +1,126 @@
+/*************************************************************************/
+/*  gdscript_test_runner.h                                               */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef GDSCRIPT_TEST_H
+#define GDSCRIPT_TEST_H
+
+#include "../gdscript.h"
+#include "core/error/error_macros.h"
+#include "core/string/print_string.h"
+#include "core/string/ustring.h"
+#include "core/templates/vector.h"
+
+namespace GDScriptTests {
+
+void init_autoloads();
+void init_language(const String &p_base_path);
+void finish_language();
+
+// Single test instance in a suite.
+class GDScriptTest {
+public:
+	enum TestStatus {
+		GDTEST_OK,
+		GDTEST_LOAD_ERROR,
+		GDTEST_PARSER_ERROR,
+		GDTEST_ANALYZER_ERROR,
+		GDTEST_COMPILER_ERROR,
+		GDTEST_RUNTIME_ERROR,
+	};
+
+	struct TestResult {
+		TestStatus status;
+		String output;
+		bool passed;
+	};
+
+private:
+	struct ErrorHandlerData {
+		TestResult *result;
+		GDScriptTest *self;
+		ErrorHandlerData(TestResult *p_result, GDScriptTest *p_this) {
+			result = p_result;
+			self = p_this;
+		}
+	};
+
+	String source_file;
+	String output_file;
+	String base_dir;
+
+	PrintHandlerList _print_handler;
+	ErrorHandlerList _error_handler;
+
+	void enable_stdout();
+	void disable_stdout();
+	bool check_output(const String &p_output) const;
+	String get_text_for_status(TestStatus p_status) const;
+
+	TestResult execute_test_code(bool p_is_generating);
+
+public:
+	static void print_handler(void *p_this, const String &p_message, bool p_error);
+	static void error_handler(void *p_this, const char *p_function, const char *p_file, int p_line, const char *p_error, const char *p_explanation, ErrorHandlerType p_type);
+	TestResult run_test();
+	bool generate_output();
+
+	const String &get_source_file() const { return source_file; }
+	const String &get_output_file() const { return output_file; }
+
+	GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir);
+	GDScriptTest() :
+			GDScriptTest(String(), String(), String()) {} // Needed to use in Vector.
+};
+
+class GDScriptTestRunner {
+	String source_dir;
+	Vector<GDScriptTest> tests;
+
+	bool is_generating = false;
+	bool do_init_languages = false;
+
+	bool make_tests();
+	bool make_tests_for_dir(const String &p_dir);
+	bool generate_class_index();
+
+public:
+	static StringName test_function_name;
+
+	static void handle_cmdline();
+	int run_tests();
+	bool generate_outputs();
+
+	GDScriptTestRunner(const String &p_source_dir, bool p_init_language);
+	~GDScriptTestRunner();
+};
+
+} // namespace GDScriptTests
+
+#endif // GDSCRIPT_TEST_H

modules/gdscript/tests/gdscript_test_runner_suite.h

@@ -0,0 +1,53 @@
+/*************************************************************************/
+/*  gdscript_test_runner_suite.h                                         */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef GDSCRIPT_TEST_RUNNER_SUITE_H
+#define GDSCRIPT_TEST_RUNNER_SUITE_H
+
+#include "gdscript_test_runner.h"
+#include "tests/test_macros.h"
+
+namespace GDScriptTests {
+
+TEST_SUITE("[Modules][GDScript]") {
+	// GDScript 2.0 is still under heavy construction.
+	// Allow the tests to fail, but do not ignore errors during development.
+	// Update the scripts and expected output as needed.
+	TEST_CASE("Script compilation and runtime") {
+		GDScriptTestRunner runner("modules/gdscript/tests/scripts", true);
+		int fail_count = runner.run_tests();
+		INFO("Make sure `*.out` files have expected results.");
+		REQUIRE_MESSAGE(fail_count == 0, "All GDScript tests should pass.");
+	}
+}
+
+} // namespace GDScriptTests
+
+#endif // GDSCRIPT_TEST_RUNNER_SUITE_H

modules/gdscript/tests/scripts/.gitignore

@@ -0,0 +1,2 @@
+# Ignore metadata if someone open this on Godot.
+/.godot

modules/gdscript/tests/scripts/parser-errors/missing-argument.gd

@@ -0,0 +1,6 @@
+func args(a, b):
+    print(a)
+    print(b)
+
+func test():
+    args(1,)

modules/gdscript/tests/scripts/parser-errors/missing-argument.out

@@ -0,0 +1,2 @@
+GDTEST_ANALYZER_ERROR
+Too few arguments for "args()" call. Expected at least 2 but received 1.

modules/gdscript/tests/scripts/parser-errors/missing-closing-expr-paren.gd

@@ -0,0 +1,2 @@
+func test():
+    var a = ("missing paren ->"

modules/gdscript/tests/scripts/parser-errors/missing-closing-expr-paren.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expected closing ")" after grouping expression.

modules/gdscript/tests/scripts/parser-errors/missing-colon.gd

@@ -0,0 +1,3 @@
+func test():
+    if true # Missing colon here.
+        print("true")

modules/gdscript/tests/scripts/parser-errors/missing-colon.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expected ":" after "if" condition.

modules/gdscript/tests/scripts/parser-errors/missing-paren-after-args.gd

@@ -0,0 +1,6 @@
+func args(a, b):
+    print(a)
+    print(b)
+
+func test():
+    args(1,2

modules/gdscript/tests/scripts/parser-errors/missing-paren-after-args.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expected closing ")" after call arguments.

modules/gdscript/tests/scripts/parser-errors/mixing-tabs-spaces.gd

@@ -0,0 +1,3 @@
+func test():
+    print("Using spaces")
+	print("Using tabs")

modules/gdscript/tests/scripts/parser-errors/mixing-tabs-spaces.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Used "\t" for indentation instead " " as used before in the file.

modules/gdscript/tests/scripts/parser-errors/nothing-after-dollar.gd

@@ -0,0 +1,3 @@
+extends Node
+func test():
+    var a = $ # Expected some node path.

modules/gdscript/tests/scripts/parser-errors/nothing-after-dollar.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expect node path as string or identifier after "$".

modules/gdscript/tests/scripts/parser-errors/wrong-value-after-dollar-slash.gd

@@ -0,0 +1,3 @@
+extends Node
+func test():
+    $MyNode/23 # Can't use number here.

modules/gdscript/tests/scripts/parser-errors/wrong-value-after-dollar-slash.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expect node path after "/".

modules/gdscript/tests/scripts/parser-errors/wrong-value-after-dollar.gd

@@ -0,0 +1,3 @@
+extends Node
+func test():
+    $23 # Can't use number here.

modules/gdscript/tests/scripts/parser-errors/wrong-value-after-dollar.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Expect node path as string or identifier after "$".

modules/gdscript/tests/scripts/parser-features/semicolon-as-end-statement.gd

@@ -0,0 +1,2 @@
+func test():
+    print("A"); print("B")

modules/gdscript/tests/scripts/parser-features/semicolon-as-end-statement.out

@@ -0,0 +1,3 @@
+GDTEST_OK
+A
+B

modules/gdscript/tests/scripts/parser-features/trailing-comma-in-function-args.gd

@@ -0,0 +1,7 @@
+# See https://github.com/godotengine/godot/issues/41066.
+
+func f(p, ): ## <-- no errors
+	print(p)
+
+func test():
+	f(0, ) ## <-- no error

modules/gdscript/tests/scripts/parser-features/trailing-comma-in-function-args.out

@@ -0,0 +1,2 @@
+GDTEST_OK
+0

modules/gdscript/tests/scripts/parser-features/variable-declaration.gd

@@ -0,0 +1,12 @@
+var a # No init.
+var b = 42 # Init.
+
+func test():
+	var c # No init, local.
+	var d = 23 # Init, local.
+
+	a = 1
+	c = 2
+
+	prints(a, b, c, d)
+	print("OK")

modules/gdscript/tests/scripts/parser-features/variable-declaration.out

@@ -0,0 +1,7 @@
+GDTEST_OK
+>> WARNING
+>> Line: 5
+>> UNASSIGNED_VARIABLE
+>> The variable 'c' was used but never assigned a value.
+1 42 2 23
+OK

modules/gdscript/tests/scripts/parser-warnings/unused-variable.gd

@@ -0,0 +1,2 @@
+func test():
+    var unused = "not used"

modules/gdscript/tests/scripts/parser-warnings/unused-variable.out

@@ -0,0 +1,5 @@
+GDTEST_OK
+>> WARNING
+>> Line: 2
+>> UNUSED_VARIABLE
+>> The local variable 'unused' is declared but never used in the block. If this is intended, prefix it with an underscore: '_unused'

modules/gdscript/tests/scripts/project.godot

@@ -0,0 +1,10 @@
+; This is not an actual project.
+; This config only exists to properly set up the test environment.
+; It also helps for opening Godot to edit the scripts, but please don't
+; let the editor changes be saved.
+
+config_version=4
+
+[application]
+
+config/name="GDScript Integration Test Suite"

modules/gdscript/tests/test_gdscript.cpp

@@ -47,7 +47,7 @@
 #include "editor/editor_settings.h"
 #endif
 
-namespace TestGDScript {
+namespace GDScriptTests {
 
 static void test_tokenizer(const String &p_code, const Vector<String> &p_lines) {
 	GDScriptTokenizer tokenizer;
@@ -183,60 +183,6 @@ static void test_compiler(const String &p_code, const String &p_script_path, con
 	}
 }
 
-void init_autoloads() {
-	Map<StringName, ProjectSettings::AutoloadInfo> autoloads = ProjectSettings::get_singleton()->get_autoload_list();
-
-	// First pass, add the constants so they exist before any script is loaded.
-	for (Map<StringName, ProjectSettings::AutoloadInfo>::Element *E = autoloads.front(); E; E = E->next()) {
-		const ProjectSettings::AutoloadInfo &info = E->get();
-
-		if (info.is_singleton) {
-			for (int i = 0; i < ScriptServer::get_language_count(); i++) {
-				ScriptServer::get_language(i)->add_global_constant(info.name, Variant());
-			}
-		}
-	}
-
-	// Second pass, load into global constants.
-	for (Map<StringName, ProjectSettings::AutoloadInfo>::Element *E = autoloads.front(); E; E = E->next()) {
-		const ProjectSettings::AutoloadInfo &info = E->get();
-
-		if (!info.is_singleton) {
-			// Skip non-singletons since we don't have a scene tree here anyway.
-			continue;
-		}
-
-		RES res = ResourceLoader::load(info.path);
-		ERR_CONTINUE_MSG(res.is_null(), "Can't autoload: " + info.path);
-		Node *n = nullptr;
-		if (res->is_class("PackedScene")) {
-			Ref<PackedScene> ps = res;
-			n = ps->instance();
-		} else if (res->is_class("Script")) {
-			Ref<Script> script_res = res;
-			StringName ibt = script_res->get_instance_base_type();
-			bool valid_type = ClassDB::is_parent_class(ibt, "Node");
-			ERR_CONTINUE_MSG(!valid_type, "Script does not inherit a Node: " + info.path);
-
-			Object *obj = ClassDB::instance(ibt);
-
-			ERR_CONTINUE_MSG(obj == nullptr,
-					"Cannot instance script for autoload, expected 'Node' inheritance, got: " +
-							String(ibt));
-
-			n = Object::cast_to<Node>(obj);
-			n->set_script(script_res);
-		}
-
-		ERR_CONTINUE_MSG(!n, "Path in autoload not a node or script: " + info.path);
-		n->set_name(info.name);
-
-		for (int i = 0; i < ScriptServer::get_language_count(); i++) {
-			ScriptServer::get_language(i)->add_global_constant(info.name, n);
-		}
-	}
-}
-
 void test(TestType p_type) {
 	List<String> cmdlargs = OS::get_singleton()->get_cmdline_args();
 
@@ -253,20 +199,8 @@ void test(TestType p_type) {
 	FileAccessRef fa = FileAccess::open(test, FileAccess::READ);
 	ERR_FAIL_COND_MSG(!fa, "Could not open file: " + test);
 
-	// Init PackedData since it's used by ProjectSettings.
-	PackedData *packed_data = memnew(PackedData);
-
-	// Setup project settings since it's needed by the languages to get the global scripts.
-	// This also sets up the base resource path.
-	Error err = ProjectSettings::get_singleton()->setup(fa->get_path_absolute().get_base_dir(), String(), true);
-	if (err) {
-		print_line("Could not load project settings.");
-		// Keep going since some scripts still work without this.
-	}
-
 	// Initialize the language for the test routine.
-	ScriptServer::init_languages();
+	init_language(fa->get_path_absolute().get_base_dir());
-	init_autoloads();
 
 	Vector<uint8_t> buf;
 	int flen = fa->get_len();
@@ -300,8 +234,6 @@ void test(TestType p_type) {
 			print_line("Not implemented.");
 	}
 
-	// Destroy stuff we set up earlier.
+	finish_language();
-	ScriptServer::finish_languages();
-	memdelete(packed_data);
 }
-} // namespace TestGDScript
+} // namespace GDScriptTests

modules/gdscript/tests/test_gdscript.h

@@ -31,7 +31,10 @@
 #ifndef TEST_GDSCRIPT_H
 #define TEST_GDSCRIPT_H
 
-namespace TestGDScript {
+#include "gdscript_test_runner.h"
+#include "tests/test_macros.h"
+
+namespace GDScriptTests {
 
 enum TestType {
 	TEST_TOKENIZER,
@@ -41,6 +44,7 @@ enum TestType {
 };
 
 void test(TestType p_type);
-} // namespace TestGDScript
+
+} // namespace GDScriptTests
 
 #endif // TEST_GDSCRIPT_H