diff --git a/bin/update_test_runner b/bin/update_test_runner new file mode 100755 index 0000000..8690388 --- /dev/null +++ b/bin/update_test_runner @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +self=$(readlink -e "$0") +bin=${self%/*} + +# shellcheck disable=SC1091 +. "${bin}/utils.bash" + +content=$(curl -s https://raw.githubusercontent.com/exercism/gdscript-test-runner/refs/heads/main/bin/test_runner.gd) +readarray -t exercises < <( active_exercises ) +for exercise in "${exercises[@]}"; do + write_test_runner "${exercise}" "${content}" +done diff --git a/bin/utils.bash b/bin/utils.bash new file mode 100644 index 0000000..a87d4d0 --- /dev/null +++ b/bin/utils.bash @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +self=$(readlink -e "$0") +bin=${self%/*} +REPO_ROOT=$(git -C "${bin}" rev-parse --show-toplevel) +readonly REPO_ROOT + +active_exercises () { + jq -r ' + .exercises | + to_entries | + map( + select(.key != "foregone") | + .key as $key | + .value | + map( + select(.status != "deprecated") | + "exercises/\($key)/\(.slug)" + ) + ) | + add | + sort | + .[] + ' "${REPO_ROOT}/config.json" +} + +test_script_content() { + printf '#!/usr/bin/env bash\n' + printf '\n' + # shellcheck disable=SC2016 + printf 'slug=${PWD##*/}\n' + # shellcheck disable=SC2016 + printf '[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing"\n' + # shellcheck disable=SC2016 + printf '[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing"\n' + # shellcheck disable=SC2016 + printf 'exec godot --headless --script ./lib/test_runner.gd -- "${PWD}"\n' +} + +write_test_runner () { + local exercise="$1" + local content + if (( $# == 2 )); then + content="$2" + else + content=$(curl -s https://raw.githubusercontent.com/exercism/gdscript-test-runner/refs/heads/main/bin/test_runner.gd) + fi + [[ -e "${exercise}/lib" ]] || mkdir "${exercise}/lib" + printf '%s\n' "${content}" > "${exercise}/lib/test_runner.gd" + test_script_content > "${exercise}/run_tests" + chmod 755 "${exercise}/run_tests" +} diff --git a/config.json b/config.json index 6db2ef6..e4ecbec 100644 --- a/config.json +++ b/config.json @@ -23,7 +23,9 @@ "%{snake_slug}.gd" ], "test": [ - "%{snake_slug}_test.gd" + "%{snake_slug}_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/allergies/.meta/config.json b/exercises/practice/allergies/.meta/config.json index ad43e6b..2251349 100644 --- a/exercises/practice/allergies/.meta/config.json +++ b/exercises/practice/allergies/.meta/config.json @@ -7,7 +7,9 @@ "allergies.gd" ], "test": [ - "allergies_test.gd" + "allergies_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/allergies/lib/test_runner.gd b/exercises/practice/allergies/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/allergies/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/allergies/run_tests b/exercises/practice/allergies/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/allergies/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/anagram/.meta/config.json b/exercises/practice/anagram/.meta/config.json index 0d7fcf2..22ff967 100644 --- a/exercises/practice/anagram/.meta/config.json +++ b/exercises/practice/anagram/.meta/config.json @@ -7,7 +7,9 @@ "anagram.gd" ], "test": [ - "anagram_test.gd" + "anagram_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/anagram/lib/test_runner.gd b/exercises/practice/anagram/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/anagram/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/anagram/run_tests b/exercises/practice/anagram/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/anagram/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/armstrong-numbers/.meta/config.json b/exercises/practice/armstrong-numbers/.meta/config.json index 14e50f8..0b96488 100644 --- a/exercises/practice/armstrong-numbers/.meta/config.json +++ b/exercises/practice/armstrong-numbers/.meta/config.json @@ -7,7 +7,9 @@ "armstrong_numbers.gd" ], "test": [ - "armstrong_numbers_test.gd" + "armstrong_numbers_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/armstrong-numbers/lib/test_runner.gd b/exercises/practice/armstrong-numbers/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/armstrong-numbers/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/armstrong-numbers/run_tests b/exercises/practice/armstrong-numbers/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/armstrong-numbers/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/atbash-cipher/.meta/config.json b/exercises/practice/atbash-cipher/.meta/config.json index 035cfc3..0e59bad 100644 --- a/exercises/practice/atbash-cipher/.meta/config.json +++ b/exercises/practice/atbash-cipher/.meta/config.json @@ -7,7 +7,9 @@ "atbash_cipher.gd" ], "test": [ - "atbash_cipher_test.gd" + "atbash_cipher_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/atbash-cipher/lib/test_runner.gd b/exercises/practice/atbash-cipher/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/atbash-cipher/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/atbash-cipher/run_tests b/exercises/practice/atbash-cipher/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/atbash-cipher/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/binary-search/.meta/config.json b/exercises/practice/binary-search/.meta/config.json index e3db785..5ed2cee 100644 --- a/exercises/practice/binary-search/.meta/config.json +++ b/exercises/practice/binary-search/.meta/config.json @@ -7,7 +7,9 @@ "binary_search.gd" ], "test": [ - "binary_search_test.gd" + "binary_search_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/binary-search/lib/test_runner.gd b/exercises/practice/binary-search/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/binary-search/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/binary-search/run_tests b/exercises/practice/binary-search/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/binary-search/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/bob/.meta/config.json b/exercises/practice/bob/.meta/config.json index 8d08075..e35ee12 100644 --- a/exercises/practice/bob/.meta/config.json +++ b/exercises/practice/bob/.meta/config.json @@ -7,7 +7,9 @@ "bob.gd" ], "test": [ - "bob_test.gd" + "bob_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/bob/lib/test_runner.gd b/exercises/practice/bob/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/bob/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/bob/run_tests b/exercises/practice/bob/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/bob/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/collatz-conjecture/.meta/config.json b/exercises/practice/collatz-conjecture/.meta/config.json index 1d40633..d186bb9 100644 --- a/exercises/practice/collatz-conjecture/.meta/config.json +++ b/exercises/practice/collatz-conjecture/.meta/config.json @@ -7,7 +7,9 @@ "collatz_conjecture.gd" ], "test": [ - "collatz_conjecture_test.gd" + "collatz_conjecture_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/collatz-conjecture/lib/test_runner.gd b/exercises/practice/collatz-conjecture/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/collatz-conjecture/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/collatz-conjecture/run_tests b/exercises/practice/collatz-conjecture/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/collatz-conjecture/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/darts/.meta/config.json b/exercises/practice/darts/.meta/config.json index 4c1503b..e21c705 100644 --- a/exercises/practice/darts/.meta/config.json +++ b/exercises/practice/darts/.meta/config.json @@ -7,7 +7,9 @@ "darts.gd" ], "test": [ - "darts_test.gd" + "darts_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/darts/lib/test_runner.gd b/exercises/practice/darts/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/darts/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/darts/run_tests b/exercises/practice/darts/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/darts/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/difference-of-squares/.meta/config.json b/exercises/practice/difference-of-squares/.meta/config.json index ea2f8d0..419e110 100644 --- a/exercises/practice/difference-of-squares/.meta/config.json +++ b/exercises/practice/difference-of-squares/.meta/config.json @@ -7,7 +7,9 @@ "difference_of_squares.gd" ], "test": [ - "difference_of_squares_test.gd" + "difference_of_squares_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/difference-of-squares/lib/test_runner.gd b/exercises/practice/difference-of-squares/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/difference-of-squares/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/difference-of-squares/run_tests b/exercises/practice/difference-of-squares/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/difference-of-squares/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/eliuds-eggs/.meta/config.json b/exercises/practice/eliuds-eggs/.meta/config.json index a39de7f..1b40836 100644 --- a/exercises/practice/eliuds-eggs/.meta/config.json +++ b/exercises/practice/eliuds-eggs/.meta/config.json @@ -7,7 +7,9 @@ "eliuds_eggs.gd" ], "test": [ - "eliuds_eggs_test.gd" + "eliuds_eggs_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/eliuds-eggs/lib/test_runner.gd b/exercises/practice/eliuds-eggs/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/eliuds-eggs/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/eliuds-eggs/run_tests b/exercises/practice/eliuds-eggs/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/eliuds-eggs/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/etl/.meta/config.json b/exercises/practice/etl/.meta/config.json index 2a47b7f..c02f520 100644 --- a/exercises/practice/etl/.meta/config.json +++ b/exercises/practice/etl/.meta/config.json @@ -7,7 +7,9 @@ "etl.gd" ], "test": [ - "etl_test.gd" + "etl_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/etl/lib/test_runner.gd b/exercises/practice/etl/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/etl/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/etl/run_tests b/exercises/practice/etl/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/etl/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/flatten-array/.meta/config.json b/exercises/practice/flatten-array/.meta/config.json index 6d1695c..4a2fec6 100644 --- a/exercises/practice/flatten-array/.meta/config.json +++ b/exercises/practice/flatten-array/.meta/config.json @@ -7,7 +7,9 @@ "flatten_array.gd" ], "test": [ - "flatten_array_test.gd" + "flatten_array_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/flatten-array/lib/test_runner.gd b/exercises/practice/flatten-array/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/flatten-array/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/flatten-array/run_tests b/exercises/practice/flatten-array/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/flatten-array/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/hamming/.meta/config.json b/exercises/practice/hamming/.meta/config.json index db41e17..de40d2a 100644 --- a/exercises/practice/hamming/.meta/config.json +++ b/exercises/practice/hamming/.meta/config.json @@ -7,7 +7,9 @@ "hamming.gd" ], "test": [ - "hamming_test.gd" + "hamming_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/hamming/lib/test_runner.gd b/exercises/practice/hamming/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/hamming/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/hamming/run_tests b/exercises/practice/hamming/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/hamming/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/hello-world/.meta/config.json b/exercises/practice/hello-world/.meta/config.json index 83d9b22..5fbd05d 100644 --- a/exercises/practice/hello-world/.meta/config.json +++ b/exercises/practice/hello-world/.meta/config.json @@ -10,7 +10,9 @@ "hello_world.gd" ], "test": [ - "hello_world_test.gd" + "hello_world_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/hello-world/lib/test_runner.gd b/exercises/practice/hello-world/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/hello-world/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/hello-world/run_tests b/exercises/practice/hello-world/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/hello-world/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/isogram/.meta/config.json b/exercises/practice/isogram/.meta/config.json index 0f8000f..9469a37 100644 --- a/exercises/practice/isogram/.meta/config.json +++ b/exercises/practice/isogram/.meta/config.json @@ -7,7 +7,9 @@ "isogram.gd" ], "test": [ - "isogram_test.gd" + "isogram_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/isogram/lib/test_runner.gd b/exercises/practice/isogram/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/isogram/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/isogram/run_tests b/exercises/practice/isogram/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/isogram/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/leap/.meta/config.json b/exercises/practice/leap/.meta/config.json index edbeb9a..2e3e90c 100644 --- a/exercises/practice/leap/.meta/config.json +++ b/exercises/practice/leap/.meta/config.json @@ -7,7 +7,9 @@ "leap.gd" ], "test": [ - "leap_test.gd" + "leap_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/leap/lib/test_runner.gd b/exercises/practice/leap/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/leap/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/leap/run_tests b/exercises/practice/leap/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/leap/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/luhn/.meta/config.json b/exercises/practice/luhn/.meta/config.json index 54db742..3a6c7ea 100644 --- a/exercises/practice/luhn/.meta/config.json +++ b/exercises/practice/luhn/.meta/config.json @@ -7,7 +7,9 @@ "luhn.gd" ], "test": [ - "luhn_test.gd" + "luhn_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/luhn/lib/test_runner.gd b/exercises/practice/luhn/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/luhn/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/luhn/run_tests b/exercises/practice/luhn/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/luhn/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/matching-brackets/.meta/config.json b/exercises/practice/matching-brackets/.meta/config.json index 2f62a05..ce3e917 100644 --- a/exercises/practice/matching-brackets/.meta/config.json +++ b/exercises/practice/matching-brackets/.meta/config.json @@ -7,7 +7,9 @@ "matching_brackets.gd" ], "test": [ - "matching_brackets_test.gd" + "matching_brackets_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/matching-brackets/lib/test_runner.gd b/exercises/practice/matching-brackets/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/matching-brackets/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/matching-brackets/run_tests b/exercises/practice/matching-brackets/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/matching-brackets/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/pangram/.meta/config.json b/exercises/practice/pangram/.meta/config.json index 5425378..e4dd53a 100644 --- a/exercises/practice/pangram/.meta/config.json +++ b/exercises/practice/pangram/.meta/config.json @@ -7,7 +7,9 @@ "pangram.gd" ], "test": [ - "pangram_test.gd" + "pangram_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/pangram/lib/test_runner.gd b/exercises/practice/pangram/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/pangram/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/pangram/run_tests b/exercises/practice/pangram/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/pangram/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/raindrops/.meta/config.json b/exercises/practice/raindrops/.meta/config.json index 39d0ba5..53edb0d 100644 --- a/exercises/practice/raindrops/.meta/config.json +++ b/exercises/practice/raindrops/.meta/config.json @@ -7,7 +7,9 @@ "raindrops.gd" ], "test": [ - "raindrops_test.gd" + "raindrops_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/raindrops/lib/test_runner.gd b/exercises/practice/raindrops/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/raindrops/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/raindrops/run_tests b/exercises/practice/raindrops/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/raindrops/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/resistor-color-duo/.meta/config.json b/exercises/practice/resistor-color-duo/.meta/config.json index f65d615..0d84e22 100644 --- a/exercises/practice/resistor-color-duo/.meta/config.json +++ b/exercises/practice/resistor-color-duo/.meta/config.json @@ -7,7 +7,9 @@ "resistor_color_duo.gd" ], "test": [ - "resistor_color_duo_test.gd" + "resistor_color_duo_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/resistor-color-duo/lib/test_runner.gd b/exercises/practice/resistor-color-duo/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/resistor-color-duo/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/resistor-color-duo/run_tests b/exercises/practice/resistor-color-duo/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/resistor-color-duo/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/resistor-color-expert/.meta/config.json b/exercises/practice/resistor-color-expert/.meta/config.json index 3b6d067..f2d8101 100644 --- a/exercises/practice/resistor-color-expert/.meta/config.json +++ b/exercises/practice/resistor-color-expert/.meta/config.json @@ -7,7 +7,9 @@ "resistor_color_expert.gd" ], "test": [ - "resistor_color_expert_test.gd" + "resistor_color_expert_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/resistor-color-expert/lib/test_runner.gd b/exercises/practice/resistor-color-expert/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/resistor-color-expert/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/resistor-color-expert/run_tests b/exercises/practice/resistor-color-expert/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/resistor-color-expert/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/resistor-color-trio/.meta/config.json b/exercises/practice/resistor-color-trio/.meta/config.json index 0691fb7..1215e7f 100644 --- a/exercises/practice/resistor-color-trio/.meta/config.json +++ b/exercises/practice/resistor-color-trio/.meta/config.json @@ -7,7 +7,9 @@ "resistor_color_trio.gd" ], "test": [ - "resistor_color_trio_test.gd" + "resistor_color_trio_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/resistor-color-trio/lib/test_runner.gd b/exercises/practice/resistor-color-trio/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/resistor-color-trio/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/resistor-color-trio/run_tests b/exercises/practice/resistor-color-trio/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/resistor-color-trio/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/resistor-color/.meta/config.json b/exercises/practice/resistor-color/.meta/config.json index af8dec4..f4d2f87 100644 --- a/exercises/practice/resistor-color/.meta/config.json +++ b/exercises/practice/resistor-color/.meta/config.json @@ -7,7 +7,9 @@ "resistor_color.gd" ], "test": [ - "resistor_color_test.gd" + "resistor_color_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/resistor-color/lib/test_runner.gd b/exercises/practice/resistor-color/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/resistor-color/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/resistor-color/run_tests b/exercises/practice/resistor-color/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/resistor-color/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/reverse-string/.meta/config.json b/exercises/practice/reverse-string/.meta/config.json index f29878c..21f4a79 100644 --- a/exercises/practice/reverse-string/.meta/config.json +++ b/exercises/practice/reverse-string/.meta/config.json @@ -7,7 +7,9 @@ "reverse_string.gd" ], "test": [ - "reverse_string_test.gd" + "reverse_string_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/reverse-string/lib/test_runner.gd b/exercises/practice/reverse-string/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/reverse-string/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/reverse-string/run_tests b/exercises/practice/reverse-string/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/reverse-string/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/robot-simulator/.meta/config.json b/exercises/practice/robot-simulator/.meta/config.json index 712bc05..9cbb66c 100644 --- a/exercises/practice/robot-simulator/.meta/config.json +++ b/exercises/practice/robot-simulator/.meta/config.json @@ -10,7 +10,9 @@ "robot_simulator.gd" ], "test": [ - "robot_simulator_test.gd" + "robot_simulator_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/robot-simulator/lib/test_runner.gd b/exercises/practice/robot-simulator/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/robot-simulator/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/robot-simulator/run_tests b/exercises/practice/robot-simulator/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/robot-simulator/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/space-age/.meta/config.json b/exercises/practice/space-age/.meta/config.json index 0ecba48..b05ed84 100644 --- a/exercises/practice/space-age/.meta/config.json +++ b/exercises/practice/space-age/.meta/config.json @@ -7,7 +7,9 @@ "space_age.gd" ], "test": [ - "space_age_test.gd" + "space_age_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/space-age/lib/test_runner.gd b/exercises/practice/space-age/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/space-age/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/space-age/run_tests b/exercises/practice/space-age/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/space-age/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/spiral-matrix/.meta/config.json b/exercises/practice/spiral-matrix/.meta/config.json index d270bbf..aad7b1a 100644 --- a/exercises/practice/spiral-matrix/.meta/config.json +++ b/exercises/practice/spiral-matrix/.meta/config.json @@ -7,7 +7,9 @@ "spiral_matrix.gd" ], "test": [ - "spiral_matrix_test.gd" + "spiral_matrix_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/spiral-matrix/lib/test_runner.gd b/exercises/practice/spiral-matrix/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/spiral-matrix/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/spiral-matrix/run_tests b/exercises/practice/spiral-matrix/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/spiral-matrix/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/triangle/.meta/config.json b/exercises/practice/triangle/.meta/config.json index f9bc584..51f5f3b 100644 --- a/exercises/practice/triangle/.meta/config.json +++ b/exercises/practice/triangle/.meta/config.json @@ -7,7 +7,9 @@ "triangle.gd" ], "test": [ - "triangle_test.gd" + "triangle_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/triangle/lib/test_runner.gd b/exercises/practice/triangle/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/triangle/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/triangle/run_tests b/exercises/practice/triangle/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/triangle/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}" diff --git a/exercises/practice/two-fer/.meta/config.json b/exercises/practice/two-fer/.meta/config.json index f9a81f3..efc078a 100644 --- a/exercises/practice/two-fer/.meta/config.json +++ b/exercises/practice/two-fer/.meta/config.json @@ -7,7 +7,9 @@ "two_fer.gd" ], "test": [ - "two_fer_test.gd" + "two_fer_test.gd", + "run_tests", + "lib/test_runner.gd" ], "example": [ ".meta/example.gd" diff --git a/exercises/practice/two-fer/lib/test_runner.gd b/exercises/practice/two-fer/lib/test_runner.gd new file mode 100644 index 0000000..5de0e6b --- /dev/null +++ b/exercises/practice/two-fer/lib/test_runner.gd @@ -0,0 +1,467 @@ +class_name TestRunner +extends SceneTree + +class ErrorReader: + """ErrorReader reads from a STDERR file (cmd 2>/file). + + It filters out the less useful Godot output and handles retrieving lines iteratively. + """ + + const EXPECTED_GODOT_ERRORS = [ + "ERROR: Could not create directory: /root/.local\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Error attempting to create data dir: /root/.local/share/godot/app_userdata/[unnamed project].\n at: ensure_user_data_dir (core/os/os.cpp:344)\n", + "ERROR: Could not create editor data directory: /root/.local/share/godot\n at: EditorPaths (editor/editor_paths.cpp:183)\n", + "ERROR: Could not create directory: /root/.config\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor config directory: /root/.config/godot\n at: EditorPaths (editor/editor_paths.cpp:198)\n", + "ERROR: Could not create directory: /root/.cache\n at: make_dir_recursive (core/io/dir_access.cpp:180)\n", + "ERROR: Could not create editor cache directory: /root/.cache/godot\n at: EditorPaths (editor/editor_paths.cpp:219)\n", + "ERROR: Can't save resource to empty path. Provide non-empty path or a Resource with non-empty resource_path.\n at: save (core/io/resource_saver.cpp:105)\n", + "ERROR: Error saving editor settings to \n at: save (editor/editor_settings.cpp:1013)\n", + ] + var already_processed_bytes: int = 0 + var filename: String = "" + var color_sequence = RegEx.create_from_string("\\e\\[\\d[\\d;]*m") + + func _init(filename: String) -> void: + self.already_processed_bytes = 0 + self.filename = filename + if not self.filename.is_empty() and ResourceLoader.exists(self.filename): + self.get_error_message() + + func check() -> Error: + if not FileAccess.file_exists(self.filename): + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + if OS.get_stderr_type() != OS.STD_HANDLE_FILE: + push_error("STDERR must be redirected to '%s' when using '--json'" % self.filename) + return ERR_FILE_NOT_FOUND + return OK + + func get_error_message() -> String: + """ + Checks the contents of the STDERR file, where error output from the + currently running test suite should be stored. If there is any relevant output + there, this method will return it. If the file is empty, an empty string is returned. + + This method will remove previously read output from the returned message. It should + be called after executing every test, to ensure that the message contains only output + relevant for the given test. + + NOTE: running a Godot instance without full access to certain folders in the + home directory (`~/.config`, `~/.local`, `~/.cache`) will result in additional + errors. This happens when GDScript test runner is executed in Docker. However, + these errors do not stop the execution of the program, so they can be ignored. + This method filters them out, leaving only the errors caused by the actual tests. + """ + if self.filename.is_empty(): + return "" + + var error_output = FileAccess.get_file_as_string(self.filename) + error_output = error_output.erase(0, self.already_processed_bytes) + self.already_processed_bytes += len(error_output) + + # By default, Godot's error messages are printed with colors. To include the output in the + # `results.json` file, color markers need to be removed first. + error_output = self.color_sequence.sub(error_output, "", true) + + # Filter out the expected error messages + for expected_error in EXPECTED_GODOT_ERRORS: + error_output = error_output.replace(expected_error, "") + + return error_output + + +class ResultWriter: + + var json_file: String = "" + var stderr: ErrorReader = null + + func _init(json_file: String, stderr: ErrorReader) -> void: + self.json_file = json_file + self.stderr = stderr + + func delete_existing() -> Error: + """ + Removes the previous `results.json` file, to ensure that the current test suite will not use it. + """ + if self.json_file.is_empty(): + return OK + if FileAccess.file_exists(self.json_file): + DirAccess.remove_absolute(self.json_file) + var error_message: String = self.stderr.get_error_message() + if not error_message.is_empty(): + push_error(error_message) + return FAILED + return OK + + func write(results: Dictionary) -> Error: + """ + If output_dir_path is empty, outputs friendly-formatted test results to + stdout. + + Otherwise, writes JSON results to results.json in the given path. + """ + if self.json_file.is_empty(): + return print_friendly_results(results) + else: + return write_results_file(results) + + func print_friendly_results(results: Dictionary) -> Error: + """ + Prints rich results to stdout. + """ + var rich_results = [""] + if results.status == "pass": + rich_results.push_back("[color=yellow]Exercise passed! Nicely done![/color]") + else: + rich_results.push_back("Exercise has at least one failed test:") + for test in results.tests: + if test.status == "pass": + rich_results.push_back(" %s [color=yellow]passed ✔[/color]" % test.name) + else: + rich_results.push_back(" %s [color=pink]failed: %s[/color]" % [test.name, test.message]) + print_rich("\n".join(rich_results)) + return OK + + func write_results_file(results: Dictionary) -> Error: + """ + Saves a dictionary as a `results.json` file in the output directory. The file + is indented with 4 spaces, keys are not sorted. + + The `results` dictionary represents the full output of a single exercise's + test suite, according to the Test Runner Interface documentation: + + https://exercism.org/docs/building/tooling/test-runners/interface + + This method automatically adds `'version' = 2` to the results. + """ + # Due to keys not being sorted, `version` should be inserted first + var full_results = {"version": 2} + full_results.merge(results) + + var pretty_results = JSON.stringify(full_results, " ", false) + + var results_json = FileAccess.open(self.json_file, FileAccess.WRITE) + if results_json == null: + var err = FileAccess.get_open_error() + push_error("Failed to write the file: %s (%s)" % [self.json_file, error_string(err)]) + return err + + results_json.store_string(pretty_results + "\n") + return OK + + +class ArgParser: + """ArgParser reads the cmdline and parses inputs and options.""" + + var solution_script_path: String = "" + var test_suite_script_path: String = "" + var run_all: bool = false + var json: bool = false + var json_file: String = "" + + func usage(message: String) -> void: + print("Usage:") + print("test_runner [--all] [--json filename.json] ") + if not message.is_empty(): + print() + print(message) + + func parse_args() -> Error: + """ + Validates the number of args passed to the script and sets values of the following + global variables, based on the given args: + * `solution_script_path` + * `test_suite_script_path` + """ + var args = OS.get_cmdline_user_args() + var positional: Array = [] + + var idx := 0 + while idx < len(args): + var arg = args[idx] + if arg == "--all": + run_all = true + elif arg.begins_with("--json"): + if arg.contains("="): + var parts = arg.split("=", true, 1) + if parts[0] == "--json": + self.json = true + self.json_file = parts[1] + else: + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + elif idx + 1 < len(args): + self.json = true + self.json_file = args[idx + 1] + idx += 1 + else: + usage("Flag %s is missing a filename" % arg) + return ERR_INVALID_PARAMETER + elif arg.begins_with("--"): + usage("Invalid flag %s" % arg) + return ERR_INVALID_PARAMETER + else: + positional.append(arg) + idx += 1 + + if len(positional) != 1: + usage("Expecting 1 positional args but got %d" % len(positional)) + return ERR_INVALID_PARAMETER + + var solution_dir_path = positional[0] + var sep = "/" + if OS.get_name() == "Windows": + sep = "\\" + var slug = solution_dir_path.rsplit(sep, true, 1)[1].replace("-", "_") + + self.solution_script_path = solution_dir_path.path_join(slug + ".gd") + self.test_suite_script_path = solution_dir_path.path_join(slug + "_test.gd") + + return OK + + func files() -> Array: + return [self.solution_script_path, self.test_suite_script_path] + + +class Runner: + + var test_suite: Object = null + var solution: Object = null + var stderr: ErrorReader = null + var result_writer: ResultWriter = null + + func _init(test_suite: Object, solution: Object, stderr: ErrorReader, result_writer: ResultWriter) -> void: + self.test_suite = test_suite + self.solution = solution + self.stderr = stderr + self.result_writer = result_writer + + func extract_methods() -> Dictionary: + """Extract the source code of a method from a script.""" + var in_func: bool = false + var parts = [] + var methods: Dictionary = {} + var name: String = "" + var func_re: RegEx = RegEx.create_from_string('^func ([^(]+)\\(') + for line in self.test_suite.get_script().source_code.split("\n"): + if not in_func: + var matched = func_re.search(line) + if matched: + in_func = true + name = matched.get_strings()[1] + parts = [] + elif not line.begins_with("\t") and not line.begins_with(" "): + if in_func: + in_func = false + methods[name] = "\n".join(parts) + else: + parts.append(line) + + return methods + + func get_failed_test_message(actual, expected) -> String: + """ + Generates a message for a failed test. If given values are strings, + this method will wrap them in single quotes, to distinguish from + other types, like integers (e.g. '3' vs 3). + + This method should be used if the actual and expected values for a + given test are different, or if they have different types. + """ + var values = [] + + for value in [actual, expected]: + var formatted_value = str(value) + if typeof(value) == TYPE_STRING: + formatted_value = "'{0}'".format([formatted_value]) + values.append(formatted_value) + + return "Expected output was {1}, actual output was {0}.".format(values) + + func run_tests() -> Array: + """ + Runs the given test suite agains the given solution script. Returns an Array + representing the results of each test case, in the same order that they are defined + in the test suite. Each test result is a Dictionary containing 3 values: + + * `name`: the name of the test case + * `status`: 'pass', 'fail', or 'error' + * `message`: null if the test passed, otherwise it contains the details of the failure/error + + This method will look for methods starting with `test_` in the test script, and + then call them with the solution script as the only argument. The `test_` method's + name will be used as the `name` value for each test result. + + If any stderr output was generated during the execution of the `test_` method, + `status` will be set to 'error'. The test will also receive an 'error' status + if calling the `test_` method returned null (which indicates execution issues). + + Otherwise, the `test_` method should return an Array of 2 elements: the actual value, + and the expected value. The values will be compared and if they are the same, the test will + pass. Otherwise, it will fail, with a relevant message being set by this method. + """ + var test_results = [] + var test_codes = self.extract_methods() + + for method in self.test_suite.get_method_list(): + if not method["name"].begins_with("test_"): + continue + var test_method_name = method["name"] + print("-> " + test_method_name) + var test_method = self.test_suite.get(test_method_name) + + var output = test_method.call(self.solution) + var error_message = self.stderr.get_error_message() + + if output == null: + test_results.append({ + "name": test_method_name, + "status": "error", + "message": error_message, + }) + else: + var actual = output[0] + var expected = output[1] + + var passed = ( + typeof(actual) == typeof(expected) and + actual == expected and + error_message.is_empty() + ) + + var status = "error" + if error_message.is_empty(): + status = "pass" if passed else "fail" + error_message = null if passed else self.get_failed_test_message( + actual, expected + ) + + var test_code = test_codes[method["name"]] + # The code would be cleaner if the test suite actually checked the values. + # Instead, the test suite returns an array with two values which are compared (above). + # This `replace()` hides some of that implementation detail by communicating that the + # array elements are being compared for equality. + test_code = test_code.replace("return [", "assert_equal [") + test_code = test_code.replace("\t", " ") + test_code = test_code.dedent().strip_edges() + + test_results.append({ + "name": test_method_name, + "status": status, + "message": error_message, + "test_code": test_code, + }) + + return test_results + + func run(run_all: bool) -> void: + """ + Executes the current suite against the current solution. Stores the results in the + `results.json` file in the output dir. + """ + var test_results = run_tests() + var results = {} + + # Check if any tests were executed + if len(test_results) == 0: + results = {"status": "error", "message": "No tests were executed.", "tests": []} + else: + results = {"status": "pass", "tests": test_results} + + # If any of the tests failed, change the global status to `fail` + for test_result in test_results: + if test_result["status"] != "pass": + results["status"] = "fail" + break + + if not run_all and results["status"] != "pass": + var success_count: int = 0 + for test_result in test_results: + if test_result["status"] == "pass": + success_count += 1 + + var has_failure = false + var filtered_results = [] + for test_result in test_results: + if test_result["status"] == "pass": + filtered_results.append(test_result) + elif not has_failure: + has_failure = true + test_result["message"] = "Passes %d/%d tests. First failure: %s" % [ + success_count, len(test_results), test_result["message"], + ] + filtered_results.append(test_result) + results["tests"] = filtered_results + + self.result_writer.write(results) + + +func load_script(solution_script_path: String) -> Object: + """Load a script. This helper is needed to catch failures without aborting the error handling.""" + return load(solution_script_path).new() + + +func load_solution_script(solution_script_path: String, result_writer: ResultWriter, stderr: ErrorReader) -> Array: + """ + Loads the solution script and saves it in a global variable. In case of any + issues, this method will save the `results.json` file with `error` status. + """ + # Make sure the STDERR file is empty. + stderr.get_error_message() + var solution_script = load_script(solution_script_path) + + if solution_script == null: + var message: String = "The solution file could not be parsed." + # Extract the error message, up until the "Failed to load" line + # which is specific to the test runner, not the user's solution. + var error_message: String = stderr.get_error_message() + if not error_message.is_empty(): + var msg = [] + var stop = 'ERROR: Failed to load script "%s" with error "Parse error".' % solution_script_path + for line in error_message.split("\n"): + if line == stop: + break + msg.append(line) + message += "\n" + "\n".join(msg) + var results = {"status": "error", "message": message, "tests": []} + result_writer.write(results) + return [null, ERR_PARSE_ERROR] + + return [solution_script, OK] + + +func _init(): + # Calling `quit(1)` doesn't stop the program immediately, so a `return` is necessary. + # That's why errors are checked directly in `_init()`, instead of calling `quit(1)` + # in each method. + var args = ArgParser.new() + if args.parse_args() != OK: + quit(1) + return + for file in args.files(): + if not ResourceLoader.exists(file): + push_error("File %s not found." % file) + quit(1) + return + + var stderr = ErrorReader.new("/tmp/stderr") + if args.json and stderr.check() != OK: + quit(1) + return + + var result_writer = ResultWriter.new(args.json_file, stderr) + if result_writer.delete_existing() != OK: + quit(1) + return + + var r = load_solution_script(args.solution_script_path, result_writer, stderr) + if r[1] != OK: + quit(1) + return + var solution = r[0] + + var test_suite = load(args.test_suite_script_path).new() + + Runner.new(test_suite, solution, stderr, result_writer).run(args.run_all) + quit() diff --git a/exercises/practice/two-fer/run_tests b/exercises/practice/two-fer/run_tests new file mode 100755 index 0000000..66be879 --- /dev/null +++ b/exercises/practice/two-fer/run_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +slug=${PWD##*/} +[[ -f "${slug//-/_}.gd" ]] || die "Error! solution file is missing" +[[ -f "${slug//-/_}_test.gd" ]] || die "Error! test file is missing" +exec godot --headless --script ./lib/test_runner.gd -- "${PWD}"