GDScript: Add support for tracking multiple runtime errors in tests

modules/gdscript/tests/gdscript_test_runner.cpp

@@ -275,6 +275,7 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 				return false;
 			}
 		} else {
+			// `*.notest.gd` files are skipped.
 			if (next.ends_with(".notest.gd")) {
 				next = dir->get_next();
 				continue;
@@ -490,7 +491,7 @@ void GDScriptTest::error_handler(void *p_this, const char *p_function, const cha
 	}
 	builder.append("\n");
 
-	result->output = builder.as_string();
+	result->output += builder.as_string();
 }
 
 bool GDScriptTest::check_output(const String &p_output) const {
@@ -628,12 +629,18 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 		}
 		return result;
 	}
-	// Script files matching this pattern are allowed to not contain a test() function.
-	if (source_file.match("*.notest.gd")) {
+
+	// `*.norun.gd` files are allowed to not contain a `test()` function (no runtime testing).
+	if (source_file.ends_with(".norun.gd")) {
 		enable_stdout();
-		result.passed = check_output(result.output);
+		result.status = GDTEST_OK;
+		result.output = get_text_for_status(result.status) + "\n" + result.output;
+		if (!p_is_generating) {
+			result.passed = check_output(result.output);
+		}
 		return result;
 	}
+
 	// Test running.
 	const HashMap<StringName, GDScriptFunction *>::ConstIterator test_function_element = script->get_member_functions().find(GDScriptTestRunner::test_function_name);
 	if (!test_function_element) {

modules/gdscript/tests/scripts/.editorconfig

@@ -3,8 +3,8 @@
 [parser/features/mixed_indentation_on_blank_lines.gd]
 trim_trailing_whitespace = false
 
-[parser/warnings/empty_file_newline.notest.gd]
+[parser/warnings/empty_file_newline.norun.gd]
 insert_final_newline = false
 
-[parser/warnings/empty_file_newline_comment.notest.gd]
+[parser/warnings/empty_file_newline_comment.norun.gd]
 insert_final_newline = false

modules/gdscript/tests/scripts/parser/warnings/empty_file.notest.out → modules/gdscript/tests/scripts/parser/warnings/empty_file.norun.out

@@ -1,3 +1,4 @@
+GDTEST_OK
 >> WARNING
 >> Line: 1
 >> EMPTY_FILE

modules/gdscript/tests/scripts/parser/warnings/empty_file_comment.norun.gd

@@ -0,0 +1 @@
+# A comment.

modules/gdscript/tests/scripts/parser/warnings/empty_file_comment.notest.out → modules/gdscript/tests/scripts/parser/warnings/empty_file_comment.norun.out

@@ -1,3 +1,4 @@
+GDTEST_OK
 >> WARNING
 >> Line: 1
 >> EMPTY_FILE

modules/gdscript/tests/scripts/parser/warnings/empty_file_newline.notest.out → modules/gdscript/tests/scripts/parser/warnings/empty_file_newline.norun.out

@@ -1,3 +1,4 @@
+GDTEST_OK
 >> WARNING
 >> Line: 1
 >> EMPTY_FILE

modules/gdscript/tests/scripts/parser/warnings/empty_file_newline_comment.norun.gd

@@ -0,0 +1,4 @@
+# A comment, followed by a bunch of newlines.
+
+
+

modules/gdscript/tests/scripts/parser/warnings/empty_file_newline_comment.notest.out → modules/gdscript/tests/scripts/parser/warnings/empty_file_newline_comment.norun.out

@@ -1,3 +1,4 @@
+GDTEST_OK
 >> WARNING
 >> Line: 1
 >> EMPTY_FILE

modules/gdscript/tests/scripts/runtime/errors/assign_freed_instance.gd

@@ -0,0 +1,22 @@
+class A extends Node:
+	pass
+
+func subtest_native():
+	var x = Node.new()
+
+	x.free()
+
+	var _ok = x
+	var _bad: Node = x
+
+func subtest_script():
+	var x = A.new()
+
+	x.free()
+
+	var _ok = x
+	var _bad: A = x
+
+func test():
+	subtest_native()
+	subtest_script()

modules/gdscript/tests/scripts/runtime/errors/assign_freed_instance.out

@@ -0,0 +1,11 @@
+GDTEST_RUNTIME_ERROR
+>> SCRIPT ERROR
+>> on function: subtest_native()
+>> runtime/errors/assign_freed_instance.gd
+>> 10
+>> Trying to assign invalid previously freed instance.
+>> SCRIPT ERROR
+>> on function: subtest_script()
+>> runtime/errors/assign_freed_instance.gd
+>> 18
+>> Trying to assign invalid previously freed instance.

modules/gdscript/tests/scripts/runtime/errors/assign_to_read_only_property.gd

@@ -0,0 +1,11 @@
+func subtest_attribute(state):
+	state.center_of_mass.x -= 1.0
+
+func subtest_variable_index(state, prop):
+	state[prop].x = 1.0
+
+func test():
+	var state = PhysicsDirectBodyState3DExtension.new()
+	subtest_attribute(state)
+	subtest_variable_index(state, &"center_of_mass")
+	state.free()

modules/gdscript/tests/scripts/runtime/errors/assign_to_read_only_property.out

@@ -0,0 +1,21 @@
+GDTEST_RUNTIME_ERROR
+>> WARNING
+>> Line: 9
+>> UNSAFE_CALL_ARGUMENT
+>> The argument 1 of the function "subtest_attribute()" requires the subtype "Variant" but the supertype "Variant" was provided.
+>> WARNING
+>> Line: 10
+>> UNSAFE_CALL_ARGUMENT
+>> The argument 1 of the function "subtest_variable_index()" requires the subtype "Variant" but the supertype "Variant" was provided.
+>> ERROR
+>> Required virtual method PhysicsDirectBodyState3DExtension::_get_center_of_mass must be overridden before calling.
+>> SCRIPT ERROR
+>> on function: subtest_attribute()
+>> runtime/errors/assign_to_read_only_property.gd
+>> 2
+>> Cannot set value into property "center_of_mass" (on base "PhysicsDirectBodyState3DExtension") because it is read-only.
+>> SCRIPT ERROR
+>> on function: subtest_variable_index()
+>> runtime/errors/assign_to_read_only_property.gd
+>> 5
+>> Cannot set value into property "center_of_mass" (on base "PhysicsDirectBodyState3DExtension") because it is read-only.

modules/gdscript/tests/scripts/runtime/errors/constant_array_is_deep.out

@@ -1,4 +1,7 @@
 GDTEST_RUNTIME_ERROR
+>> ERROR
+>> Condition "_p->read_only" is true. Returning: false
+>> Dictionary is in read-only state.
 >> SCRIPT ERROR
 >> on function: test()
 >> runtime/errors/constant_array_is_deep.gd

modules/gdscript/tests/scripts/runtime/errors/non_static_method_call_on_native_class.out

@@ -1,4 +1,8 @@
 GDTEST_RUNTIME_ERROR
+>> WARNING
+>> Line: 6
+>> UNSAFE_METHOD_ACCESS
+>> The method "has_method()" is not present on the inferred type "Variant" (but may be present on a subtype).
 >> SCRIPT ERROR
 >> on function: example()
 >> runtime/errors/non_static_method_call_on_native_class.gd

modules/gdscript/tests/scripts/runtime/errors/outer_class_constants.gd

@@ -0,0 +1,28 @@
+class Outer:
+	const OUTER_CONST := 0
+	class Inner:
+		pass
+
+func subtest_type_hard():
+	var type := Outer.Inner
+	print(type.OUTER_CONST)
+
+func subtest_type_weak():
+	var type := Outer.Inner
+	var type_v: Variant = type
+	print(type_v.OUTER_CONST)
+
+func subtest_instance_hard():
+	var instance := Outer.Inner.new()
+	print(instance.OUTER_CONST)
+
+func subtest_instance_weak():
+	var instance := Outer.Inner.new()
+	var instance_v: Variant = instance
+	print(instance_v.OUTER_CONST)
+
+func test():
+	subtest_type_hard()
+	subtest_type_weak()
+	subtest_instance_hard()
+	subtest_instance_weak()

modules/gdscript/tests/scripts/runtime/errors/outer_class_constants.out

@@ -0,0 +1,29 @@
+GDTEST_RUNTIME_ERROR
+>> WARNING
+>> Line: 8
+>> UNSAFE_PROPERTY_ACCESS
+>> The property "OUTER_CONST" is not present on the inferred type "Inner" (but may be present on a subtype).
+>> WARNING
+>> Line: 17
+>> UNSAFE_PROPERTY_ACCESS
+>> The property "OUTER_CONST" is not present on the inferred type "Inner" (but may be present on a subtype).
+>> SCRIPT ERROR
+>> on function: subtest_type_hard()
+>> runtime/errors/outer_class_constants.gd
+>> 8
+>> Invalid access to property or key 'OUTER_CONST' on a base object of type 'GDScript'.
+>> SCRIPT ERROR
+>> on function: subtest_type_weak()
+>> runtime/errors/outer_class_constants.gd
+>> 13
+>> Invalid access to property or key 'OUTER_CONST' on a base object of type 'GDScript'.
+>> SCRIPT ERROR
+>> on function: subtest_instance_hard()
+>> runtime/errors/outer_class_constants.gd
+>> 17
+>> Invalid access to property or key 'OUTER_CONST' on a base object of type 'RefCounted (Inner)'.
+>> SCRIPT ERROR
+>> on function: subtest_instance_weak()
+>> runtime/errors/outer_class_constants.gd
+>> 22
+>> Invalid access to property or key 'OUTER_CONST' on a base object of type 'RefCounted (Inner)'.

modules/gdscript/tests/scripts/runtime/errors/read_only_dictionary.out

@@ -1,4 +1,7 @@
 GDTEST_RUNTIME_ERROR
+>> ERROR
+>> Condition "_p->read_only" is true. Returning: false
+>> Dictionary is in read-only state.
 >> SCRIPT ERROR
 >> on function: test()
 >> runtime/errors/read_only_dictionary.gd

modules/gdscript/tests/scripts/runtime/errors/typed_array_assign_basic_to_typed.gd

@@ -1,4 +1,4 @@
 func test():
 	var basic := [1]
-	var typed: Array[int] = basic
+	var _typed: Array[int] = basic
 	print('not ok')

modules/gdscript/tests/scripts/runtime/errors/typed_array_assign_differently_typed.gd

@@ -1,4 +1,4 @@
 func test():
 	var differently: Variant = [1.0] as Array[float]
-	var typed: Array[int] = differently
+	var _typed: Array[int] = differently
 	print('not ok')

modules/gdscript/tests/scripts/runtime/errors/typed_array_assign_wrong_to_typed.gd

@@ -3,5 +3,5 @@ class Bar extends Foo: pass
 class Baz extends Foo: pass
 
 func test():
-	var typed: Array[Bar] = [Baz.new() as Foo]
+	var _typed: Array[Bar] = [Baz.new() as Foo]
 	print('not ok')

modules/gdscript/tests/scripts/runtime/errors/typed_array_assign_wrong_to_typed.out

@@ -1,5 +1,8 @@
 GDTEST_RUNTIME_ERROR
 >> ERROR
+>> Condition "!other_script->inherits_script(script)" is true. Returning: false
+>> Attempted to assign an object into a TypedArray, that does not inherit from 'GDScript'.
+>> ERROR
 >> Method/function failed.
 >> Unable to convert array index 0 from "Object" to "Object".
 not ok