Skip to content

Commit

Permalink
GDScript: Support tracking multiple analyzer and runtime errors in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
dalexeev committed Nov 21, 2024
1 parent 9e60984 commit 1d7e920
Show file tree
Hide file tree
Showing 293 changed files with 537 additions and 1,083 deletions.
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ repos:
.*\.patch$|
.*\.out$|
modules/gdscript/tests/scripts/parser/features/mixed_indentation_on_blank_lines\.gd$|
modules/gdscript/tests/scripts/parser/warnings/empty_file_newline_comment\.notest\.gd$|
modules/gdscript/tests/scripts/parser/warnings/empty_file_newline\.notest\.gd$|
modules/gdscript/tests/scripts/parser/warnings/empty_file_newline_comment\.norun\.gd$|
modules/gdscript/tests/scripts/parser/warnings/empty_file_newline\.norun\.gd$|
platform/android/java/editor/src/main/java/com/android/.*|
platform/android/java/lib/src/com/google/.*
)
Expand Down
75 changes: 35 additions & 40 deletions modules/gdscript/tests/gdscript_test_runner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ static String strip_warnings(const String &p_expected) {
// so it doesn't fail just because of difference in warnings.
String expected_no_warnings;
for (String line : p_expected.split("\n")) {
if (line.begins_with(">> ")) {
if (line.begins_with("~~ ")) {
continue;
}
expected_no_warnings += line + "\n";
Expand Down Expand Up @@ -275,6 +275,7 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
return false;
}
} else {
// `*.notest.gd` files are skipped.
if (next.ends_with(".notest.gd")) {
next = dir->get_next();
continue;
Expand Down Expand Up @@ -450,47 +451,43 @@ void GDScriptTest::error_handler(void *p_this, const char *p_function, const cha

result->status = GDTEST_RUNTIME_ERROR;

StringBuilder builder;
builder.append(">> ");
// Only include the function, file and line for script errors, otherwise the
// test outputs changes based on the platform/compiler.
// Only include the file, line, and function for script errors,
// otherwise the test outputs changes based on the platform/compiler.
String header;
bool include_source_info = false;
switch (p_type) {
case ERR_HANDLER_ERROR:
builder.append("ERROR");
header = "ERROR";
break;
case ERR_HANDLER_WARNING:
builder.append("WARNING");
header = "WARNING";
break;
case ERR_HANDLER_SCRIPT:
builder.append("SCRIPT ERROR");
header = "SCRIPT ERROR";
include_source_info = true;
break;
case ERR_HANDLER_SHADER:
builder.append("SHADER ERROR");
header = "SHADER ERROR";
break;
default:
builder.append("Unknown error type");
header = "UNKNOWN ERROR";
break;
}

if (include_source_info) {
builder.append("\n>> on function: ");
builder.append(String::utf8(p_function));
builder.append("()\n>> ");
builder.append(String::utf8(p_file).trim_prefix(self->base_dir).replace("\\", "/"));
builder.append("\n>> ");
builder.append(itos(p_line));
}
builder.append("\n>> ");
builder.append(String::utf8(p_error));
header += vformat(" at %s:%d on %s()",
String::utf8(p_file).trim_prefix(self->base_dir).replace("\\", "/"),
p_line,
String::utf8(p_function));
}

StringBuilder error_string;
error_string.append(vformat(">> %s: %s\n", header, String::utf8(p_error)));
if (strlen(p_explanation) > 0) {
builder.append("\n>> ");
builder.append(String::utf8(p_explanation));
error_string.append(vformat(">> %s\n", String::utf8(p_explanation)));
}
builder.append("\n");

result->output = builder.as_string();
result->output += error_string.as_string();
}

bool GDScriptTest::check_output(const String &p_output) const {
Expand Down Expand Up @@ -588,11 +585,11 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
result.status = GDTEST_ANALYZER_ERROR;
result.output = get_text_for_status(result.status) + "\n";

const List<GDScriptParser::ParserError> &errors = parser.get_errors();
if (!errors.is_empty()) {
// Only the first error since the following might be cascading.
result.output += errors.front()->get().message + "\n"; // TODO: line, column?
StringBuilder error_string;
for (const GDScriptParser::ParserError &error : parser.get_errors()) {
error_string.append(vformat(">> ERROR at line %d: %s\n", error.line, error.message));
}
result.output += error_string.as_string();
if (!p_is_generating) {
result.passed = check_output(result.output);
}
Expand All @@ -601,16 +598,8 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {

#ifdef DEBUG_ENABLED
StringBuilder warning_string;
for (const GDScriptWarning &E : parser.get_warnings()) {
const GDScriptWarning warning = E;
warning_string.append(">> WARNING");
warning_string.append("\n>> Line: ");
warning_string.append(itos(warning.start_line));
warning_string.append("\n>> ");
warning_string.append(warning.get_name());
warning_string.append("\n>> ");
warning_string.append(warning.get_message());
warning_string.append("\n");
for (const GDScriptWarning &warning : parser.get_warnings()) {
warning_string.append(vformat("~~ WARNING at line %d: (%s) %s\n", warning.start_line, warning.get_name(), warning.get_message()));
}
result.output += warning_string.as_string();
#endif
Expand All @@ -628,12 +617,18 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
}
return result;
}
// Script files matching this pattern are allowed to not contain a test() function.
if (source_file.match("*.notest.gd")) {

// `*.norun.gd` files are allowed to not contain a `test()` function (no runtime testing).
if (source_file.ends_with(".norun.gd")) {
enable_stdout();
result.passed = check_output(result.output);
result.status = GDTEST_OK;
result.output = get_text_for_status(result.status) + "\n" + result.output;
if (!p_is_generating) {
result.passed = check_output(result.output);
}
return result;
}

// Test running.
const HashMap<StringName, GDScriptFunction *>::ConstIterator test_function_element = script->get_member_functions().find(GDScriptTestRunner::test_function_name);
if (!test_function_element) {
Expand Down
5 changes: 3 additions & 2 deletions modules/gdscript/tests/scripts/.editorconfig
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# Some tests handle invalid syntax deliberately; exclude relevant attributes.
# See also the `file-format` section in `.pre-commit-config.yaml`.

[parser/features/mixed_indentation_on_blank_lines.gd]
trim_trailing_whitespace = false

[parser/warnings/empty_file_newline.notest.gd]
[parser/warnings/empty_file_newline.norun.gd]
insert_final_newline = false

[parser/warnings/empty_file_newline_comment.notest.gd]
[parser/warnings/empty_file_newline_comment.norun.gd]
insert_final_newline = false
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
class A extends InstancePlaceholder:
func _init():
print('no')

class B extends A:
pass

func test():
InstancePlaceholder.new()
B.new()
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
GDTEST_ANALYZER_ERROR
Native class "InstancePlaceholder" cannot be constructed as it is abstract.
>> ERROR at line 9: Native class "InstancePlaceholder" cannot be constructed as it is abstract.
>> ERROR at line 9: Name "new" is a Callable. You can call it with "new.call()" instead.
>> ERROR at line 10: Class "abstract_script_instantiate.gd::B" cannot be constructed as it is based on abstract native class "InstancePlaceholder".
>> ERROR at line 10: Name "new" is a Callable. You can call it with "new.call()" instead.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Argument 1 of annotation "@export_range" isn't a constant expression.
>> ERROR at line 3: Argument 1 of annotation "@export_range" isn't a constant expression.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a constant.
>> ERROR at line 3: Cannot assign a new value to a constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a constant.
>> ERROR at line 3: Cannot assign a new value to a constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a constant.
>> ERROR at line 4: Cannot assign a new value to a constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a read-only property.
>> ERROR at line 3: Cannot assign a new value to a read-only property.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a read-only property.
>> ERROR at line 3: Cannot assign a new value to a read-only property.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "Color" as "String".
>> ERROR at line 2: Cannot assign a value of type "Color" as "String".
>> ERROR at line 2: Cannot assign a value of type Color to variable "var_color" with specified type String.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot infer the type of "_a" variable because the value doesn't have a set type.
>> ERROR at line 4: Cannot infer the type of "_a" variable because the value doesn't have a set type.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Invalid operands to operator <<, float and int.
>> ERROR at line 3: Invalid operands to operator <<, float and int.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Invalid operands to operator >>, int and float.
>> ERROR at line 3: Invalid operands to operator >>, int and float.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Static function "not_existing_method()" not found in base "MyClass".
>> ERROR at line 7: Static function "not_existing_method()" not found in base "MyClass".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Invalid cast. Cannot convert from "int" to "Array".
>> ERROR at line 3: Invalid cast. Cannot convert from "int" to "Array".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Invalid cast. Cannot convert from "int" to "Node".
>> ERROR at line 3: Invalid cast. Cannot convert from "int" to "Node".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Invalid cast. Cannot convert from "RefCounted" to "int".
>> ERROR at line 3: Invalid cast. Cannot convert from "RefCounted" to "int".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Class "Vector2" hides a built-in type.
>> ERROR at line 1: Class "Vector2" hides a built-in type.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a constant.
>> ERROR at line 5: Cannot assign a new value to a constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot assign a new value to a constant.
>> ERROR at line 5: Cannot assign a new value to a constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
The member "Vector2" cannot have the same name as a builtin type.
>> ERROR at line 1: The member "Vector2" cannot have the same name as a builtin type.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Expression is of type "int" so it can't be of type "String".
>> ERROR at line 5: Expression is of type "int" so it can't be of type "String".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Member "CONSTANT" is not a function.
>> ERROR at line 5: Member "CONSTANT" is not a function.
>> ERROR at line 5: Name "CONSTANT" called as a function but is a "int".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Expression is of type "B" so it can't be of type "C".
>> ERROR at line 10: Expression is of type "B" so it can't be of type "C".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cyclic inheritance.
>> ERROR at line 4: Cyclic inheritance.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "c1": Cyclic reference.
>> ERROR at line 5: Could not resolve member "c1": Cyclic reference.
>> ERROR at line 5: Could not resolve type for constant "c2".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "E1": Cyclic reference.
>> ERROR at line 5: Could not resolve member "E1": Cyclic reference.
>> ERROR at line 5: Enum values must be constant.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "EV1": Cyclic reference.
>> ERROR at line 5: Could not resolve member "EV1": Cyclic reference.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve external class member "v".
>> ERROR at line 4: Could not resolve external class member "v".
>> ERROR at line 4: Cannot find member "v" in base "/home/danil/godot-src/modules/gdscript/tests/scripts/analyzer/errors/cyclic_ref_external_a.notest.gd".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "f1": Cyclic reference.
>> ERROR at line 8: Could not resolve member "f1": Cyclic reference.
>> ERROR at line 8: Cannot infer the type of "p" parameter because the value doesn't have a set type.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "f": Cyclic reference.
>> ERROR at line 11: Could not resolve member "f": Cyclic reference.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "v1": Cyclic reference.
>> ERROR at line 5: Could not resolve member "v1": Cyclic reference.
>> ERROR at line 5: Cannot infer the type of "v2" variable because the value doesn't have a set type.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Could not resolve member "v1": Cyclic reference.
>> ERROR at line 1: Could not resolve member "v1": Cyclic reference.
>> ERROR at line 1: Could not resolve type for variable "v1".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Key "a" was already used in this dictionary (at line 3).
>> ERROR at line 5: Key "a" was already used in this dictionary (at line 3).
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Key "a" was already used in this dictionary (at line 3).
>> ERROR at line 5: Key "a" was already used in this dictionary (at line 3).
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Key "a" was already used in this dictionary (at line 3).
>> ERROR at line 5: Key "a" was already used in this dictionary (at line 3).
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Key "key" was already used in this dictionary (at line 5).
>> ERROR at line 6: Key "key" was already used in this dictionary (at line 5).
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot construct native class "Time" because it is an engine singleton.
>> ERROR at line 2: Cannot construct native class "Time" because it is an engine singleton.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot call non-const Dictionary function "clear()" on enum "Enum".
>> ERROR at line 4: Cannot call non-const Dictionary function "clear()" on enum "Enum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot find member "V3" in base "enum_bad_value.gd.Enum".
>> ERROR at line 4: Cannot find member "V3" in base "enum_bad_value.gd.Enum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "enum_class_var_assign_with_wrong_enum_type.gd.MyOtherEnum" as "enum_class_var_assign_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 9: Cannot assign a value of type "enum_class_var_assign_with_wrong_enum_type.gd.MyOtherEnum" as "enum_class_var_assign_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 9: Value of type "enum_class_var_assign_with_wrong_enum_type.gd.MyOtherEnum" cannot be assigned to a variable of type "enum_class_var_assign_with_wrong_enum_type.gd.MyEnum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "enum_class_var_init_with_wrong_enum_type.gd.MyOtherEnum" as "enum_class_var_init_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 5: Cannot assign a value of type "enum_class_var_init_with_wrong_enum_type.gd.MyOtherEnum" as "enum_class_var_init_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 5: Cannot assign a value of type enum_class_var_init_with_wrong_enum_type.gd.MyOtherEnum to variable "class_var" with specified type enum_class_var_init_with_wrong_enum_type.gd.MyEnum.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Cannot call non-const Dictionary function "clear()" on enum "Enum".
>> ERROR at line 5: Cannot call non-const Dictionary function "clear()" on enum "Enum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
Enum values must be integers.
>> ERROR at line 3: Enum values must be integers.
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot pass a value of type "enum_function_parameter_wrong_type.gd.MyOtherEnum" as "enum_function_parameter_wrong_type.gd.MyEnum".
>> ERROR at line 8: Cannot pass a value of type "enum_function_parameter_wrong_type.gd.MyOtherEnum" as "enum_function_parameter_wrong_type.gd.MyEnum".
>> ERROR at line 8: Invalid argument for "enum_func()" function: argument 1 should be "enum_function_parameter_wrong_type.gd.MyEnum" but is "enum_function_parameter_wrong_type.gd.MyOtherEnum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot return a value of type "enum_function_return_wrong_type.gd.MyOtherEnum" as "enum_function_return_wrong_type.gd.MyEnum".
>> ERROR at line 5: Cannot return a value of type "enum_function_return_wrong_type.gd.MyOtherEnum" as "enum_function_return_wrong_type.gd.MyEnum".
>> ERROR at line 5: Cannot return value of type "enum_function_return_wrong_type.gd.MyOtherEnum" because the function return type is "enum_function_return_wrong_type.gd.MyEnum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "enum_local_var_assign_outer_with_wrong_enum_type.gd::InnerClass.MyEnum" as "enum_local_var_assign_outer_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 9: Cannot assign a value of type "enum_local_var_assign_outer_with_wrong_enum_type.gd::InnerClass.MyEnum" as "enum_local_var_assign_outer_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 9: Value of type "enum_local_var_assign_outer_with_wrong_enum_type.gd::InnerClass.MyEnum" cannot be assigned to a variable of type "enum_local_var_assign_outer_with_wrong_enum_type.gd.MyEnum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "enum_local_var_assign_with_wrong_enum_type.gd.MyOtherEnum" as "enum_local_var_assign_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 7: Cannot assign a value of type "enum_local_var_assign_with_wrong_enum_type.gd.MyOtherEnum" as "enum_local_var_assign_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 7: Value of type "enum_local_var_assign_with_wrong_enum_type.gd.MyOtherEnum" cannot be assigned to a variable of type "enum_local_var_assign_with_wrong_enum_type.gd.MyEnum".
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GDTEST_ANALYZER_ERROR
Cannot assign a value of type "enum_local_var_init_with_wrong_enum_type.gd.MyOtherEnum" as "enum_local_var_init_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 5: Cannot assign a value of type "enum_local_var_init_with_wrong_enum_type.gd.MyOtherEnum" as "enum_local_var_init_with_wrong_enum_type.gd.MyEnum".
>> ERROR at line 5: Cannot assign a value of type enum_local_var_init_with_wrong_enum_type.gd.MyOtherEnum to variable "local_var" with specified type enum_local_var_init_with_wrong_enum_type.gd.MyEnum.
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
GDTEST_ANALYZER_ERROR
The member "Vector2" cannot have the same name as a builtin type.
>> ERROR at line 1: The member "Vector2" cannot have the same name as a builtin type.
Loading

0 comments on commit 1d7e920

Please sign in to comment.