Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release-23.2: roachtest: add flaky django/asyncpg/npgsql tests to ignore list #120573

Merged
merged 3 commits into from
Mar 16, 2024
+28 −74
Merged

release-23.2: roachtest: add flaky django/asyncpg/npgsql tests to ignore list #120573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
02dfabc
roachtest: mark a django test as flaky
rafiss Mar 15, 2024
a0bc794
roachtest: add flaky tests to asyncpg ignorelist
rafiss Mar 15, 2024
a97dcc9
roachtest: add flaky tests to npgsql blocklist
rafiss Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion pkg/cmd/roachtest/tests/asyncpg_blocklist.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,4 +86,15 @@ var asyncpgBlocklist = blocklist{
`test_utils.TestUtils.test_mogrify_simple`: "multi-dim arrays - https://github.com/cockroachdb/cockroach/issues/32552",
}

var asyncpgIgnoreList = blocklist{}
var asyncpgIgnoreList = blocklist{
`test_copy.TestCopyFrom.test_copy_from_table_large_rows`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_records_to_table_1`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_records_to_table_no_binary_codec`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_basics`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_fail_in_source_1`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_fail_in_source_2`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_from_bytes_like`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_from_file_path`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_large_rows`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
`test_copy.TestCopyTo.test_copy_to_table_timeout`: "flaky; see #119291 and https://github.com/MagicStack/asyncpg/issues/240",
}
3 changes: 1 addition & 2 deletions pkg/cmd/roachtest/tests/django_blocklist.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,5 @@ var enabledDjangoTests = []string{
var djangoBlocklist = blocklist{}

var djangoIgnoreList = blocklist{
"schema.tests.SchemaTests.test_add_auto_field": "unneeded once django-cockroachdb 4.1.1 is released",
"schema.tests.SchemaTests.test_autofield_to_o2o": "unneeded once django-cockroachdb 4.1.1 is released",
`select_for_update.tests.SelectForUpdateTests.test_nowait_raises_error_on_block`: "flaky; see #120196",
}
16 changes: 14 additions & 2 deletions pkg/cmd/roachtest/tests/dotnet_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,16 +81,28 @@ func extractFailureFromTRX(contents []byte) ([]string, []status, map[string]stri
idToFullName[testDef.TestID] = fmt.Sprintf("%s.%s", testDef.TestMethod.ClassName, testDef.TestMethod.Name)
}

npgsqlFlakeErrors := []string{
"Received backend message ReadyForQuery while expecting",
"Got idle connector but State is Copy",
}
isAnyFlakeError := func(s string) bool {
for _, e := range npgsqlFlakeErrors {
if strings.Contains(s, e) {
return true
}
}
return false
}

// Check each result.
for _, testCase := range testRun.Results.UnitTestResults {
const npgsqlFlakeError = "Received backend message ReadyForQuery while expecting"
testName := idToFullName[testCase.TestID]
tests = append(tests, testName)
if testCase.Outcome == "Skipped" || testCase.Outcome == "NotExecuted" {
testStatuses = append(testStatuses, statusSkip)
} else if testCase.Outcome == "Passed" {
testStatuses = append(testStatuses, statusPass)
} else if strings.Contains(testCase.Output.ErrorInfo.Message, npgsqlFlakeError) {
} else if isAnyFlakeError(testCase.Output.ErrorInfo.Message) {
// npgsql tests frequently flake with this error. Until we resolve this
// specific error, we will ignore all tests that failed for that reason.
// See https://github.com/cockroachdb/cockroach/issues/108414.
Expand Down
70 changes: 1 addition & 69 deletions pkg/cmd/roachtest/tests/npgsql_blocklist.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,72 +713,4 @@ var npgsqlBlocklist = blocklist{
`Npgsql.Tests.Types.TextTests(NonMultiplexing).Null_character`: "unknown",
}

var npgsqlIgnoreList = blocklist{
`Npgsql.Tests.CommandTests(Multiplexing).Command_Dispose_does_not_close_reader`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Cursor_move_RecordsAffected `: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Multiple_statements_with_parameters(NotPrepared)`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).QueryNonQuery`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Same_command_different_param_instances`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Same_command_different_param_values`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).SingleNonQuery`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).SingleQuery`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Statement_mapped_output_parameters(Default)`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).TableDirect`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Unreferenced_positional_parameter_works`: "flaky",
`Npgsql.Tests.CommandTests(Multiplexing).Use_across_connection_change(NotPrepared)`: "flaky",
`Npgsql.Tests.CommandTests(NonMultiplexing).Cached_command_clears_parameters_placeholder_type`: "flaky",
`Npgsql.Tests.CommandTests(NonMultiplexing).CloseConnection_with_exception`: "flaky",
`Npgsql.Tests.CommandTests(NonMultiplexing).Cursor_move_RecordsAffected `: "flaky",
`Npgsql.Tests.CommandTests(NonMultiplexing).Statement_mapped_output_parameters(SequentialAccess)`: "flaky",
`Npgsql.Tests.CommandTests(NonMultiplexing).Use_across_connection_change(Prepared)`: "flaky",
`Npgsql.Tests.ConnectionTests(Multiplexing).Fail_connect_then_succeed(True)`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).PostgreSqlVersion_ServerVersion`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).Connector_not_initialized_exception`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).Many_open_close_with_transaction`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).ManyOpenClose`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).PhysicalConnectionInitializer_async`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).PhysicalConnectionInitializer_sync`: "flaky",
`Npgsql.Tests.ConnectionTests(NonMultiplexing).Timezone_connection_param`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Cancel_binary_import`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Cancel_raw_binary_import`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Cancel_text_export_when_not_consumed_and_then_Dispose`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Cancel_text_import`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_bytea`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_commit_in_middle_of_row`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_direct_buffer`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_exception_does_not_commit`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_numeric`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_string_array`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Import_string_with_buffer_length`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Prepended_messages`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Text_import(False)`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Text_import(True)`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Text_import_empty`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Write_column_out_of_bounds_throws`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Write_null_values`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Wrong_table_definition_binary_export`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Wrong_table_definition_binary_import`: "flaky",
`Npgsql.Tests.CopyTests(Multiplexing).Wrong_table_definition_raw_binary_copy`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Cancel_text_export_when_not_consumed_and_then_Dispose`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Import_numeric`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Import_string_array`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Import_string_with_buffer_length`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Prepended_messages`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Text_import_empty`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Undefined_table_throws`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Write_column_out_of_bounds_throws`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Write_null_values`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Wrong_format_binary_export`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Wrong_format_binary_import`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Wrong_format_raw_binary_copy`: "flaky",
`Npgsql.Tests.CopyTests(NonMultiplexing).Wrong_format_text_import`: "flaky",
`Npgsql.Tests.NotificationTests.WaitAsync_with_timeout`: "flaky",
`Npgsql.Tests.NotificationTests.Wait_with_timeout`: "flaky",
`Npgsql.Tests.ReaderTests(Multiplexing,Default).Cleans_up_ok_with_dispose_calls(NotPrepared)`: "flaky",
`Npgsql.Tests.TaskTimeoutAndCancellationTest.DelayedFaultedTaskCancellation("CancelOnly")`: "flaky",
`Npgsql.Tests.TaskTimeoutAndCancellationTest.DelayedFaultedTaskCancellation("TimeoutOnly")`: "flaky",
`Npgsql.Tests.TransactionTests(Multiplexing).Failed_transaction_on_close_with_custom_timeout`: "flaky",
`Npgsql.Tests.TransactionTests(NonMultiplexing).CommitAsync(Prepared)`: "flaky",
`Npgsql.Tests.TransactionTests(NonMultiplexing).Rollback(Prepared)`: "flaky",
`Npgsql.Tests.TransactionTests(NonMultiplexing).RollbackAsync(NotPrepared)`: "flaky",
}
var npgsqlIgnoreList = blocklist{}