From 539c265006da95b6e384a82d951dcdbc9151edb2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 12:24:03 +0300 Subject: [PATCH 01/12] fix tablegc flaky test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanager/tablegc/tablegc_test.go | 75 ++++++++----------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index bb286775be2..8ff060f6a36 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -16,6 +16,7 @@ limitations under the License. package tablegc import ( + "context" "flag" "os" "testing" @@ -160,7 +161,27 @@ func tableExists(tableExpr string) (exists bool, tableName string, err error) { return true, row.AsString("Name", ""), nil } -// tableExists sees that a given table exists in MySQL +func validateTableDoesNotExist(t *testing.T, tableExpr string) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + ticker := time.NewTicker(time.Second) + for { + select { + case <-ticker.C: + exists, _, err := tableExists(tableExpr) + require.NoError(t, err) + if !exists { + return + } + case <-ctx.Done(): + assert.NoError(t, ctx.Err(), "validateTableDoesNotExist timed out, table %v still exists", tableExpr) + return + } + } +} + +// dropTable drops a table func dropTable(tableName string) (err error) { query := `drop table if exists %a` parsed := sqlparser.BuildParsedQuery(query, tableName) @@ -175,11 +196,7 @@ func TestPopulateTable(t *testing.T) { assert.NoError(t, err) assert.True(t, exists) } - { - exists, _, err := tableExists("no_such_table") - assert.NoError(t, err) - assert.False(t, exists) - } + validateTableDoesNotExist(t, "no_such_table") } func TestHold(t *testing.T) { @@ -190,11 +207,7 @@ func TestHold(t *testing.T) { _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) assert.NoError(t, err) - { - exists, _, err := tableExists("t1") - assert.NoError(t, err) - assert.False(t, exists) - } + validateTableDoesNotExist(t, "t1") { exists, _, err := tableExists(tableName) assert.NoError(t, err) @@ -214,9 +227,7 @@ func TestHold(t *testing.T) { time.Sleep(10 * time.Second) { // We're now both beyond table's timestamp as well as a tableGC interval - exists, _, err := tableExists(tableName) - assert.NoError(t, err) - assert.False(t, exists) + validateTableDoesNotExist(t, tableName) } { // Table should be renamed as _vt_PURGE_... @@ -236,11 +247,7 @@ func TestEvac(t *testing.T) { _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) assert.NoError(t, err) - { - exists, _, err := tableExists("t1") - assert.NoError(t, err) - assert.False(t, exists) - } + validateTableDoesNotExist(t, "t1") { exists, _, err := tableExists(tableName) assert.NoError(t, err) @@ -258,19 +265,11 @@ func TestEvac(t *testing.T) { } time.Sleep(10 * time.Second) - { - // We're now both beyond table's timestamp as well as a tableGC interval - exists, _, err := tableExists(tableName) - assert.NoError(t, err) - assert.False(t, exists) - } + // We're now both beyond table's timestamp as well as a tableGC interval + validateTableDoesNotExist(t, tableName) time.Sleep(5 * time.Second) - { - // Table should be renamed as _vt_DROP_... and then dropped! - exists, _, err := tableExists(`\_vt\_DROP\_%`) - assert.NoError(t, err) - assert.False(t, exists) - } + // Table should be renamed as _vt_DROP_... and then dropped! + validateTableDoesNotExist(t, `\_vt\_DROP\_%`) } func TestDrop(t *testing.T) { @@ -281,19 +280,11 @@ func TestDrop(t *testing.T) { _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) assert.NoError(t, err) - { - exists, _, err := tableExists("t1") - assert.NoError(t, err) - assert.False(t, exists) - } + validateTableDoesNotExist(t, "t1") time.Sleep(20 * time.Second) // 10s for timestamp to pass, then 10s for checkTables and drop of table - { - // We're now both beyond table's timestamp as well as a tableGC interval - exists, _, err := tableExists(tableName) - assert.NoError(t, err) - assert.False(t, exists) - } + // We're now both beyond table's timestamp as well as a tableGC interval + validateTableDoesNotExist(t, tableName) } func TestPurge(t *testing.T) { From d58999fc8258fe2a63ec904da136616fab0cdc04 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 14:29:37 +0300 Subject: [PATCH 02/12] Pseudo GC state: TableDroppedGCState Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schema/tablegc.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/schema/tablegc.go b/go/vt/schema/tablegc.go index 80dbb6b773a..27f6f0712e9 100644 --- a/go/vt/schema/tablegc.go +++ b/go/vt/schema/tablegc.go @@ -40,6 +40,8 @@ const ( EvacTableGCState TableGCState = "EVAC" // DropTableGCState is the state where the table is to be dropped. Probably ASAP DropTableGCState TableGCState = "DROP" + // TableDroppedGCState is a pseudo state; a hint that the table does not exists anymore + TableDroppedGCState TableGCState = "" ) var ( From 0e4cf35f3bde2be98f511a46b652af127982dd01 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 14:29:46 +0300 Subject: [PATCH 03/12] allow next transition and any transition that follows; validate rowcount where table exists, validate drop table Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanager/tablegc/tablegc_test.go | 90 ++++++++++--------- 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 8ff060f6a36..13aef5bdf32 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -181,12 +181,46 @@ func validateTableDoesNotExist(t *testing.T, tableExpr string) { } } +func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableGCState) (exists bool, dropFunc func()) { + for _, state := range states { + searchExpr := "" + switch state { + case schema.HoldTableGCState: + searchExpr = `\_vt\_HOLD\_%` + case schema.PurgeTableGCState: + searchExpr = `\_vt\_PURGE\_%` + case schema.EvacTableGCState: + searchExpr = `\_vt\_EVAC\_%` + case schema.DropTableGCState: + searchExpr = `\_vt\_DROP\_%` + case schema.TableDroppedGCState: + validateTableDoesNotExist(t, `\_vt\_%`) + return + default: + t.Log("Unknown state") + t.Fail() + } + exists, tableName, err := tableExists(searchExpr) + require.NoError(t, err) + + if exists { + if expectNumRows >= 0 { + checkTableRows(t, tableName, expectNumRows) + } + return exists, func() { + dropTable(t, tableName) + } + } + } + return false, nil +} + // dropTable drops a table -func dropTable(tableName string) (err error) { +func dropTable(t *testing.T, tableName string) { query := `drop table if exists %a` parsed := sqlparser.BuildParsedQuery(query, tableName) - _, err = primaryTablet.VttabletProcess.QueryTablet(parsed.Query, keyspaceName, true) - return err + _, err := primaryTablet.VttabletProcess.QueryTablet(parsed.Query, keyspaceName, true) + require.NoError(t, err) } func TestPopulateTable(t *testing.T) { @@ -229,13 +263,9 @@ func TestHold(t *testing.T) { // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) } - { - // Table should be renamed as _vt_PURGE_... - exists, purgeTableName, err := tableExists(`\_vt\_PURGE\_%`) - assert.NoError(t, err) - assert.True(t, exists) - err = dropTable(purgeTableName) - assert.NoError(t, err) + _, dropFunc := validateAnyState(t, -1, schema.PurgeTableGCState, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + if dropFunc != nil { + dropFunc() } } @@ -269,7 +299,10 @@ func TestEvac(t *testing.T) { validateTableDoesNotExist(t, tableName) time.Sleep(5 * time.Second) // Table should be renamed as _vt_DROP_... and then dropped! - validateTableDoesNotExist(t, `\_vt\_DROP\_%`) + _, dropFunc := validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) + if dropFunc != nil { + dropFunc() + } } func TestDrop(t *testing.T) { @@ -323,14 +356,9 @@ func TestPurge(t *testing.T) { require.NoError(t, err) require.False(t, exists) } - { - // Table should be renamed as _vt_EVAC_... - exists, evacTableName, err := tableExists(`\_vt\_EVAC\_%`) - require.NoError(t, err) - require.True(t, exists) - checkTableRows(t, evacTableName, 0) - err = dropTable(evacTableName) - require.NoError(t, err) + _, dropFunc := validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + if dropFunc != nil { + dropFunc() } } @@ -383,26 +411,8 @@ func TestPurgeView(t *testing.T) { require.NoError(t, err) require.True(t, exists) } - { - // View should be renamed as _vt_EVAC_ or _vt_DROP: views only spend a fraction of a second in "EVAC" - // because evacuation is irrelevant to views. They are immediately renamed to DROP. - // Because there might be a race condition, we allow both cases - evacTableExists, evacTableName, err := tableExists(`\_vt\_EVAC\_%`) - require.NoError(t, err) - - dropTableExists, dropTableName, err := tableExists(`\_vt\_DROP\_%`) - require.NoError(t, err) - - require.True(t, evacTableExists || dropTableExists) - switch { - case evacTableExists: - checkTableRows(t, evacTableName, 1024) // the renamed view still points to t1's data - err = dropTable(evacTableName) - require.NoError(t, err) - case dropTableExists: - checkTableRows(t, dropTableName, 1024) // the renamed view still points to t1's data - err = dropTable(dropTableName) - require.NoError(t, err) - } + _, dropFunc := validateAnyState(t, 1024, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) + if dropFunc != nil { + dropFunc() } } From 2b91f95c8890acdc9a11cb013f15176f8e712961 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 14:34:21 +0300 Subject: [PATCH 04/12] perform validation Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/tabletmanager/tablegc/tablegc_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 13aef5bdf32..9bcd40b3c73 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -182,6 +182,7 @@ func validateTableDoesNotExist(t *testing.T, tableExpr string) { } func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableGCState) (exists bool, dropFunc func()) { + stateMatched := false for _, state := range states { searchExpr := "" switch state { @@ -195,6 +196,7 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG searchExpr = `\_vt\_DROP\_%` case schema.TableDroppedGCState: validateTableDoesNotExist(t, `\_vt\_%`) + // No need for further validations return default: t.Log("Unknown state") @@ -202,6 +204,7 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG } exists, tableName, err := tableExists(searchExpr) require.NoError(t, err) + stateMatched = true if exists { if expectNumRows >= 0 { @@ -212,6 +215,7 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG } } } + assert.True(t, stateMatched, "could not match any of the states: %v", states) return false, nil } From 948f2cd56c0e2c6476d9df910ce379136fd804a5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 14:43:04 +0300 Subject: [PATCH 05/12] simplify Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanager/tablegc/tablegc_test.go | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 9bcd40b3c73..2af05668d5c 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -181,7 +181,7 @@ func validateTableDoesNotExist(t *testing.T, tableExpr string) { } } -func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableGCState) (exists bool, dropFunc func()) { +func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableGCState) { stateMatched := false for _, state := range states { searchExpr := "" @@ -210,13 +210,12 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG if expectNumRows >= 0 { checkTableRows(t, tableName, expectNumRows) } - return exists, func() { - dropTable(t, tableName) - } + // Now that the table is validated, we can drop it + dropTable(t, tableName) + return } } assert.True(t, stateMatched, "could not match any of the states: %v", states) - return false, nil } // dropTable drops a table @@ -267,10 +266,7 @@ func TestHold(t *testing.T) { // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) } - _, dropFunc := validateAnyState(t, -1, schema.PurgeTableGCState, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) - if dropFunc != nil { - dropFunc() - } + validateAnyState(t, -1, schema.PurgeTableGCState, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } func TestEvac(t *testing.T) { @@ -303,10 +299,7 @@ func TestEvac(t *testing.T) { validateTableDoesNotExist(t, tableName) time.Sleep(5 * time.Second) // Table should be renamed as _vt_DROP_... and then dropped! - _, dropFunc := validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) - if dropFunc != nil { - dropFunc() - } + validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) } func TestDrop(t *testing.T) { @@ -360,10 +353,7 @@ func TestPurge(t *testing.T) { require.NoError(t, err) require.False(t, exists) } - _, dropFunc := validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) - if dropFunc != nil { - dropFunc() - } + validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } func TestPurgeView(t *testing.T) { @@ -415,8 +405,5 @@ func TestPurgeView(t *testing.T) { require.NoError(t, err) require.True(t, exists) } - _, dropFunc := validateAnyState(t, 1024, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) - if dropFunc != nil { - dropFunc() - } + validateAnyState(t, 1024, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } From f5f68a701bfab569606c62633ca9aac9bb81bcb2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 15:19:47 +0300 Subject: [PATCH 06/12] remove redundant check Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/tabletmanager/tablegc/tablegc_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 2af05668d5c..4df0bb0fe54 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -347,12 +347,6 @@ func TestPurge(t *testing.T) { } time.Sleep(15 * time.Second) // purgeReentranceInterval - { - // We're now both beyond table's timestamp as well as a tableGC interval - exists, _, err := tableExists(tableName) - require.NoError(t, err) - require.False(t, exists) - } validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } From fde461d9ed565b48c37f98c20d1083de49f5c48c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 17:28:46 +0300 Subject: [PATCH 07/12] more debug info Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/tabletmanager/tablegc/tablegc_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 4df0bb0fe54..a75eea6ebfa 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -166,16 +166,19 @@ func validateTableDoesNotExist(t *testing.T, tableExpr string) { defer cancel() ticker := time.NewTicker(time.Second) + var foundTableName string + var exists bool + var err error for { select { case <-ticker.C: - exists, _, err := tableExists(tableExpr) + exists, foundTableName, err = tableExists(tableExpr) require.NoError(t, err) if !exists { return } case <-ctx.Done(): - assert.NoError(t, ctx.Err(), "validateTableDoesNotExist timed out, table %v still exists", tableExpr) + assert.NoError(t, ctx.Err(), "validateTableDoesNotExist timed out, table %v still exists (%v)", tableExpr, foundTableName) return } } From 136a8cf38366f714e8bea931ac220e2fe021d493 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 19:24:27 +0300 Subject: [PATCH 08/12] reduce intervals Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/tabletmanager/tablegc/tablegc_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index a75eea6ebfa..431ce7942da 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -91,8 +91,8 @@ func TestMain(m *testing.M) { "--enable_replication_reporter", "--heartbeat_enable", "--heartbeat_interval", "250ms", - "--gc_check_interval", "5s", - "--gc_purge_check_interval", "5s", + "--gc_check_interval", "2s", + "--gc_purge_check_interval", "2s", "--table_gc_lifecycle", "hold,purge,evac,drop", } // We do not need semiSync for this test case. From ad7c7586423f28c41f6dc5089904c0246de8a7e0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 8 May 2022 19:31:05 +0300 Subject: [PATCH 09/12] do not waste time waiting Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../endtoend/tabletmanager/tablegc/tablegc_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index 431ce7942da..ac4a0323a69 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -185,8 +185,8 @@ func validateTableDoesNotExist(t *testing.T, tableExpr string) { } func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableGCState) { - stateMatched := false for _, state := range states { + expectTableToExist := true searchExpr := "" switch state { case schema.HoldTableGCState: @@ -198,16 +198,14 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG case schema.DropTableGCState: searchExpr = `\_vt\_DROP\_%` case schema.TableDroppedGCState: - validateTableDoesNotExist(t, `\_vt\_%`) - // No need for further validations - return + searchExpr = `\_vt\_%` + expectTableToExist = false default: t.Log("Unknown state") t.Fail() } exists, tableName, err := tableExists(searchExpr) require.NoError(t, err) - stateMatched = true if exists { if expectNumRows >= 0 { @@ -215,10 +213,13 @@ func validateAnyState(t *testing.T, expectNumRows int64, states ...schema.TableG } // Now that the table is validated, we can drop it dropTable(t, tableName) + } + if exists == expectTableToExist { + // condition met return } } - assert.True(t, stateMatched, "could not match any of the states: %v", states) + assert.Fail(t, "could not match any of the states: %v", states) } // dropTable drops a table From cc05440623cef637fc9a1bb5f0d6533a977c30a6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 9 May 2022 08:44:47 +0300 Subject: [PATCH 10/12] parameterize all time values Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletmanager/tablegc/tablegc_test.go | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index ac4a0323a69..afc4e76b485 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -68,6 +68,10 @@ var ( } } }` + + tableTransitionExpiration = 10 * time.Second + gcCheckInterval = 2 * time.Second + gcPurgeCheckInterval = 2 * time.Second ) func TestMain(m *testing.M) { @@ -91,8 +95,8 @@ func TestMain(m *testing.M) { "--enable_replication_reporter", "--heartbeat_enable", "--heartbeat_interval", "250ms", - "--gc_check_interval", "2s", - "--gc_purge_check_interval", "2s", + "--gc_check_interval", gcCheckInterval.String(), + "--gc_purge_check_interval", gcPurgeCheckInterval.String(), "--table_gc_lifecycle", "hold,purge,evac,drop", } // We do not need semiSync for this test case. @@ -242,7 +246,7 @@ func TestPopulateTable(t *testing.T) { func TestHold(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.HoldTableGCState, time.Now().UTC().Add(10*time.Second)) + query, tableName, err := schema.GenerateRenameStatement("t1", schema.HoldTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) assert.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) @@ -255,7 +259,7 @@ func TestHold(t *testing.T) { assert.True(t, exists) } - time.Sleep(5 * time.Second) + time.Sleep(tableTransitionExpiration / 2) { // Table was created with +10s timestamp, so it should still exist exists, _, err := tableExists(tableName) @@ -265,7 +269,7 @@ func TestHold(t *testing.T) { checkTableRows(t, tableName, 1024) } - time.Sleep(10 * time.Second) + time.Sleep(tableTransitionExpiration) { // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) @@ -275,7 +279,7 @@ func TestHold(t *testing.T) { func TestEvac(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.EvacTableGCState, time.Now().UTC().Add(10*time.Second)) + query, tableName, err := schema.GenerateRenameStatement("t1", schema.EvacTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) assert.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) @@ -288,7 +292,7 @@ func TestEvac(t *testing.T) { assert.True(t, exists) } - time.Sleep(5 * time.Second) + time.Sleep(tableTransitionExpiration / 2) { // Table was created with +10s timestamp, so it should still exist exists, _, err := tableExists(tableName) @@ -298,17 +302,16 @@ func TestEvac(t *testing.T) { checkTableRows(t, tableName, 1024) } - time.Sleep(10 * time.Second) + time.Sleep(tableTransitionExpiration) // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) - time.Sleep(5 * time.Second) // Table should be renamed as _vt_DROP_... and then dropped! validateAnyState(t, 0, schema.DropTableGCState, schema.TableDroppedGCState) } func TestDrop(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.DropTableGCState, time.Now().UTC().Add(10*time.Second)) + query, tableName, err := schema.GenerateRenameStatement("t1", schema.DropTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) assert.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) @@ -316,14 +319,15 @@ func TestDrop(t *testing.T) { validateTableDoesNotExist(t, "t1") - time.Sleep(20 * time.Second) // 10s for timestamp to pass, then 10s for checkTables and drop of table + time.Sleep(tableTransitionExpiration) + time.Sleep(2 * gcCheckInterval) // We're now both beyond table's timestamp as well as a tableGC interval validateTableDoesNotExist(t, tableName) } func TestPurge(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("t1", schema.PurgeTableGCState, time.Now().UTC().Add(10*time.Second)) + query, tableName, err := schema.GenerateRenameStatement("t1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) require.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) @@ -340,7 +344,7 @@ func TestPurge(t *testing.T) { require.True(t, exists) } - time.Sleep(5 * time.Second) + time.Sleep(tableTransitionExpiration / 2) { // Table was created with +10s timestamp, so it should still exist exists, _, err := tableExists(tableName) @@ -350,13 +354,14 @@ func TestPurge(t *testing.T) { checkTableRows(t, tableName, 1024) } - time.Sleep(15 * time.Second) // purgeReentranceInterval + time.Sleep(2 * gcPurgeCheckInterval) // wwait for table to be purged + time.Sleep(2 * gcCheckInterval) // wait for GC state transition validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } func TestPurgeView(t *testing.T) { populateTable(t) - query, tableName, err := schema.GenerateRenameStatement("v1", schema.PurgeTableGCState, time.Now().UTC().Add(10*time.Second)) + query, tableName, err := schema.GenerateRenameStatement("v1", schema.PurgeTableGCState, time.Now().UTC().Add(tableTransitionExpiration)) require.NoError(t, err) _, err = primaryTablet.VttabletProcess.QueryTablet(query, keyspaceName, true) @@ -379,7 +384,7 @@ func TestPurgeView(t *testing.T) { require.True(t, exists) } - time.Sleep(5 * time.Second) + time.Sleep(tableTransitionExpiration / 2) { // View was created with +10s timestamp, so it should still exist exists, _, err := tableExists(tableName) @@ -390,7 +395,8 @@ func TestPurgeView(t *testing.T) { checkTableRows(t, tableName, 1024) } - time.Sleep(15 * time.Second) // purgeReentranceInterval + time.Sleep(2 * gcPurgeCheckInterval) // wwait for table to be purged + time.Sleep(2 * gcCheckInterval) // wait for GC state transition { // We're now both beyond view's timestamp as well as a tableGC interval exists, _, err := tableExists(tableName) From 09feae2870accc567bc61da1f985713ea5f5c5a2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 9 May 2022 09:45:53 +0300 Subject: [PATCH 11/12] give the table more chance to get purged Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/tabletmanager/tablegc/tablegc_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go index afc4e76b485..c46e242c773 100644 --- a/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go +++ b/go/test/endtoend/tabletmanager/tablegc/tablegc_test.go @@ -72,6 +72,7 @@ var ( tableTransitionExpiration = 10 * time.Second gcCheckInterval = 2 * time.Second gcPurgeCheckInterval = 2 * time.Second + waitForTransitionTimeout = 30 * time.Second ) func TestMain(m *testing.M) { @@ -166,7 +167,7 @@ func tableExists(tableExpr string) (exists bool, tableName string, err error) { } func validateTableDoesNotExist(t *testing.T, tableExpr string) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), waitForTransitionTimeout) defer cancel() ticker := time.NewTicker(time.Second) @@ -354,7 +355,7 @@ func TestPurge(t *testing.T) { checkTableRows(t, tableName, 1024) } - time.Sleep(2 * gcPurgeCheckInterval) // wwait for table to be purged + time.Sleep(5 * gcPurgeCheckInterval) // wwait for table to be purged time.Sleep(2 * gcCheckInterval) // wait for GC state transition validateAnyState(t, 0, schema.EvacTableGCState, schema.DropTableGCState, schema.TableDroppedGCState) } From ea62fde754f083c138b60a8ef286667761f6f00f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 9 May 2022 15:17:49 +0300 Subject: [PATCH 12/12] flakiness: remove --heartbeat_on_demand_duration Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/onlineddl/ghost/onlineddl_ghost_test.go | 1 - go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/go/test/endtoend/onlineddl/ghost/onlineddl_ghost_test.go b/go/test/endtoend/onlineddl/ghost/onlineddl_ghost_test.go index 3a895f7d6e9..734dff23c6a 100644 --- a/go/test/endtoend/onlineddl/ghost/onlineddl_ghost_test.go +++ b/go/test/endtoend/onlineddl/ghost/onlineddl_ghost_test.go @@ -167,7 +167,6 @@ func TestMain(m *testing.M) { "--throttle_threshold", "1s", "--heartbeat_enable", "--heartbeat_interval", "250ms", - "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "5s", "--gh-ost-path", os.Getenv("VITESS_ENDTOEND_GH_OST_PATH"), // leave env variable empty/unset to get the default behavior. Override in Mac. } diff --git a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go index ccee23d4aa3..fe2c06cad73 100644 --- a/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go +++ b/go/test/endtoend/onlineddl/vrepl/onlineddl_vrepl_test.go @@ -171,7 +171,6 @@ func TestMain(m *testing.M) { "--throttle_threshold", "1s", "--heartbeat_enable", "--heartbeat_interval", "250ms", - "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "5s", } clusterInstance.VtGateExtraArgs = []string{