From 59ed5270e992eb0e78e1dd625397017369c7ea5b Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Wed, 2 Aug 2023 11:15:05 +0530 Subject: [PATCH 1/3] feat(spanner): add integration tests for Bit Reversed Sequences --- spanner/integration_test.go | 293 +++++++++++++++++++++++++++++++++--- 1 file changed, 275 insertions(+), 18 deletions(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index f8cae0c1e281..236fdef6c77f 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -29,6 +29,7 @@ import ( "os/exec" "reflect" "regexp" + "strconv" "strings" "sync" "testing" @@ -57,12 +58,13 @@ const ( directPathIPV6Prefix = "[2001:4860:8040" directPathIPV4Prefix = "34.126" - singerDDLStatements = "SINGER_DDL_STATEMENTS" - simpleDDLStatements = "SIMPLE_DDL_STATEMENTS" - readDDLStatements = "READ_DDL_STATEMENTS" - backupDDLStatements = "BACKUP_DDL_STATEMENTS" - testTableDDLStatements = "TEST_TABLE_DDL_STATEMENTS" - fkdcDDLStatements = "FKDC_DDL_STATEMENTS" + singerDDLStatements = "SINGER_DDL_STATEMENTS" + simpleDDLStatements = "SIMPLE_DDL_STATEMENTS" + readDDLStatements = "READ_DDL_STATEMENTS" + backupDDLStatements = "BACKUP_DDL_STATEMENTS" + testTableDDLStatements = "TEST_TABLE_DDL_STATEMENTS" + fkdcDDLStatements = "FKDC_DDL_STATEMENTS" + testTableBitReversedSeqStatements = "TEST_TABLE_BIT_REVERSED_SEQUENCE_STATEMENTS" ) var ( @@ -288,22 +290,51 @@ var ( PRIMARY KEY (CartId))`, } + bitReverseSeqDBStatments = []string{ + `CREATE SEQUENCE seqT OPTIONS (sequence_kind = "bit_reversed_positive")`, + `CREATE TABLE T ( + id INT64 DEFAULT (GET_NEXT_SEQUENCE_VALUE('seqT')), + value INT64, + counter INT64 DEFAULT (GET_INTERNAL_SEQUENCE_STATE('seqT')), + br_id INT64 AS (BIT_REVERSE(id, true)) STORED, + CONSTRAINT id_gt_0 CHECK (id > 0), + CONSTRAINT counter_gt_br_id CHECK (counter >= br_id), + CONSTRAINT br_id_true CHECK (id = BIT_REVERSE(br_id, true)), + ) PRIMARY KEY (id)`, + } + + bitReverseSeqDBPGStatments = []string{ + `CREATE SEQUENCE seqT BIT_REVERSED_POSITIVE`, + `CREATE TABLE T ( + id BIGINT DEFAULT nextval('seqT'), + value BIGINT, + counter BIGINT DEFAULT spanner.get_internal_sequence_state('seqT'), + br_id bigint GENERATED ALWAYS AS (spanner.bit_reverse(id, true)) STORED, + CHECK (id > 0), + CHECK (counter >= br_id), + CHECK (id = spanner.bit_reverse(br_id, true)), + PRIMARY KEY (id) + )`, + } + statements = map[adminpb.DatabaseDialect]map[string][]string{ adminpb.DatabaseDialect_GOOGLE_STANDARD_SQL: { - singerDDLStatements: singerDBStatements, - simpleDDLStatements: simpleDBStatements, - readDDLStatements: readDBStatements, - backupDDLStatements: backupDBStatements, - testTableDDLStatements: readDBStatements, - fkdcDDLStatements: fkdcDBStatements, + singerDDLStatements: singerDBStatements, + simpleDDLStatements: simpleDBStatements, + readDDLStatements: readDBStatements, + backupDDLStatements: backupDBStatements, + testTableDDLStatements: readDBStatements, + fkdcDDLStatements: fkdcDBStatements, + testTableBitReversedSeqStatements: bitReverseSeqDBStatments, }, adminpb.DatabaseDialect_POSTGRESQL: { - singerDDLStatements: singerDBPGStatements, - simpleDDLStatements: simpleDBPGStatements, - readDDLStatements: readDBPGStatements, - backupDDLStatements: backupDBPGStatements, - testTableDDLStatements: readDBPGStatements, - fkdcDDLStatements: fkdcDBPGStatements, + singerDDLStatements: singerDBPGStatements, + simpleDDLStatements: simpleDBPGStatements, + readDDLStatements: readDBPGStatements, + backupDDLStatements: backupDBPGStatements, + testTableDDLStatements: readDBPGStatements, + fkdcDDLStatements: fkdcDBPGStatements, + testTableBitReversedSeqStatements: bitReverseSeqDBPGStatments, }, } @@ -4632,6 +4663,204 @@ func TestIntegration_GFE_Latency(t *testing.T) { DisableGfeLatencyAndHeaderMissingCountViews() } +func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { + skipEmulatorTest(t) + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + // create table with bit reverse seq options + client, dbPath, cleanup := prepareIntegrationTest(ctx, t, DefaultSessionPoolConfig, statements[testDialect][testTableBitReversedSeqStatements]) + defer cleanup() + + var tests = []struct { + name string + test func() error + wantErr error + }{ + { + name: "success: inserted rows should have auto generated keys", + test: func() error { + var ( + values [][]interface{} + err error + ) + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (100), (200), (300) THEN RETURN id, counter")) + values, err = readAllBitReversedSeqTable(iter, true) + return err + }) + if len(values) != 3 { + return errors.New("expected 3 rows to be inserted") + } + counter := int64(0) + for i := 0; i < 3; i++ { + newID, newCounter := values[i][0].(int64), values[i][1].(int64) + if newID <= 0 { + return errors.New("expected id1, id2, id3 > 0") + + } + if newCounter < counter { + return errors.New("expected c3 >= c2 >= c1") + } + counter = newCounter + } + iter := client.Single().Query(ctx, NewStatement("SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')")) + r, err := iter.Next() + if err != nil { + return err + } + var c3 int64 + err = r.Columns(&c3) + if err != nil { + return err + } + if c3 > counter { + return errors.New("expected c3 <= SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')") + } + return err + }, + }, + { + name: "success: reduce ranges to half", + test: func() error { + op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbPath, + Statements: []string{`ALTER SEQUENCE seqT SET OPTIONS ( + skip_range_min = 0, + skip_range_max = 4611686018427387904 + )`}, + }) + if err != nil { + t.Fatalf("cannot modify sequence %v: %v", dbPath, err) + } + if err := op.Wait(ctx); err != nil { + t.Fatalf("timeout modifying sequence %v: %v", dbPath, err) + } + var values [][]interface{} + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { + var v string + for i := 1; i <= 100; i++ { + v = v + " (" + strconv.Itoa(i) + ")" + if i != 100 { + v = v + ", " + } + } + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES "+v+" THEN RETURN id, counter")) + values, err = readAllBitReversedSeqTable(iter, true) + return err + }) + if err != nil { + return err + } + if len(values) != 100 { + return errors.New("expected 100 rows to be inserted") + } + counter := int64(0) + for i := 0; i < 100; i++ { + newID, newCounter := values[i][0].(int64), values[i][1].(int64) + if newID <= 0 || newID < 4611686018427387904 { + return errors.New("expected d1, id2, id3, …., id100 > 4611686018427387904") + + } + if newCounter < counter { + return errors.New("expected c3 >= c2 >= c1") + } + counter = newCounter + } + return err + }, + }, + { + name: "success: move start with counter forward", + test: func() error { + op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbPath, + Statements: []string{`ALTER SEQUENCE seqT SET OPTIONS (start_with_counter=10001)`}, + }) + if err != nil { + t.Fatalf("cannot modify sequence %v: %v", dbPath, err) + } + if err := op.Wait(ctx); err != nil { + t.Fatalf("timeout modifying sequence %v: %v", dbPath, err) + } + var values [][]interface{} + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (4), (5), (6) THEN RETURN id, br_id, counter")) + values, err = readAllBitReversedSeqTable(iter, false) + return err + }) + if err != nil { + return err + } + if len(values) != 3 { + return errors.New("expected 3 rows to be inserted") + } + counter := int64(0) + for i := 0; i < 3; i++ { + newID, newBrID, newCounter := values[i][0].(int64), values[i][1].(int64), values[i][2].(int64) + if newID <= 0 { + return errors.New("expected id > 0") + } + if newBrID < 10001 { + return errors.New("expected br_idi >= 10001") + } + if newCounter < 10001 { + return errors.New("expected ci >= 10001") + } + if counter < newCounter { + counter = newCounter + } + } + iter := client.Single().Query(ctx, NewStatement("SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')")) + r, err := iter.Next() + if err != nil { + return err + } + var c3 int64 + err = r.Columns(&c3) + if err != nil { + return err + } + if c3 > counter { + return errors.New("expected max(ci) <= SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')") + } + return err + }, + }, + { + name: "success: drop the sequences", + test: func() error { + op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbPath, + Statements: []string{ + "ALTER TABLE T ALTER COLUMN id DROP DEFAULT", + "ALTER TABLE T ALTER COLUMN counter DROP DEFAULT", + "ALTER TABLE T DROP CONSTRAINT id_gt_0", + "ALTER TABLE T DROP CONSTRAINT counter_gt_br_id", + "ALTER TABLE T DROP CONSTRAINT br_id_true", + "DROP SEQUENCE seqT", + }, + }) + if err != nil { + t.Fatalf("cannot delete sequences %v: %v", dbPath, err) + } + if err := op.Wait(ctx); err != nil { + t.Fatalf("timeout deleting sequence %v: %v", dbPath, err) + } + return nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := tt.test() + if gotErr != nil && strings.ToLower(gotErr.Error()) != strings.ToLower(tt.wantErr.Error()) { + t.Errorf("BIT REVERSED SEQUENECES error=%v, wantErr: %v", gotErr, tt.wantErr) + } + }) + } +} + // Prepare initializes Cloud Spanner testing DB and clients. func prepareIntegrationTest(ctx context.Context, t *testing.T, spc SessionPoolConfig, statements []string) (*Client, string, func()) { return prepareDBAndClient(ctx, t, spc, statements, testDialect) @@ -4976,6 +5205,34 @@ func readAllAccountsTable(iter *RowIterator) ([][]interface{}, error) { } } +func readAllBitReversedSeqTable(iter *RowIterator, onlyIDCounter bool) ([][]interface{}, error) { + defer iter.Stop() + var vals [][]interface{} + for { + row, err := iter.Next() + if err == iterator.Done { + return vals, nil + } + if err != nil { + return nil, err + } + var id, brID, counter int64 + if onlyIDCounter { + err = row.Columns(&id, &counter) + if err != nil { + return nil, err + } + vals = append(vals, []interface{}{id, counter}) + continue + } + err = row.Columns(&id, &brID, &counter) + if err != nil { + return nil, err + } + vals = append(vals, []interface{}{id, brID, counter}) + } +} + func maxDuration(a, b time.Duration) time.Duration { if a > b { return a From 93685bde7f5c2685b6d35787ecc4d5f8037f29c4 Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Thu, 3 Aug 2023 15:03:12 +0530 Subject: [PATCH 2/3] fix tests --- spanner/integration_test.go | 63 +++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index 236fdef6c77f..6b838ce0c8ed 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -293,9 +293,9 @@ var ( bitReverseSeqDBStatments = []string{ `CREATE SEQUENCE seqT OPTIONS (sequence_kind = "bit_reversed_positive")`, `CREATE TABLE T ( - id INT64 DEFAULT (GET_NEXT_SEQUENCE_VALUE('seqT')), + id INT64 DEFAULT (GET_NEXT_SEQUENCE_VALUE(Sequence seqT)), value INT64, - counter INT64 DEFAULT (GET_INTERNAL_SEQUENCE_STATE('seqT')), + counter INT64 DEFAULT (GET_INTERNAL_SEQUENCE_STATE(Sequence seqT)), br_id INT64 AS (BIT_REVERSE(id, true)) STORED, CONSTRAINT id_gt_0 CHECK (id > 0), CONSTRAINT counter_gt_br_id CHECK (counter >= br_id), @@ -310,9 +310,9 @@ var ( value BIGINT, counter BIGINT DEFAULT spanner.get_internal_sequence_state('seqT'), br_id bigint GENERATED ALWAYS AS (spanner.bit_reverse(id, true)) STORED, - CHECK (id > 0), - CHECK (counter >= br_id), - CHECK (id = spanner.bit_reverse(br_id, true)), + CONSTRAINT id_gt_0 CHECK (id > 0), + CONSTRAINT counter_gt_br_id CHECK (counter >= br_id), + CONSTRAINT br_id_true CHECK (id = spanner.bit_reverse(br_id, true)), PRIMARY KEY (id) )`, } @@ -387,7 +387,7 @@ const ( func TestMain(m *testing.M) { cleanup := initIntegrationTests() defer cleanup() - for _, dialect := range []adminpb.DatabaseDialect{adminpb.DatabaseDialect_GOOGLE_STANDARD_SQL, adminpb.DatabaseDialect_POSTGRESQL} { + for _, dialect := range []adminpb.DatabaseDialect{adminpb.DatabaseDialect_POSTGRESQL} { if isEmulatorEnvSet() && dialect == adminpb.DatabaseDialect_POSTGRESQL { // PG tests are not supported in emulator continue @@ -4672,6 +4672,13 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { client, dbPath, cleanup := prepareIntegrationTest(ctx, t, DefaultSessionPoolConfig, statements[testDialect][testTableBitReversedSeqStatements]) defer cleanup() + returningSQL := `THEN RETURN` + internalStateSQL := `GET_INTERNAL_SEQUENCE_STATE(Sequence seqT)` + if testDialect == adminpb.DatabaseDialect_POSTGRESQL { + returningSQL = `RETURNING` + internalStateSQL = `spanner.get_internal_sequence_state('seqT')` + } + var tests = []struct { name string test func() error @@ -4685,7 +4692,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { err error ) _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { - iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (100), (200), (300) THEN RETURN id, counter")) + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (100), (200), (300) "+returningSQL+" id, counter")) values, err = readAllBitReversedSeqTable(iter, true) return err }) @@ -4704,7 +4711,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { } counter = newCounter } - iter := client.Single().Query(ctx, NewStatement("SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')")) + iter := client.Single().Query(ctx, NewStatement("SELECT "+internalStateSQL)) r, err := iter.Next() if err != nil { return err @@ -4715,7 +4722,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { return err } if c3 > counter { - return errors.New("expected c3 <= SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')") + return errors.New("expected c3 <= SELECT GET_INTERNAL_SEQUENCE_STATE(Sequence seqT)") } return err }, @@ -4723,18 +4730,22 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { { name: "success: reduce ranges to half", test: func() error { - op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ - Database: dbPath, - Statements: []string{`ALTER SEQUENCE seqT SET OPTIONS ( + ddl := `ALTER SEQUENCE seqT SET OPTIONS ( skip_range_min = 0, skip_range_max = 4611686018427387904 - )`}, + )` + if testDialect == adminpb.DatabaseDialect_POSTGRESQL { + ddl = `ALTER SEQUENCE seqT SKIP RANGE 0 4611686018427387904` + } + op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ + Database: dbPath, + Statements: []string{ddl}, }) if err != nil { - t.Fatalf("cannot modify sequence %v: %v", dbPath, err) + return err } if err := op.Wait(ctx); err != nil { - t.Fatalf("timeout modifying sequence %v: %v", dbPath, err) + return err } var values [][]interface{} _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { @@ -4745,7 +4756,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { v = v + ", " } } - iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES "+v+" THEN RETURN id, counter")) + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES "+v+" "+returningSQL+" id, counter")) values, err = readAllBitReversedSeqTable(iter, true) return err }) @@ -4773,19 +4784,23 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { { name: "success: move start with counter forward", test: func() error { + ddl := `ALTER SEQUENCE seqT SET OPTIONS (start_with_counter=10001)` + if testDialect == adminpb.DatabaseDialect_POSTGRESQL { + ddl = `ALTER SEQUENCE seqT RESTART COUNTER WITH 10001` + } op, err := databaseAdmin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ Database: dbPath, - Statements: []string{`ALTER SEQUENCE seqT SET OPTIONS (start_with_counter=10001)`}, + Statements: []string{ddl}, }) if err != nil { - t.Fatalf("cannot modify sequence %v: %v", dbPath, err) + return err } if err := op.Wait(ctx); err != nil { - t.Fatalf("timeout modifying sequence %v: %v", dbPath, err) + return err } var values [][]interface{} _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { - iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (4), (5), (6) THEN RETURN id, br_id, counter")) + iter := tx.Query(ctx, NewStatement("INSERT INTO T (value) VALUES (4), (5), (6) "+returningSQL+" id, br_id, counter")) values, err = readAllBitReversedSeqTable(iter, false) return err }) @@ -4811,7 +4826,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { counter = newCounter } } - iter := client.Single().Query(ctx, NewStatement("SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')")) + iter := client.Single().Query(ctx, NewStatement("SELECT "+internalStateSQL)) r, err := iter.Next() if err != nil { return err @@ -4822,7 +4837,7 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { return err } if c3 > counter { - return errors.New("expected max(ci) <= SELECT GET_INTERNAL_SEQUENCE_STATE('seqT')") + return errors.New("expected max(ci) <= SELECT GET_INTERNAL_SEQUENCE_STATE(Sequence seqT)") } return err }, @@ -4842,10 +4857,10 @@ func TestIntegration_Bit_Reversed_Sequence(t *testing.T) { }, }) if err != nil { - t.Fatalf("cannot delete sequences %v: %v", dbPath, err) + return err } if err := op.Wait(ctx); err != nil { - t.Fatalf("timeout deleting sequence %v: %v", dbPath, err) + return err } return nil }, From fb67ec3152f62762d3290b1434114f0404810cd8 Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Thu, 3 Aug 2023 19:33:58 +0530 Subject: [PATCH 3/3] fix typo --- spanner/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spanner/integration_test.go b/spanner/integration_test.go index 6b838ce0c8ed..0b94fb481513 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -387,7 +387,7 @@ const ( func TestMain(m *testing.M) { cleanup := initIntegrationTests() defer cleanup() - for _, dialect := range []adminpb.DatabaseDialect{adminpb.DatabaseDialect_POSTGRESQL} { + for _, dialect := range []adminpb.DatabaseDialect{adminpb.DatabaseDialect_GOOGLE_STANDARD_SQL, adminpb.DatabaseDialect_POSTGRESQL} { if isEmulatorEnvSet() && dialect == adminpb.DatabaseDialect_POSTGRESQL { // PG tests are not supported in emulator continue