-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
bulk/kv write jobID to each MVCCValue's MVCCValueHeader during IMPORT
This patch makes IMPORT write the import job ID to each ingested MVCC Value, via the SSTBatcher. In a future PR, the import jobID will be used to track and rollback an IMPORT. This additional information will allow IMPORTing tables to be backed up and restored, as described in this [RFC](https://docs.google.com/document/d/16TbkFznqbsu3mialSw6o1sxOEn1pKhhJ9HTxNdw0-WE/edit#heading=h.bpox0bmkz77i). Informs #76722 Release note: None
- Loading branch information
Showing
8 changed files
with
242 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| // Copyright 2022 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package importer_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/base" | ||
| "github.com/cockroachdb/cockroach/pkg/keys" | ||
| "github.com/cockroachdb/cockroach/pkg/kv" | ||
| "github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord" | ||
| "github.com/cockroachdb/cockroach/pkg/roachpb" | ||
| "github.com/cockroachdb/cockroach/pkg/storage" | ||
| "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" | ||
| "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" | ||
| "github.com/cockroachdb/cockroach/pkg/util/hlc" | ||
| "github.com/cockroachdb/cockroach/pkg/util/leaktest" | ||
| "github.com/cockroachdb/cockroach/pkg/util/log" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestMVCCValueHeaderImportJobId tests that the import job ID is properly | ||
| // stored in the MVCCValueHeader in an imported key's MVCCValue. | ||
| func TestMVCCValueHeaderImportJobID(t *testing.T) { | ||
| defer leaktest.AfterTest(t)() | ||
| defer log.Scope(t).Close(t) | ||
|
|
||
| defer leaktest.AfterTest(t)() | ||
| defer log.Scope(t).Close(t) | ||
|
|
||
| s, db, _ := serverutils.StartServer(t, base.TestServerArgs{}) | ||
| ctx := context.Background() | ||
| defer s.Stopper().Stop(ctx) | ||
| sqlDB := sqlutils.MakeSQLRunner(db) | ||
|
|
||
| sqlDB.Exec(t, `CREATE DATABASE d`) | ||
|
|
||
| srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == "GET" { | ||
| fmt.Fprint(w, "1") | ||
| } | ||
| })) | ||
| defer srv.Close() | ||
|
|
||
| // Create a table where the first row ( in sort order) comes from an IMPORT | ||
| // while the second comes from an INSERT. | ||
| sqlDB.Exec(t, `CREATE TABLE d.t (a INT8)`) | ||
| sqlDB.Exec(t, `INSERT INTO d.t VALUES ('2')`) | ||
| sqlDB.Exec(t, `IMPORT INTO d.t CSV DATA ($1)`, srv.URL) | ||
|
|
||
| // Conduct an export request to iterator over the keys in the table. | ||
| var tableID uint32 | ||
| sqlDB.QueryRow(t, `SELECT id FROM system.namespace WHERE name = $1`, | ||
| "t").Scan(&tableID) | ||
|
|
||
| startKey := keys.SystemSQLCodec.TablePrefix(tableID) | ||
| endKey := startKey.PrefixEnd() | ||
|
|
||
| req := &roachpb.ExportRequest{ | ||
| RequestHeader: roachpb.RequestHeader{ | ||
| Key: startKey, | ||
| EndKey: endKey, | ||
| }, | ||
| MVCCFilter: roachpb.MVCCFilter_All, | ||
| StartTime: hlc.Timestamp{}, | ||
| ReturnSST: true, | ||
| } | ||
|
|
||
| header := roachpb.Header{Timestamp: s.Clock().Now()} | ||
| resp, roachErr := kv.SendWrappedWith(ctx, | ||
| s.DistSenderI().(*kvcoord.DistSender), header, req) | ||
| require.NoError(t, roachErr.GoError()) | ||
|
|
||
| iterOpts := storage.IterOptions{ | ||
| KeyTypes: storage.IterKeyTypePointsOnly, | ||
| LowerBound: startKey, | ||
| UpperBound: endKey, | ||
| } | ||
|
|
||
| // Ensure there are 2 keys in the span, and only the first one contains job ID metadata | ||
| keyCount := 0 | ||
| for _, file := range resp.(*roachpb.ExportResponse).Files { | ||
| it, err := storage.NewPebbleMemSSTIterator(file.SST, false /* verify */, iterOpts) | ||
| require.NoError(t, err) | ||
| defer it.Close() | ||
| for it.SeekGE(storage.NilKey); ; it.Next() { | ||
| ok, err := it.Valid() | ||
| require.NoError(t, err) | ||
| if !ok { | ||
| break | ||
| } | ||
| val, err := storage.DecodeMVCCValue(it.UnsafeValue()) | ||
| require.NoError(t, err) | ||
| if keyCount == 0 { | ||
| require.NotEqual(t, int64(0), val.ClientMeta.ImportJobId) | ||
| } else if keyCount == 1 { | ||
| require.Equal(t, int64(0), val.ClientMeta.ImportJobId) | ||
| } else { | ||
| t.Fatal("more than 2 keys in the table") | ||
| } | ||
| require.Equal(t, hlc.ClockTimestamp{}, val.LocalTimestamp) | ||
| keyCount++ | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters