Skip to content

Commit

Permalink
Merge #98206 #98287
Browse files Browse the repository at this point in the history 
98206: deps: update go-libedit r=rickystewart a=knz

I am expecting this will be one of the last updates to go-libedit since we plan to replace it with bubbline for good. However, this imports a few patches and reduces build warnings, which folk find useful.

xref knz/go-libedit#24

Release note: None
Epic: None

98287: roachtest: mvcc_gc disable lease rebalancing for stability r=erikgrinaker a=aliher1911

Test sometimes flakes if lease is moved from under GC and cleanup is not performed in time. This commit disables rebalance as it is not required to test GC functionality.
This commit also improves reliability of test by preventing a race between workload stop and table data deletion which could cause test to fail if workload doesn't terminate before clear range is performed.

Release note: None

Fixes #97340

Co-authored-by: Raphael 'kena' Poss <[email protected]>
Co-authored-by: Oleg Afanasyev <[email protected]>
  • Loading branch information
@knz @aliher1911
3 people committed Mar 10, 2023
3 parents 01e14f6 + 57bbe58 + 27f5166 commit d336a62
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 63 deletions.
6 changes: 3 additions & 3 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5367,10 +5367,10 @@ def go_deps():
patches = [
"@com_github_cockroachdb_cockroach//build/patches:com_github_knz_go_libedit.patch",
],
sha256 = "d2ae0f8e43c49f917a2cadf52178c0efe1336fda5b8410a3d0f1270ae05d2532",
strip_prefix = "github.com/otan-cockroach/[email protected].20201030151939-7cced08450e7",
sha256 = "d603b4baf8ab6608a79f56edf2e5a24c5bf96af06f665002f47267d8401a7614",
strip_prefix = "github.com/knz/[email protected].20230308124748-6f1b59dd42bc",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/otan-cockroach/go-libedit/com_github_otan_cockroach_go_libedit-v1.10.2-0.20201030151939-7cced08450e7.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/go-libedit/com_github_knz_go_libedit-v1.10.2-0.20230308124748-6f1b59dd42bc.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion build/bazelutil/distdir_files.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,7 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/klauspost/pgzip/com_github_klauspost_pgzip-v1.2.5.zip": "1143b6417d4bb46d26dc8e6223407b84b6cd5f32e5d705cd4a9fb142220ce4ba",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/bubbline/com_github_knz_bubbline-v0.0.0-20230205122847-05558f88fdc4.zip": "0922c6fae4190a73e2023b1a56401347e29afddf1174990b8fd881433b9c75a9",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/catwalk/com_github_knz_catwalk-v0.1.4.zip": "f422f7974090494e54226262586c7b34fe57b33ab7d668151ca55eba8e309c1e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/go-libedit/com_github_knz_go_libedit-v1.10.2-0.20230308124748-6f1b59dd42bc.zip": "d603b4baf8ab6608a79f56edf2e5a24c5bf96af06f665002f47267d8401a7614",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/lipgloss-convert/com_github_knz_lipgloss_convert-v0.1.0.zip": "f9f9ffa12e7df4007cc60c87327d47ad42d1f71a80e360af4014674138de8bef",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/knz/strtime/com_github_knz_strtime-v0.0.0-20200318182718-be999391ffa9.zip": "c1e1b06c339798387413af1444f06f31a483d4f5278ab3a91b6cd5d7cd8d91a1",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/konsorten/go-windows-terminal-sequences/com_github_konsorten_go_windows_terminal_sequences-v1.0.3.zip": "429b01413b972b108ea86bbde3d5e660913f3e8099190d07ccfb2f186bc6d837",
Expand Down Expand Up @@ -823,7 +824,6 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/openzipkin-contrib/zipkin-go-opentracing/com_github_openzipkin_contrib_zipkin_go_opentracing-v0.4.5.zip": "74763b01a30fa2f7116f0408c792b4db50bb01200cfe5f3f8b351ac638d1adb4",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/openzipkin/zipkin-go/com_github_openzipkin_zipkin_go-v0.2.5.zip": "337535c088bd6f7a479e21747044286f66490871948989d52f7812bc4cca955e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/ory/dockertest/v3/com_github_ory_dockertest_v3-v3.9.1.zip": "2c9dabed798ccf07ac92653d8895ab50991ecc5f578806f7b856360bcc2087a9",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/otan-cockroach/go-libedit/com_github_otan_cockroach_go_libedit-v1.10.2-0.20201030151939-7cced08450e7.zip": "d2ae0f8e43c49f917a2cadf52178c0efe1336fda5b8410a3d0f1270ae05d2532",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/otan/gopgkrb5/com_github_otan_gopgkrb5-v1.0.3.zip": "8d7241eed0a03a88654c49ee0a0c5db4f5bd37682011ee953cddb623f16bb349",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/oxtoacart/bpool/com_github_oxtoacart_bpool-v0.0.0-20190530202638-03653db5a59c.zip": "6816ec3a6f197cbee0ba6ddb9ec70958bc28870e59864b24e43da0c858079a1b",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/pact-foundation/pact-go/com_github_pact_foundation_pact_go-v1.0.4.zip": "e753f63d70bf56300c60fe87817d04935bd41693fef06d273ec70014cccabd3b",
Expand Down
4 changes: 1 addition & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ require (
github.com/klauspost/compress v1.15.15
github.com/klauspost/pgzip v1.2.5
github.com/knz/bubbline v0.0.0-20230205122847-05558f88fdc4
github.com/knz/go-libedit v1.10.1
github.com/knz/go-libedit v1.10.2-0.20230308124748-6f1b59dd42bc
github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9
github.com/kr/pretty v0.3.0
github.com/kr/text v0.2.0
Expand Down Expand Up @@ -423,6 +423,4 @@ replace vitess.io/vitess => github.com/cockroachdb/vitess v0.0.0-20210218160543-

replace gopkg.in/yaml.v2 => github.com/cockroachdb/yaml v0.0.0-20210825132133-2d6955c8edbc

replace github.com/knz/go-libedit => github.com/otan-cockroach/go-libedit v1.10.2-0.20201030151939-7cced08450e7

replace github.com/docker/docker => github.com/moby/moby v20.10.6+incompatible
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1496,6 +1496,8 @@ github.com/knz/bubbline v0.0.0-20230205122847-05558f88fdc4 h1:WT5NrWC3UParTXCijO
github.com/knz/bubbline v0.0.0-20230205122847-05558f88fdc4/go.mod h1:ucXvyrucVy4jp/4afdKWNW1TVO73GMI72VNINzyT678=
github.com/knz/catwalk v0.1.4 h1:GgCxHbPp+nzyZBJcNL/CJd1aba4ACoeuI1lnsshAPkY=
github.com/knz/catwalk v0.1.4/go.mod h1:Q+Yj4ny4AXgrOOyWyDGY/HJzmbGH8MFnsUqvCAiUT5s=
github.com/knz/go-libedit v1.10.2-0.20230308124748-6f1b59dd42bc h1:I7VuCOZXntHLQVZhEj/nrDqUl853RzJhef3Mwx7/iqI=
github.com/knz/go-libedit v1.10.2-0.20230308124748-6f1b59dd42bc/go.mod h1:dmDChGdWopkB61HsdDN0/fxKAMIYljKTu+AG9uc4qVY=
github.com/knz/lipgloss-convert v0.1.0 h1:qUPUt6r8mqvi9DIV3nBPu3kEmFyHrZtXzv0BlPBPLNQ=
github.com/knz/lipgloss-convert v0.1.0/go.mod h1:S14GmtoiW/VAHqB7xEzuZOt0/G6GQ2dfjJN0fHpm30Q=
github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9 h1:GQE1iatYDRrIidq4Zf/9ZzKWyrTk2sXOYc1JADbkAjQ=
Expand Down Expand Up @@ -1843,8 +1845,6 @@ github.com/openzipkin/zipkin-go v0.2.5 h1:UwtQQx2pyPIgWYHRg+epgdx1/HnBQTgN3/oIYE
github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE=
github.com/ory/dockertest/v3 v3.8.1/go.mod h1:wSRQ3wmkz+uSARYMk7kVJFDBGm8x5gSxIhI7NDc+BAQ=
github.com/ory/dockertest/v3 v3.9.1/go.mod h1:42Ir9hmvaAPm0Mgibk6mBPi7SFvTXxEcnztDYOJ//uM=
github.com/otan-cockroach/go-libedit v1.10.2-0.20201030151939-7cced08450e7 h1:+sIdymRXD4aKCvmVMBLL7/bO95KZFYrbz0EzQ1Jlj4A=
github.com/otan-cockroach/go-libedit v1.10.2-0.20201030151939-7cced08450e7/go.mod h1:X8+oa2glxe4aJyviH7x8FG5AoSdqZ0VJYOxt83gh6m8=
github.com/otan/gopgkrb5 v1.0.3 h1:iDZlYPC8mxvKvpIvDu66j48Q4WqW15HuWUX+MIwjF0U=
github.com/otan/gopgkrb5 v1.0.3/go.mod h1:aamIwpVk0oxQLpA6drHPvqvcPuJ7lzCSZ4NUkC+69MQ=
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
Expand Down
155 changes: 101 additions & 54 deletions pkg/cmd/roachtest/tests/mvcc_gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,21 @@ func runMVCCGC(ctx context.Context, t test.Test, c cluster.Cluster) {
execSQLOrFail(fmt.Sprintf(`SET CLUSTER SETTING %s = $1`, name), value)
}

// Explicitly enable range tombstones. Can be removed once ranges tombstones
// are enabled by default.
setClusterSetting("storage.mvcc.range_tombstones.enabled", true)
// Protected timestamps prevent GC from collecting data, even with low ttl
// we need to wait for protected ts to be moved. By reducing this interval
// we ensure that data will always be collectable after ttl + 5s.
setClusterSetting("kv.protectedts.poll_interval", "5s")
// Disable mvcc_gc queue throttling, we always manually enqueue replicas as
// fast as possible.
setClusterSetting("kv.mvcc_gc.queue_interval", "0s")
// Load based lease balancing could move leaseholders from under the GC. If
// that happens gc will end up running on non lease-holder store and its
// requests would be rejected. This is causing test to flake. Disabling
// rebalancing is better than increasing wait time further.
setClusterSetting("kv.allocator.load_based_lease_rebalancing.enabled", false)

if err := WaitFor3XReplication(ctx, t, conn); err != nil {
t.Fatalf("failed to up-replicate cluster: %s", err)
Expand Down Expand Up @@ -135,8 +147,7 @@ func runMVCCGC(ctx context.Context, t test.Test, c cluster.Cluster) {
t.L().Printf("partially deleted some data using tombstones")

assertRangesWithGCRetry(ctx, t, c, gcRetryTimeout, m, func() error {
totals, rangeCount := collectTableMVCCStatsOrFatal(t, conn, m)
return checkRangesHaveNoRangeTombstones(totals, rangeCount)
return assertTableMVCCStatsOrFatal(t, conn, m, checkRangesHaveNoRangeTombstones)
})

t.L().Printf("all range tombstones were garbage collected")
Expand All @@ -152,21 +163,38 @@ func runMVCCGC(ctx context.Context, t test.Test, c cluster.Cluster) {
wlCancel()

if err := retry.WithMaxAttempts(ctx, retry.Options{}, 3, func() error {
return deleteAllTableDataWithOverlappingTombstones(ctx, t, c, conn, rng, m, 5)
err := deleteAllTableDataWithOverlappingTombstones(ctx, t, c, conn, rng, m, 5)
if err != nil {
return err
}
// We must check if table is empty because workload termination is only
// cancelling context and termination will happen in the background.
return checkTableIsEmpty(t, conn, m)
}); err != nil {
t.Fatal(err)
}

t.L().Printf("deleted all table data using tombstones")

assertRangesWithGCRetry(ctx, t, c, gcRetryTimeout, m, func() error {
totals, details := collectStatsAndConsistencyOrFail(t, conn, m)
return checkRangesConsistentAndHaveNoData(totals, details)
return assertStatsAndConsistencyOrFatal(t, conn, m, checkRangesConsistentAndHaveNoData)
})

return nil
})
m.Wait()
}

func checkTableIsEmpty(t test.Test, conn *gosql.DB, m tableMetadata) error {
t.Helper()
var rows int
queryRowOrFatal(t, conn, fmt.Sprintf("select count(*) from %s.%s", m.databaseName, m.tableName), nil, []any{&rows})
if rows > 0 {
return errors.Newf("table still has %d rows", rows)
}
return nil
}

func assertRangesWithGCRetry(
ctx context.Context,
t test.Test,
Expand All @@ -178,17 +206,26 @@ func assertRangesWithGCRetry(
t.Helper()
deadline := timeutil.Now().Add(timeout)
gcRetryTime := timeutil.Now()
// Track number of ranges in the table to handle cases when merges run
// concurrently with GC and doesn't allow it to collect all data.
var rangeCount int
lastMergeDeadline := timeutil.Now()
for {
if timeutil.Now().After(gcRetryTime) {
enqueueAllTableRangesForGC(ctx, t, c, m)
t.L().Printf("enqueued ranges for GC")
rc := enqueueAllTableRangesForGC(ctx, t, c, m)
t.L().Printf("enqueued %d ranges for GC", rc)
gcRetryTime = timeutil.Now().Add(2 * time.Minute)
if rc != rangeCount {
// Wait at least one more GC run if number of ranges changed.
lastMergeDeadline = gcRetryTime.Add(time.Minute)
rangeCount = rc
}
}
err := assertion()
if err == nil {
return
}
if timeutil.Now().After(deadline) {
if now := timeutil.Now(); now.After(deadline) && now.After(lastMergeDeadline) {
t.Fatalf("assertion still failing after %s: %s", timeout, err)
}
select {
Expand All @@ -200,12 +237,9 @@ func assertRangesWithGCRetry(
}
}

func checkRangesHaveNoRangeTombstones(totals enginepb.MVCCStats, c int) error {
if c == 0 {
return errors.New("failed to find any ranges for table")
}
if totals.RangeKeyCount > 0 || totals.RangeKeyBytes > 0 {
return errors.Errorf("table ranges contain range tombstones %s", totals.String())
func checkRangesHaveNoRangeTombstones(rangeStats enginepb.MVCCStats) error {
if rangeStats.RangeKeyCount > 0 || rangeStats.RangeKeyBytes > 0 {
return errors.Errorf("range contain range tombstones %s", rangeStats.String())
}
return nil
}
Expand Down Expand Up @@ -238,12 +272,7 @@ func getSplitKey(index, fragments int) int64 {
return math.MinInt64 + int64(getSplitPoint(index, fragments))
}

func checkRangesConsistentAndHaveNoData(
totals enginepb.MVCCStats, details map[int]rangeDetails,
) error {
if len(details) == 0 {
return errors.New("failed to find any ranges for table")
}
func checkRangesConsistentAndHaveNoData(totals enginepb.MVCCStats, details rangeDetails) error {
if totals.RangeKeyCount > 0 || totals.RangeKeyBytes > 0 {
return errors.Errorf("table ranges contain range tombstones %s", totals.String())
}
Expand All @@ -254,21 +283,19 @@ func checkRangesConsistentAndHaveNoData(
totals.IntentBytes > 0 || totals.IntentCount > 0 || totals.SeparatedIntentCount > 0 {
return errors.Errorf("table ranges contain live data %s", totals.String())
}
for id, d := range details {
if d.status != kvpb.CheckConsistencyResponse_RANGE_CONSISTENT.String() {
return errors.Errorf("consistency check failed for r%d: %s detail: %s", id, d.status,
d.detail)
}
if details.status != kvpb.CheckConsistencyResponse_RANGE_CONSISTENT.String() {
return errors.Errorf("consistency check failed %s detail: %s", details.status,
details.detail)
}
return nil
}

func collectTableMVCCStatsOrFatal(
t test.Test, conn *gosql.DB, m tableMetadata,
) (enginepb.MVCCStats, int) {
func assertTableMVCCStatsOrFatal(
t test.Test, conn *gosql.DB, m tableMetadata, assert func(enginepb.MVCCStats) error,
) error {
t.Helper()
rows, err := conn.Query(fmt.Sprintf(`
SELECT range_id, raw_start_key, crdb_internal.range_stats(raw_start_key)
SELECT range_id, raw_start_key, raw_end_key, crdb_internal.range_stats(raw_start_key)
FROM [SHOW RANGES FROM TABLE %s.%s WITH KEYS]
ORDER BY start_key`,
tree.NameString(m.databaseName), tree.NameString(m.tableName)))
Expand All @@ -278,25 +305,31 @@ ORDER BY start_key`,
defer rows.Close()

jsonpb := protoutil.JSONPb{}
var tableStats enginepb.MVCCStats
var rangeCount int
for rows.Next() {
var (
rangeID int
rangeStartKey roachpb.Key
rangeEndKey roachpb.Key
jsonb []byte
rangeStats enginepb.MVCCStats
)
if err := rows.Scan(&rangeID, &rangeStartKey, &jsonb); err != nil {
if err := rows.Scan(&rangeID, &rangeStartKey, &rangeEndKey, &jsonb); err != nil {
t.Fatalf("failed to run consistency check query on table: %s", err)
}
if err := jsonpb.Unmarshal(jsonb, &rangeStats); err != nil {
t.Fatalf("failed to unmarshal json %s stats: %s", string(jsonb), err)
}
tableStats.Add(rangeStats)
if err := assert(rangeStats); err != nil {
return errors.Wrapf(err, "assertion failed on range r%d %s", rangeID,
roachpb.Span{Key: rangeStartKey, EndKey: rangeEndKey})
}
rangeCount++
}
return tableStats, rangeCount
if rangeCount == 0 {
return errors.New("failed to find any ranges for table")
}
return nil
}

// rangeDetails contains results of check_consistency for a single range.
Expand All @@ -306,9 +339,9 @@ type rangeDetails struct {
detail string
}

func collectStatsAndConsistencyOrFail(
t test.Test, conn *gosql.DB, m tableMetadata,
) (enginepb.MVCCStats, map[int]rangeDetails) {
func assertStatsAndConsistencyOrFatal(
t test.Test, conn *gosql.DB, m tableMetadata, assert func(enginepb.MVCCStats, rangeDetails) error,
) error {
t.Helper()
var startKey, endKey roachpb.Key
queryRowOrFatal(t, conn,
Expand All @@ -323,9 +356,8 @@ FROM [SHOW RANGES FROM TABLE %s.%s WITH KEYS]`,
}
defer rows.Close()

var details = make(map[int]rangeDetails)
jsonpb := protoutil.JSONPb{}
var tableStats enginepb.MVCCStats
var tableRanges int
for rows.Next() {
var (
rangeID int
Expand All @@ -341,14 +373,19 @@ FROM [SHOW RANGES FROM TABLE %s.%s WITH KEYS]`,
if err := jsonpb.Unmarshal(jsonb, &rangeStats); err != nil {
t.Fatalf("failed to unmarshal json %s stats: %s", string(jsonb), err)
}
tableStats.Add(rangeStats)
details[rangeID] = rangeDetails{
if err := assert(rangeStats, rangeDetails{
stats: rangeStats,
status: status,
detail: detail,
}); err != nil {
return errors.Wrapf(err, "assertion failed on range r%d %s", rangeID, rangeStartKey)
}
tableRanges++
}
if tableRanges == 0 {
return errors.New("failed to find any ranges for table")
}
return tableStats, details
return nil
}

type tableMetadata struct {
Expand Down Expand Up @@ -380,44 +417,54 @@ func queryTableMetaOrFatal(

func enqueueAllTableRangesForGC(
ctx context.Context, t test.Test, c cluster.Cluster, m tableMetadata,
) {
) int {
t.Helper()
var conns []*gosql.DB
for _, node := range c.All() {
conns = append(conns, c.Conn(ctx, t.L(), node))
}
if err := visitTableRanges(ctx, t, conns[0], m, func(rangeID int) error {
var tableRanges int
if err := visitTableRanges(ctx, t, conns[0], m, func(info rangeInfo) error {
t.L().Printf("enqueuing range r%d [%s,%s) for GC", info.id, info.startKey, info.endKey)
for _, c := range conns {
_, _ = c.ExecContext(ctx, `select crdb_internal.kv_enqueue_replica($1, 'mvccGC', true)`, rangeID)
_, _ = c.ExecContext(ctx, `select crdb_internal.kv_enqueue_replica($1, 'mvccGC', true)`, info.id)
}
tableRanges++
return nil
}); err != nil {
t.Fatalf("failed to enqueue all ranges for GC: %s", err)
t.Fatalf("failed to enqueue table ranges for GC: %s", err)
}
return tableRanges
}

type rangeInfo struct {
id int
startKey, endKey string
}

func visitTableRanges(
ctx context.Context, t test.Test, conn *gosql.DB, m tableMetadata, f func(rangeID int) error,
ctx context.Context, t test.Test, conn *gosql.DB, m tableMetadata, f func(info rangeInfo) error,
) error {
t.Helper()
rows, err := conn.QueryContext(
ctx, fmt.Sprintf(`SELECT range_id FROM [ SHOW RANGES FROM TABLE %s.%s ]`, m.databaseName, m.tableName),
ctx, fmt.Sprintf(`SELECT range_id, start_key, end_key FROM [ SHOW RANGES FROM TABLE %s.%s ]`, m.databaseName, m.tableName),
)
if err != nil {
t.Fatalf("failed to run consistency check query on table: %s", err)
t.Fatalf("failed to query ranges for table: %s", err)
}
defer rows.Close()
var rangeIDs []int

var ranges []rangeInfo
for rows.Next() {
var rangeID int
if err := rows.Scan(&rangeID); err != nil {
t.Fatalf("failed to run consistency check query on table: %s", err)
var info rangeInfo
if err := rows.Scan(&info.id, &info.startKey, &info.endKey); err != nil {
t.Fatalf("failed to query ranges for table: %s", err)
}
rangeIDs = append(rangeIDs, rangeID)
ranges = append(ranges, info)
}
rows.Close()

for _, id := range rangeIDs {
for _, id := range ranges {
if err := f(id); err != nil {
return err
}
Expand Down

0 comments on commit d336a62

Please sign in to comment.