Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kvserver: repro L0 snap ingestion #80611

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions DEPS.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -1347,10 +1347,10 @@ def go_deps():
patches = [
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
],
sha256 = "5b3237c6bcbfc1085831ddf636ce4b4243211f754ceabe9b8fd9bf8962629a12",
strip_prefix = "github.com/cockroachdb/[email protected]20220322040433-6164579cf2cb",
sha256 = "dfa5ce136f7d8d40ddf24077323df27de81f0e1889c1058e4453c77cd8c2cb75",
strip_prefix = "github.com/cockroachdb/[email protected]20220426173801-b33d6e173cae",
urls = [
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220322040433-6164579cf2cb.zip",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220426173801-b33d6e173cae.zip",
],
)
go_repository(
Expand Down
2 changes: 1 addition & 1 deletion build/bazelutil/distdir_files.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.13.0.zip": "b3d43d8f95edf65f73a5348f29e1159823cac64b148f8d3bb48340bf55d70872",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20211118104740-dabe8e521a4f.zip": "1972c3f171f118add3fd9e64bcea6cbb9959a3b7fa0ada308e8a7310813fea74",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/panicparse/v2/com_github_cockroachdb_panicparse_v2-v2.0.0-20211103220158-604c82a44f1e.zip": "a51ff19f83c9a1e468b5d722d0857eb468d0f9baa29365e700c2e6ec24bd47a7",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220322040433-6164579cf2cb.zip": "5b3237c6bcbfc1085831ddf636ce4b4243211f754ceabe9b8fd9bf8962629a12",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20220426173801-b33d6e173cae.zip": "dfa5ce136f7d8d40ddf24077323df27de81f0e1889c1058e4453c77cd8c2cb75",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.3.zip": "7778b1e4485e4f17f35e5e592d87eb99c29e173ac9507801d000ad76dd0c261e",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9",
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ require (
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
github.com/cockroachdb/gostdlib v1.13.0
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f
github.com/cockroachdb/pebble v0.0.0-20220322040433-6164579cf2cb
github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae
github.com/cockroachdb/redact v1.1.3
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
github.com/cockroachdb/stress v0.0.0-20220310203902-58fb4627376e
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -454,8 +454,8 @@ github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0n
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/panicparse/v2 v2.0.0-20211103220158-604c82a44f1e h1:FrERdkPlRj+v7fc+PGpey3GUiDGuTR5CsmLCA54YJ8I=
github.com/cockroachdb/panicparse/v2 v2.0.0-20211103220158-604c82a44f1e/go.mod h1:pMxsKyCewnV3xPaFvvT9NfwvDTcIx2Xqg0qL5Gq0SjM=
github.com/cockroachdb/pebble v0.0.0-20220322040433-6164579cf2cb h1:1JgeoLiHDlpa+AV6MU2qvDctffM9zoHiPRXOCvgOX2k=
github.com/cockroachdb/pebble v0.0.0-20220322040433-6164579cf2cb/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU=
github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae h1:7lGpwt2wTBh7FApXTEdTDX5OFWsEg9CCe8wMlHJZDwA=
github.com/cockroachdb/pebble v0.0.0-20220426173801-b33d6e173cae/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU=
github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
Expand Down
10 changes: 10 additions & 0 deletions log.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
W220428 13:55:46.934554 491 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n3,s3,r11/2:{-}] 60 XXX ingestion into L0: {Bytes:8063 ApproxIngestedIntoL0Bytes:2782}
W220428 13:55:47.954636 506 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n2,s2,r11/3:{-}] 63 XXX ingestion into L0: {Bytes:8219 ApproxIngestedIntoL0Bytes:2939}
W220428 13:55:54.088449 603 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n3,s3,r45/2:{-}] 65 XXX ingestion into L0: {Bytes:129007245 ApproxIngestedIntoL0Bytes:6311}
W220428 13:55:59.208826 200 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n2,s2,r45/3:{-}] 67 XXX ingestion into L0: {Bytes:129007393 ApproxIngestedIntoL0Bytes:6459}
W220428 13:56:00.112429 803 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n3,s3,r12/2:{-}] 69 XXX ingestion into L0: {Bytes:7420 ApproxIngestedIntoL0Bytes:4977}
W220428 13:56:01.141574 632 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n2,s2,r12/3:{-}] 70 XXX ingestion into L0: {Bytes:7670 ApproxIngestedIntoL0Bytes:5227}
W220428 13:56:02.059377 805 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n2,s2,r1/2:{-}] 74 XXX ingestion into L0: {Bytes:9112 ApproxIngestedIntoL0Bytes:5009}
W220428 13:56:03.125276 833 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n3,s3,r1/3:{-}] 72 XXX ingestion into L0: {Bytes:9439 ApproxIngestedIntoL0Bytes:5257}
W220428 13:56:04.759412 910 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n3,s3,r19/2:{-}] 75 XXX ingestion into L0: {Bytes:7657 ApproxIngestedIntoL0Bytes:6357}
W220428 13:56:06.062855 846 kv/kvserver/pkg/kv/kvserver/replica_raftstorage.go:952 ⋮ [n2,s2,r19/3:{-}] 78 XXX ingestion into L0: {Bytes:7813 ApproxIngestedIntoL0Bytes:5135}
9 changes: 8 additions & 1 deletion pkg/kv/kvserver/replica_raftstorage.go
Original file line number Diff line number Diff line change
Expand Up @@ -943,9 +943,16 @@ func (r *Replica) applySnapshot(
return err
}
}
if err := r.store.engine.IngestExternalFiles(ctx, inSnap.SSTStorageScratch.SSTs()); err != nil {
eng := r.store.engine.(*storage.Pebble)
ss, err := eng.IngestExternalFilesWithStats(ctx, inSnap.SSTStorageScratch.SSTs())
if err != nil {
return errors.Wrapf(err, "while ingesting %s", inSnap.SSTStorageScratch.SSTs())
}
if ss.ApproxIngestedIntoL0Bytes > 0 {
log.Warningf(ctx, "XXX ingestion into L0: %+v", ss)
} else {
log.Infof(ctx, "YYY ingestion below L0: %+v", ss)
}
stats.ingestion = timeutil.Now()

state, err := stateloader.Make(desc.RangeID).Load(ctx, r.store.engine, desc)
Expand Down
7 changes: 7 additions & 0 deletions pkg/storage/pebble.go
Original file line number Diff line number Diff line change
Expand Up @@ -1557,6 +1557,13 @@ func (p *Pebble) IngestExternalFiles(ctx context.Context, paths []string) error
return p.db.Ingest(paths)
}

// IngestExternalFilesWithStats .
func (p *Pebble) IngestExternalFilesWithStats(
ctx context.Context, paths []string,
) (pebble.IngestOperationStats, error) {
return p.db.IngestWithStats(paths)
}

// PreIngestDelay implements the Engine interface.
func (p *Pebble) PreIngestDelay(ctx context.Context) {
preIngestDelay(ctx, p, p.settings)
Expand Down
34 changes: 34 additions & 0 deletions repro.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

set -euo pipefail

# For real cluster: roachprod create -n 7 $USER-snaps and fill
# in the name here
clus=local

# roachprod create -n 7 local
# roachprod wipe $clus
roachprod stop $clus:1-7 || true
roachprod put $clus ./cockroach-repro ./cockroach

roachprod start $clus:1-7

# Wait for >=3x replication for all replicas
# NB: this takes minutes, don't be alarmed.
# Probably worth reusing the cluster (which is
# why the `wipe` above is commented out).
time roachprod sql $clus:1 -- -e "
select crdb_internal.force_retry('3100ms') from crdb_internal.ranges_no_leases where array_length(replicas, 1) < 3;
"

# NB: this should be more like 1000 on a real cluster.
roachprod run $clus:1 -- ./cockroach workload fixtures import tpcc --warehouses=10 --checks=false.

# Start moving replicas off n[12] to n[34567].
# --wait makes sure we don't permanently remove the nodes & don't block here.
roachprod run $clus:1 ./cockroach node decommission --insecure --wait=none 1 2

# Can run this to have replicas move back to n[12]:
# Can then decommission other nodes, etc. This is to be run manually.
# roachprod run $clus:1 -- ./cockroach node recommission --insecure 1 2

2 changes: 1 addition & 1 deletion vendor
Submodule vendor updated 57 files
+75 −51 github.com/cockroachdb/pebble/batch.go
+88 −45 github.com/cockroachdb/pebble/compaction.go
+16 −16 github.com/cockroachdb/pebble/compaction_iter.go
+35 −4 github.com/cockroachdb/pebble/compaction_picker.go
+180 −101 github.com/cockroachdb/pebble/db.go
+24 −20 github.com/cockroachdb/pebble/error_iter.go
+14 −9 github.com/cockroachdb/pebble/external_iterator.go
+65 −32 github.com/cockroachdb/pebble/ingest.go
+8 −3 github.com/cockroachdb/pebble/internal/arenaskl/node.go
+1 −1 github.com/cockroachdb/pebble/internal/base/filenames.go
+25 −5 github.com/cockroachdb/pebble/internal/base/internal.go
+76 −0 github.com/cockroachdb/pebble/internal/base/metrics.go
+11 −0 github.com/cockroachdb/pebble/internal/base/options.go
+0 −61 github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go
+56 −170 github.com/cockroachdb/pebble/internal/keyspan/bound_iter.go
+416 −0 github.com/cockroachdb/pebble/internal/keyspan/defragment.go
+2 −2 github.com/cockroachdb/pebble/internal/keyspan/doc.go
+98 −0 github.com/cockroachdb/pebble/internal/keyspan/filter.go
+66 −57 github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go
+14 −47 github.com/cockroachdb/pebble/internal/keyspan/get.go
+791 −0 github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go
+19 −16 github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go
+98 −152 github.com/cockroachdb/pebble/internal/keyspan/iter.go
+407 −0 github.com/cockroachdb/pebble/internal/keyspan/level_iter.go
+196 −303 github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go
+51 −189 github.com/cockroachdb/pebble/internal/keyspan/seek.go
+232 −53 github.com/cockroachdb/pebble/internal/keyspan/span.go
+43 −28 github.com/cockroachdb/pebble/internal/keyspan/truncate.go
+2 −0 github.com/cockroachdb/pebble/internal/manifest/version.go
+43 −0 github.com/cockroachdb/pebble/internal/rangedel/rangedel.go
+182 −138 github.com/cockroachdb/pebble/internal/rangekey/coalesce.go
+0 −450 github.com/cockroachdb/pebble/internal/rangekey/defragment.go
+0 −782 github.com/cockroachdb/pebble/internal/rangekey/interleaving_iter.go
+0 −231 github.com/cockroachdb/pebble/internal/rangekey/iter.go
+173 −172 github.com/cockroachdb/pebble/internal/rangekey/rangekey.go
+274 −62 github.com/cockroachdb/pebble/iterator.go
+18 −21 github.com/cockroachdb/pebble/level_checker.go
+1 −4 github.com/cockroachdb/pebble/level_iter.go
+29 −37 github.com/cockroachdb/pebble/mem_table.go
+17 −15 github.com/cockroachdb/pebble/merging_iter.go
+44 −0 github.com/cockroachdb/pebble/metrics.go
+1 −0 github.com/cockroachdb/pebble/open.go
+55 −11 github.com/cockroachdb/pebble/options.go
+35 −56 github.com/cockroachdb/pebble/range_keys.go
+100 −18 github.com/cockroachdb/pebble/record/log_writer.go
+211 −82 github.com/cockroachdb/pebble/sstable/block.go
+1 −7 github.com/cockroachdb/pebble/sstable/block_property.go
+5 −0 github.com/cockroachdb/pebble/sstable/options.go
+6 −5 github.com/cockroachdb/pebble/sstable/reader.go
+186 −144 github.com/cockroachdb/pebble/sstable/writer.go
+10 −8 github.com/cockroachdb/pebble/table_cache.go
+137 −125 github.com/cockroachdb/pebble/table_stats.go
+1 −1 github.com/cockroachdb/pebble/tool/db.go
+22 −18 github.com/cockroachdb/pebble/tool/find.go
+21 −19 github.com/cockroachdb/pebble/tool/sstable.go
+47 −2 github.com/cockroachdb/pebble/vfs/clone.go
+2 −2 modules.txt