From 0df3a03e781cf780f291cd4302d74083011b5bc1 Mon Sep 17 00:00:00 2001 From: Stan Rosenberg Date: Wed, 31 May 2023 22:46:15 -0400 Subject: [PATCH] roachtest: require perf. tests to opt in via TestSpec.Benchmark Previously, roachtests which benchmark performance (cf. correctness) were indistinguishable from correctness tests. That is, a performance test is like any other test with the exception of _optionally_ writing stats.json under 'Test.PerfArtifactsDir'; these artifacts are automatically exported to a gcs bucket, used in conjunction with the roachperf dashboard. Having no direct way to distinguish a performance test from a correctness test has several challenges. E.g., performance tests may require a specific machine type or architecture; background workloads like incremental backup may cause a performance regression; new metamorphic configurations like arm64 and fips may require a "bake-in" time before performance tests can be enabled. In future, the test runner may make specialized decisions (e.g., don't reuse a cluster) when executing a performance test. Thus, we need a (standard) mechanism to enumerate all performance tests. Given their specific requirements, the test author must explicitly opt in, by setting TestSpec.Benchmark to 'true'. This PR applies the above change retroactively, i.e., setting 'TestSpec.Benchmark' for all _known_ performance tests, including those which _assert_ on performance instead of exporting stats.json. It also fixes `roachtest list --bench` and `roachtest bench`, which were out-of-date, albeit not actively used. Epic: none Release note: None --- pkg/cmd/roachtest/github_test.go | 4 +-- pkg/cmd/roachtest/main.go | 28 +++++++---------- pkg/cmd/roachtest/registry/test_spec.go | 8 +++++ pkg/cmd/roachtest/test_registry.go | 16 +++++++--- pkg/cmd/roachtest/test_registry_test.go | 4 +-- pkg/cmd/roachtest/test_runner.go | 4 ++- pkg/cmd/roachtest/test_test.go | 8 ++--- pkg/cmd/roachtest/tests/allocation_bench.go | 5 +-- pkg/cmd/roachtest/tests/allocator.go | 31 ++++++++++--------- pkg/cmd/roachtest/tests/backup.go | 1 + pkg/cmd/roachtest/tests/cdc.go | 1 + pkg/cmd/roachtest/tests/connection_latency.go | 19 +++++++----- pkg/cmd/roachtest/tests/copyfrom.go | 27 +++++++++------- pkg/cmd/roachtest/tests/decommissionbench.go | 3 +- pkg/cmd/roachtest/tests/failover.go | 31 ++++++++++++------- pkg/cmd/roachtest/tests/import.go | 2 ++ .../roachtest/tests/import_cancellation.go | 11 ++++--- pkg/cmd/roachtest/tests/indexes.go | 7 +++-- pkg/cmd/roachtest/tests/kv.go | 18 ++++++----- pkg/cmd/roachtest/tests/ledger.go | 7 +++-- .../tests/loss_of_quorum_recovery.go | 2 ++ pkg/cmd/roachtest/tests/registry.go | 11 ------- pkg/cmd/roachtest/tests/restore.go | 22 +++++++------ pkg/cmd/roachtest/tests/schemachange.go | 22 +++++++------ .../tests/schemachange_random_load.go | 6 ++-- pkg/cmd/roachtest/tests/scrub.go | 9 +++--- pkg/cmd/roachtest/tests/tpcc.go | 1 + pkg/cmd/roachtest/tests/tpch_concurrency.go | 18 ++++++----- pkg/cmd/roachtest/tests/tpchbench.go | 7 +++-- pkg/cmd/roachtest/tests/ycsb.go | 21 +++++++------ 30 files changed, 198 insertions(+), 156 deletions(-) diff --git a/pkg/cmd/roachtest/github_test.go b/pkg/cmd/roachtest/github_test.go index 6d62abb3232d..2d69716f4d84 100644 --- a/pkg/cmd/roachtest/github_test.go +++ b/pkg/cmd/roachtest/github_test.go @@ -76,7 +76,7 @@ func TestShouldPost(t *testing.T) { {false, 1, "token", "master", true, ""}, } - reg := makeTestRegistry(spec.GCE, "", "", false) + reg := makeTestRegistry(spec.GCE, "", "", false, false) for _, c := range testCases { t.Setenv("GITHUB_API_TOKEN", c.envGithubAPIToken) t.Setenv("TC_BUILD_BRANCH", c.envTcBuildBranch) @@ -146,7 +146,7 @@ func TestCreatePostRequest(t *testing.T) { {true, false, true, false, otherErr, false, false, nil}, } - reg := makeTestRegistry(spec.GCE, "", "", false) + reg := makeTestRegistry(spec.GCE, "", "", false, false) for _, c := range testCases { clusterSpec := reg.MakeClusterSpec(1) diff --git a/pkg/cmd/roachtest/main.go b/pkg/cmd/roachtest/main.go index 88b0e0aed96b..9901b32bde0a 100644 --- a/pkg/cmd/roachtest/main.go +++ b/pkg/cmd/roachtest/main.go @@ -200,14 +200,10 @@ Examples: roachtest list tag:owner-kv,weekly tag:aws `, RunE: func(_ *cobra.Command, args []string) error { - r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg) - if !listBench { - tests.RegisterTests(&r) - } else { - tests.RegisterBenchmarks(&r) - } + r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg, listBench) + tests.RegisterTests(&r) - matchedTests := r.List(context.Background(), args) + matchedTests := r.List(args) for _, test := range matchedTests { var skip string if test.Skip != "" && !runSkipped { @@ -258,7 +254,7 @@ runner itself. clusterID: clusterID, versionsBinaryOverride: versionsBinaryOverride, enableFIPS: enableFIPS, - }) + }, false /* benchOnly */) }, } @@ -282,7 +278,7 @@ runner itself. if literalArtifacts == "" { literalArtifacts = artifacts } - return runTests(tests.RegisterBenchmarks, cliCfg{ + return runTests(tests.RegisterTests, cliCfg{ args: args, count: count, cpuQuota: cpuQuota, @@ -296,7 +292,7 @@ runner itself. clusterID: clusterID, versionsBinaryOverride: versionsBinaryOverride, enableFIPS: enableFIPS, - }) + }, true /* benchOnly */) }, } @@ -404,11 +400,11 @@ type cliCfg struct { enableFIPS bool } -func runTests(register func(registry.Registry), cfg cliCfg) error { +func runTests(register func(registry.Registry), cfg cliCfg, benchOnly bool) error { if cfg.count <= 0 { return fmt.Errorf("--count (%d) must by greater than 0", cfg.count) } - r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg) + r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg, benchOnly) // actual registering of tests // TODO: don't register if we can't run on the specified registry cloud @@ -448,7 +444,7 @@ func runTests(register func(registry.Registry), cfg cliCfg) error { return err } - tests := testsToRun(context.Background(), r, filter) + tests := testsToRun(r, filter) n := len(tests) if n*cfg.count < cfg.parallelism { // Don't spin up more workers than necessary. This has particular @@ -594,10 +590,8 @@ func testRunnerLogger( return l, teeOpt } -func testsToRun( - ctx context.Context, r testRegistryImpl, filter *registry.TestFilter, -) []registry.TestSpec { - tests, tagMismatch := r.GetTests(ctx, filter) +func testsToRun(r testRegistryImpl, filter *registry.TestFilter) []registry.TestSpec { + tests, tagMismatch := r.GetTests(filter) var notSkipped []registry.TestSpec for _, s := range tests { diff --git a/pkg/cmd/roachtest/registry/test_spec.go b/pkg/cmd/roachtest/registry/test_spec.go index 406d2963750f..0b71394a2790 100644 --- a/pkg/cmd/roachtest/registry/test_spec.go +++ b/pkg/cmd/roachtest/registry/test_spec.go @@ -46,6 +46,14 @@ type TestSpec struct { // associated cluster expires. The timeout is always truncated to 10m before // the test's cluster expires. Timeout time.Duration + // Denotes whether the test is a roachperf benchmark. If true, the test is expected, but not required, to produce + // artifacts in Test.PerfArtifactsDir(), which get exported to the roachperf dashboard + // (see getPerfArtifacts() in test_runner.go). + // N.B. performance tests may choose to _assert_ on latency, throughput, or some other perf. metric, _without_ + // exporting artifacts to the dashboard. E.g., see 'registerKVContention' and 'verifyTxnPerSecond'. + // N.B. performance tests may have different requirements than correctness tests, e.g., machine type/architecture. + // Thus, they must be opted into explicitly via this field. + Benchmark bool // Tags is a set of tags associated with the test that allow grouping // tests. If no tags are specified, the set ["default"] is automatically // given. diff --git a/pkg/cmd/roachtest/test_registry.go b/pkg/cmd/roachtest/test_registry.go index 4749b96ae324..680a4343b7c4 100644 --- a/pkg/cmd/roachtest/test_registry.go +++ b/pkg/cmd/roachtest/test_registry.go @@ -11,7 +11,6 @@ package main import ( - "context" "fmt" "os" "regexp" @@ -43,11 +42,13 @@ type testRegistryImpl struct { snapshotPrefixes map[string]struct{} promRegistry *prometheus.Registry + // benchOnly is true iff the registry is being used to run benchmarks only. + benchOnly bool } // makeTestRegistry constructs a testRegistryImpl and configures it with opts. func makeTestRegistry( - cloud string, instanceType string, zones string, preferSSD bool, + cloud string, instanceType string, zones string, preferSSD bool, benchOnly bool, ) testRegistryImpl { return testRegistryImpl{ cloud: cloud, @@ -57,6 +58,7 @@ func makeTestRegistry( m: make(map[string]*registry.TestSpec), snapshotPrefixes: make(map[string]struct{}), promRegistry: prometheus.NewRegistry(), + benchOnly: benchOnly, } } @@ -66,6 +68,10 @@ func (r *testRegistryImpl) Add(spec registry.TestSpec) { fmt.Fprintf(os.Stderr, "test %s already registered\n", spec.Name) os.Exit(1) } + if r.benchOnly && !spec.Benchmark { + // Skip non-benchmarks. + return + } if spec.SnapshotPrefix != "" { for existingPrefix := range r.snapshotPrefixes { if strings.HasPrefix(existingPrefix, spec.SnapshotPrefix) { @@ -160,7 +166,7 @@ func (r *testRegistryImpl) PromFactory() promauto.Factory { // Skipped tests are included, and tests that don't match their minVersion spec // are also included but marked as skipped. func (r testRegistryImpl) GetTests( - ctx context.Context, filter *registry.TestFilter, + filter *registry.TestFilter, ) ([]registry.TestSpec, []registry.TestSpec) { var tests []registry.TestSpec var tagMismatch []registry.TestSpec @@ -183,9 +189,9 @@ func (r testRegistryImpl) GetTests( } // List lists tests that match one of the filters. -func (r testRegistryImpl) List(ctx context.Context, filters []string) []registry.TestSpec { +func (r testRegistryImpl) List(filters []string) []registry.TestSpec { filter := registry.NewTestFilter(filters, true) - tests, _ := r.GetTests(ctx, filter) + tests, _ := r.GetTests(filter) sort.Slice(tests, func(i, j int) bool { return tests[i].Name < tests[j].Name }) return tests } diff --git a/pkg/cmd/roachtest/test_registry_test.go b/pkg/cmd/roachtest/test_registry_test.go index 21002046bfd0..719dc5b64bc4 100644 --- a/pkg/cmd/roachtest/test_registry_test.go +++ b/pkg/cmd/roachtest/test_registry_test.go @@ -22,7 +22,7 @@ import ( func TestMakeTestRegistry(t *testing.T) { testutils.RunTrueAndFalse(t, "preferSSD", func(t *testing.T, preferSSD bool) { - r := makeTestRegistry(spec.AWS, "foo", "zone123", preferSSD) + r := makeTestRegistry(spec.AWS, "foo", "zone123", preferSSD, false) require.Equal(t, preferSSD, r.preferSSD) require.Equal(t, "zone123", r.zones) require.Equal(t, "foo", r.instanceType) @@ -48,7 +48,7 @@ func TestMakeTestRegistry(t *testing.T) { // TestPrometheusMetricParser tests that the registry.PromSub() // helper properly converts a string into a metric name that Prometheus can read. func TestPrometheusMetricParser(t *testing.T) { - r := makeTestRegistry(spec.AWS, "foo", "zone123", true) + r := makeTestRegistry(spec.AWS, "foo", "zone123", true, false) f := r.PromFactory() rawName := "restore/nodes=4/duration" diff --git a/pkg/cmd/roachtest/test_runner.go b/pkg/cmd/roachtest/test_runner.go index 7cf8ebb54ee5..08b84d724df9 100644 --- a/pkg/cmd/roachtest/test_runner.go +++ b/pkg/cmd/roachtest/test_runner.go @@ -779,7 +779,9 @@ func (r *testRunner) runWorker( } } else { // Upon success fetch the perf artifacts from the remote hosts. - getPerfArtifacts(ctx, l, c, t) + if t.spec.Benchmark { + getPerfArtifacts(ctx, l, c, t) + } if debugMode == DebugKeepAlways { c.Save(ctx, "cluster saved since --debug-always set", l) c = nil diff --git a/pkg/cmd/roachtest/test_test.go b/pkg/cmd/roachtest/test_test.go index 04bd1c882b6d..ad8dfceb7834 100644 --- a/pkg/cmd/roachtest/test_test.go +++ b/pkg/cmd/roachtest/test_test.go @@ -44,7 +44,7 @@ const defaultParallelism = 10 func mkReg(t *testing.T) testRegistryImpl { t.Helper() - return makeTestRegistry(spec.GCE, "", "", false /* preferSSD */) + return makeTestRegistry(spec.GCE, "", "", false /* preferSSD */, false /* benchOnly */) } func TestMatchOrSkip(t *testing.T) { @@ -248,7 +248,7 @@ type runnerTest struct { func setupRunnerTest(t *testing.T, r testRegistryImpl, testFilters []string) *runnerTest { ctx := context.Background() - tests := testsToRun(ctx, r, registry.NewTestFilter(testFilters, false)) + tests := testsToRun(r, registry.NewTestFilter(testFilters, false)) cr := newClusterRegistry() stopper := stop.NewStopper() @@ -405,7 +405,7 @@ func TestRegistryPrepareSpec(t *testing.T) { } for _, c := range testCases { t.Run("", func(t *testing.T) { - r := makeTestRegistry(spec.GCE, "", "", false /* preferSSD */) + r := makeTestRegistry(spec.GCE, "", "", false /* preferSSD */, false /* benchOnly */) err := r.prepareSpec(&c.spec) if !testutils.IsError(err, c.expectedErr) { t.Fatalf("expected %q, but found %q", c.expectedErr, err.Error()) @@ -439,7 +439,7 @@ func runExitCodeTest(t *testing.T, injectedError error) error { } }, }) - tests := testsToRun(ctx, r, registry.NewTestFilter(nil, false)) + tests := testsToRun(r, registry.NewTestFilter(nil, false)) lopt := loggingOpt{ l: nilLogger(), tee: logger.NoTee, diff --git a/pkg/cmd/roachtest/tests/allocation_bench.go b/pkg/cmd/roachtest/tests/allocation_bench.go index a319b4d6cb6d..77e866672b40 100644 --- a/pkg/cmd/roachtest/tests/allocation_bench.go +++ b/pkg/cmd/roachtest/tests/allocation_bench.go @@ -268,8 +268,9 @@ func registerAllocationBench(r registry.Registry) { func registerAllocationBenchSpec(r registry.Registry, allocSpec allocationBenchSpec) { specOptions := []spec.Option{spec.CPU(allocSpec.cpus)} r.Add(registry.TestSpec{ - Name: fmt.Sprintf("allocbench/nodes=%d/cpu=%d/%s", allocSpec.nodes, allocSpec.cpus, allocSpec.load.desc), - Owner: registry.OwnerKV, + Name: fmt.Sprintf("allocbench/nodes=%d/cpu=%d/%s", allocSpec.nodes, allocSpec.cpus, allocSpec.load.desc), + Owner: registry.OwnerKV, + Benchmark: true, Cluster: r.MakeClusterSpec( allocSpec.nodes+1, specOptions..., diff --git a/pkg/cmd/roachtest/tests/allocator.go b/pkg/cmd/roachtest/tests/allocator.go index a1597bb57d13..39c1bc502030 100644 --- a/pkg/cmd/roachtest/tests/allocator.go +++ b/pkg/cmd/roachtest/tests/allocator.go @@ -148,30 +148,33 @@ func registerAllocator(r registry.Registry) { } r.Add(registry.TestSpec{ - Name: `replicate/up/1to3`, - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(4), - Leases: registry.MetamorphicLeases, + Name: `replicate/up/1to3`, + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(4), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runAllocator(ctx, t, c, 1, 10.0) }, }) r.Add(registry.TestSpec{ - Name: `replicate/rebalance/3to5`, - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(6), - Leases: registry.MetamorphicLeases, + Name: `replicate/rebalance/3to5`, + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(6), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runAllocator(ctx, t, c, 3, 42.0) }, }) r.Add(registry.TestSpec{ - Name: `replicate/wide`, - Owner: registry.OwnerKV, - Timeout: 10 * time.Minute, - Cluster: r.MakeClusterSpec(9, spec.CPU(1)), - Leases: registry.MetamorphicLeases, - Run: runWideReplication, + Name: `replicate/wide`, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 10 * time.Minute, + Cluster: r.MakeClusterSpec(9, spec.CPU(1)), + Leases: registry.MetamorphicLeases, + Run: runWideReplication, }) } diff --git a/pkg/cmd/roachtest/tests/backup.go b/pkg/cmd/roachtest/tests/backup.go index 84b80075ca9a..e0c23dc693d8 100644 --- a/pkg/cmd/roachtest/tests/backup.go +++ b/pkg/cmd/roachtest/tests/backup.go @@ -385,6 +385,7 @@ func registerBackup(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("backup/2TB/%s", backup2TBSpec), Owner: registry.OwnerDisasterRecovery, + Benchmark: true, Cluster: backup2TBSpec, EncryptionSupport: registry.EncryptionAlwaysDisabled, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { diff --git a/pkg/cmd/roachtest/tests/cdc.go b/pkg/cmd/roachtest/tests/cdc.go index 274c36ddccba..7a40681cb402 100644 --- a/pkg/cmd/roachtest/tests/cdc.go +++ b/pkg/cmd/roachtest/tests/cdc.go @@ -941,6 +941,7 @@ func registerCDC(r registry.Registry) { r.Add(registry.TestSpec{ Name: "cdc/initial-scan-only", Owner: registry.OwnerCDC, + Benchmark: true, Cluster: r.MakeClusterSpec(4, spec.CPU(16)), RequiresLicense: true, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { diff --git a/pkg/cmd/roachtest/tests/connection_latency.go b/pkg/cmd/roachtest/tests/connection_latency.go index fa937c7ff9c9..a7a01ff34115 100644 --- a/pkg/cmd/roachtest/tests/connection_latency.go +++ b/pkg/cmd/roachtest/tests/connection_latency.go @@ -118,8 +118,9 @@ func registerConnectionLatencyTest(r registry.Registry) { // Single region test. numNodes := 3 r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/certs", numNodes), - Owner: registry.OwnerSQLFoundations, + Name: fmt.Sprintf("connection_latency/nodes=%d/certs", numNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, // Add one more node for load node. Cluster: r.MakeClusterSpec(numNodes+1, spec.Zones(regionUsCentral)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { @@ -134,18 +135,20 @@ func registerConnectionLatencyTest(r registry.Registry) { loadNodes := numZones r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/certs", numMultiRegionNodes), - Owner: registry.OwnerSQLFoundations, - Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/certs", numMultiRegionNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runConnectionLatencyTest(ctx, t, c, numMultiRegionNodes, numZones, false /*password*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/password", numMultiRegionNodes), - Owner: registry.OwnerSQLFoundations, - Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/password", numMultiRegionNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runConnectionLatencyTest(ctx, t, c, numMultiRegionNodes, numZones, true /*password*/) }, diff --git a/pkg/cmd/roachtest/tests/copyfrom.go b/pkg/cmd/roachtest/tests/copyfrom.go index b55a9a92760e..a51f520119a8 100644 --- a/pkg/cmd/roachtest/tests/copyfrom.go +++ b/pkg/cmd/roachtest/tests/copyfrom.go @@ -156,28 +156,31 @@ func registerCopyFrom(r registry.Registry) { for _, tc := range testcases { tc := tc r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/crdb-atomic/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/crdb-atomic/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromCRDB(ctx, t, c, tc.sf, true /*atomic*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/crdb-nonatomic/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/crdb-nonatomic/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromCRDB(ctx, t, c, tc.sf, false /*atomic*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/pg/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/pg/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromPG(ctx, t, c, tc.sf) }, diff --git a/pkg/cmd/roachtest/tests/decommissionbench.go b/pkg/cmd/roachtest/tests/decommissionbench.go index 721bb3923ebf..db34ed58d01e 100644 --- a/pkg/cmd/roachtest/tests/decommissionbench.go +++ b/pkg/cmd/roachtest/tests/decommissionbench.go @@ -287,7 +287,8 @@ func registerDecommissionBenchSpec(r registry.Registry, benchSpec decommissionBe r.Add(registry.TestSpec{ Name: fmt.Sprintf("decommissionBench/nodes=%d/warehouses=%d%s", benchSpec.nodes, benchSpec.warehouses, extraName), - Owner: registry.OwnerKV, + Owner: registry.OwnerKV, + Benchmark: true, Cluster: r.MakeClusterSpec( benchSpec.nodes+addlNodeCount+1, specOptions..., diff --git a/pkg/cmd/roachtest/tests/failover.go b/pkg/cmd/roachtest/tests/failover.go index f3e03f6a35c8..cfdb1de9d907 100644 --- a/pkg/cmd/roachtest/tests/failover.go +++ b/pkg/cmd/roachtest/tests/failover.go @@ -48,6 +48,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: "failover/chaos" + suffix, Owner: registry.OwnerKV, + Benchmark: true, Timeout: 60 * time.Minute, Cluster: r.MakeClusterSpec(10, spec.CPU(4), spec.PreferLocalSSD(false)), SkipPostValidations: registry.PostValidationNoDeadNodes, // cleanup kills nodes @@ -58,30 +59,33 @@ func registerFailover(r registry.Registry) { } r.Add(registry.TestSpec{ - Name: "failover/partial/lease-gateway" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(8, spec.CPU(4)), + Name: "failover/partial/lease-gateway" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(8, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseGateway(ctx, t, c, expirationLeases) }, }) r.Add(registry.TestSpec{ - Name: "failover/partial/lease-leader" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(7, spec.CPU(4)), + Name: "failover/partial/lease-leader" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(7, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseLeader(ctx, t, c, expirationLeases) }, }) r.Add(registry.TestSpec{ - Name: "failover/partial/lease-liveness" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(8, spec.CPU(4)), + Name: "failover/partial/lease-liveness" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(8, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseLiveness(ctx, t, c, expirationLeases) }, @@ -101,6 +105,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/non-system/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(7, spec.CPU(4), spec.PreferLocalSSD(!usePD)), @@ -111,6 +116,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/liveness/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(5, spec.CPU(4), spec.PreferLocalSSD(!usePD)), @@ -121,6 +127,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/system-non-liveness/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(7, spec.CPU(4), spec.PreferLocalSSD(!usePD)), diff --git a/pkg/cmd/roachtest/tests/import.go b/pkg/cmd/roachtest/tests/import.go index 8b07bdbb2509..135e7cabeebc 100644 --- a/pkg/cmd/roachtest/tests/import.go +++ b/pkg/cmd/roachtest/tests/import.go @@ -167,6 +167,7 @@ func registerImportTPCC(r registry.Registry) { r.Add(registry.TestSpec{ Name: testName, Owner: registry.OwnerSQLQueries, + Benchmark: true, Cluster: r.MakeClusterSpec(numNodes), Timeout: timeout, EncryptionSupport: registry.EncryptionMetamorphic, @@ -213,6 +214,7 @@ func registerImportTPCH(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf(`import/tpch/nodes=%d`, item.nodes), Owner: registry.OwnerSQLQueries, + Benchmark: true, Cluster: r.MakeClusterSpec(item.nodes), Timeout: item.timeout, EncryptionSupport: registry.EncryptionMetamorphic, diff --git a/pkg/cmd/roachtest/tests/import_cancellation.go b/pkg/cmd/roachtest/tests/import_cancellation.go index 87cc7f801508..19f2606609c3 100644 --- a/pkg/cmd/roachtest/tests/import_cancellation.go +++ b/pkg/cmd/roachtest/tests/import_cancellation.go @@ -31,11 +31,12 @@ import ( func registerImportCancellation(r registry.Registry) { for _, rangeTombstones := range []bool{true, false} { r.Add(registry.TestSpec{ - Name: fmt.Sprintf(`import-cancellation/rangeTs=%t`, rangeTombstones), - Owner: registry.OwnerDisasterRecovery, - Timeout: 4 * time.Hour, - Cluster: r.MakeClusterSpec(6, spec.CPU(32)), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf(`import-cancellation/rangeTs=%t`, rangeTombstones), + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Timeout: 4 * time.Hour, + Cluster: r.MakeClusterSpec(6, spec.CPU(32)), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runImportCancellation(ctx, t, c, rangeTombstones) }, diff --git a/pkg/cmd/roachtest/tests/indexes.go b/pkg/cmd/roachtest/tests/indexes.go index f72354fd073b..73df4e5f937c 100644 --- a/pkg/cmd/roachtest/tests/indexes.go +++ b/pkg/cmd/roachtest/tests/indexes.go @@ -34,9 +34,10 @@ func registerNIndexes(r registry.Registry, secondaryIndexes int) { } geoZonesStr := strings.Join(geoZones, ",") r.Add(registry.TestSpec{ - Name: fmt.Sprintf("indexes/%d/nodes=%d/multi-region", secondaryIndexes, nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("indexes/%d/nodes=%d/multi-region", secondaryIndexes, nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(geoZonesStr)), // Uses CONFIGURE ZONE USING ... COPY FROM PARENT syntax. Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { firstAZ := geoZones[0] diff --git a/pkg/cmd/roachtest/tests/kv.go b/pkg/cmd/roachtest/tests/kv.go index 2f30cab6971e..864ef2a600f2 100644 --- a/pkg/cmd/roachtest/tests/kv.go +++ b/pkg/cmd/roachtest/tests/kv.go @@ -325,10 +325,11 @@ func registerKV(r registry.Registry) { skip = fmt.Sprintf("multi-store tests are not supported on cloud %s", cSpec.Cloud) } r.Add(registry.TestSpec{ - Skip: skip, - Name: strings.Join(nameParts, "/"), - Owner: owner, - Cluster: cSpec, + Skip: skip, + Name: strings.Join(nameParts, "/"), + Owner: owner, + Benchmark: true, + Cluster: cSpec, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runKV(ctx, t, c, opts) }, @@ -341,10 +342,11 @@ func registerKV(r registry.Registry) { func registerKVContention(r registry.Registry) { const nodes = 4 r.Add(registry.TestSpec{ - Name: fmt.Sprintf("kv/contention/nodes=%d", nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes + 1), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("kv/contention/nodes=%d", nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes + 1), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) diff --git a/pkg/cmd/roachtest/tests/ledger.go b/pkg/cmd/roachtest/tests/ledger.go index 8313a1ecbb22..fbef17403313 100644 --- a/pkg/cmd/roachtest/tests/ledger.go +++ b/pkg/cmd/roachtest/tests/ledger.go @@ -28,9 +28,10 @@ func registerLedger(r registry.Registry) { // https://github.com/cockroachdb/cockroach/issues/66184 const azs = "us-central1-f,us-central1-b,us-central1-c" r.Add(registry.TestSpec{ - Name: fmt.Sprintf("ledger/nodes=%d/multi-az", nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(azs)), + Name: fmt.Sprintf("ledger/nodes=%d/multi-az", nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(azs)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { roachNodes := c.Range(1, nodes) gatewayNodes := c.Range(1, nodes/3) diff --git a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go index d1e9912e295c..50480952541f 100644 --- a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go +++ b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go @@ -71,6 +71,7 @@ func registerLOQRecovery(r registry.Registry) { r.Add(registry.TestSpec{ Name: s.testName(""), Owner: registry.OwnerReplication, + Benchmark: true, Tags: registry.Tags(`default`), Cluster: spec, Leases: registry.MetamorphicLeases, @@ -83,6 +84,7 @@ func registerLOQRecovery(r registry.Registry) { r.Add(registry.TestSpec{ Name: s.testName("half-online"), Owner: registry.OwnerReplication, + Benchmark: true, Tags: registry.Tags(`default`), Cluster: spec, Leases: registry.MetamorphicLeases, diff --git a/pkg/cmd/roachtest/tests/registry.go b/pkg/cmd/roachtest/tests/registry.go index 6c2428198fc5..98c88cc9c76a 100644 --- a/pkg/cmd/roachtest/tests/registry.go +++ b/pkg/cmd/roachtest/tests/registry.go @@ -148,14 +148,3 @@ func RegisterTests(r registry.Registry) { registerDeclarativeSchemaChangerJobCompatibilityInMixedVersion(r) registerTenantSpanStatsMixedVersion(r) } - -// RegisterBenchmarks registers all benchmarks to the registry. This powers `roachtest bench`. -// -// TODO(tbg): it's unclear that `roachtest bench` is that useful, perhaps we make everything -// a roachtest but use a `bench` tag to determine what tests to understand as benchmarks. -func RegisterBenchmarks(r registry.Registry) { - registerIndexesBench(r) - registerTPCCBench(r) - registerKVBench(r) - registerTPCHBench(r) -} diff --git a/pkg/cmd/roachtest/tests/restore.go b/pkg/cmd/roachtest/tests/restore.go index 4b265fe37303..f019f2fb9a34 100644 --- a/pkg/cmd/roachtest/tests/restore.go +++ b/pkg/cmd/roachtest/tests/restore.go @@ -118,11 +118,12 @@ func registerRestore(r registry.Registry) { withPauseSpecs.initTestName() r.Add(registry.TestSpec{ - Name: withPauseSpecs.testName, - Owner: registry.OwnerDisasterRecovery, - Cluster: withPauseSpecs.hardware.makeClusterSpecs(r, withPauseSpecs.backup.cloud), - Timeout: withPauseSpecs.timeout, - Tags: registry.Tags("aws"), + Name: withPauseSpecs.testName, + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Cluster: withPauseSpecs.hardware.makeClusterSpecs(r, withPauseSpecs.backup.cloud), + Timeout: withPauseSpecs.timeout, + Tags: registry.Tags("aws"), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { rd := makeRestoreDriver(t, c, withPauseSpecs) @@ -390,10 +391,11 @@ func registerRestore(r registry.Registry) { sp := sp sp.initTestName() r.Add(registry.TestSpec{ - Name: sp.testName, - Owner: registry.OwnerDisasterRecovery, - Cluster: sp.hardware.makeClusterSpecs(r, sp.backup.cloud), - Timeout: sp.timeout, + Name: sp.testName, + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Cluster: sp.hardware.makeClusterSpecs(r, sp.backup.cloud), + Timeout: sp.timeout, // These tests measure performance. To ensure consistent perf, // disable metamorphic encryption. EncryptionSupport: registry.EncryptionAlwaysDisabled, @@ -724,7 +726,7 @@ func (rd *restoreDriver) prepareCluster(ctx context.Context) { if rd.c.Spec().Cloud != rd.sp.backup.cloud { // For now, only run the test on the cloud provider that also stores the backup. - rd.t.Skip("test configured to run on %s", rd.sp.backup.cloud) + rd.t.Skipf("test configured to run on %s", rd.sp.backup.cloud) } rd.c.Put(ctx, rd.t.Cockroach(), "./cockroach") diff --git a/pkg/cmd/roachtest/tests/schemachange.go b/pkg/cmd/roachtest/tests/schemachange.go index da2a7ed19563..52b3f25fbfd3 100644 --- a/pkg/cmd/roachtest/tests/schemachange.go +++ b/pkg/cmd/roachtest/tests/schemachange.go @@ -307,11 +307,12 @@ func makeIndexAddTpccTest( spec spec.ClusterSpec, warehouses int, length time.Duration, ) registry.TestSpec { return registry.TestSpec{ - Name: fmt.Sprintf("schemachange/index/tpcc/w=%d", warehouses), - Owner: registry.OwnerSQLFoundations, - Cluster: spec, - Leases: registry.MetamorphicLeases, - Timeout: length * 3, + Name: fmt.Sprintf("schemachange/index/tpcc/w=%d", warehouses), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: spec, + Leases: registry.MetamorphicLeases, + Timeout: length * 3, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, @@ -427,11 +428,12 @@ func makeSchemaChangeDuringTPCC( spec spec.ClusterSpec, warehouses int, length time.Duration, ) registry.TestSpec { return registry.TestSpec{ - Name: "schemachange/during/tpcc", - Owner: registry.OwnerSQLFoundations, - Cluster: spec, - Leases: registry.MetamorphicLeases, - Timeout: length * 3, + Name: "schemachange/during/tpcc", + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: spec, + Leases: registry.MetamorphicLeases, + Timeout: length * 3, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, diff --git a/pkg/cmd/roachtest/tests/schemachange_random_load.go b/pkg/cmd/roachtest/tests/schemachange_random_load.go index cd6cb29a9cdb..3e150490c6a7 100644 --- a/pkg/cmd/roachtest/tests/schemachange_random_load.go +++ b/pkg/cmd/roachtest/tests/schemachange_random_load.go @@ -43,8 +43,9 @@ func registerSchemaChangeRandomLoad(r registry.Registry) { } geoZonesStr := strings.Join(geoZones, ",") r.Add(registry.TestSpec{ - Name: "schemachange/random-load", - Owner: registry.OwnerSQLFoundations, + Name: "schemachange/random-load", + Owner: registry.OwnerSQLFoundations, + Benchmark: true, Cluster: r.MakeClusterSpec( 3, spec.Geo(), @@ -91,6 +92,7 @@ func registerRandomLoadBenchSpec(r registry.Registry, b randomLoadBenchSpec) { r.Add(registry.TestSpec{ Name: name, Owner: registry.OwnerSQLFoundations, + Benchmark: true, Cluster: r.MakeClusterSpec(b.Nodes), NativeLibs: registry.LibGEOS, Skip: "https://github.com/cockroachdb/cockroach/issues/56230", diff --git a/pkg/cmd/roachtest/tests/scrub.go b/pkg/cmd/roachtest/tests/scrub.go index 23d5c72ea6aa..404d154b1a83 100644 --- a/pkg/cmd/roachtest/tests/scrub.go +++ b/pkg/cmd/roachtest/tests/scrub.go @@ -54,10 +54,11 @@ func makeScrubTPCCTest( } return registry.TestSpec{ - Name: fmt.Sprintf("scrub/%s/tpcc/w=%d", optionName, warehouses), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(numNodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("scrub/%s/tpcc/w=%d", optionName, warehouses), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(numNodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, diff --git a/pkg/cmd/roachtest/tests/tpcc.go b/pkg/cmd/roachtest/tests/tpcc.go index abf0e63e41b2..5333c7626e29 100644 --- a/pkg/cmd/roachtest/tests/tpcc.go +++ b/pkg/cmd/roachtest/tests/tpcc.go @@ -1112,6 +1112,7 @@ func registerTPCCBenchSpec(r registry.Registry, b tpccBenchSpec) { r.Add(registry.TestSpec{ Name: name, Owner: owner, + Benchmark: true, Cluster: nodes, Timeout: 7 * time.Hour, Tags: b.Tags, diff --git a/pkg/cmd/roachtest/tests/tpch_concurrency.go b/pkg/cmd/roachtest/tests/tpch_concurrency.go index 3808923315af..7a240c1751b7 100644 --- a/pkg/cmd/roachtest/tests/tpch_concurrency.go +++ b/pkg/cmd/roachtest/tests/tpch_concurrency.go @@ -208,20 +208,22 @@ func registerTPCHConcurrency(r registry.Registry) { const timeout = 18 * time.Hour r.Add(registry.TestSpec{ - Name: "tpch_concurrency", - Owner: registry.OwnerSQLQueries, - Timeout: timeout, - Cluster: r.MakeClusterSpec(numNodes), + Name: "tpch_concurrency", + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Timeout: timeout, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHConcurrency(ctx, t, c, false /* disableStreamer */) }, }) r.Add(registry.TestSpec{ - Name: "tpch_concurrency/no_streamer", - Owner: registry.OwnerSQLQueries, - Timeout: timeout, - Cluster: r.MakeClusterSpec(numNodes), + Name: "tpch_concurrency/no_streamer", + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Timeout: timeout, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHConcurrency(ctx, t, c, true /* disableStreamer */) }, diff --git a/pkg/cmd/roachtest/tests/tpchbench.go b/pkg/cmd/roachtest/tests/tpchbench.go index 49f0f9e8501f..d0538b18019e 100644 --- a/pkg/cmd/roachtest/tests/tpchbench.go +++ b/pkg/cmd/roachtest/tests/tpchbench.go @@ -172,9 +172,10 @@ func registerTPCHBenchSpec(r registry.Registry, b tpchBenchSpec) { } r.Add(registry.TestSpec{ - Name: strings.Join(nameParts, "/"), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(numNodes), + Name: strings.Join(nameParts, "/"), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHBench(ctx, t, c, b) }, diff --git a/pkg/cmd/roachtest/tests/ycsb.go b/pkg/cmd/roachtest/tests/ycsb.go index 5d8c12c5fce8..3145b33a08e2 100644 --- a/pkg/cmd/roachtest/tests/ycsb.go +++ b/pkg/cmd/roachtest/tests/ycsb.go @@ -102,9 +102,10 @@ func registerYCSB(r registry.Registry) { } wl, cpus := wl, cpus r.Add(registry.TestSpec{ - Name: name, - Owner: registry.OwnerTestEng, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), + Name: name, + Owner: registry.OwnerTestEng, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, false /* rangeTombstone */) }, @@ -113,9 +114,10 @@ func registerYCSB(r registry.Registry) { if wl == "A" { r.Add(registry.TestSpec{ - Name: fmt.Sprintf("zfs/ycsb/%s/nodes=3/cpu=%d", wl, cpus), - Owner: registry.OwnerStorage, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus), spec.SetFileSystem(spec.Zfs)), + Name: fmt.Sprintf("zfs/ycsb/%s/nodes=3/cpu=%d", wl, cpus), + Owner: registry.OwnerStorage, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus), spec.SetFileSystem(spec.Zfs)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, false /* rangeTombstone */) }, @@ -124,9 +126,10 @@ func registerYCSB(r registry.Registry) { if cpus == cpusWithGlobalMVCCRangeTombstone { r.Add(registry.TestSpec{ - Name: fmt.Sprintf("%s/mvcc-range-keys=global", name), - Owner: registry.OwnerTestEng, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), + Name: fmt.Sprintf("%s/mvcc-range-keys=global", name), + Owner: registry.OwnerTestEng, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, true /* rangeTombstone */) },