diff --git a/pkg/cmd/roachtest/github_test.go b/pkg/cmd/roachtest/github_test.go index 6d62abb3232d..2d69716f4d84 100644 --- a/pkg/cmd/roachtest/github_test.go +++ b/pkg/cmd/roachtest/github_test.go @@ -76,7 +76,7 @@ func TestShouldPost(t *testing.T) { {false, 1, "token", "master", true, ""}, } - reg := makeTestRegistry(spec.GCE, "", "", false) + reg := makeTestRegistry(spec.GCE, "", "", false, false) for _, c := range testCases { t.Setenv("GITHUB_API_TOKEN", c.envGithubAPIToken) t.Setenv("TC_BUILD_BRANCH", c.envTcBuildBranch) @@ -146,7 +146,7 @@ func TestCreatePostRequest(t *testing.T) { {true, false, true, false, otherErr, false, false, nil}, } - reg := makeTestRegistry(spec.GCE, "", "", false) + reg := makeTestRegistry(spec.GCE, "", "", false, false) for _, c := range testCases { clusterSpec := reg.MakeClusterSpec(1) diff --git a/pkg/cmd/roachtest/main.go b/pkg/cmd/roachtest/main.go index 88b0e0aed96b..9901b32bde0a 100644 --- a/pkg/cmd/roachtest/main.go +++ b/pkg/cmd/roachtest/main.go @@ -200,14 +200,10 @@ Examples: roachtest list tag:owner-kv,weekly tag:aws `, RunE: func(_ *cobra.Command, args []string) error { - r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg) - if !listBench { - tests.RegisterTests(&r) - } else { - tests.RegisterBenchmarks(&r) - } + r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg, listBench) + tests.RegisterTests(&r) - matchedTests := r.List(context.Background(), args) + matchedTests := r.List(args) for _, test := range matchedTests { var skip string if test.Skip != "" && !runSkipped { @@ -258,7 +254,7 @@ runner itself. clusterID: clusterID, versionsBinaryOverride: versionsBinaryOverride, enableFIPS: enableFIPS, - }) + }, false /* benchOnly */) }, } @@ -282,7 +278,7 @@ runner itself. if literalArtifacts == "" { literalArtifacts = artifacts } - return runTests(tests.RegisterBenchmarks, cliCfg{ + return runTests(tests.RegisterTests, cliCfg{ args: args, count: count, cpuQuota: cpuQuota, @@ -296,7 +292,7 @@ runner itself. clusterID: clusterID, versionsBinaryOverride: versionsBinaryOverride, enableFIPS: enableFIPS, - }) + }, true /* benchOnly */) }, } @@ -404,11 +400,11 @@ type cliCfg struct { enableFIPS bool } -func runTests(register func(registry.Registry), cfg cliCfg) error { +func runTests(register func(registry.Registry), cfg cliCfg, benchOnly bool) error { if cfg.count <= 0 { return fmt.Errorf("--count (%d) must by greater than 0", cfg.count) } - r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg) + r := makeTestRegistry(cloud, instanceType, zonesF, localSSDArg, benchOnly) // actual registering of tests // TODO: don't register if we can't run on the specified registry cloud @@ -448,7 +444,7 @@ func runTests(register func(registry.Registry), cfg cliCfg) error { return err } - tests := testsToRun(context.Background(), r, filter) + tests := testsToRun(r, filter) n := len(tests) if n*cfg.count < cfg.parallelism { // Don't spin up more workers than necessary. This has particular @@ -594,10 +590,8 @@ func testRunnerLogger( return l, teeOpt } -func testsToRun( - ctx context.Context, r testRegistryImpl, filter *registry.TestFilter, -) []registry.TestSpec { - tests, tagMismatch := r.GetTests(ctx, filter) +func testsToRun(r testRegistryImpl, filter *registry.TestFilter) []registry.TestSpec { + tests, tagMismatch := r.GetTests(filter) var notSkipped []registry.TestSpec for _, s := range tests { diff --git a/pkg/cmd/roachtest/registry/test_spec.go b/pkg/cmd/roachtest/registry/test_spec.go index 406d2963750f..0b71394a2790 100644 --- a/pkg/cmd/roachtest/registry/test_spec.go +++ b/pkg/cmd/roachtest/registry/test_spec.go @@ -46,6 +46,14 @@ type TestSpec struct { // associated cluster expires. The timeout is always truncated to 10m before // the test's cluster expires. Timeout time.Duration + // Denotes whether the test is a roachperf benchmark. If true, the test is expected, but not required, to produce + // artifacts in Test.PerfArtifactsDir(), which get exported to the roachperf dashboard + // (see getPerfArtifacts() in test_runner.go). + // N.B. performance tests may choose to _assert_ on latency, throughput, or some other perf. metric, _without_ + // exporting artifacts to the dashboard. E.g., see 'registerKVContention' and 'verifyTxnPerSecond'. + // N.B. performance tests may have different requirements than correctness tests, e.g., machine type/architecture. + // Thus, they must be opted into explicitly via this field. + Benchmark bool // Tags is a set of tags associated with the test that allow grouping // tests. If no tags are specified, the set ["default"] is automatically // given. diff --git a/pkg/cmd/roachtest/test_registry.go b/pkg/cmd/roachtest/test_registry.go index 4749b96ae324..680a4343b7c4 100644 --- a/pkg/cmd/roachtest/test_registry.go +++ b/pkg/cmd/roachtest/test_registry.go @@ -11,7 +11,6 @@ package main import ( - "context" "fmt" "os" "regexp" @@ -43,11 +42,13 @@ type testRegistryImpl struct { snapshotPrefixes map[string]struct{} promRegistry *prometheus.Registry + // benchOnly is true iff the registry is being used to run benchmarks only. + benchOnly bool } // makeTestRegistry constructs a testRegistryImpl and configures it with opts. func makeTestRegistry( - cloud string, instanceType string, zones string, preferSSD bool, + cloud string, instanceType string, zones string, preferSSD bool, benchOnly bool, ) testRegistryImpl { return testRegistryImpl{ cloud: cloud, @@ -57,6 +58,7 @@ func makeTestRegistry( m: make(map[string]*registry.TestSpec), snapshotPrefixes: make(map[string]struct{}), promRegistry: prometheus.NewRegistry(), + benchOnly: benchOnly, } } @@ -66,6 +68,10 @@ func (r *testRegistryImpl) Add(spec registry.TestSpec) { fmt.Fprintf(os.Stderr, "test %s already registered\n", spec.Name) os.Exit(1) } + if r.benchOnly && !spec.Benchmark { + // Skip non-benchmarks. + return + } if spec.SnapshotPrefix != "" { for existingPrefix := range r.snapshotPrefixes { if strings.HasPrefix(existingPrefix, spec.SnapshotPrefix) { @@ -160,7 +166,7 @@ func (r *testRegistryImpl) PromFactory() promauto.Factory { // Skipped tests are included, and tests that don't match their minVersion spec // are also included but marked as skipped. func (r testRegistryImpl) GetTests( - ctx context.Context, filter *registry.TestFilter, + filter *registry.TestFilter, ) ([]registry.TestSpec, []registry.TestSpec) { var tests []registry.TestSpec var tagMismatch []registry.TestSpec @@ -183,9 +189,9 @@ func (r testRegistryImpl) GetTests( } // List lists tests that match one of the filters. -func (r testRegistryImpl) List(ctx context.Context, filters []string) []registry.TestSpec { +func (r testRegistryImpl) List(filters []string) []registry.TestSpec { filter := registry.NewTestFilter(filters, true) - tests, _ := r.GetTests(ctx, filter) + tests, _ := r.GetTests(filter) sort.Slice(tests, func(i, j int) bool { return tests[i].Name < tests[j].Name }) return tests } diff --git a/pkg/cmd/roachtest/test_registry_test.go b/pkg/cmd/roachtest/test_registry_test.go index 21002046bfd0..719dc5b64bc4 100644 --- a/pkg/cmd/roachtest/test_registry_test.go +++ b/pkg/cmd/roachtest/test_registry_test.go @@ -22,7 +22,7 @@ import ( func TestMakeTestRegistry(t *testing.T) { testutils.RunTrueAndFalse(t, "preferSSD", func(t *testing.T, preferSSD bool) { - r := makeTestRegistry(spec.AWS, "foo", "zone123", preferSSD) + r := makeTestRegistry(spec.AWS, "foo", "zone123", preferSSD, false) require.Equal(t, preferSSD, r.preferSSD) require.Equal(t, "zone123", r.zones) require.Equal(t, "foo", r.instanceType) @@ -48,7 +48,7 @@ func TestMakeTestRegistry(t *testing.T) { // TestPrometheusMetricParser tests that the registry.PromSub() // helper properly converts a string into a metric name that Prometheus can read. func TestPrometheusMetricParser(t *testing.T) { - r := makeTestRegistry(spec.AWS, "foo", "zone123", true) + r := makeTestRegistry(spec.AWS, "foo", "zone123", true, false) f := r.PromFactory() rawName := "restore/nodes=4/duration" diff --git a/pkg/cmd/roachtest/test_runner.go b/pkg/cmd/roachtest/test_runner.go index 7cf8ebb54ee5..08b84d724df9 100644 --- a/pkg/cmd/roachtest/test_runner.go +++ b/pkg/cmd/roachtest/test_runner.go @@ -779,7 +779,9 @@ func (r *testRunner) runWorker( } } else { // Upon success fetch the perf artifacts from the remote hosts. - getPerfArtifacts(ctx, l, c, t) + if t.spec.Benchmark { + getPerfArtifacts(ctx, l, c, t) + } if debugMode == DebugKeepAlways { c.Save(ctx, "cluster saved since --debug-always set", l) c = nil diff --git a/pkg/cmd/roachtest/test_test.go b/pkg/cmd/roachtest/test_test.go index 04bd1c882b6d..ad8dfceb7834 100644 --- a/pkg/cmd/roachtest/test_test.go +++ b/pkg/cmd/roachtest/test_test.go @@ -44,7 +44,7 @@ const defaultParallelism = 10 func mkReg(t *testing.T) testRegistryImpl { t.Helper() - return makeTestRegistry(spec.GCE, "", "", false /* preferSSD */) + return makeTestRegistry(spec.GCE, "", "", false /* preferSSD */, false /* benchOnly */) } func TestMatchOrSkip(t *testing.T) { @@ -248,7 +248,7 @@ type runnerTest struct { func setupRunnerTest(t *testing.T, r testRegistryImpl, testFilters []string) *runnerTest { ctx := context.Background() - tests := testsToRun(ctx, r, registry.NewTestFilter(testFilters, false)) + tests := testsToRun(r, registry.NewTestFilter(testFilters, false)) cr := newClusterRegistry() stopper := stop.NewStopper() @@ -405,7 +405,7 @@ func TestRegistryPrepareSpec(t *testing.T) { } for _, c := range testCases { t.Run("", func(t *testing.T) { - r := makeTestRegistry(spec.GCE, "", "", false /* preferSSD */) + r := makeTestRegistry(spec.GCE, "", "", false /* preferSSD */, false /* benchOnly */) err := r.prepareSpec(&c.spec) if !testutils.IsError(err, c.expectedErr) { t.Fatalf("expected %q, but found %q", c.expectedErr, err.Error()) @@ -439,7 +439,7 @@ func runExitCodeTest(t *testing.T, injectedError error) error { } }, }) - tests := testsToRun(ctx, r, registry.NewTestFilter(nil, false)) + tests := testsToRun(r, registry.NewTestFilter(nil, false)) lopt := loggingOpt{ l: nilLogger(), tee: logger.NoTee, diff --git a/pkg/cmd/roachtest/tests/allocation_bench.go b/pkg/cmd/roachtest/tests/allocation_bench.go index a319b4d6cb6d..77e866672b40 100644 --- a/pkg/cmd/roachtest/tests/allocation_bench.go +++ b/pkg/cmd/roachtest/tests/allocation_bench.go @@ -268,8 +268,9 @@ func registerAllocationBench(r registry.Registry) { func registerAllocationBenchSpec(r registry.Registry, allocSpec allocationBenchSpec) { specOptions := []spec.Option{spec.CPU(allocSpec.cpus)} r.Add(registry.TestSpec{ - Name: fmt.Sprintf("allocbench/nodes=%d/cpu=%d/%s", allocSpec.nodes, allocSpec.cpus, allocSpec.load.desc), - Owner: registry.OwnerKV, + Name: fmt.Sprintf("allocbench/nodes=%d/cpu=%d/%s", allocSpec.nodes, allocSpec.cpus, allocSpec.load.desc), + Owner: registry.OwnerKV, + Benchmark: true, Cluster: r.MakeClusterSpec( allocSpec.nodes+1, specOptions..., diff --git a/pkg/cmd/roachtest/tests/allocator.go b/pkg/cmd/roachtest/tests/allocator.go index a1597bb57d13..39c1bc502030 100644 --- a/pkg/cmd/roachtest/tests/allocator.go +++ b/pkg/cmd/roachtest/tests/allocator.go @@ -148,30 +148,33 @@ func registerAllocator(r registry.Registry) { } r.Add(registry.TestSpec{ - Name: `replicate/up/1to3`, - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(4), - Leases: registry.MetamorphicLeases, + Name: `replicate/up/1to3`, + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(4), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runAllocator(ctx, t, c, 1, 10.0) }, }) r.Add(registry.TestSpec{ - Name: `replicate/rebalance/3to5`, - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(6), - Leases: registry.MetamorphicLeases, + Name: `replicate/rebalance/3to5`, + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(6), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runAllocator(ctx, t, c, 3, 42.0) }, }) r.Add(registry.TestSpec{ - Name: `replicate/wide`, - Owner: registry.OwnerKV, - Timeout: 10 * time.Minute, - Cluster: r.MakeClusterSpec(9, spec.CPU(1)), - Leases: registry.MetamorphicLeases, - Run: runWideReplication, + Name: `replicate/wide`, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 10 * time.Minute, + Cluster: r.MakeClusterSpec(9, spec.CPU(1)), + Leases: registry.MetamorphicLeases, + Run: runWideReplication, }) } diff --git a/pkg/cmd/roachtest/tests/backup.go b/pkg/cmd/roachtest/tests/backup.go index 84b80075ca9a..e0c23dc693d8 100644 --- a/pkg/cmd/roachtest/tests/backup.go +++ b/pkg/cmd/roachtest/tests/backup.go @@ -385,6 +385,7 @@ func registerBackup(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("backup/2TB/%s", backup2TBSpec), Owner: registry.OwnerDisasterRecovery, + Benchmark: true, Cluster: backup2TBSpec, EncryptionSupport: registry.EncryptionAlwaysDisabled, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { diff --git a/pkg/cmd/roachtest/tests/cdc.go b/pkg/cmd/roachtest/tests/cdc.go index 274c36ddccba..7a40681cb402 100644 --- a/pkg/cmd/roachtest/tests/cdc.go +++ b/pkg/cmd/roachtest/tests/cdc.go @@ -941,6 +941,7 @@ func registerCDC(r registry.Registry) { r.Add(registry.TestSpec{ Name: "cdc/initial-scan-only", Owner: registry.OwnerCDC, + Benchmark: true, Cluster: r.MakeClusterSpec(4, spec.CPU(16)), RequiresLicense: true, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { diff --git a/pkg/cmd/roachtest/tests/connection_latency.go b/pkg/cmd/roachtest/tests/connection_latency.go index fa937c7ff9c9..a7a01ff34115 100644 --- a/pkg/cmd/roachtest/tests/connection_latency.go +++ b/pkg/cmd/roachtest/tests/connection_latency.go @@ -118,8 +118,9 @@ func registerConnectionLatencyTest(r registry.Registry) { // Single region test. numNodes := 3 r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/certs", numNodes), - Owner: registry.OwnerSQLFoundations, + Name: fmt.Sprintf("connection_latency/nodes=%d/certs", numNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, // Add one more node for load node. Cluster: r.MakeClusterSpec(numNodes+1, spec.Zones(regionUsCentral)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { @@ -134,18 +135,20 @@ func registerConnectionLatencyTest(r registry.Registry) { loadNodes := numZones r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/certs", numMultiRegionNodes), - Owner: registry.OwnerSQLFoundations, - Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/certs", numMultiRegionNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runConnectionLatencyTest(ctx, t, c, numMultiRegionNodes, numZones, false /*password*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/password", numMultiRegionNodes), - Owner: registry.OwnerSQLFoundations, - Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("connection_latency/nodes=%d/multiregion/password", numMultiRegionNodes), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: r.MakeClusterSpec(numMultiRegionNodes+loadNodes, spec.Geo(), spec.Zones(geoZonesStr)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runConnectionLatencyTest(ctx, t, c, numMultiRegionNodes, numZones, true /*password*/) }, diff --git a/pkg/cmd/roachtest/tests/copyfrom.go b/pkg/cmd/roachtest/tests/copyfrom.go index b55a9a92760e..a51f520119a8 100644 --- a/pkg/cmd/roachtest/tests/copyfrom.go +++ b/pkg/cmd/roachtest/tests/copyfrom.go @@ -156,28 +156,31 @@ func registerCopyFrom(r registry.Registry) { for _, tc := range testcases { tc := tc r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/crdb-atomic/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/crdb-atomic/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromCRDB(ctx, t, c, tc.sf, true /*atomic*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/crdb-nonatomic/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/crdb-nonatomic/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromCRDB(ctx, t, c, tc.sf, false /*atomic*/) }, }) r.Add(registry.TestSpec{ - Name: fmt.Sprintf("copyfrom/pg/sf=%d/nodes=%d", tc.sf, tc.nodes), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(tc.nodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("copyfrom/pg/sf=%d/nodes=%d", tc.sf, tc.nodes), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(tc.nodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runCopyFromPG(ctx, t, c, tc.sf) }, diff --git a/pkg/cmd/roachtest/tests/decommissionbench.go b/pkg/cmd/roachtest/tests/decommissionbench.go index 721bb3923ebf..db34ed58d01e 100644 --- a/pkg/cmd/roachtest/tests/decommissionbench.go +++ b/pkg/cmd/roachtest/tests/decommissionbench.go @@ -287,7 +287,8 @@ func registerDecommissionBenchSpec(r registry.Registry, benchSpec decommissionBe r.Add(registry.TestSpec{ Name: fmt.Sprintf("decommissionBench/nodes=%d/warehouses=%d%s", benchSpec.nodes, benchSpec.warehouses, extraName), - Owner: registry.OwnerKV, + Owner: registry.OwnerKV, + Benchmark: true, Cluster: r.MakeClusterSpec( benchSpec.nodes+addlNodeCount+1, specOptions..., diff --git a/pkg/cmd/roachtest/tests/failover.go b/pkg/cmd/roachtest/tests/failover.go index f3e03f6a35c8..cfdb1de9d907 100644 --- a/pkg/cmd/roachtest/tests/failover.go +++ b/pkg/cmd/roachtest/tests/failover.go @@ -48,6 +48,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: "failover/chaos" + suffix, Owner: registry.OwnerKV, + Benchmark: true, Timeout: 60 * time.Minute, Cluster: r.MakeClusterSpec(10, spec.CPU(4), spec.PreferLocalSSD(false)), SkipPostValidations: registry.PostValidationNoDeadNodes, // cleanup kills nodes @@ -58,30 +59,33 @@ func registerFailover(r registry.Registry) { } r.Add(registry.TestSpec{ - Name: "failover/partial/lease-gateway" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(8, spec.CPU(4)), + Name: "failover/partial/lease-gateway" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(8, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseGateway(ctx, t, c, expirationLeases) }, }) r.Add(registry.TestSpec{ - Name: "failover/partial/lease-leader" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(7, spec.CPU(4)), + Name: "failover/partial/lease-leader" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(7, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseLeader(ctx, t, c, expirationLeases) }, }) r.Add(registry.TestSpec{ - Name: "failover/partial/lease-liveness" + suffix, - Owner: registry.OwnerKV, - Timeout: 30 * time.Minute, - Cluster: r.MakeClusterSpec(8, spec.CPU(4)), + Name: "failover/partial/lease-liveness" + suffix, + Owner: registry.OwnerKV, + Benchmark: true, + Timeout: 30 * time.Minute, + Cluster: r.MakeClusterSpec(8, spec.CPU(4)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runFailoverPartialLeaseLiveness(ctx, t, c, expirationLeases) }, @@ -101,6 +105,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/non-system/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(7, spec.CPU(4), spec.PreferLocalSSD(!usePD)), @@ -111,6 +116,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/liveness/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(5, spec.CPU(4), spec.PreferLocalSSD(!usePD)), @@ -121,6 +127,7 @@ func registerFailover(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf("failover/system-non-liveness/%s%s", failureMode, suffix), Owner: registry.OwnerKV, + Benchmark: true, Timeout: 30 * time.Minute, SkipPostValidations: postValidation, Cluster: r.MakeClusterSpec(7, spec.CPU(4), spec.PreferLocalSSD(!usePD)), diff --git a/pkg/cmd/roachtest/tests/import.go b/pkg/cmd/roachtest/tests/import.go index 8b07bdbb2509..135e7cabeebc 100644 --- a/pkg/cmd/roachtest/tests/import.go +++ b/pkg/cmd/roachtest/tests/import.go @@ -167,6 +167,7 @@ func registerImportTPCC(r registry.Registry) { r.Add(registry.TestSpec{ Name: testName, Owner: registry.OwnerSQLQueries, + Benchmark: true, Cluster: r.MakeClusterSpec(numNodes), Timeout: timeout, EncryptionSupport: registry.EncryptionMetamorphic, @@ -213,6 +214,7 @@ func registerImportTPCH(r registry.Registry) { r.Add(registry.TestSpec{ Name: fmt.Sprintf(`import/tpch/nodes=%d`, item.nodes), Owner: registry.OwnerSQLQueries, + Benchmark: true, Cluster: r.MakeClusterSpec(item.nodes), Timeout: item.timeout, EncryptionSupport: registry.EncryptionMetamorphic, diff --git a/pkg/cmd/roachtest/tests/import_cancellation.go b/pkg/cmd/roachtest/tests/import_cancellation.go index 87cc7f801508..19f2606609c3 100644 --- a/pkg/cmd/roachtest/tests/import_cancellation.go +++ b/pkg/cmd/roachtest/tests/import_cancellation.go @@ -31,11 +31,12 @@ import ( func registerImportCancellation(r registry.Registry) { for _, rangeTombstones := range []bool{true, false} { r.Add(registry.TestSpec{ - Name: fmt.Sprintf(`import-cancellation/rangeTs=%t`, rangeTombstones), - Owner: registry.OwnerDisasterRecovery, - Timeout: 4 * time.Hour, - Cluster: r.MakeClusterSpec(6, spec.CPU(32)), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf(`import-cancellation/rangeTs=%t`, rangeTombstones), + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Timeout: 4 * time.Hour, + Cluster: r.MakeClusterSpec(6, spec.CPU(32)), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runImportCancellation(ctx, t, c, rangeTombstones) }, diff --git a/pkg/cmd/roachtest/tests/indexes.go b/pkg/cmd/roachtest/tests/indexes.go index f72354fd073b..73df4e5f937c 100644 --- a/pkg/cmd/roachtest/tests/indexes.go +++ b/pkg/cmd/roachtest/tests/indexes.go @@ -34,9 +34,10 @@ func registerNIndexes(r registry.Registry, secondaryIndexes int) { } geoZonesStr := strings.Join(geoZones, ",") r.Add(registry.TestSpec{ - Name: fmt.Sprintf("indexes/%d/nodes=%d/multi-region", secondaryIndexes, nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(geoZonesStr)), + Name: fmt.Sprintf("indexes/%d/nodes=%d/multi-region", secondaryIndexes, nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(geoZonesStr)), // Uses CONFIGURE ZONE USING ... COPY FROM PARENT syntax. Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { firstAZ := geoZones[0] diff --git a/pkg/cmd/roachtest/tests/kv.go b/pkg/cmd/roachtest/tests/kv.go index 2f30cab6971e..864ef2a600f2 100644 --- a/pkg/cmd/roachtest/tests/kv.go +++ b/pkg/cmd/roachtest/tests/kv.go @@ -325,10 +325,11 @@ func registerKV(r registry.Registry) { skip = fmt.Sprintf("multi-store tests are not supported on cloud %s", cSpec.Cloud) } r.Add(registry.TestSpec{ - Skip: skip, - Name: strings.Join(nameParts, "/"), - Owner: owner, - Cluster: cSpec, + Skip: skip, + Name: strings.Join(nameParts, "/"), + Owner: owner, + Benchmark: true, + Cluster: cSpec, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runKV(ctx, t, c, opts) }, @@ -341,10 +342,11 @@ func registerKV(r registry.Registry) { func registerKVContention(r registry.Registry) { const nodes = 4 r.Add(registry.TestSpec{ - Name: fmt.Sprintf("kv/contention/nodes=%d", nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes + 1), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("kv/contention/nodes=%d", nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes + 1), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { c.Put(ctx, t.Cockroach(), "./cockroach", c.Range(1, nodes)) c.Put(ctx, t.DeprecatedWorkload(), "./workload", c.Node(nodes+1)) diff --git a/pkg/cmd/roachtest/tests/ledger.go b/pkg/cmd/roachtest/tests/ledger.go index 8313a1ecbb22..fbef17403313 100644 --- a/pkg/cmd/roachtest/tests/ledger.go +++ b/pkg/cmd/roachtest/tests/ledger.go @@ -28,9 +28,10 @@ func registerLedger(r registry.Registry) { // https://github.com/cockroachdb/cockroach/issues/66184 const azs = "us-central1-f,us-central1-b,us-central1-c" r.Add(registry.TestSpec{ - Name: fmt.Sprintf("ledger/nodes=%d/multi-az", nodes), - Owner: registry.OwnerKV, - Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(azs)), + Name: fmt.Sprintf("ledger/nodes=%d/multi-az", nodes), + Owner: registry.OwnerKV, + Benchmark: true, + Cluster: r.MakeClusterSpec(nodes+1, spec.CPU(16), spec.Geo(), spec.Zones(azs)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { roachNodes := c.Range(1, nodes) gatewayNodes := c.Range(1, nodes/3) diff --git a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go index d1e9912e295c..50480952541f 100644 --- a/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go +++ b/pkg/cmd/roachtest/tests/loss_of_quorum_recovery.go @@ -71,6 +71,7 @@ func registerLOQRecovery(r registry.Registry) { r.Add(registry.TestSpec{ Name: s.testName(""), Owner: registry.OwnerReplication, + Benchmark: true, Tags: registry.Tags(`default`), Cluster: spec, Leases: registry.MetamorphicLeases, @@ -83,6 +84,7 @@ func registerLOQRecovery(r registry.Registry) { r.Add(registry.TestSpec{ Name: s.testName("half-online"), Owner: registry.OwnerReplication, + Benchmark: true, Tags: registry.Tags(`default`), Cluster: spec, Leases: registry.MetamorphicLeases, diff --git a/pkg/cmd/roachtest/tests/registry.go b/pkg/cmd/roachtest/tests/registry.go index 6c2428198fc5..98c88cc9c76a 100644 --- a/pkg/cmd/roachtest/tests/registry.go +++ b/pkg/cmd/roachtest/tests/registry.go @@ -148,14 +148,3 @@ func RegisterTests(r registry.Registry) { registerDeclarativeSchemaChangerJobCompatibilityInMixedVersion(r) registerTenantSpanStatsMixedVersion(r) } - -// RegisterBenchmarks registers all benchmarks to the registry. This powers `roachtest bench`. -// -// TODO(tbg): it's unclear that `roachtest bench` is that useful, perhaps we make everything -// a roachtest but use a `bench` tag to determine what tests to understand as benchmarks. -func RegisterBenchmarks(r registry.Registry) { - registerIndexesBench(r) - registerTPCCBench(r) - registerKVBench(r) - registerTPCHBench(r) -} diff --git a/pkg/cmd/roachtest/tests/restore.go b/pkg/cmd/roachtest/tests/restore.go index 4b265fe37303..f019f2fb9a34 100644 --- a/pkg/cmd/roachtest/tests/restore.go +++ b/pkg/cmd/roachtest/tests/restore.go @@ -118,11 +118,12 @@ func registerRestore(r registry.Registry) { withPauseSpecs.initTestName() r.Add(registry.TestSpec{ - Name: withPauseSpecs.testName, - Owner: registry.OwnerDisasterRecovery, - Cluster: withPauseSpecs.hardware.makeClusterSpecs(r, withPauseSpecs.backup.cloud), - Timeout: withPauseSpecs.timeout, - Tags: registry.Tags("aws"), + Name: withPauseSpecs.testName, + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Cluster: withPauseSpecs.hardware.makeClusterSpecs(r, withPauseSpecs.backup.cloud), + Timeout: withPauseSpecs.timeout, + Tags: registry.Tags("aws"), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { rd := makeRestoreDriver(t, c, withPauseSpecs) @@ -390,10 +391,11 @@ func registerRestore(r registry.Registry) { sp := sp sp.initTestName() r.Add(registry.TestSpec{ - Name: sp.testName, - Owner: registry.OwnerDisasterRecovery, - Cluster: sp.hardware.makeClusterSpecs(r, sp.backup.cloud), - Timeout: sp.timeout, + Name: sp.testName, + Owner: registry.OwnerDisasterRecovery, + Benchmark: true, + Cluster: sp.hardware.makeClusterSpecs(r, sp.backup.cloud), + Timeout: sp.timeout, // These tests measure performance. To ensure consistent perf, // disable metamorphic encryption. EncryptionSupport: registry.EncryptionAlwaysDisabled, @@ -724,7 +726,7 @@ func (rd *restoreDriver) prepareCluster(ctx context.Context) { if rd.c.Spec().Cloud != rd.sp.backup.cloud { // For now, only run the test on the cloud provider that also stores the backup. - rd.t.Skip("test configured to run on %s", rd.sp.backup.cloud) + rd.t.Skipf("test configured to run on %s", rd.sp.backup.cloud) } rd.c.Put(ctx, rd.t.Cockroach(), "./cockroach") diff --git a/pkg/cmd/roachtest/tests/schemachange.go b/pkg/cmd/roachtest/tests/schemachange.go index da2a7ed19563..52b3f25fbfd3 100644 --- a/pkg/cmd/roachtest/tests/schemachange.go +++ b/pkg/cmd/roachtest/tests/schemachange.go @@ -307,11 +307,12 @@ func makeIndexAddTpccTest( spec spec.ClusterSpec, warehouses int, length time.Duration, ) registry.TestSpec { return registry.TestSpec{ - Name: fmt.Sprintf("schemachange/index/tpcc/w=%d", warehouses), - Owner: registry.OwnerSQLFoundations, - Cluster: spec, - Leases: registry.MetamorphicLeases, - Timeout: length * 3, + Name: fmt.Sprintf("schemachange/index/tpcc/w=%d", warehouses), + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: spec, + Leases: registry.MetamorphicLeases, + Timeout: length * 3, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, @@ -427,11 +428,12 @@ func makeSchemaChangeDuringTPCC( spec spec.ClusterSpec, warehouses int, length time.Duration, ) registry.TestSpec { return registry.TestSpec{ - Name: "schemachange/during/tpcc", - Owner: registry.OwnerSQLFoundations, - Cluster: spec, - Leases: registry.MetamorphicLeases, - Timeout: length * 3, + Name: "schemachange/during/tpcc", + Owner: registry.OwnerSQLFoundations, + Benchmark: true, + Cluster: spec, + Leases: registry.MetamorphicLeases, + Timeout: length * 3, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, diff --git a/pkg/cmd/roachtest/tests/schemachange_random_load.go b/pkg/cmd/roachtest/tests/schemachange_random_load.go index cd6cb29a9cdb..3e150490c6a7 100644 --- a/pkg/cmd/roachtest/tests/schemachange_random_load.go +++ b/pkg/cmd/roachtest/tests/schemachange_random_load.go @@ -43,8 +43,9 @@ func registerSchemaChangeRandomLoad(r registry.Registry) { } geoZonesStr := strings.Join(geoZones, ",") r.Add(registry.TestSpec{ - Name: "schemachange/random-load", - Owner: registry.OwnerSQLFoundations, + Name: "schemachange/random-load", + Owner: registry.OwnerSQLFoundations, + Benchmark: true, Cluster: r.MakeClusterSpec( 3, spec.Geo(), @@ -91,6 +92,7 @@ func registerRandomLoadBenchSpec(r registry.Registry, b randomLoadBenchSpec) { r.Add(registry.TestSpec{ Name: name, Owner: registry.OwnerSQLFoundations, + Benchmark: true, Cluster: r.MakeClusterSpec(b.Nodes), NativeLibs: registry.LibGEOS, Skip: "https://github.com/cockroachdb/cockroach/issues/56230", diff --git a/pkg/cmd/roachtest/tests/scrub.go b/pkg/cmd/roachtest/tests/scrub.go index 23d5c72ea6aa..404d154b1a83 100644 --- a/pkg/cmd/roachtest/tests/scrub.go +++ b/pkg/cmd/roachtest/tests/scrub.go @@ -54,10 +54,11 @@ func makeScrubTPCCTest( } return registry.TestSpec{ - Name: fmt.Sprintf("scrub/%s/tpcc/w=%d", optionName, warehouses), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(numNodes), - Leases: registry.MetamorphicLeases, + Name: fmt.Sprintf("scrub/%s/tpcc/w=%d", optionName, warehouses), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(numNodes), + Leases: registry.MetamorphicLeases, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCC(ctx, t, c, tpccOptions{ Warehouses: warehouses, diff --git a/pkg/cmd/roachtest/tests/tpcc.go b/pkg/cmd/roachtest/tests/tpcc.go index abf0e63e41b2..5333c7626e29 100644 --- a/pkg/cmd/roachtest/tests/tpcc.go +++ b/pkg/cmd/roachtest/tests/tpcc.go @@ -1112,6 +1112,7 @@ func registerTPCCBenchSpec(r registry.Registry, b tpccBenchSpec) { r.Add(registry.TestSpec{ Name: name, Owner: owner, + Benchmark: true, Cluster: nodes, Timeout: 7 * time.Hour, Tags: b.Tags, diff --git a/pkg/cmd/roachtest/tests/tpch_concurrency.go b/pkg/cmd/roachtest/tests/tpch_concurrency.go index 3808923315af..7a240c1751b7 100644 --- a/pkg/cmd/roachtest/tests/tpch_concurrency.go +++ b/pkg/cmd/roachtest/tests/tpch_concurrency.go @@ -208,20 +208,22 @@ func registerTPCHConcurrency(r registry.Registry) { const timeout = 18 * time.Hour r.Add(registry.TestSpec{ - Name: "tpch_concurrency", - Owner: registry.OwnerSQLQueries, - Timeout: timeout, - Cluster: r.MakeClusterSpec(numNodes), + Name: "tpch_concurrency", + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Timeout: timeout, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHConcurrency(ctx, t, c, false /* disableStreamer */) }, }) r.Add(registry.TestSpec{ - Name: "tpch_concurrency/no_streamer", - Owner: registry.OwnerSQLQueries, - Timeout: timeout, - Cluster: r.MakeClusterSpec(numNodes), + Name: "tpch_concurrency/no_streamer", + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Timeout: timeout, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHConcurrency(ctx, t, c, true /* disableStreamer */) }, diff --git a/pkg/cmd/roachtest/tests/tpchbench.go b/pkg/cmd/roachtest/tests/tpchbench.go index 49f0f9e8501f..d0538b18019e 100644 --- a/pkg/cmd/roachtest/tests/tpchbench.go +++ b/pkg/cmd/roachtest/tests/tpchbench.go @@ -172,9 +172,10 @@ func registerTPCHBenchSpec(r registry.Registry, b tpchBenchSpec) { } r.Add(registry.TestSpec{ - Name: strings.Join(nameParts, "/"), - Owner: registry.OwnerSQLQueries, - Cluster: r.MakeClusterSpec(numNodes), + Name: strings.Join(nameParts, "/"), + Owner: registry.OwnerSQLQueries, + Benchmark: true, + Cluster: r.MakeClusterSpec(numNodes), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runTPCHBench(ctx, t, c, b) }, diff --git a/pkg/cmd/roachtest/tests/ycsb.go b/pkg/cmd/roachtest/tests/ycsb.go index 5d8c12c5fce8..3145b33a08e2 100644 --- a/pkg/cmd/roachtest/tests/ycsb.go +++ b/pkg/cmd/roachtest/tests/ycsb.go @@ -102,9 +102,10 @@ func registerYCSB(r registry.Registry) { } wl, cpus := wl, cpus r.Add(registry.TestSpec{ - Name: name, - Owner: registry.OwnerTestEng, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), + Name: name, + Owner: registry.OwnerTestEng, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, false /* rangeTombstone */) }, @@ -113,9 +114,10 @@ func registerYCSB(r registry.Registry) { if wl == "A" { r.Add(registry.TestSpec{ - Name: fmt.Sprintf("zfs/ycsb/%s/nodes=3/cpu=%d", wl, cpus), - Owner: registry.OwnerStorage, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus), spec.SetFileSystem(spec.Zfs)), + Name: fmt.Sprintf("zfs/ycsb/%s/nodes=3/cpu=%d", wl, cpus), + Owner: registry.OwnerStorage, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus), spec.SetFileSystem(spec.Zfs)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, false /* rangeTombstone */) }, @@ -124,9 +126,10 @@ func registerYCSB(r registry.Registry) { if cpus == cpusWithGlobalMVCCRangeTombstone { r.Add(registry.TestSpec{ - Name: fmt.Sprintf("%s/mvcc-range-keys=global", name), - Owner: registry.OwnerTestEng, - Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), + Name: fmt.Sprintf("%s/mvcc-range-keys=global", name), + Owner: registry.OwnerTestEng, + Benchmark: true, + Cluster: r.MakeClusterSpec(4, spec.CPU(cpus)), Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { runYCSB(ctx, t, c, wl, cpus, true /* rangeTombstone */) },