From 5bfdd0eb0c5b23a0482453ee4a5a88f6d0ee94fb Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Sat, 1 Oct 2022 19:01:05 -0400 Subject: [PATCH 1/6] roachprod: skip login step for `roachprod grafana-start` Release note: None --- pkg/roachprod/prometheus/prometheus.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/roachprod/prometheus/prometheus.go b/pkg/roachprod/prometheus/prometheus.go index adc994e51f72..31e0009c32db 100644 --- a/pkg/roachprod/prometheus/prometheus.go +++ b/pkg/roachprod/prometheus/prometheus.go @@ -294,12 +294,12 @@ sudo apt-get update -qqy && sudo apt-get install -qqy grafana-enterprise && sudo if err := c.RepeatRun(ctx, l, os.Stdout, os.Stderr, cfg.PrometheusNode, "permissions", - `sudo chmod 777 /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards /var/lib/grafana/dashboards`, + `sudo chmod 777 /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards /var/lib/grafana/dashboards /etc/grafana/grafana.ini`, ); err != nil { return nil, err } - // Set up grafana config + // Set up grafana config. if err := c.PutString(ctx, l, cfg.PrometheusNode, `apiVersion: 1 datasources: @@ -310,7 +310,6 @@ datasources: `, "/etc/grafana/provisioning/datasources/prometheus.yaml", 0777); err != nil { return nil, err } - if err := c.PutString(ctx, l, cfg.PrometheusNode, `apiVersion: 1 providers: @@ -324,6 +323,15 @@ providers: `, "/etc/grafana/provisioning/dashboards/cockroach.yaml", 0777); err != nil { return nil, err } + if err := c.PutString(ctx, l, cfg.PrometheusNode, ` +[auth.anonymous] +enabled = true +org_role = Admin +`, + "/etc/grafana/grafana.ini", 0777); err != nil { + return nil, err + } + for idx, u := range cfg.Grafana.DashboardURLs { cmd := fmt.Sprintf("curl -fsSL %s -o /var/lib/grafana/dashboards/%d.json", u, idx) if err := c.Run(ctx, l, os.Stdout, os.Stderr, cfg.PrometheusNode, "download dashboard", From 15aa7558600ff1b2680ca2f1c692f079aadb6edf Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Sun, 2 Oct 2022 07:48:03 -0400 Subject: [PATCH 2/6] roachtest: introduce a `grafana-dump` subcommand This lets one create data dumps off of roachprod clusters that have been collecting data using `roachprod grafana-start`. To use: roachprod grafana-dump $CLUSTER --dump-dir ./ ./prometheus-docker-run.sh The latter sets up a prometheus server on localhost:9090. To point your grafana server to it, you can: brew install grafana brew services start grafana # navigate to localhost:3000 --- pkg/cmd/roachprod/flags.go | 8 ++----- pkg/cmd/roachprod/main.go | 22 +++++++++++++------ pkg/roachprod/prometheus/prometheus.go | 3 ++- pkg/roachprod/roachprod.go | 29 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go index 248e6681a926..09319c6bd66e 100644 --- a/pkg/cmd/roachprod/flags.go +++ b/pkg/cmd/roachprod/flags.go @@ -235,18 +235,14 @@ func initFlags() { cachedHostsCmd.Flags().StringVar(&cachedHostsCluster, "cluster", "", "print hosts matching cluster") - // TODO (msbutler): this flag should instead point to a relative file path that's check into - // the repo, not some random URL. grafanaStartCmd.Flags().StringVar(&grafanaConfig, "grafana-config", "", "URL to grafana json config") grafanaURLCmd.Flags().BoolVar(&grafanaurlOpen, "open", false, "open the grafana dashboard url on the browser") - grafanaStopCmd.Flags().StringVar(&grafanaDumpDir, "dump-dir", "", - "the absolute path, on the machine running roachprod, to dump prometheus data to.\n"+ - "In the dump-dir, the 'prometheus-docker-run.sh' script spins up a prometheus UI accessible on \n"+ - " 0.0.0.0:9090. If dump-dir is empty, no data will get dumped.") + grafanaDumpCmd.Flags().StringVar(&grafanaDumpDir, "dump-dir", "", + "the absolute path to dump prometheus data to (use the contained 'prometheus-docker-run.sh' to visualize") for _, cmd := range []*cobra.Command{createCmd, destroyCmd, extendCmd, logsCmd} { cmd.Flags().StringVarP(&username, "username", "u", os.Getenv("ROACHPROD_USER"), diff --git a/pkg/cmd/roachprod/main.go b/pkg/cmd/roachprod/main.go index b8de605e2803..9f9dc03816e3 100644 --- a/pkg/cmd/roachprod/main.go +++ b/pkg/cmd/roachprod/main.go @@ -900,10 +900,8 @@ var getProvidersCmd = &cobra.Command{ var grafanaStartCmd = &cobra.Command{ Use: `grafana-start `, - Short: `spins up a prometheus and grafana instances on the last node in the cluster`, - Long: `spins up a prometheus and grafana instances on the highest numbered node in the cluster -and will scrape from all nodes in the cluster`, - Args: cobra.ExactArgs(1), + Short: `spins up a prometheus and grafana instance on the last node in the cluster`, + Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { return roachprod.StartGrafana(context.Background(), roachprodLibraryLogger, args[0], grafanaConfig, nil) @@ -913,10 +911,21 @@ and will scrape from all nodes in the cluster`, var grafanaStopCmd = &cobra.Command{ Use: `grafana-stop `, Short: `spins down prometheus and grafana instances on the last node in the cluster`, - Long: `spins down the prometheus and grafana instances on the last node in the cluster`, Args: cobra.ExactArgs(1), Run: wrap(func(cmd *cobra.Command, args []string) error { - return roachprod.StopGrafana(context.Background(), roachprodLibraryLogger, args[0], grafanaDumpDir) + return roachprod.StopGrafana(context.Background(), roachprodLibraryLogger, args[0], "") + }), +} + +var grafanaDumpCmd = &cobra.Command{ + Use: `grafana-dump `, + Short: `dump prometheus data to the specified directory`, + Args: cobra.ExactArgs(1), + Run: wrap(func(cmd *cobra.Command, args []string) error { + if grafanaDumpDir == "" { + return errors.New("--dump-dir unspecified") + } + return roachprod.PrometheusSnapshot(context.Background(), roachprodLibraryLogger, args[0], grafanaDumpDir) }), } @@ -990,6 +999,7 @@ func main() { getProvidersCmd, grafanaStartCmd, grafanaStopCmd, + grafanaDumpCmd, grafanaURLCmd, ) setBashCompletionFunction() diff --git a/pkg/roachprod/prometheus/prometheus.go b/pkg/roachprod/prometheus/prometheus.go index 31e0009c32db..9624bf6ca741 100644 --- a/pkg/roachprod/prometheus/prometheus.go +++ b/pkg/roachprod/prometheus/prometheus.go @@ -367,7 +367,7 @@ func Snapshot( os.Stderr, promNode, "prometheus snapshot", - `curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot && + `sudo rm -rf /tmp/prometheus/data/snapshots/* && curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot && cd /tmp/prometheus && tar cvf prometheus-snapshot.tar.gz data/snapshots`, ); err != nil { return err @@ -375,6 +375,7 @@ func Snapshot( if err := os.WriteFile(filepath.Join(dir, "prometheus-docker-run.sh"), []byte(`#!/bin/sh set -eu +rm -rf data/snapshots tar xf prometheus-snapshot.tar.gz snapdir=$(find data/snapshots -mindepth 1 -maxdepth 1 -type d) promyml=$(mktemp) diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index 1382f4b9bb93..9fd4fd0970e5 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -1457,3 +1457,32 @@ func GrafanaURL( } return urlGenerator(c, l, grafanaNode, uConfig) } + +// PrometheusSnapshot takes a snapshot of prometheus and stores the snapshot and +// a script to spin up a docker instance for it to the given directory. We +// assume the last node contains the prometheus server. +func PrometheusSnapshot( + ctx context.Context, l *logger.Logger, clusterName string, dumpDir string, +) error { + if err := LoadClusters(); err != nil { + return err + } + c, err := newCluster(l, clusterName) + if err != nil { + return err + } + nodes, err := install.ListNodes("all", len(c.VMs)) + if err != nil { + return err + } + + promNode := install.Nodes{nodes[len(nodes)-1]} + + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + if err := prometheus.Snapshot(ctx, c, l, promNode, dumpDir); err != nil { + l.Printf("failed to get prometheus snapshot: %v", err) + return err + } + return nil +} From 850fd7475204a22a9e4f0f629acab6ac9cfb1962 Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Sun, 2 Oct 2022 07:47:50 -0400 Subject: [PATCH 3/6] prometheus: allow editing pre-loaded grafana dashboards It's possible to pre-load a roachprod grafana instance with specific dashboards using --grafana-config. Consider the following for ex. roachprod grafana-start $CLUSTER \ --grafana-config http://go.crdb.dev/p/snapshot-admission-control-grafana By default these dashboards are not editable. Make sure they are. Release note: None --- pkg/roachprod/prometheus/prometheus.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/roachprod/prometheus/prometheus.go b/pkg/roachprod/prometheus/prometheus.go index 9624bf6ca741..ab9278586eef 100644 --- a/pkg/roachprod/prometheus/prometheus.go +++ b/pkg/roachprod/prometheus/prometheus.go @@ -318,6 +318,7 @@ providers: folder: '' folderUid: '' type: file + allowUiUpdates: true options: path: /var/lib/grafana/dashboards `, "/etc/grafana/provisioning/dashboards/cockroach.yaml", 0777); err != nil { From d7d7d851eaf0f3a36ed718aaaaae2a51eea59206 Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Sun, 2 Oct 2022 14:16:46 -0400 Subject: [PATCH 4/6] roachtest: remove admission param from decomm tests All variants set it to true so it's not teaching as much. We don't need/want to run decommissioning variants with AC being switched off; we don't foresee switching it off in production clusters either. Release note: None --- .../admission_control_multi_store_overload.go | 2 +- pkg/cmd/roachtest/tests/decommissionbench.go | 104 ++++++++---------- .../roachtest/tests/multitenant_fairness.go | 2 +- pkg/cmd/roachtest/tests/util.go | 4 +- 4 files changed, 49 insertions(+), 63 deletions(-) diff --git a/pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go b/pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go index 518c1c0ddcc8..8626753c2d82 100644 --- a/pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go +++ b/pkg/cmd/roachtest/tests/admission_control_multi_store_overload.go @@ -53,7 +53,7 @@ func registerMultiStoreOverload(r registry.Registry) { } // Defensive, since admission control is enabled by default. This test can // fail if admission control is disabled. - SetAdmissionControl(ctx, t, c, true) + setAdmissionControl(ctx, t, c, true) if _, err := db.ExecContext(ctx, "SET CLUSTER SETTING kv.range_split.by_load_enabled = 'false'"); err != nil { t.Fatalf("failed to disable load based splitting: %v", err) diff --git a/pkg/cmd/roachtest/tests/decommissionbench.go b/pkg/cmd/roachtest/tests/decommissionbench.go index 7c4321ad62be..31fd83f947f2 100644 --- a/pkg/cmd/roachtest/tests/decommissionbench.go +++ b/pkg/cmd/roachtest/tests/decommissionbench.go @@ -60,14 +60,13 @@ const ( ) type decommissionBenchSpec struct { - nodes int - cpus int - warehouses int - load bool - admissionControl bool - multistore bool - snapshotRate int - duration time.Duration + nodes int + cpus int + warehouses int + load bool + multistore bool + snapshotRate int + duration time.Duration // When true, the test will attempt to stop the node prior to decommission. whileDown bool @@ -95,34 +94,30 @@ func registerDecommissionBench(r registry.Registry) { for _, benchSpec := range []decommissionBenchSpec{ // Basic benchmark configurations, to be run nightly. { - nodes: 4, - cpus: 16, - warehouses: 1000, - load: true, - admissionControl: true, + nodes: 4, + cpus: 16, + warehouses: 1000, + load: true, }, { - nodes: 4, - cpus: 16, - warehouses: 1000, - load: true, - admissionControl: true, - duration: 1 * time.Hour, + nodes: 4, + cpus: 16, + warehouses: 1000, + load: true, + duration: 1 * time.Hour, }, { - nodes: 4, - cpus: 16, - warehouses: 1000, - load: true, - admissionControl: true, - whileDown: true, + nodes: 4, + cpus: 16, + warehouses: 1000, + load: true, + whileDown: true, }, { - nodes: 8, - cpus: 16, - warehouses: 3000, - load: true, - admissionControl: true, + nodes: 8, + cpus: 16, + warehouses: 3000, + load: true, // This test can take nearly an hour to import and achieve balance, so // we extend the timeout to let it complete. timeout: 4 * time.Hour, @@ -134,7 +129,6 @@ func registerDecommissionBench(r registry.Registry) { cpus: 16, warehouses: 3000, load: true, - admissionControl: true, whileUpreplicating: true, // This test can take nearly an hour to import and achieve balance, so // we extend the timeout to let it complete. @@ -143,12 +137,11 @@ func registerDecommissionBench(r registry.Registry) { }, { // Drain before decommission, without adding a new node. - nodes: 8, - cpus: 16, - warehouses: 3000, - load: true, - admissionControl: true, - drainFirst: true, + nodes: 8, + cpus: 16, + warehouses: 3000, + load: true, + drainFirst: true, // This test can take nearly an hour to import and achieve balance, so // we extend the timeout to let it complete. timeout: 4 * time.Hour, @@ -160,7 +153,6 @@ func registerDecommissionBench(r registry.Registry) { cpus: 16, warehouses: 3000, load: true, - admissionControl: true, whileUpreplicating: true, drainFirst: true, // This test can take nearly an hour to import and achieve balance, so @@ -169,13 +161,12 @@ func registerDecommissionBench(r registry.Registry) { skip: manualBenchmarkingOnly, }, { - nodes: 4, - cpus: 16, - warehouses: 1000, - load: true, - admissionControl: true, - drainFirst: true, - skip: manualBenchmarkingOnly, + nodes: 4, + cpus: 16, + warehouses: 1000, + load: true, + drainFirst: true, + skip: manualBenchmarkingOnly, }, { nodes: 4, @@ -197,12 +188,11 @@ func registerDecommissionBench(r registry.Registry) { skip: manualBenchmarkingOnly, }, { - nodes: 12, - cpus: 16, - warehouses: 3000, - load: true, - admissionControl: true, - multistore: true, + nodes: 12, + cpus: 16, + warehouses: 3000, + load: true, + multistore: true, // This test can take nearly an hour to import and achieve balance, so // we extend the timeout to let it complete. timeout: 3 * time.Hour, @@ -210,11 +200,10 @@ func registerDecommissionBench(r registry.Registry) { }, { // Test to compare 12 4-store nodes vs 48 single-store nodes - nodes: 48, - cpus: 16, - warehouses: 3000, - load: true, - admissionControl: true, + nodes: 48, + cpus: 16, + warehouses: 3000, + load: true, // This test can take nearly an hour to import and achieve balance, so // we extend the timeout to let it complete. timeout: 3 * time.Hour, @@ -268,8 +257,6 @@ func registerDecommissionBenchSpec(r registry.Registry, benchSpec decommissionBe if benchSpec.slowWrites { extraNameParts = append(extraNameParts, "hi-read-amp") - } else if !benchSpec.admissionControl { - extraNameParts = append(extraNameParts, "no-admission") } if benchSpec.duration > 0 { @@ -382,7 +369,6 @@ func setupDecommissionBench( t.Status(fmt.Sprintf("initializing cluster with %d warehouses", benchSpec.warehouses)) c.Run(ctx, c.Node(pinnedNode), importCmd) - SetAdmissionControl(ctx, t, c, benchSpec.admissionControl) { db := c.Conn(ctx, t.L(), pinnedNode) defer db.Close() diff --git a/pkg/cmd/roachtest/tests/multitenant_fairness.go b/pkg/cmd/roachtest/tests/multitenant_fairness.go index f447b93399f8..7ea6474cc8c5 100644 --- a/pkg/cmd/roachtest/tests/multitenant_fairness.go +++ b/pkg/cmd/roachtest/tests/multitenant_fairness.go @@ -144,7 +144,7 @@ func runMultiTenantFairness( c.Put(ctx, t.Cockroach(), "./cockroach") c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(install.SecureOption(true)), c.Node(1)) - SetAdmissionControl(ctx, t, c, s.acEnabled) + setAdmissionControl(ctx, t, c, s.acEnabled) setRateLimit := func(ctx context.Context, val int, node int) { db := c.Conn(ctx, t.L(), node) diff --git a/pkg/cmd/roachtest/tests/util.go b/pkg/cmd/roachtest/tests/util.go index dc0221584266..71ab5f628b28 100644 --- a/pkg/cmd/roachtest/tests/util.go +++ b/pkg/cmd/roachtest/tests/util.go @@ -105,9 +105,9 @@ func WaitForUpdatedReplicationReport(ctx context.Context, t test.Test, db *gosql } } -// SetAdmissionControl sets the admission control cluster settings on the +// setAdmissionControl sets the admission control cluster settings on the // given cluster. -func SetAdmissionControl(ctx context.Context, t test.Test, c cluster.Cluster, enabled bool) { +func setAdmissionControl(ctx context.Context, t test.Test, c cluster.Cluster, enabled bool) { db := c.Conn(ctx, t.L(), 1) defer db.Close() val := "true" From 7fc79fb03b57a51c0e7117657954982f3cb05e81 Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Sun, 2 Oct 2022 19:25:29 -0400 Subject: [PATCH 5/6] roachtest: introduce a --skip-init flag For some tests to do have expensive initialization steps (large imports for ex.), when iterating on them it's often useful to recycle roachtest created clusters without re-running the full gamut. Introduce this flag to let test authors define such fast paths. Release note: None --- pkg/cmd/roachtest/cluster_test.go | 4 ++++ pkg/cmd/roachtest/main.go | 11 ++++++++++- pkg/cmd/roachtest/test/test_interface.go | 1 + pkg/cmd/roachtest/test_impl.go | 5 +++++ pkg/cmd/roachtest/test_runner.go | 2 ++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/roachtest/cluster_test.go b/pkg/cmd/roachtest/cluster_test.go index 3e0f0c8368ef..787d705f9649 100644 --- a/pkg/cmd/roachtest/cluster_test.go +++ b/pkg/cmd/roachtest/cluster_test.go @@ -86,6 +86,10 @@ func (t testWrapper) VersionsBinaryOverride() map[string]string { panic("implement me") } +func (t testWrapper) SkipInit() bool { + panic("implement me") +} + func (t testWrapper) Progress(f float64) { panic("implement me") } diff --git a/pkg/cmd/roachtest/main.go b/pkg/cmd/roachtest/main.go index 2ef3a680e96f..5e6465dfa0be 100644 --- a/pkg/cmd/roachtest/main.go +++ b/pkg/cmd/roachtest/main.go @@ -85,6 +85,7 @@ func main() { var literalArtifacts string var httpPort int var debugEnabled bool + var skipInit bool var clusterID string var count = 1 var versionsBinaryOverride map[string]string @@ -221,6 +222,7 @@ runner itself. count: count, cpuQuota: cpuQuota, debugEnabled: debugEnabled, + skipInit: skipInit, httpPort: httpPort, parallelism: parallelism, artifactsDir: artifacts, @@ -260,6 +262,7 @@ runner itself. count: count, cpuQuota: cpuQuota, debugEnabled: debugEnabled, + skipInit: skipInit, httpPort: httpPort, parallelism: parallelism, artifactsDir: artifacts, @@ -284,6 +287,8 @@ runner itself. &count, "count", 1, "the number of times to run each test") cmd.Flags().BoolVarP( &debugEnabled, "debug", "d", debugEnabled, "don't wipe and destroy cluster if test fails") + cmd.Flags().BoolVar( + &skipInit, "skip-init", false, "skip initialization step (imports, table creation, etc.) for tests that support it, useful when re-using clusters with --wipe=false") cmd.Flags().IntVarP( ¶llelism, "parallelism", "p", parallelism, "number of tests to run in parallel") cmd.Flags().StringVar( @@ -351,6 +356,7 @@ type cliCfg struct { count int cpuQuota int debugEnabled bool + skipInit bool httpPort int parallelism int artifactsDir string @@ -426,7 +432,10 @@ func runTests(register func(registry.Registry), cfg cliCfg) error { CtrlC(ctx, l, cancel, cr) err = runner.Run( ctx, tests, cfg.count, cfg.parallelism, opt, - testOpts{versionsBinaryOverride: cfg.versionsBinaryOverride}, + testOpts{ + versionsBinaryOverride: cfg.versionsBinaryOverride, + skipInit: cfg.skipInit, + }, lopt, nil /* clusterAllocator */) // Make sure we attempt to clean up. We run with a non-canceled ctx; the diff --git a/pkg/cmd/roachtest/test/test_interface.go b/pkg/cmd/roachtest/test/test_interface.go index 7da9b1c030d5..582802a8f5a9 100644 --- a/pkg/cmd/roachtest/test/test_interface.go +++ b/pkg/cmd/roachtest/test/test_interface.go @@ -35,6 +35,7 @@ type Test interface { // through all registered roachtests to change how they register the test. Spec() interface{} VersionsBinaryOverride() map[string]string + SkipInit() bool Skip(args ...interface{}) Skipf(format string, args ...interface{}) Error(args ...interface{}) diff --git a/pkg/cmd/roachtest/test_impl.go b/pkg/cmd/roachtest/test_impl.go index 49d18125c021..7355dbb7a084 100644 --- a/pkg/cmd/roachtest/test_impl.go +++ b/pkg/cmd/roachtest/test_impl.go @@ -105,6 +105,7 @@ type testImpl struct { // // Version strings look like "20.1.4". versionsBinaryOverride map[string]string + skipInit bool } // BuildVersion exposes the build version of the cluster @@ -129,6 +130,10 @@ func (t *testImpl) VersionsBinaryOverride() map[string]string { return t.versionsBinaryOverride } +func (t *testImpl) SkipInit() bool { + return t.skipInit +} + // Spec returns the TestSpec. func (t *testImpl) Spec() interface{} { return t.spec diff --git a/pkg/cmd/roachtest/test_runner.go b/pkg/cmd/roachtest/test_runner.go index 2e325dd98924..d3dbfff22c77 100644 --- a/pkg/cmd/roachtest/test_runner.go +++ b/pkg/cmd/roachtest/test_runner.go @@ -164,6 +164,7 @@ func (c clustersOpt) validate() error { type testOpts struct { versionsBinaryOverride map[string]string + skipInit bool } // Run runs tests. @@ -607,6 +608,7 @@ func (r *testRunner) runWorker( artifactsSpec: artifactsSpec, l: testL, versionsBinaryOverride: topt.versionsBinaryOverride, + skipInit: topt.skipInit, debug: debug, } // Now run the test. From d8ada8b66d680fd19ad8da3467ba7176cdf7115f Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Thu, 29 Sep 2022 22:20:54 -0400 Subject: [PATCH 6/6] roachtests: introduce admission-control/snapshot-overload This test sets up a 3-node CRDB cluster on 8vCPU machines, loads it up with a large TPC-C dataset, and sets up a foreground load of kv95/1b. The TPC-C dataset has a replication of factor of 1 and is used as the large lump we bus around through high-rate snapshots -- snapshots that are send to the node containing leaseholders serving kv95 traffic. Snapshots follow where the leases travel, cycling through each node, evaluating performance isolation in the presence of snapshots. This test comes 'battery-included', shipping with a custom grafana dashboard and requisite prometheus/grafana plumbing. At the end of test run a prometheus dump is kept around for later inspection. It also integrates with --local and --skip-init for fast iteration. We're going to cargo cult such things in future integration tests for admission control. Release note: None --- pkg/cmd/roachtest/tests/BUILD.bazel | 1 + pkg/cmd/roachtest/tests/admission_control.go | 17 ++ .../admission_control_snapshot_overload.go | 196 ++++++++++++++++++ pkg/cmd/roachtest/tests/tpcc.go | 29 ++- 4 files changed, 228 insertions(+), 15 deletions(-) create mode 100644 pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index b9562f8c4f79..fdc508143029 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "activerecord_blocklist.go", "admission_control.go", "admission_control_multi_store_overload.go", + "admission_control_snapshot_overload.go", "admission_control_tpcc_overload.go", "allocator.go", "alterpk.go", diff --git a/pkg/cmd/roachtest/tests/admission_control.go b/pkg/cmd/roachtest/tests/admission_control.go index e51d4acdb9d8..b8e092247826 100644 --- a/pkg/cmd/roachtest/tests/admission_control.go +++ b/pkg/cmd/roachtest/tests/admission_control.go @@ -13,8 +13,25 @@ package tests import "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" func registerAdmission(r registry.Registry) { + // TODO(irfansharif): Can we write these tests using cgroups instead? + // Limiting CPU/bandwidth directly? + + // TODO(irfansharif): Some of these tests hooks into prometheus/grafana. + // It'd be nice to use the grafana annotations API to explicitly annotate + // the points at which we do cluster-level things, like set zone configs to + // trigger a round of snapshots. + + // TODO(irfansharif): Integrate with probabilistic tracing machinery, + // capturing outliers automatically for later analysis. + + // TODO(irfansharif): Look into clusterstats and what that emits to + // roachperf. Need to munge with histogram data to compute % test run spent + // over some latency threshold. Will be Useful to track over time. + registerMultiStoreOverload(r) + registerSnapshotOverload(r) registerTPCCOverload(r) + // TODO(irfansharif): Once registerMultiTenantFairness is unskipped and // observed to be non-flaky for 3-ish months, transfer ownership to the AC // group + re-home it here. diff --git a/pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go b/pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go new file mode 100644 index 000000000000..216b58fbd26d --- /dev/null +++ b/pkg/cmd/roachtest/tests/admission_control_snapshot_overload.go @@ -0,0 +1,196 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tests + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/cockroachdb/cockroach/pkg/roachprod/prometheus" +) + +// This test sets up a 3-node CRDB cluster on 8vCPU machines, loads +// it up with a large TPC-C dataset, and sets up a foreground load of kv95/1b. +// The TPC-C dataset has a replication of factor of 1 and is used as the large +// lump we bus around through snapshots -- snapshots that are send to the node +// containing leaseholders serving kv95 traffic. Snapshots follow where the +// leases travel, cycling through each node, evaluating performance isolation in +// the presence of snapshots. +// +// TODO(irfansharif): This primarily stresses CPU; write an equivalent for IO. +// TODO(irfansharif): This is a relatively long-running test (takes ~3.5hrs); +// make it shorter. +func registerSnapshotOverload(r registry.Registry) { + r.Add(registry.TestSpec{ + Name: "admission-control/snapshot-overload", + Owner: registry.OwnerAdmissionControl, + // TODO(irfansharif): After two weeks of nightly baking time, reduce + // this to a weekly cadence. This is a long-running test and serves only + // as a coarse-grained benchmark. + // Tags: []string{`weekly`}, + Cluster: r.MakeClusterSpec(4, spec.CPU(8)), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + if c.Spec().NodeCount < 4 { + t.Fatalf("expected at least 4 nodes, found %d", c.Spec().NodeCount) + } + + c.Put(ctx, t.Cockroach(), "./cockroach", c.All()) + crdbNodes := c.Spec().NodeCount - 1 + workloadNode := crdbNodes + 1 + for i := 1; i <= crdbNodes; i++ { + startOpts := option.DefaultStartOpts() + startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--attrs=n%d", i)) + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Node(i)) + } + + db := c.Conn(ctx, t.L(), crdbNodes) + defer db.Close() + + // Set a replication factor of 1 and pin replicas to n1 by default. + // For the active dataset (we use kv further below) we'll use a + // different config. + t.Status(fmt.Sprintf("configuring default zone and settings (<%s)", 30*time.Second)) + { + if _, err := db.ExecContext(ctx, + "ALTER RANGE DEFAULT CONFIGURE ZONE USING num_replicas = 1, constraints = '[+n1]';", + ); err != nil { + t.Fatalf("failed to configure zone for RANGE DEFAULT: %v", err) + } + + // Defensive, since admission control is enabled by default. This + // test can fail if admission control is disabled. + setAdmissionControl(ctx, t, c, true) + + // Set high snapshot rates. + if _, err := db.ExecContext( + ctx, "SET CLUSTER SETTING kv.snapshot_rebalance.max_rate = '256MiB'"); err != nil { + t.Fatalf("failed to set kv.snapshot_rebalance.max_rate: %v", err) + } + if _, err := db.ExecContext( + ctx, "SET CLUSTER SETTING kv.snapshot_recovery.max_rate = '256MiB'"); err != nil { + t.Fatalf("failed to set kv.snapshot_recovery.max_rate: %v", err) + } + } + + t.Status(fmt.Sprintf("setting up prometheus/grafana (<%s)", 2*time.Minute)) + promCfg := &prometheus.Config{} + promCfg.WithPrometheusNode(c.Node(workloadNode).InstallNodes()[0]) + promCfg.WithNodeExporter(c.Range(1, c.Spec().NodeCount-1).InstallNodes()) + promCfg.WithCluster(c.Range(1, c.Spec().NodeCount-1).InstallNodes()) + promCfg.ScrapeConfigs = append(promCfg.ScrapeConfigs, prometheus.MakeWorkloadScrapeConfig("workload", + "/", makeWorkloadScrapeNodes(c.Node(workloadNode).InstallNodes()[0], []workloadInstance{ + {nodes: c.Node(workloadNode)}, + }))) + promCfg.WithGrafanaDashboard("http://go.crdb.dev/p/snapshot-admission-control-grafana") + _, cleanupFunc := setupPrometheusForRoachtest(ctx, t, c, promCfg, nil) + defer cleanupFunc() + + var constraints []string + for i := 1; i <= crdbNodes; i++ { + constraints = append(constraints, fmt.Sprintf("+n%d: 1", i)) + } + constraint := strings.Join(constraints, ",") + + // Initialize the kv database with replicas on all nodes but + // leaseholders on just n1. + t.Status(fmt.Sprintf("initializing kv dataset (<%s)", time.Minute)) + if !t.SkipInit() { + splits := ifLocal(c, " --splits=10", " --splits=100") + c.Run(ctx, c.Node(workloadNode), "./cockroach workload init kv "+splits+" {pgurl:1}") + + if _, err := db.ExecContext(ctx, fmt.Sprintf( + "ALTER DATABASE kv CONFIGURE ZONE USING num_replicas = %d, constraints = '{%s}', lease_preferences = '[[+n1]]'", + crdbNodes, constraint), + ); err != nil { + t.Fatalf("failed to configure zone for DATABASE kv: %v", err) + } + } + + t.Status(fmt.Sprintf("initializing tpcc dataset (<%s)", 20*time.Minute)) + if !t.SkipInit() { + warehouses := ifLocal(c, " --warehouses=10", " --warehouses=2000") + c.Run(ctx, c.Node(workloadNode), "./cockroach workload fixtures import tpcc --checks=false"+warehouses+" {pgurl:1}") + } + + const iters = 4 + padDuration, err := time.ParseDuration(ifLocal(c, "20s", "5m")) + if err != nil { + t.Fatal(err) + } + leaseWaitDuration, err := time.ParseDuration(ifLocal(c, "20s", "5m")) + if err != nil { + t.Fatal(err) + } + replicaWaitDuration, err := time.ParseDuration(ifLocal(c, "40s", "25m")) + if err != nil { + t.Fatal(err) + } + + transferDuration := leaseWaitDuration + replicaWaitDuration + totalTransferDuration := transferDuration * iters + totalWorkloadDuration := totalTransferDuration + (2 * padDuration) + + t.Status(fmt.Sprintf("starting kv workload thread to run for %s (<%s)", totalWorkloadDuration, time.Minute)) + m := c.NewMonitor(ctx, c.Range(1, crdbNodes)) + m.Go(func(ctx context.Context) error { + duration := " --duration=" + totalWorkloadDuration.String() + histograms := " --histograms=" + t.PerfArtifactsDir() + "/stats.json" + concurrency := ifLocal(c, " --concurrency=8", " --concurrency=256") + maxRate := ifLocal(c, " --max-rate=100", " --max-rate=12000") + splits := ifLocal(c, " --splits=10", " --splits=100") + c.Run(ctx, c.Node(crdbNodes+1), + "./cockroach workload run kv --max-block-bytes=1 --read-percent=95 "+ + histograms+duration+concurrency+maxRate+splits+fmt.Sprintf(" {pgurl:1-%d}", crdbNodes), + ) + return nil + }) + + t.Status(fmt.Sprintf("setting performance baseline (<%s)", padDuration)) + time.Sleep(padDuration) + + t.Status(fmt.Sprintf("starting snapshot transfers for %s (<%s)", totalTransferDuration, time.Minute)) + m.Go(func(ctx context.Context) error { + for i := 0; i < iters; i++ { + nextDestinationNode := 1 + ((i + 1) % crdbNodes) // if crdbNodes = 3, this cycles through 2, 3, 1, 2, 3, 1, ... + t.Status(fmt.Sprintf("snapshot round %d/%d: inert data and active leases routing to n%d (<%s)", + i+1, iters, nextDestinationNode, transferDuration)) + + if _, err := db.ExecContext(ctx, fmt.Sprintf( + "ALTER DATABASE kv CONFIGURE ZONE USING num_replicas = %d, constraints = '{%s}', lease_preferences = '[[+n%d]]'", + crdbNodes, constraint, nextDestinationNode), + ); err != nil { + t.Fatalf("failed to configure zone for DATABASE kv: %v", err) + } + time.Sleep(leaseWaitDuration) + + if _, err := db.ExecContext(ctx, + fmt.Sprintf("ALTER DATABASE tpcc CONFIGURE ZONE USING num_replicas = 1, constraints = '[+n%d]';", nextDestinationNode), + ); err != nil { + t.Fatalf("failed to configure zone for RANGE DEFAULT: %v", err) + } + time.Sleep(replicaWaitDuration) + } + return nil + }) + + t.Status(fmt.Sprintf("waiting for workload/snapshot transfers to finish (<%s)", totalWorkloadDuration-padDuration)) + m.Wait() + }, + }) +} diff --git a/pkg/cmd/roachtest/tests/tpcc.go b/pkg/cmd/roachtest/tests/tpcc.go index f6764f7fda03..3d06ad4fd750 100644 --- a/pkg/cmd/roachtest/tests/tpcc.go +++ b/pkg/cmd/roachtest/tests/tpcc.go @@ -192,8 +192,12 @@ func runTPCC(ctx context.Context, t test.Test, c cluster.Cluster, opts tpccOptio } var ep *tpccChaosEventProcessor - promCfg, cleanupFunc := setupPrometheusForTPCC(ctx, t, c, opts, workloadInstances) - defer cleanupFunc() + var promCfg *prometheus.Config + if !opts.DisablePrometheus { + var cleanupFunc func() + promCfg, cleanupFunc = setupPrometheusForRoachtest(ctx, t, c, opts.PrometheusConfig, workloadInstances) + defer cleanupFunc() + } if opts.ChaosEventsProcessor != nil { if promCfg == nil { t.Skip("skipping test as prometheus is needed, but prometheus does not yet work locally") @@ -1413,39 +1417,34 @@ func makeWorkloadScrapeNodes( return workloadScrapeNodes } -// setupPrometheusForTPCC initializes prometheus to run against the provided +// setupPrometheusForRoachtest initializes prometheus to run against the provided // PrometheusConfig. If no PrometheusConfig is provided, it creates a prometheus // scraper for all CockroachDB nodes in the TPC-C setup, as well as one for // each workloadInstance. // Returns the created PrometheusConfig if prometheus is initialized, as well // as a cleanup function which should be called in a defer statement. -func setupPrometheusForTPCC( +func setupPrometheusForRoachtest( ctx context.Context, t test.Test, c cluster.Cluster, - opts tpccOptions, + promCfg *prometheus.Config, workloadInstances []workloadInstance, ) (*prometheus.Config, func()) { - cfg := opts.PrometheusConfig + cfg := promCfg if cfg == nil { // Avoid setting prometheus automatically up for local clusters. if c.IsLocal() { return nil, func() {} } - if opts.DisablePrometheus { - return nil, func() {} - } cfg = &prometheus.Config{} workloadNode := c.Node(c.Spec().NodeCount).InstallNodes()[0] cfg.WithPrometheusNode(workloadNode) cfg.WithNodeExporter(c.Range(1, c.Spec().NodeCount-1).InstallNodes()) cfg.WithCluster(c.Range(1, c.Spec().NodeCount-1).InstallNodes()) - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, prometheus.MakeWorkloadScrapeConfig("workload", - "/", makeWorkloadScrapeNodes(workloadNode, workloadInstances))) - - } - if opts.DisablePrometheus { - t.Fatal("test has PrometheusConfig but DisablePrometheus was on") + if len(workloadInstances) > 0 { + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, prometheus.MakeWorkloadScrapeConfig("workload", + "/", makeWorkloadScrapeNodes(workloadNode, workloadInstances))) + } } if c.IsLocal() { t.Skip("skipping test as prometheus is needed, but prometheus does not yet work locally")