From 5f9c51f9005302aa2341729643f0fff7f195582b Mon Sep 17 00:00:00 2001 From: wenyihu6 Date: Wed, 2 Aug 2023 16:46:02 -0400 Subject: [PATCH 1/2] asim: sort before iterating over maps when printing Previously, the simulator iterates over an unordered map when formatting and printing states, stores, and ranges, resulting in non-deterministic output. This patch addresses the issue by sorting the maps by key before printing and formatting. Release note: none Epic: none --- pkg/kv/kvserver/asim/gen/generator.go | 4 +-- pkg/kv/kvserver/asim/state/impl.go | 49 +++++++++++++++++++-------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/pkg/kv/kvserver/asim/gen/generator.go b/pkg/kv/kvserver/asim/gen/generator.go index fb2c5746e5f4..447bf8236ae5 100644 --- a/pkg/kv/kvserver/asim/gen/generator.go +++ b/pkg/kv/kvserver/asim/gen/generator.go @@ -166,8 +166,8 @@ type BasicCluster struct { // nodes having the same locality and with the specified number of stores/nodes // created. The cluster is created based on the stores and stores-per-node // values the basic cluster generator is created with. -func (lc BasicCluster) Generate(seed int64, settings *config.SimulationSettings) state.State { - info := state.ClusterInfoWithStoreCount(lc.Nodes, lc.StoresPerNode) +func (bc BasicCluster) Generate(seed int64, settings *config.SimulationSettings) state.State { + info := state.ClusterInfoWithStoreCount(bc.Nodes, bc.StoresPerNode) return state.LoadClusterInfo(info, settings) } diff --git a/pkg/kv/kvserver/asim/state/impl.go b/pkg/kv/kvserver/asim/state/impl.go index 5bfe7f781192..90d03958acef 100644 --- a/pkg/kv/kvserver/asim/state/impl.go +++ b/pkg/kv/kvserver/asim/state/impl.go @@ -149,14 +149,22 @@ func (s *state) String() string { }) nStores := len(s.stores) - iterStores := 0 builder.WriteString(fmt.Sprintf("stores(%d)=[", nStores)) - for _, store := range s.stores { + + // Sort the unordered map storeIDs by its key to ensure deterministic + // printing. + var storeIDs StoreIDSlice + for storeID := range s.stores { + storeIDs = append(storeIDs, storeID) + } + sort.Sort(storeIDs) + + for i, storeID := range storeIDs { + store := s.stores[storeID] builder.WriteString(store.String()) - if iterStores < nStores-1 { + if i < nStores-1 { builder.WriteString(",") } - iterStores++ } builder.WriteString("] ") @@ -1320,14 +1328,20 @@ func (s *store) String() string { builder := &strings.Builder{} builder.WriteString(fmt.Sprintf("s%dn%d=(", s.storeID, s.nodeID)) - nRepls := len(s.replicas) - iterRepls := 0 - for rangeID, replicaID := range s.replicas { + // Sort the unordered map rangeIDs by its key to ensure deterministic + // printing. + var rangeIDs RangeIDSlice + for rangeID := range s.replicas { + rangeIDs = append(rangeIDs, rangeID) + } + sort.Sort(rangeIDs) + + for i, rangeID := range rangeIDs { + replicaID := s.replicas[rangeID] builder.WriteString(fmt.Sprintf("r%d:%d", rangeID, replicaID)) - if iterRepls < nRepls-1 { + if i < len(s.replicas)-1 { builder.WriteString(",") } - iterRepls++ } builder.WriteString(")") return builder.String() @@ -1381,17 +1395,24 @@ func (r *rng) String() string { builder := &strings.Builder{} builder.WriteString(fmt.Sprintf("r%d(%d)=(", r.rangeID, r.startKey)) - nRepls := len(r.replicas) - iterRepls := 0 - for storeID, replica := range r.replicas { + // Sort the unordered map storeIDs by its key to ensure deterministic + // printing. + var storeIDs StoreIDSlice + for storeID := range r.replicas { + storeIDs = append(storeIDs, storeID) + } + sort.Sort(storeIDs) + + for i, storeID := range storeIDs { + replica := r.replicas[storeID] builder.WriteString(fmt.Sprintf("s%d:r%d", storeID, replica.replicaID)) if r.leaseholder == replica.replicaID { builder.WriteString("*") } - if iterRepls < nRepls-1 { + if i < len(r.replicas)-1 { builder.WriteString(",") } - iterRepls++ + i++ } builder.WriteString(")") From 238578742c3774b8f88a788cfa04d577a9224b9b Mon Sep 17 00:00:00 2001 From: wenyihu6 Date: Wed, 2 Aug 2023 10:19:36 -0400 Subject: [PATCH 2/2] asim: better outputs for data-driven tests Previously, the randomized testing framework's output only indicates whether each iteration passes. This lack of of detail makes checking the randomized framework and debugging challenging. This patch adds more info to the output, including the selected configurations, the initial state of each simulation. Additionally, this patch removes the verbosity flag for printing history plots as it does not seem to have any practical use cases. New verbosity flags for eval are now supported. ``` "eval" [verbose=(<[]("result_only","test_settings","initial_state","config_gen","topology","all")>)] - verbose(default value is OutputResultOnly): used to set flags on what to show in the test output messages. By default, all details are displayed upon assertion failure. - result_only: only shows whether the test passed or failed, along with any failure messages - test_settings: displays settings used for the repeated tests - initial_state: displays the initial state of each test iteration - config_gen: displays the input configurations generated for each test iteration - topology: displays the topology of cluster configurations - all: displays everything above ``` Part Of: [#106311](https://github.com/kvoli/cockroach/issues/106311) Release Note: none (cherry picked from commit 27e9bd438a6ef1f6f0ed7205d91f28c9b54fd314) --- pkg/kv/kvserver/asim/asim.go | 8 + pkg/kv/kvserver/asim/gen/generator.go | 35 ++ pkg/kv/kvserver/asim/state/config_loader.go | 22 ++ pkg/kv/kvserver/asim/state/impl.go | 30 ++ pkg/kv/kvserver/asim/state/state.go | 3 + pkg/kv/kvserver/asim/tests/BUILD.bazel | 2 +- .../kvserver/asim/tests/default_settings.go | 20 +- pkg/kv/kvserver/asim/tests/helpers_test.go | 4 + pkg/kv/kvserver/asim/tests/output.go | 164 +++++++++ pkg/kv/kvserver/asim/tests/rand_framework.go | 132 ++----- pkg/kv/kvserver/asim/tests/rand_gen.go | 8 + pkg/kv/kvserver/asim/tests/rand_test.go | 25 +- .../asim/tests/testdata/rand/default_settings | 217 ++++++++++- .../asim/tests/testdata/rand/rand_cluster | 193 ++++++++++ .../asim/tests/testdata/rand/rand_ranges | 347 ++++++------------ .../asim/tests/testdata/rand/weighted_rand | 62 ++++ 16 files changed, 893 insertions(+), 379 deletions(-) create mode 100644 pkg/kv/kvserver/asim/tests/output.go create mode 100644 pkg/kv/kvserver/asim/tests/testdata/rand/rand_cluster create mode 100644 pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand diff --git a/pkg/kv/kvserver/asim/asim.go b/pkg/kv/kvserver/asim/asim.go index 9c7dd9bc8f0f..4156ece3d903 100644 --- a/pkg/kv/kvserver/asim/asim.go +++ b/pkg/kv/kvserver/asim/asim.go @@ -63,6 +63,14 @@ type Simulator struct { history History } +func (s *Simulator) Curr() time.Time { + return s.curr +} + +func (s *Simulator) State() state.State { + return s.state +} + // History contains recorded information that summarizes a simulation run. // Currently it only contains the store metrics of the run. // TODO(kvoli): Add a range log like structure to the history. diff --git a/pkg/kv/kvserver/asim/gen/generator.go b/pkg/kv/kvserver/asim/gen/generator.go index 447bf8236ae5..bb28ce19883b 100644 --- a/pkg/kv/kvserver/asim/gen/generator.go +++ b/pkg/kv/kvserver/asim/gen/generator.go @@ -37,6 +37,7 @@ type LoadGen interface { // Generate returns a workload generator that is parameterized randomly by // the seed and simulation settings provided. Generate(seed int64, settings *config.SimulationSettings) []workload.Generator + String() string } // ClusterGen provides a method to generate the initial cluster state, given a @@ -46,6 +47,7 @@ type ClusterGen interface { // Generate returns a new State that is parameterized randomly by the seed // and simulation settings provided. Generate(seed int64, settings *config.SimulationSettings) state.State + String() string } // RangeGen provides a method to generate the initial range splits, range @@ -55,6 +57,7 @@ type RangeGen interface { // simulation settings provided. In the updated state, ranges will have been // created, replicas and leases assigned to stores in the cluster. Generate(seed int64, settings *config.SimulationSettings, s state.State) state.State + String() string } // EventGen provides a method to generate a list of events that will apply to @@ -63,6 +66,7 @@ type RangeGen interface { type EventGen interface { // Generate returns a list of events, which should be exectued at the delay specified. Generate(seed int64) event.DelayedEventList + String() string } // GenerateSimulation is a utility function that creates a new allocation @@ -113,6 +117,13 @@ type BasicLoad struct { MinKey, MaxKey int64 } +func (bl BasicLoad) String() string { + return fmt.Sprintf( + "basic load with rw_ratio=%0.2f, rate=%0.2f, skewed_access=%t, min_block_size=%d, max_block_size=%d, "+ + "min_key=%d, max_key=%d", + bl.RWRatio, bl.Rate, bl.SkewedAccess, bl.MinBlockSize, bl.MaxBlockSize, bl.MinKey, bl.MaxKey) +} + // Generate returns a new list of workload generators where the generator // parameters are populated with the parameters from the generator and either a // uniform or zipfian key generator is created depending on whether @@ -156,12 +167,20 @@ func (lc LoadedCluster) Generate(seed int64, settings *config.SimulationSettings return state.LoadClusterInfo(lc.Info, settings) } +func (lc LoadedCluster) String() string { + return fmt.Sprintf("loaded cluster with\n %v", lc.Info) +} + // BasicCluster implements the ClusterGen interace. type BasicCluster struct { Nodes int StoresPerNode int } +func (bc BasicCluster) String() string { + return fmt.Sprintf("basic cluster with nodes=%d, stores_per_node=%d", bc.Nodes, bc.StoresPerNode) +} + // Generate returns a new simulator state, where the cluster is created with all // nodes having the same locality and with the specified number of stores/nodes // created. The cluster is created based on the stores and stores-per-node @@ -176,6 +195,10 @@ type LoadedRanges struct { Info state.RangesInfo } +func (lr LoadedRanges) String() string { + return fmt.Sprintf("loaded ranges with ranges=%d", len(lr.Info)) +} + // Generate returns an updated simulator state, where the cluster is loaded // with the range info that the generator was created with. There is no // randomness in this cluster generation. @@ -238,6 +261,10 @@ type BaseRanges struct { Bytes int64 } +func (b BaseRanges) String() string { + return fmt.Sprintf("ranges=%d, key_space=%d, replication_factor=%d, bytes=%d", b.Ranges, b.KeySpace, b.ReplicationFactor, b.Bytes) +} + // GetRangesInfo generates and distributes ranges across stores based on // PlacementType while using other BaseRanges fields for range configuration. func (b BaseRanges) GetRangesInfo( @@ -272,6 +299,10 @@ type BasicRanges struct { PlacementType PlacementType } +func (br BasicRanges) String() string { + return fmt.Sprintf("basic ranges with placement_type=%v, %v", br.PlacementType, br.BaseRanges) +} + // Generate returns an updated simulator state, where the cluster is loaded with // ranges generated based on the parameters specified in the fields of // BasicRanges. @@ -292,6 +323,10 @@ type StaticEvents struct { DelayedEvents event.DelayedEventList } +func (se StaticEvents) String() string { + return fmt.Sprintf("number of static events generated=%d", len(se.DelayedEvents)) +} + // Generate returns a list of events, exactly the same as the events // StaticEvents was created with. func (se StaticEvents) Generate(seed int64) event.DelayedEventList { diff --git a/pkg/kv/kvserver/asim/state/config_loader.go b/pkg/kv/kvserver/asim/state/config_loader.go index a04ab3e304b6..84ce47a35c67 100644 --- a/pkg/kv/kvserver/asim/state/config_loader.go +++ b/pkg/kv/kvserver/asim/state/config_loader.go @@ -12,6 +12,7 @@ package state import ( "fmt" + "strings" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/config" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -263,6 +264,27 @@ type ClusterInfo struct { Regions []Region } +func (c ClusterInfo) String() (s string) { + buf := &strings.Builder{} + for i, r := range c.Regions { + buf.WriteString(fmt.Sprintf("\t\tregion:%s [", r.Name)) + if len(r.Zones) == 0 { + panic(fmt.Sprintf("number of zones within region %s is zero", r.Name)) + } + for j, z := range r.Zones { + buf.WriteString(fmt.Sprintf("zone=%s(nodes=%d,stores=%d)", z.Name, z.NodeCount, z.StoresPerNode)) + if j != len(r.Zones)-1 { + buf.WriteString(", ") + } + } + buf.WriteString("]") + if i != len(c.Regions)-1 { + buf.WriteString("\n") + } + } + return buf.String() +} + type RangeInfo struct { Descriptor roachpb.RangeDescriptor Config *roachpb.SpanConfig diff --git a/pkg/kv/kvserver/asim/state/impl.go b/pkg/kv/kvserver/asim/state/impl.go index 90d03958acef..6fc92ddfeb87 100644 --- a/pkg/kv/kvserver/asim/state/impl.go +++ b/pkg/kv/kvserver/asim/state/impl.go @@ -136,6 +136,29 @@ func (rm *rmap) initFirstRange() { rm.rangeMap[rangeID] = rng } +// PrettyPrint returns a pretty formatted string representation of the +// state (more concise than String()). +func (s *state) PrettyPrint() string { + builder := &strings.Builder{} + nStores := len(s.stores) + builder.WriteString(fmt.Sprintf("stores(%d)=[", nStores)) + var storeIDs StoreIDSlice + for storeID := range s.stores { + storeIDs = append(storeIDs, storeID) + } + sort.Sort(storeIDs) + + for i, storeID := range storeIDs { + store := s.stores[storeID] + builder.WriteString(store.PrettyPrint()) + if i < nStores-1 { + builder.WriteString(",") + } + } + builder.WriteString("]") + return builder.String() +} + // String returns a string containing a compact representation of the state. // TODO(kvoli,lidorcarmel): Add a unit test for this function. func (s *state) String() string { @@ -1323,6 +1346,13 @@ type store struct { replicas map[RangeID]ReplicaID } +// PrettyPrint returns pretty formatted string representation of the store. +func (s *store) PrettyPrint() string { + builder := &strings.Builder{} + builder.WriteString(fmt.Sprintf("s%dn%d=(replicas(%d))", s.storeID, s.nodeID, len(s.replicas))) + return builder.String() +} + // String returns a compact string representing the current state of the store. func (s *store) String() string { builder := &strings.Builder{} diff --git a/pkg/kv/kvserver/asim/state/state.go b/pkg/kv/kvserver/asim/state/state.go index d4be4b264839..05acaa6cf5e6 100644 --- a/pkg/kv/kvserver/asim/state/state.go +++ b/pkg/kv/kvserver/asim/state/state.go @@ -55,6 +55,9 @@ type State interface { // TODO(kvoli): Unit test this fn. // String returns string containing a compact representation of the state. String() string + // PrettyPrint returns a pretty formatted string representation of the + // state (more concise than String()). + PrettyPrint() string // ClusterInfo returns the info of the cluster represented in state. ClusterInfo() ClusterInfo // Store returns the Store with ID StoreID. This fails if no Store exists diff --git a/pkg/kv/kvserver/asim/tests/BUILD.bazel b/pkg/kv/kvserver/asim/tests/BUILD.bazel index 2b0faf6098dd..c5585ac9b335 100644 --- a/pkg/kv/kvserver/asim/tests/BUILD.bazel +++ b/pkg/kv/kvserver/asim/tests/BUILD.bazel @@ -34,6 +34,7 @@ go_library( srcs = [ "assert.go", "default_settings.go", + "output.go", "rand_framework.go", "rand_gen.go", ], @@ -49,7 +50,6 @@ go_library( "//pkg/roachpb", "//pkg/spanconfig/spanconfigtestutils", "//pkg/util/log", - "@com_github_guptarohit_asciigraph//:asciigraph", "@com_github_montanaflynn_stats//:stats", ], ) diff --git a/pkg/kv/kvserver/asim/tests/default_settings.go b/pkg/kv/kvserver/asim/tests/default_settings.go index e2a692676d37..4905a4fa98df 100644 --- a/pkg/kv/kvserver/asim/tests/default_settings.go +++ b/pkg/kv/kvserver/asim/tests/default_settings.go @@ -61,7 +61,7 @@ func defaultLoadGen() gen.BasicLoad { } const ( - defaultRanges = 1 + defaultRanges = 10 defaultPlacementType = gen.Even defaultReplicationFactor = 3 defaultBytes = 0 @@ -89,21 +89,3 @@ func defaultAssertions() []SimulationAssertion { }, } } - -const ( - defaultStat = "replicas" - defaultHeight, defaultWidth = 15, 80 -) - -type plotSettings struct { - stat string - height, width int -} - -func defaultPlotSettings() plotSettings { - return plotSettings{ - stat: defaultStat, - height: defaultHeight, - width: defaultWidth, - } -} diff --git a/pkg/kv/kvserver/asim/tests/helpers_test.go b/pkg/kv/kvserver/asim/tests/helpers_test.go index 17f481fa1b12..43778b208803 100644 --- a/pkg/kv/kvserver/asim/tests/helpers_test.go +++ b/pkg/kv/kvserver/asim/tests/helpers_test.go @@ -24,6 +24,10 @@ func scanArg(t *testing.T, d *datadriven.TestData, key string, dest interface{}) switch dest := dest.(type) { case *string, *int, *int64, *uint64, *bool, *time.Duration, *float64, *[]int, *[]float64: d.ScanArgs(t, key, dest) + case *OutputFlags: + var flagsTmp []string + d.ScanArgs(t, key, &flagsTmp) + *dest = dest.ScanFlags(flagsTmp) case *gen.PlacementType: d.ScanArgs(t, key, &tmp) *dest = gen.GetRangePlacementType(tmp) diff --git a/pkg/kv/kvserver/asim/tests/output.go b/pkg/kv/kvserver/asim/tests/output.go new file mode 100644 index 000000000000..6008cd748b9f --- /dev/null +++ b/pkg/kv/kvserver/asim/tests/output.go @@ -0,0 +1,164 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tests + +import ( + "fmt" + "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/gen" + "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/state" +) + +// OutputFlags sets flags for what to output in tests. If you want to add a flag +// here, please add after OutputResultOnly and before OutputAll. +type OutputFlags int + +const ( + // OutputResultOnly only shows whether the test passed or failed, along with + // any failure messages. + OutputResultOnly OutputFlags = 0 + // OutputTestSettings displays settings used for the repeated tests. + OutputTestSettings = 1 << (iota - 1) // 1 << 0: 0000 0001 + // OutputInitialState displays the initial state of each test iteration. + OutputInitialState // 1 << 1: 0000 0010 + // OutputConfigGen displays the input configurations generated for each test + // iteration. + OutputConfigGen // 1 << 2: 0000 0100 + // OutputTopology displays the topology of cluster configurations. + OutputTopology // 1 << 3: 0000 1000 + // OutputAll shows everything above. + OutputAll = (1 << (iota - 1)) - 1 // (1 << 4) - 1: 0000 1111 +) + +// ScanFlags converts an array of input strings into a single flag. +func (o OutputFlags) ScanFlags(inputs []string) OutputFlags { + dict := map[string]OutputFlags{"result_only": OutputResultOnly, "test_settings": OutputTestSettings, + "initial_state": OutputInitialState, "config_gen": OutputConfigGen, "topology": OutputTopology, "all": OutputAll} + flag := OutputResultOnly + for _, input := range inputs { + flag = flag.set(dict[input]) + } + return flag +} + +// set returns a flag with f set on. +func (o OutputFlags) set(f OutputFlags) OutputFlags { + return o | f +} + +// Has returns true if this flag has the given f OutputFlags on. +func (o OutputFlags) Has(f OutputFlags) bool { + return o&f != 0 +} + +type testResult struct { + seed int64 + failed bool + reason string + clusterGen gen.ClusterGen + rangeGen gen.RangeGen + loadGen gen.LoadGen + eventGen gen.EventGen + initialTime time.Time + initialState state.State +} + +type testResultsReport struct { + flags OutputFlags + settings testSettings + outputSlice []testResult +} + +func printTestSettings(t testSettings, buf *strings.Builder) { + buf.WriteString(fmt.Sprintln("test settings")) + buf.WriteString(fmt.Sprintf("\tnum_iterations=%v duration=%s\n", t.numIterations, t.duration.Round(time.Second))) + divider := fmt.Sprintln("----------------------------------") + buf.WriteString(divider) + + configStr := func(configType string, optionType string) string { + return fmt.Sprintf("generating %s configurations using %s option\n", configType, optionType) + } + + if t.randOptions.cluster { + buf.WriteString(configStr("cluster", "randomized")) + buf.WriteString(fmt.Sprintf("\t%v\n", t.clusterGen)) + } else { + buf.WriteString(configStr("cluster", "static")) + } + + if t.randOptions.ranges { + buf.WriteString(configStr("ranges", "randomized")) + buf.WriteString(fmt.Sprintf("\t%v\n", t.rangeGen)) + } else { + buf.WriteString(configStr("ranges", "static")) + } + + if t.randOptions.load { + buf.WriteString(configStr("load", "randomized")) + } else { + buf.WriteString(configStr("load", "static")) + } + + if t.randOptions.staticEvents { + buf.WriteString(configStr("events", "randomized")) + } else { + buf.WriteString(configStr("events", "static")) + } + + if t.randOptions.staticSettings { + buf.WriteString(configStr("settings", "randomized")) + } else { + buf.WriteString(configStr("settings", "static")) + } +} + +// String() formats testResultsReport into a string for output including: 1. +// test settings 2. configurations generated 3. initial state 4. topology 5. +// assertion result +func (tr testResultsReport) String() string { + buf := strings.Builder{} + if tr.flags.Has(OutputTestSettings) { + printTestSettings(tr.settings, &buf) + } + divider := fmt.Sprintln("----------------------------------") + for i, output := range tr.outputSlice { + nthSample := i + 1 + failed := output.failed + if i == 0 { + buf.WriteString(divider) + } + buf.WriteString(fmt.Sprintf("sample%d: start running\n", nthSample)) + if failed || tr.flags.Has(OutputConfigGen) { + buf.WriteString(fmt.Sprintf("configurations generated using seed %d\n", output.seed)) + buf.WriteString(fmt.Sprintf("\t%v\n", output.clusterGen)) + buf.WriteString(fmt.Sprintf("\t%v\n", output.rangeGen)) + buf.WriteString(fmt.Sprintf("\t%v\n", output.loadGen)) + buf.WriteString(fmt.Sprintf("\t%v\n", output.eventGen)) + } + if failed || tr.flags.Has(OutputInitialState) { + buf.WriteString(fmt.Sprintf("initial state at %s:\n", output.initialTime.String())) + buf.WriteString(fmt.Sprintf("\t%v\n", output.initialState.PrettyPrint())) + } + if failed || tr.flags.Has(OutputTopology) { + topology := output.initialState.Topology() + buf.WriteString(fmt.Sprintf("topology:\n%s", topology.String())) + } + if failed { + buf.WriteString(fmt.Sprintf("sample%d: failed assertion\n%s", nthSample, output.reason)) + } else { + buf.WriteString(fmt.Sprintf("sample%d: pass\n", nthSample)) + } + buf.WriteString(divider) + } + return buf.String() +} diff --git a/pkg/kv/kvserver/asim/tests/rand_framework.go b/pkg/kv/kvserver/asim/tests/rand_framework.go index 1a1c45e4517c..9f10cd0f126c 100644 --- a/pkg/kv/kvserver/asim/tests/rand_framework.go +++ b/pkg/kv/kvserver/asim/tests/rand_framework.go @@ -20,9 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/gen" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/metrics" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/state" - "github.com/guptarohit/asciigraph" ) type testRandOptions struct { @@ -36,7 +34,7 @@ type testRandOptions struct { type testSettings struct { numIterations int duration time.Duration - verbose bool + verbose OutputFlags randSource *rand.Rand assertions []SimulationAssertion randOptions testRandOptions @@ -44,64 +42,7 @@ type testSettings struct { rangeGen rangeGenSettings } -// String converts the test setting to a string for output. -// For example, -// test settings -// num_iterations=10 duration=5m0s -// ---------------------------------- -// generating cluster configurations using randomized option -// clusterGenType=multi_region -// generating ranges configurations using static option -// generating load configurations using static option -// generating events configurations using static option -// generating settings configurations using static option -func (t testSettings) String() string { - buf := strings.Builder{} - buf.WriteString(fmt.Sprintln("test settings")) - buf.WriteString(fmt.Sprintf("\tnum_iterations=%v duration=%s\n", t.numIterations, t.duration.Round(time.Second))) - divider := fmt.Sprintln("----------------------------------") - buf.WriteString(divider) - - configStr := func(configType string, optionType string) string { - return fmt.Sprintf("generating %s configurations using %s option\n", configType, optionType) - } - - if t.randOptions.cluster { - buf.WriteString(configStr("cluster", "randomized")) - buf.WriteString(fmt.Sprintf("\t%v\n", t.clusterGen)) - } else { - buf.WriteString(configStr("cluster", "static")) - } - - if t.randOptions.ranges { - buf.WriteString(configStr("ranges", "randomized")) - buf.WriteString(fmt.Sprintf("\t%v\n", t.rangeGen)) - } else { - buf.WriteString(configStr("ranges", "static")) - } - - if t.randOptions.load { - buf.WriteString(configStr("load", "randomized")) - } else { - buf.WriteString(configStr("load", "static")) - } - - if t.randOptions.staticEvents { - buf.WriteString(configStr("events", "randomized")) - } else { - buf.WriteString(configStr("events", "static")) - } - - if t.randOptions.staticSettings { - buf.WriteString(configStr("settings", "randomized")) - } else { - buf.WriteString(configStr("settings", "static")) - } - return buf.String() -} - type randTestingFramework struct { - recordBuf *strings.Builder s testSettings rangeGenerator generator keySpaceGenerator generator @@ -121,10 +62,8 @@ func newRandTestingFramework(settings testSettings) randTestingFramework { } rangeGenerator := newGenerator(settings.randSource, defaultMinRange, defaultMaxRange, settings.rangeGen.rangeGenType) keySpaceGenerator := newGenerator(settings.randSource, defaultMinKeySpace, defaultMaxKeySpace, settings.rangeGen.keySpaceGenType) - var buf strings.Builder return randTestingFramework{ - recordBuf: &buf, s: settings, rangeGenerator: rangeGenerator, keySpaceGenerator: keySpaceGenerator, @@ -168,55 +107,48 @@ func (f randTestingFramework) getStaticEvents() gen.StaticEvents { // runRandTest creates randomized configurations based on the specified test // settings and runs one test using those configurations. -func (f randTestingFramework) runRandTest(nthSample int) (asim.History, string) { - buf := strings.Builder{} - divider := fmt.Sprintln("----------------------------------") - if nthSample == 1 { - buf.WriteString(divider) - } - buf.WriteString(fmt.Sprintf("sample%d: start running\n", nthSample)) +func (f randTestingFramework) runRandTest() testResult { ctx := context.Background() cluster := f.getCluster() ranges := f.getRanges() load := f.getLoad() staticSettings := f.getStaticSettings() staticEvents := f.getStaticEvents() - simulator := gen.GenerateSimulation(f.s.duration, cluster, ranges, load, staticSettings, staticEvents, f.s.randSource.Int63()) + seed := f.s.randSource.Int63() + simulator := gen.GenerateSimulation(f.s.duration, cluster, ranges, load, staticSettings, staticEvents, seed) + initialState, initialTime := simulator.State(), simulator.Curr() simulator.RunSim(ctx) history := simulator.History() failed, reason := checkAssertions(ctx, history, f.s.assertions) - if failed { - buf.WriteString(fmt.Sprintf("sample%d: failed assertion\n%s", nthSample, reason)) - } else { - buf.WriteString(fmt.Sprintf("sample%d: pass\n", nthSample)) + return testResult{ + seed: seed, + failed: failed, + reason: reason, + clusterGen: cluster, + rangeGen: ranges, + loadGen: load, + eventGen: staticEvents, + initialState: initialState, + initialTime: initialTime, } - buf.WriteString(divider) - return history, buf.String() } // runRandTestRepeated runs the test multiple times, each time with a new // randomly generated configuration. The result of each iteration is recorded in // f.recordBuf. -func (f randTestingFramework) runRandTestRepeated() { +func (f randTestingFramework) runRandTestRepeated() testResultsReport { numIterations := f.s.numIterations - runs := make([]asim.History, numIterations) - f.recordBuf.WriteString(f.s.String()) + outputs := make([]testResult, numIterations) for i := 0; i < numIterations; i++ { - history, output := f.runRandTest(i + 1) - runs[i] = history - f.recordBuf.WriteString(output) + outputs[i] = f.runRandTest() } - if f.s.verbose { - plotAllHistory(runs, f.recordBuf) + return testResultsReport{ + flags: f.s.verbose, + settings: f.s, + outputSlice: outputs, } } -// printResults outputs the following information: 1. test settings used for -// generating the tests 2. results of each randomized test -func (f randTestingFramework) printResults() string { - return f.recordBuf.String() -} - // loadClusterInfo creates a LoadedCluster from a matching ClusterInfo based on // the given configNam, or panics if no match is found in existing // configurations. @@ -227,26 +159,6 @@ func loadClusterInfo(configName string) gen.LoadedCluster { } } -// PlotAllHistory outputs stat plots for the provided asim history array into -// the given strings.Builder buf. -func plotAllHistory(runs []asim.History, buf *strings.Builder) { - settings := defaultPlotSettings() - stat, height, width := settings.stat, settings.height, settings.width - for i := 0; i < len(runs); i++ { - history := runs[i] - ts := metrics.MakeTS(history.Recorded) - statTS := ts[stat] - buf.WriteString(fmt.Sprintf("sample%d\n", i+1)) - buf.WriteString(asciigraph.PlotMany( - statTS, - asciigraph.Caption(stat), - asciigraph.Height(height), - asciigraph.Width(width), - )) - buf.WriteString("\n") - } -} - // checkAssertions checks the given history and assertions, returning (bool, // reason) indicating any failures and reasons if any assertions fail. func checkAssertions( diff --git a/pkg/kv/kvserver/asim/tests/rand_gen.go b/pkg/kv/kvserver/asim/tests/rand_gen.go index 7d61909abb38..324640231b38 100644 --- a/pkg/kv/kvserver/asim/tests/rand_gen.go +++ b/pkg/kv/kvserver/asim/tests/rand_gen.go @@ -49,6 +49,10 @@ type RandomizedBasicRanges struct { var _ gen.RangeGen = &RandomizedBasicRanges{} +func (r RandomizedBasicRanges) String() string { + return fmt.Sprintf("randomized ranges with placement_type=%v, %v", r.placementType, r.BaseRanges) +} + func (r RandomizedBasicRanges) Generate( seed int64, settings *config.SimulationSettings, s state.State, ) state.State { @@ -71,6 +75,10 @@ type WeightedRandomizedBasicRanges struct { var _ gen.RangeGen = &WeightedRandomizedBasicRanges{} +func (wr WeightedRandomizedBasicRanges) String() string { + return fmt.Sprintf("weighted randomized ranges with placement_type=%v, weighted_rand=%v, %v", wr.placementType, wr.weightedRand, wr.BaseRanges) +} + func (wr WeightedRandomizedBasicRanges) Generate( seed int64, settings *config.SimulationSettings, s state.State, ) state.State { diff --git a/pkg/kv/kvserver/asim/tests/rand_test.go b/pkg/kv/kvserver/asim/tests/rand_test.go index 74f61276e9a5..9dcd465247de 100644 --- a/pkg/kv/kvserver/asim/tests/rand_test.go +++ b/pkg/kv/kvserver/asim/tests/rand_test.go @@ -24,7 +24,7 @@ const ( defaultNumIterations = 3 defaultSeed = int64(42) defaultDuration = 10 * time.Minute - defaultVerbosity = false + defaultVerbosity = OutputResultOnly ) // TestRandomized is a randomized data-driven testing framework that validates @@ -79,8 +79,8 @@ const ( // number of stores 4. sum of weights in the array should be equal to 1 // 3. "eval" [seed=] [num_iterations=] [duration=] -// [verbose=] -// e.g. eval seed=20 duration=30m2s verbose=true +// [verbose=(<[]("result_only","test_settings","initial_state","config_gen","topology","all")>)] +// e.g. eval seed=20 duration=30m2s verbose=(test_settings,initial_state) // - eval: generates a simulation based on the configuration set with the given // commands. // - seed(default value is int64(42)): used to create a new random number @@ -88,8 +88,17 @@ const ( // - num_iterations(default value is 3): specifies the number of simulations to // run. // - duration(default value is 10m): defines duration of each iteration. -// - verbose(default value is false): if set to true, plots all stat(as -// specified by defaultStat) history. +// - verbose(default value is OutputResultOnly): used to set flags on what to +// show in the test output messages. By default, all details are displayed +// upon assertion failure. +// - result_only: only shows whether the test passed or failed, along with +// any failure messages +// - test_settings: displays settings used for the repeated tests +// - initial_state: displays the initial state of each test iteration +// - config_gen: displays the input configurations generated for each test +// iteration +// - topology: displays the topology of cluster configurations +// - all: display everything above // RandTestingFramework is initialized with specified testSetting and maintains // its state across all iterations. It repeats the test with different random @@ -98,7 +107,7 @@ const ( // 1. Generates a random configuration: based on whether randOption is on and // the specific settings for randomized generation. // 2. Executes the simulation and checks the assertions on the final state. -// 3. Stores any outputs and assertion failures in a buffer. +// 3. Stores any outputs and assertion failures in a slice. func TestRandomized(t *testing.T) { dir := datapathutils.TestDataPath(t, "rand") datadriven.Walk(t, dir, func(t *testing.T, path string) { @@ -163,8 +172,8 @@ func TestRandomized(t *testing.T) { clusterGen: cGenSettings, } f := newRandTestingFramework(settings) - f.runRandTestRepeated() - return f.printResults() + outputs := f.runRandTestRepeated() + return outputs.String() default: return fmt.Sprintf("unknown command: %s", d.Cmd) } diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/default_settings b/pkg/kv/kvserver/asim/tests/testdata/rand/default_settings index 30c15114859f..9e670f417972 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/rand/default_settings +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/default_settings @@ -3,10 +3,46 @@ # set correctly, and they should stay consistent across iterations and across # runs. -eval verbose=true duration=5m +# Default setting: we expect the output to only include test result. +eval +---- +---------------------------------- +sample1: start running +sample1: pass +---------------------------------- +sample2: start running +sample2: pass +---------------------------------- +sample3: start running +sample3: pass +---------------------------------- + +clear +---- + +# result_only flag: we expect the output to only include test result. +eval verbose=(result_only) +---- +---------------------------------- +sample1: start running +sample1: pass +---------------------------------- +sample2: start running +sample2: pass +---------------------------------- +sample3: start running +sample3: pass +---------------------------------- + +clear +---- + +# test_settings flag: we expect the output to include the test settings used to +# generate the configurations. +eval verbose=(test_settings) ---- test settings - num_iterations=3 duration=5m0s + num_iterations=3 duration=10m0s ---------------------------------- generating cluster configurations using static option generating ranges configurations using static option @@ -23,20 +59,144 @@ sample2: pass sample3: start running sample3: pass ---------------------------------- -sample1 - 1.00 ┼─────────────────────────────────────────────────────────────────────────────── - replicas -sample2 - 1.00 ┼─────────────────────────────────────────────────────────────────────────────── - replicas -sample3 - 1.00 ┼─────────────────────────────────────────────────────────────────────────────── - replicas clear ---- -eval +# config_gen flag: we expect the output to include the configurations generated +# at each iteration. +eval verbose=(config_gen) +---- +---------------------------------- +sample1: start running +configurations generated using seed 3440579354231278675 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 608747136543856411 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 5571782338101878760 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample3: pass +---------------------------------- + +clear +---- + +# initial_state flag: we expect the output to include the initial state of the +# simulation at each iteration. +eval verbose=(initial_state) +---- +---------------------------------- +sample1: start running +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample1: pass +---------------------------------- +sample2: start running +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample2: pass +---------------------------------- +sample3: start running +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample3: pass +---------------------------------- + +clear +---- + +# topology flag: we expect the output to include the topology of the cluster +# configuration generated at each iteration. +eval verbose=(topology) +---- +---------------------------------- +sample1: start running +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] +sample1: pass +---------------------------------- +sample2: start running +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] +sample2: pass +---------------------------------- +sample3: start running +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] +sample3: pass +---------------------------------- + +clear +---- + +# initial_state,test_settings,config_gen flags: we expect that the output to +# include initial state, test settings, and generated configurations. +eval verbose=(initial_state,test_settings,config_gen) +---- +test settings + num_iterations=3 duration=10m0s +---------------------------------- +generating cluster configurations using static option +generating ranges configurations using static option +generating load configurations using static option +generating events configurations using static option +generating settings configurations using static option +---------------------------------- +sample1: start running +configurations generated using seed 3440579354231278675 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 608747136543856411 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 5571782338101878760 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +sample3: pass +---------------------------------- + +# all flag: we expect that the output to include test settings, generated +# configurations, initial state, and topology. +eval verbose=(all) ---- test settings num_iterations=3 duration=10m0s @@ -48,11 +208,44 @@ generating events configurations using static option generating settings configurations using static option ---------------------------------- sample1: start running +configurations generated using seed 3440579354231278675 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] sample1: pass ---------------------------------- sample2: start running +configurations generated using seed 608747136543856411 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] sample2: pass ---------------------------------- sample3: start running +configurations generated using seed 5571782338101878760 + basic cluster with nodes=3, stores_per_node=1 + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(3)=[s1n1=(replicas(10)),s2n2=(replicas(10)),s3n3=(replicas(10))] +topology: +AU_EAST + AU_EAST_1 + └── [1 2 3] sample3: pass ---------------------------------- diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/rand_cluster b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_cluster new file mode 100644 index 000000000000..77b4392b1fbc --- /dev/null +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_cluster @@ -0,0 +1,193 @@ +# In this test, we run the randomized testing framework with rand_cluster option +# only. We expect that the range_gen configurations will be properly configured +# and remain consistent across different runs. Meanwhile, the cluster_gen +# configuration will vary according to the selected cluster_gen_type option. + +rand_cluster cluster_gen_type=single_region +---- + +eval duration=5m num_iterations=3 verbose=(config_gen) +---- +---------------------------------- +sample1: start running +configurations generated using seed 608747136543856411 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 3534334367214237261 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample3: pass +---------------------------------- + +clear +---- + +rand_cluster cluster_gen_type=multi_region +---- + +eval duration=5m num_iterations=3 verbose=(all) +---- +test settings + num_iterations=3 duration=5m0s +---------------------------------- +generating cluster configurations using randomized option + cluster_gen_type=multi_region +generating ranges configurations using static option +generating load configurations using static option +generating events configurations using static option +generating settings configurations using static option +---------------------------------- +sample1: start running +configurations generated using seed 608747136543856411 + loaded cluster with + region:US_East [zone=US_East_1(nodes=1,stores=0), zone=US_East_2(nodes=2,stores=0), zone=US_East_3(nodes=3,stores=0), zone=US_East_3(nodes=10,stores=0)] + region:US_West [zone=US_West_1(nodes=2,stores=0)] + region:EU [zone=EU_1(nodes=3,stores=0), zone=EU_2(nodes=3,stores=0), zone=EU_3(nodes=4,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(28)=[s1n1=(replicas(1)),s2n2=(replicas(1)),s3n3=(replicas(1)),s4n4=(replicas(1)),s5n5=(replicas(1)),s6n6=(replicas(1)),s7n7=(replicas(1)),s8n8=(replicas(1)),s9n9=(replicas(2)),s10n10=(replicas(1)),s11n11=(replicas(1)),s12n12=(replicas(1)),s13n13=(replicas(1)),s14n14=(replicas(1)),s15n15=(replicas(1)),s16n16=(replicas(1)),s17n17=(replicas(1)),s18n18=(replicas(1)),s19n19=(replicas(1)),s20n20=(replicas(1)),s21n21=(replicas(1)),s22n22=(replicas(2)),s23n23=(replicas(1)),s24n24=(replicas(1)),s25n25=(replicas(1)),s26n26=(replicas(1)),s27n27=(replicas(1)),s28n28=(replicas(1))] +topology: +EU + EU_1 + │ └── [19 20 21] + EU_2 + │ └── [22 23 24] + EU_3 + │ └── [25 26 27 28] +US_East + US_East_1 + │ └── [1] + US_East_2 + │ └── [2 3] + US_East_3 + │ └── [4 5 6 7 8 9 10 11 12 13 14 15 16] +US_West + US_West_1 + └── [17 18] +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US_East [zone=US_East_1(nodes=4,stores=0), zone=US_East_2(nodes=4,stores=0), zone=US_East_3(nodes=4,stores=0)] + region:US_West [zone=US_West_1(nodes=4,stores=0), zone=US_West_2(nodes=4,stores=0), zone=US_West_3(nodes=4,stores=0)] + region:EU [zone=EU_1(nodes=4,stores=0), zone=EU_2(nodes=4,stores=0), zone=EU_3(nodes=4,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(36)=[s1n1=(replicas(1)),s2n2=(replicas(1)),s3n3=(replicas(1)),s4n4=(replicas(1)),s5n5=(replicas(1)),s6n6=(replicas(1)),s7n7=(replicas(0)),s8n8=(replicas(1)),s9n9=(replicas(1)),s10n10=(replicas(1)),s11n11=(replicas(1)),s12n12=(replicas(1)),s13n13=(replicas(0)),s14n14=(replicas(1)),s15n15=(replicas(0)),s16n16=(replicas(0)),s17n17=(replicas(1)),s18n18=(replicas(1)),s19n19=(replicas(1)),s20n20=(replicas(1)),s21n21=(replicas(1)),s22n22=(replicas(1)),s23n23=(replicas(1)),s24n24=(replicas(1)),s25n25=(replicas(1)),s26n26=(replicas(1)),s27n27=(replicas(1)),s28n28=(replicas(0)),s29n29=(replicas(1)),s30n30=(replicas(1)),s31n31=(replicas(0)),s32n32=(replicas(1)),s33n33=(replicas(1)),s34n34=(replicas(1)),s35n35=(replicas(1)),s36n36=(replicas(1))] +topology: +EU + EU_1 + │ └── [25 26 27 28] + EU_2 + │ └── [29 30 31 32] + EU_3 + │ └── [33 34 35 36] +US_East + US_East_1 + │ └── [1 2 3 4] + US_East_2 + │ └── [5 6 7 8] + US_East_3 + │ └── [9 10 11 12] +US_West + US_West_1 + └── [13 14 15 16] + US_West_2 + └── [17 18 19 20] + US_West_3 + └── [21 22 23 24] +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 3534334367214237261 + loaded cluster with + region:US_East [zone=US_East_1(nodes=1,stores=0), zone=US_East_2(nodes=2,stores=0), zone=US_East_3(nodes=3,stores=0), zone=US_East_3(nodes=10,stores=0)] + region:US_West [zone=US_West_1(nodes=2,stores=0)] + region:EU [zone=EU_1(nodes=3,stores=0), zone=EU_2(nodes=3,stores=0), zone=EU_3(nodes=4,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(28)=[s1n1=(replicas(1)),s2n2=(replicas(1)),s3n3=(replicas(1)),s4n4=(replicas(1)),s5n5=(replicas(1)),s6n6=(replicas(1)),s7n7=(replicas(1)),s8n8=(replicas(1)),s9n9=(replicas(2)),s10n10=(replicas(1)),s11n11=(replicas(1)),s12n12=(replicas(1)),s13n13=(replicas(1)),s14n14=(replicas(1)),s15n15=(replicas(1)),s16n16=(replicas(1)),s17n17=(replicas(1)),s18n18=(replicas(1)),s19n19=(replicas(1)),s20n20=(replicas(1)),s21n21=(replicas(1)),s22n22=(replicas(2)),s23n23=(replicas(1)),s24n24=(replicas(1)),s25n25=(replicas(1)),s26n26=(replicas(1)),s27n27=(replicas(1)),s28n28=(replicas(1))] +topology: +EU + EU_1 + │ └── [19 20 21] + EU_2 + │ └── [22 23 24] + EU_3 + │ └── [25 26 27 28] +US_East + US_East_1 + │ └── [1] + US_East_2 + │ └── [2 3] + US_East_3 + │ └── [4 5 6 7 8 9 10 11 12 13 14 15 16] +US_West + US_West_1 + └── [17 18] +sample3: pass +---------------------------------- + +clear +---- + +rand_cluster cluster_gen_type=any_region +---- + +eval duration=5m num_iterations=3 verbose=(config_gen) +---- +---------------------------------- +sample1: start running +configurations generated using seed 608747136543856411 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 3534334367214237261 + loaded cluster with + region:US_East [zone=US_East_1(nodes=1,stores=0), zone=US_East_2(nodes=2,stores=0), zone=US_East_3(nodes=3,stores=0), zone=US_East_3(nodes=10,stores=0)] + region:US_West [zone=US_West_1(nodes=2,stores=0)] + region:EU [zone=EU_1(nodes=3,stores=0), zone=EU_2(nodes=3,stores=0), zone=EU_3(nodes=4,stores=0)] + basic ranges with placement_type=even, ranges=10, key_space=200000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +sample3: pass +---------------------------------- diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges index abf6ff055db6..f3f61adf76ef 100644 --- a/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/rand_ranges @@ -1,105 +1,35 @@ # In this test, we run the randomized testing framework with both the -# rand_ranges and rand_cluster options. +# rand_ranges and rand_cluster options. We expect that the range_gen and +# cluster_gen configurations will change across iterations and runs. +# +# When a specific generator type is used, we expect the numbers generated to +# form a distribution. Due to limited sample size, it can be hard to observe the +# corresponding distribution pattern, but we can confirm that the parameters are +# set correctly and numbers generated look roughly accurate. +# +# Other parameters, such as replication factor and placement type should also be +# set and remain consistent across iterations. rand_cluster cluster_gen_type=single_region ---- -rand_ranges placement_type=random replication_factor=3 range_gen_type=uniform keyspace_gen_type=uniform ----- - -eval duration=5m num_iterations=3 verbose=true ----- -test settings - num_iterations=3 duration=5m0s ----------------------------------- -generating cluster configurations using randomized option - cluster_gen_type=single_region -generating ranges configurations using randomized option - placement_type=random, range_gen_type=uniform, key_space=uniform, replication_factor=3, weightedRand=[] -generating load configurations using static option -generating events configurations using static option -generating settings configurations using static option ----------------------------------- -sample1: start running -sample1: pass ----------------------------------- -sample2: start running -sample2: pass ----------------------------------- -sample3: start running -sample3: pass ----------------------------------- -sample1 - 111 ┼─╮ - 104 ┤ ╰╮ - 98 ┼──╮─╮ - 91 ┤ ╰──╮╮ - 85 ┼────╮╰──╮ - 78 ┤ ╰───╰──╮ - 72 ┼─────╮ ╰─╰╰────╮ - 65 ┤ ╰─╮ ╰─╭╭╮╭╮╭╭╮╮─╮─╭╮─╭────╮─────╭─╮─╭─╮─╮ ╭──╮ ╭──────╮ ╭────╮─── - 59 ┼───╮───────╭╭────╯╰───────╯╰────────────────╯─╰──────────────────────────────── - 52 ┤ ╰───╮╭───╯╯ ╰╯ - 46 ┤ ╭╭╰╯╯╯ - 39 ┼───╭───╯ - 33 ┤ ╭╯╯ - 26 ┼──│ - 20 ┤╭─╯ - 13 ┼╯ - replicas -sample2 - 345 ┼───╮ - 322 ┤ ╰╰─────╮ - 299 ┼──────╮╰─╰─────╮ - 276 ┼──╮──╮╰──────────────╮ - 253 ┤ ╰────────────────╮─╰───────╮╮ - 230 ┤ ╰─╰─────────────────╮╮ - 207 ┤ ╰╰───────────────────────────────────────── - 184 ┼─────────────────────────────────────────╭───────────────────────────────────── - 161 ┼─────────────────────────────────────────╯╯╯ - 138 ┤ ╭─────────╯╯ - 115 ┤ ╭──────╯╯ - 92 ┤ ╭─────╭────╯╯ - 69 ┼───╯╯╭───╯ - 46 ┼───╭─╯ - 23 ┤ ╭─╯ - 0 ┼─╯ - replicas -sample3 - 180 ┼─╮╮ - 168 ┤ ╰──╮╮ - 156 ┼───────╮╮ - 144 ┤ ╰───╮╮╮ - 132 ┤ ╰───╮╮╮ - 120 ┼───────╮ ╰╰────╮╮ - 108 ┤ ╰───╰────────╮───╰───────╮ - 96 ┤ ╰───────────╰────────────────────────────────────────────────── - 84 ┼─────────────────╭╭──────╯╯ - 72 ┤ ╭────╯╯ - 60 ┤ ╭─╭───╯ - 48 ┤╭╭───────╯ - 36 ┼─╯╭──╯ - 24 ┼──╯╯ - 12 ┤ ╭╯ - 0 ┼─╯ - replicas - -clear ----- - - -rand_cluster cluster_gen_type=multi_region ----- - +# We expect the placement of ranges across stores to be randomized with a +# replication factor of 3. range∈[1, 1000] should be generated with an uniform +# distribution pattern across iterations. The numbers generated are +# (305,944,479,487,285,611,271,502,865,787) - randomly distributed within [1, +# 1000]. keyspace∈[1000, 200000] should be generated with a zipf distribution +# pattern across iterations (1015,2357,1003,2171,1060,1000,1439,1198,1427.1001) +# - numbers close to 1000 are expected to occur frequently. configurations +# should be generated with an uniform distribution pattern across iterations. rand_ranges placement_type=random replication_factor=3 range_gen_type=uniform keyspace_gen_type=zipf ---- -eval duration=5m num_iterations=10 +eval duration=5m num_iterations=10 verbose=(test_settings,config_gen) ---- test settings num_iterations=10 duration=5m0s ---------------------------------- generating cluster configurations using randomized option - cluster_gen_type=multi_region + cluster_gen_type=single_region generating ranges configurations using randomized option placement_type=random, range_gen_type=uniform, key_space=zipf, replication_factor=3, weightedRand=[] generating load configurations using static option @@ -107,152 +37,115 @@ generating events configurations using static option generating settings configurations using static option ---------------------------------- sample1: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=305, key_space=1015, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample1: pass ---------------------------------- sample2: start running +configurations generated using seed 2643318057788968173 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + randomized ranges with placement_type=random, ranges=944, key_space=1357, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample2: pass ---------------------------------- sample3: start running +configurations generated using seed 6972490225919430754 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + randomized ranges with placement_type=random, ranges=479, key_space=1003, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample3: pass ---------------------------------- sample4: start running +configurations generated using seed 8427801741804500990 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=487, key_space=2171, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample4: pass ---------------------------------- sample5: start running +configurations generated using seed 8063729658764635782 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=285, key_space=1060, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample5: pass ---------------------------------- sample6: start running +configurations generated using seed 3814222400681984302 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=611, key_space=1000, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample6: pass ---------------------------------- sample7: start running +configurations generated using seed 13013938835543503 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=271, key_space=1439, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample7: pass ---------------------------------- sample8: start running +configurations generated using seed 2207144605302255518 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=502, key_space=1198, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample8: pass ---------------------------------- sample9: start running +configurations generated using seed 5888461606762344739 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=865, key_space=1427, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample9: pass ---------------------------------- sample10: start running +configurations generated using seed 6738330972202035110 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=787, key_space=1001, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample10: pass ---------------------------------- clear ---- -rand_cluster cluster_gen_type=multi_region ----- - -rand_ranges placement_type=random replication_factor=3 range_gen_type=zipf keyspace_gen_type=uniform ----- - -eval duration=5m num_iterations=10 ----- -test settings - num_iterations=10 duration=5m0s ----------------------------------- -generating cluster configurations using randomized option - cluster_gen_type=multi_region -generating ranges configurations using randomized option - placement_type=random, range_gen_type=zipf, key_space=uniform, replication_factor=3, weightedRand=[] -generating load configurations using static option -generating events configurations using static option -generating settings configurations using static option ----------------------------------- -sample1: start running -sample1: pass ----------------------------------- -sample2: start running -sample2: pass ----------------------------------- -sample3: start running -sample3: pass ----------------------------------- -sample4: start running -sample4: pass ----------------------------------- -sample5: start running -sample5: pass ----------------------------------- -sample6: start running -sample6: pass ----------------------------------- -sample7: start running -sample7: pass ----------------------------------- -sample8: start running -sample8: pass ----------------------------------- -sample9: start running -sample9: pass ----------------------------------- -sample10: start running -sample10: pass ----------------------------------- - -clear ----- - -rand_cluster cluster_gen_type=any_region ----- - -rand_ranges placement_type=random replication_factor=3 range_gen_type=zipf keyspace_gen_type=zipf ----- - -eval duration=5m num_iterations=10 ----- -test settings - num_iterations=10 duration=5m0s ----------------------------------- -generating cluster configurations using randomized option - cluster_gen_type=any_region -generating ranges configurations using randomized option - placement_type=random, range_gen_type=zipf, key_space=zipf, replication_factor=3, weightedRand=[] -generating load configurations using static option -generating events configurations using static option -generating settings configurations using static option ----------------------------------- -sample1: start running -sample1: pass ----------------------------------- -sample2: start running -sample2: pass ----------------------------------- -sample3: start running -sample3: pass ----------------------------------- -sample4: start running -sample4: pass ----------------------------------- -sample5: start running -sample5: pass ----------------------------------- -sample6: start running -sample6: pass ----------------------------------- -sample7: start running -sample7: pass ----------------------------------- -sample8: start running -sample8: pass ----------------------------------- -sample9: start running -sample9: pass ----------------------------------- -sample10: start running -sample10: pass ----------------------------------- - -clear ----- - +# We expect ranges to be randomly allocated across stores with a replication +# factor of 1. Assertion failures on some samples are expected under this setup. +# When there is only one replica and the removal target in rebalancing is the +# leaseholder, stabilizing is hard. The system can't easily remove the replica, +# so it choose to fall back to adding another replica, hoping lease transfer +# would happen next time this range is checked. This can cause thrashing where +# settling is difficult. In addition, we expect all output details to be +# displayed upon test failure. Please see the comment in +# ReplicationChangesForRebalance for more details. rand_cluster cluster_gen_type=single_region ---- rand_ranges placement_type=random replication_factor=1 ---- -eval duration=20m num_iterations=3 +eval duration=20m num_iterations=3 verbose=(test_settings,config_gen) ---- test settings num_iterations=3 duration=20m0s @@ -266,9 +159,31 @@ generating events configurations using static option generating settings configurations using static option ---------------------------------- sample1: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US [zone=US_1(nodes=1,stores=5), zone=US_2(nodes=1,stores=5), zone=US_3(nodes=1,stores=5)] + randomized ranges with placement_type=random, ranges=305, key_space=96760, replication_factor=1, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample1: pass ---------------------------------- sample2: start running +configurations generated using seed 2643318057788968173 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + randomized ranges with placement_type=random, ranges=944, key_space=150098, replication_factor=1, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(15)=[s1n1=(replicas(65)),s2n2=(replicas(64)),s3n3=(replicas(66)),s4n4=(replicas(63)),s5n5=(replicas(63)),s6n6=(replicas(63)),s7n7=(replicas(63)),s8n8=(replicas(62)),s9n9=(replicas(64)),s10n10=(replicas(64)),s11n11=(replicas(62)),s12n12=(replicas(63)),s13n13=(replicas(63)),s14n14=(replicas(64)),s15n15=(replicas(64))] +topology: +US + US_1 + └── [1 2 3 4 5] + US_2 + └── [6 7 8 9 10] + US_3 + └── [11 12 13 14 15] sample2: failed assertion conformance unavailable=0 under=0 over=0 violating=0 actual unavailable=0 under=0, over=9 violating=0 @@ -284,37 +199,11 @@ over replicated: r875:000013{8966-9125} [(n3,s3):3, (n2,s2):4] applying ttl_seconds=0 num_replicas=1 num_voters=1 ---------------------------------- sample3: start running -sample3: pass ----------------------------------- - -clear ----- - -rand_cluster cluster_gen_type=single_region ----- - -rand_ranges replication_factor=3 placement_type=weighted_rand weighted_rand=(0.1, 0.2, 0.7) ----- - -eval duration=20m num_iterations=3 ----- -test settings - num_iterations=3 duration=20m0s ----------------------------------- -generating cluster configurations using randomized option - cluster_gen_type=single_region -generating ranges configurations using randomized option - placement_type=weighted_rand, range_gen_type=uniform, key_space=uniform, replication_factor=3, weightedRand=[0.1 0.2 0.7] -generating load configurations using static option -generating events configurations using static option -generating settings configurations using static option ----------------------------------- -sample1: start running -sample1: pass ----------------------------------- -sample2: start running -sample2: pass ----------------------------------- -sample3: start running +configurations generated using seed 6972490225919430754 + loaded cluster with + region:US [zone=US_1(nodes=5,stores=0), zone=US_2(nodes=5,stores=0), zone=US_3(nodes=5,stores=0)] + randomized ranges with placement_type=random, ranges=479, key_space=199954, replication_factor=1, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 sample3: pass ---------------------------------- diff --git a/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand b/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand new file mode 100644 index 000000000000..82645562ad50 --- /dev/null +++ b/pkg/kv/kvserver/asim/tests/testdata/rand/weighted_rand @@ -0,0 +1,62 @@ +rand_cluster cluster_gen_type=multi_region +---- + +# We expect ranges distributed to the third and fourth stores to be larger than +# the first two stores. Ranges on other stores are significantly smaller, but +# they are not zero due to the best-effort nature of the range information +# distribution. +rand_ranges replication_factor=3 placement_type=weighted_rand weighted_rand=(0.01, 0.04, 0.2, 0.75) +---- + +eval duration=5m num_iterations=3 verbose=(test_settings,config_gen,initial_state) +---- +test settings + num_iterations=3 duration=5m0s +---------------------------------- +generating cluster configurations using randomized option + cluster_gen_type=multi_region +generating ranges configurations using randomized option + placement_type=weighted_rand, range_gen_type=uniform, key_space=uniform, replication_factor=3, weightedRand=[0.01 0.04 0.2 0.75] +generating load configurations using static option +generating events configurations using static option +generating settings configurations using static option +---------------------------------- +sample1: start running +configurations generated using seed 1926012586526624009 + loaded cluster with + region:US_East [zone=US_East_1(nodes=1,stores=0), zone=US_East_2(nodes=2,stores=0), zone=US_East_3(nodes=3,stores=0), zone=US_East_3(nodes=10,stores=0)] + region:US_West [zone=US_West_1(nodes=2,stores=0)] + region:EU [zone=EU_1(nodes=3,stores=0), zone=EU_2(nodes=3,stores=0), zone=EU_3(nodes=4,stores=0)] + weighted randomized ranges with placement_type=weighted_rand, weighted_rand=[0.01 0.04 0.2 0.75], ranges=305, key_space=96760, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(28)=[s1n1=(replicas(59)),s2n2=(replicas(74)),s3n3=(replicas(75)),s4n4=(replicas(77)),s5n5=(replicas(25)),s6n6=(replicas(27)),s7n7=(replicas(26)),s8n8=(replicas(28)),s9n9=(replicas(27)),s10n10=(replicas(26)),s11n11=(replicas(27)),s12n12=(replicas(25)),s13n13=(replicas(25)),s14n14=(replicas(27)),s15n15=(replicas(26)),s16n16=(replicas(26)),s17n17=(replicas(25)),s18n18=(replicas(26)),s19n19=(replicas(25)),s20n20=(replicas(28)),s21n21=(replicas(25)),s22n22=(replicas(26)),s23n23=(replicas(27)),s24n24=(replicas(27)),s25n25=(replicas(27)),s26n26=(replicas(27)),s27n27=(replicas(26)),s28n28=(replicas(26))] +sample1: pass +---------------------------------- +sample2: start running +configurations generated using seed 6603068123710785442 + loaded cluster with + region:US_East [zone=US_East_1(nodes=4,stores=0), zone=US_East_2(nodes=4,stores=0), zone=US_East_3(nodes=4,stores=0)] + region:US_West [zone=US_West_1(nodes=4,stores=0), zone=US_West_2(nodes=4,stores=0), zone=US_West_3(nodes=4,stores=0)] + region:EU [zone=EU_1(nodes=4,stores=0), zone=EU_2(nodes=4,stores=0), zone=EU_3(nodes=4,stores=0)] + weighted randomized ranges with placement_type=weighted_rand, weighted_rand=[0.01 0.04 0.2 0.75], ranges=603, key_space=99075, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(36)=[s1n1=(replicas(159)),s2n2=(replicas(306)),s3n3=(replicas(372)),s4n4=(replicas(374)),s5n5=(replicas(19)),s6n6=(replicas(20)),s7n7=(replicas(18)),s8n8=(replicas(17)),s9n9=(replicas(18)),s10n10=(replicas(18)),s11n11=(replicas(19)),s12n12=(replicas(18)),s13n13=(replicas(18)),s14n14=(replicas(20)),s15n15=(replicas(19)),s16n16=(replicas(19)),s17n17=(replicas(18)),s18n18=(replicas(17)),s19n19=(replicas(19)),s20n20=(replicas(21)),s21n21=(replicas(21)),s22n22=(replicas(19)),s23n23=(replicas(20)),s24n24=(replicas(19)),s25n25=(replicas(18)),s26n26=(replicas(18)),s27n27=(replicas(18)),s28n28=(replicas(17)),s29n29=(replicas(19)),s30n30=(replicas(18)),s31n31=(replicas(20)),s32n32=(replicas(18)),s33n33=(replicas(19)),s34n34=(replicas(18)),s35n35=(replicas(18)),s36n36=(replicas(20))] +sample2: pass +---------------------------------- +sample3: start running +configurations generated using seed 6656619927997724782 + loaded cluster with + region:US_East [zone=US_East_1(nodes=1,stores=0), zone=US_East_2(nodes=2,stores=0), zone=US_East_3(nodes=3,stores=0), zone=US_East_3(nodes=10,stores=0)] + region:US_West [zone=US_West_1(nodes=2,stores=0)] + region:EU [zone=EU_1(nodes=3,stores=0), zone=EU_2(nodes=3,stores=0), zone=EU_3(nodes=4,stores=0)] + weighted randomized ranges with placement_type=weighted_rand, weighted_rand=[0.01 0.04 0.2 0.75], ranges=394, key_space=89281, replication_factor=3, bytes=0 + basic load with rw_ratio=0.00, rate=0.00, skewed_access=false, min_block_size=1, max_block_size=1, min_key=1, max_key=200000 + number of static events generated=0 +initial state at 2022-03-21 11:00:00 +0000 UTC: + stores(28)=[s1n1=(replicas(116)),s2n2=(replicas(117)),s3n3=(replicas(156)),s4n4=(replicas(162)),s5n5=(replicas(26)),s6n6=(replicas(28)),s7n7=(replicas(26)),s8n8=(replicas(26)),s9n9=(replicas(26)),s10n10=(replicas(26)),s11n11=(replicas(26)),s12n12=(replicas(27)),s13n13=(replicas(25)),s14n14=(replicas(27)),s15n15=(replicas(25)),s16n16=(replicas(26)),s17n17=(replicas(28)),s18n18=(replicas(26)),s19n19=(replicas(27)),s20n20=(replicas(26)),s21n21=(replicas(26)),s22n22=(replicas(26)),s23n23=(replicas(26)),s24n24=(replicas(25)),s25n25=(replicas(28)),s26n26=(replicas(26)),s27n27=(replicas(26)),s28n28=(replicas(27))] +sample3: pass +----------------------------------