From 2b26224afc95097f6840d32c2854970fce95196e Mon Sep 17 00:00:00 2001
From: Radu Berinde <radu@cockroachlabs.com>
Date: Wed, 8 May 2024 17:59:20 -0700
Subject: [PATCH] metamorphic: make a reduce parameter configurable

Make the number of attempts for each probability configurable. When
the problem doesn't reproduce most of the time, we need more attempts.
---
 internal/metamorphic/meta_test.go            |  4 +--
 internal/metamorphic/metaflags/meta_flags.go |  6 +++++
 internal/metamorphic/reduce_test.go          | 28 +++++++++++---------
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/internal/metamorphic/meta_test.go b/internal/metamorphic/meta_test.go
index a5d5da1f4d..ad2a51e824 100644
--- a/internal/metamorphic/meta_test.go
+++ b/internal/metamorphic/meta_test.go
@@ -67,7 +67,7 @@ func runTestMeta(t *testing.T, multiInstance bool) {
 		}
 		testRootDir, runSubdirs := runOnceFlags.ParseCompare()
 		if runOnceFlags.TryToReduce {
-			tryToReduceCompare(t, runOnceFlags.Dir, testRootDir, runSubdirs)
+			tryToReduceCompare(t, runOnceFlags.Dir, testRootDir, runSubdirs, runOnceFlags.ReduceAttempts)
 			return
 		}
 		metamorphic.Compare(t, testRootDir, runOnceFlags.Seed, runSubdirs, onceOpts...)
@@ -81,7 +81,7 @@ func runTestMeta(t *testing.T, multiInstance bool) {
 			onceOpts = append(onceOpts, metamorphic.MultiInstance(2))
 		}
 		if runOnceFlags.TryToReduce {
-			tryToReduce(t, runOnceFlags.Dir, runOnceFlags.RunDir)
+			tryToReduce(t, runOnceFlags.Dir, runOnceFlags.RunDir, runOnceFlags.ReduceAttempts)
 			return
 		}
 		metamorphic.RunOnce(t, runOnceFlags.RunDir, runOnceFlags.Seed, filepath.Join(runOnceFlags.RunDir, "history"), onceOpts...)
diff --git a/internal/metamorphic/metaflags/meta_flags.go b/internal/metamorphic/metaflags/meta_flags.go
index 3561062e5e..b347030b10 100644
--- a/internal/metamorphic/metaflags/meta_flags.go
+++ b/internal/metamorphic/metaflags/meta_flags.go
@@ -97,6 +97,9 @@ type RunOnceFlags struct {
 	// operations that reproduce a problem during a test run (e.g. panic or
 	// internal error).
 	TryToReduce bool
+	// ReduceAttempts is the number of attempts to reduce (for each op removal
+	// probability).
+	ReduceAttempts int
 }
 
 func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
@@ -114,6 +117,9 @@ Example, --compare '_meta/231220-164251.3552792807512/{standard-000,random-025}'
 		`if set, we will try to reduce the number of operations that cause a failure. The
 verbose flag should be used with this flag.`)
 
+	flag.IntVar(&ro.ReduceAttempts, "reduce-attempts", 100,
+		`the number of attempts to reduce, for each probability; only used with --try-to-reduce.`)
+
 	return ro
 }
 
diff --git a/internal/metamorphic/reduce_test.go b/internal/metamorphic/reduce_test.go
index 5e54ae1fa3..2addab6d82 100644
--- a/internal/metamorphic/reduce_test.go
+++ b/internal/metamorphic/reduce_test.go
@@ -32,10 +32,10 @@ import (
 //
 // The test will save the smallest reproduction found and print out the relevant
 // information.
-func tryToReduce(t *testing.T, testStateDir string, runDir string) {
+func tryToReduce(t *testing.T, testStateDir string, runDir string, reduceAttempts int) {
 	testRootDir := filepath.Dir(runDir)
 	runSubdir := filepath.Base(runDir)
-	r := makeReducer(t, testStateDir, testRootDir, []string{runSubdir})
+	r := makeReducer(t, testStateDir, testRootDir, []string{runSubdir}, reduceAttempts)
 	r.Run(t)
 }
 
@@ -52,9 +52,9 @@ func tryToReduce(t *testing.T, testStateDir string, runDir string) {
 // The test will save the smallest reproduction found and print out the relevant
 // information.
 func tryToReduceCompare(
-	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string,
+	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string, reduceAttempts int,
 ) {
-	r := makeReducer(t, testStateDir, testRootDir, runSubdirs)
+	r := makeReducer(t, testStateDir, testRootDir, runSubdirs, reduceAttempts)
 	r.Run(t)
 }
 
@@ -62,8 +62,9 @@ func tryToReduceCompare(
 // tries to reduce the number of operations.
 type reducer struct {
 	// testRootDir is the directory of the test, which contains the "ops" file.
-	testRootDir string
-	configs     []testConfig
+	testRootDir    string
+	configs        []testConfig
+	reduceAttempts int
 
 	ops []string
 
@@ -83,7 +84,7 @@ type testConfig struct {
 }
 
 func makeReducer(
-	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string,
+	t *testing.T, testStateDir string, testRootDir string, runSubdirs []string, reduceAttempts int,
 ) *reducer {
 	// All run dirs should have the same parent path.
 	opsData, err := os.ReadFile(filepath.Join(testRootDir, "ops"))
@@ -104,10 +105,11 @@ func makeReducer(
 	t.Logf("Starting with %d operations", len(ops))
 
 	return &reducer{
-		testRootDir:  testRootDir,
-		configs:      tc,
-		ops:          ops,
-		testStateDir: testStateDir,
+		testRootDir:    testRootDir,
+		configs:        tc,
+		ops:            ops,
+		reduceAttempts: reduceAttempts,
+		testStateDir:   testStateDir,
 	}
 }
 
@@ -196,8 +198,8 @@ func (r *reducer) Run(t *testing.T) {
 	// find any reductions we decrease it. This works well even if the problem is
 	// not deterministic and isn't reproduced on every run.
 	for removeProbability := 0.1; removeProbability > 1e-5 && removeProbability > 0.1/float64(len(ops)); removeProbability *= 0.5 {
-		t.Logf("removeProbability %.2f", removeProbability)
-		for i := 0; i < 100; i++ {
+		t.Logf("removeProbability %.2f%%", removeProbability*100.0)
+		for i := 0; i < r.reduceAttempts; i++ {
 			if o := randomSubset(t, ops, removeProbability); r.try(t, o) {
 				ops = o
 				// Reset the counter.