storage/engine: MVCC Metamorphic test suite, first phase

This PR adds a new test-only sub-package to engine, metamorphic,
which has one test, TestMeta, that generates and runs random MVCC
operations on rocksdb and pebble instances with default settings.

Future additions to this test suite could include:

 - A "check" mode that takes an output file as input, parses it,
   runs the operations in that sequence, and compares output
   strings.
 - Diffing test output between rocksdb and pebble and failing if
   there's a difference
 - Adding support for more operations
 - Adding a "restart" operation that closes the engine and
   restarts a different kind of engine in the store directory,
   then confirming operations after that point generate the
   same output.

First-but-biggest part of cockroachdb#43762 .

Release note: None

pkg/storage/engine/metamorphic/deck.go

@@ -0,0 +1,67 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package metamorphic
+
+import (
+	"math/rand"
+	"sync"
+)
+
+// Deck is a random number generator that generates numbers in the range
+// [0,len(weights)-1] where the probability of i is
+// weights(i)/sum(weights). Unlike Weighted, the weights are specified as
+// integers and used in a deck-of-cards style random number selection which
+// ensures that each element is returned with a desired frequency within the
+// size of the deck.
+type Deck struct {
+	rng *rand.Rand
+	mu  struct {
+		sync.Mutex
+		index int
+		deck  []int
+	}
+}
+
+// NewDeck returns a new deck random number generator.
+func NewDeck(rng *rand.Rand, weights ...int) *Deck {
+	var sum int
+	for i := range weights {
+		sum += weights[i]
+	}
+	deck := make([]int, 0, sum)
+	for i := range weights {
+		for j := 0; j < weights[i]; j++ {
+			deck = append(deck, i)
+		}
+	}
+	d := &Deck{
+		rng: rng,
+	}
+	d.mu.index = len(deck)
+	d.mu.deck = deck
+	return d
+}
+
+// Int returns a random number in the range [0,len(weights)-1] where the
+// probability of i is weights(i)/sum(weights).
+func (d *Deck) Int() int {
+	d.mu.Lock()
+	if d.mu.index == len(d.mu.deck) {
+		d.rng.Shuffle(len(d.mu.deck), func(i, j int) {
+			d.mu.deck[i], d.mu.deck[j] = d.mu.deck[j], d.mu.deck[i]
+		})
+		d.mu.index = 0
+	}
+	result := d.mu.deck[d.mu.index]
+	d.mu.index++
+	d.mu.Unlock()
+	return result
+}

pkg/storage/engine/metamorphic/generator.go

@@ -0,0 +1,181 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package metamorphic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/storage/engine"
+	"github.com/cockroachdb/cockroach/pkg/util/hlc"
+)
+
+// Object to store info corresponding to one metamorphic test run. Responsible
+// for generating and executing operations.
+type metaTestRunner struct {
+	ctx         context.Context
+	w           io.Writer
+	t           *testing.T
+	rng         *rand.Rand
+	seed        int64
+	engine      engine.Engine
+	tsGenerator tsGenerator
+	managers    map[operandType]operandManager
+	nameToOp    map[string]*mvccOp
+	weights     []int
+	ops         []*mvccOp
+}
+
+func (m *metaTestRunner) init() {
+	// Use a passed-in seed. Using the same seed for two consecutive metamorphic
+	// test runs should guarantee the same operations being generated.
+	m.rng = rand.New(rand.NewSource(m.seed))
+
+	m.managers = map[operandType]operandManager{
+		OPERAND_TRANSACTION: &txnManager{
+			rng:             m.rng,
+			tsGenerator:     &m.tsGenerator,
+			txnIdMap:        make(map[string]*roachpb.Transaction),
+			inFlightBatches: make(map[*roachpb.Transaction][]engine.Batch),
+			testRunner:      m,
+		},
+		OPERAND_READWRITER: &readWriterManager{
+			rng:          m.rng,
+			eng:          m.engine,
+			batchToIdMap: make(map[engine.Batch]int),
+		},
+		OPERAND_MVCC_KEY: &keyManager{
+			rng:         m.rng,
+			tsGenerator: &m.tsGenerator,
+		},
+		OPERAND_VALUE:       &valueManager{m.rng},
+		OPERAND_TEST_RUNNER: &testRunnerManager{m},
+		OPERAND_ITERATOR: &iteratorManager{
+			rng:          m.rng,
+			readerToIter: make(map[engine.Reader][]engine.Iterator),
+			iterToId:     make(map[engine.Iterator]uint64),
+			iterCounter:  0,
+		},
+	}
+	m.nameToOp = make(map[string]*mvccOp)
+
+	m.weights = make([]int, len(operations))
+	for i := range operations {
+		m.weights[i] = operations[i].weight
+		m.nameToOp[operations[i].name] = &operations[i]
+	}
+	m.ops = nil
+}
+
+// generateAndRun generates n operations using a TPCC-style deck shuffle with
+// weighted probabilities of each operation appearing.
+func (m *metaTestRunner) generateAndRun(n uint64) {
+	m.ops = make([]*mvccOp, n)
+	deck := NewDeck(m.rng, m.weights...)
+
+	for i := uint64(0); i < n; i++ {
+		opToAdd := &operations[deck.Int()]
+
+		m.resolveAndRunOp(opToAdd)
+	}
+
+	// Close all open objects. This should let the engine close cleanly.
+	closingOrder := []operandType{
+		OPERAND_ITERATOR,
+		OPERAND_READWRITER,
+		OPERAND_TRANSACTION,
+	}
+	for _, operandType := range closingOrder {
+		m.managers[operandType].closeAll()
+	}
+}
+func (m *metaTestRunner) parseFileAndRun(f io.Reader) {
+	// TODO(itsbilal): Implement this.
+}
+
+func (m *metaTestRunner) runOp(run opRun) string {
+	op := run.op
+
+	// This operation might require other operations to run before it runs. Call
+	// the dependentOps method to resolve these dependencies.
+	if op.dependentOps != nil {
+		for _, opRun := range op.dependentOps(m, run.args...) {
+			m.runOp(opRun)
+		}
+	}
+
+	// Running the operation could cause this operand to not exist. Build strings
+	// for arguments beforehand.
+	argStrings := make([]string, len(op.operands))
+	for i, arg := range run.args {
+		argStrings[i] = m.managers[op.operands[i]].toString(arg)
+	}
+
+	m.ops = append(m.ops, op)
+	output := op.run(m.ctx, m, run.args...)
+	m.printOp(op, argStrings, output)
+	return output
+}
+
+// Resolve all operands (including recursively running openers for operands as
+// necessary) and run the specified operation.
+func (m *metaTestRunner) resolveAndRunOp(op *mvccOp) {
+	operandInstances := make([]operand, len(op.operands))
+
+	// Operation op depends on some operands to exist in an open state.
+	// If those operands' managers report a zero count for that object's open
+	// instances, recursively call addOp with that operand type's opener.
+	for i, operand := range op.operands {
+		opManager := m.managers[operand]
+		if opManager.count() == 0 {
+			// Add this operation to the list first, so that it creates the dependency.
+			m.resolveAndRunOp(m.nameToOp[opManager.opener()])
+		}
+		operandInstances[i] = opManager.get()
+	}
+
+	m.runOp(opRun{
+		op:   op,
+		args: operandInstances,
+	})
+}
+
+// Print passed-in operation, arguments and output string to output file.
+func (m *metaTestRunner) printOp(op *mvccOp, argStrings []string, output string) {
+	fmt.Fprintf(m.w, "%s(", op.name)
+	for i, arg := range argStrings {
+		if i > 0 {
+			fmt.Fprintf(m.w, ", ")
+		}
+		fmt.Fprintf(m.w, "%s", arg)
+	}
+	fmt.Fprintf(m.w, ") -> %s\n", output)
+}
+
+// Monotonically increasing timestamp generator.
+type tsGenerator struct {
+	lastTS hlc.Timestamp
+}
+
+func (t *tsGenerator) generate() hlc.Timestamp {
+	t.lastTS.WallTime++
+	return t.lastTS
+}
+
+func (t *tsGenerator) randomPastTimestamp(rng *rand.Rand) hlc.Timestamp {
+	var result hlc.Timestamp
+	result.WallTime = int64(float64(t.lastTS.WallTime+1) * rng.Float64())
+	return result
+}

pkg/storage/engine/metamorphic/meta_test.go

@@ -0,0 +1,139 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package metamorphic
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
+	"github.com/cockroachdb/cockroach/pkg/storage/engine"
+	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
+	"github.com/cockroachdb/cockroach/pkg/testutils"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+)
+
+// createTestRocksDBEngine returns a new in-memory RocksDB engine with 1MB of
+// storage capacity.
+func createTestRocksDBEngine(path string) (engine.Engine, error) {
+	return engine.NewEngine(enginepb.EngineTypeRocksDB, 1<<20, base.StorageConfig{
+		Attrs:           roachpb.Attributes{},
+		Dir:             path,
+		MustExist:       false,
+		MaxSize:         0,
+		Settings:        cluster.MakeTestingClusterSettings(),
+		UseFileRegistry: false,
+		ExtraOptions:    nil,
+	})
+}
+
+// createTestPebbleEngine returns a new in-memory Pebble storage engine.
+func createTestPebbleEngine(path string) (engine.Engine, error) {
+	return engine.NewEngine(enginepb.EngineTypePebble, 1<<20, base.StorageConfig{
+		Attrs:           roachpb.Attributes{},
+		Dir:             path,
+		MustExist:       false,
+		MaxSize:         0,
+		Settings:        cluster.MakeTestingClusterSettings(),
+		UseFileRegistry: false,
+		ExtraOptions:    nil,
+	})
+}
+
+var mvccEngineImpls = []struct {
+	name   string
+	create func(path string) (engine.Engine, error)
+}{
+	{"rocksdb", createTestRocksDBEngine},
+	{"pebble", createTestPebbleEngine},
+}
+
+var (
+	keep  = flag.Bool("keep", false, "keep temp directories after test")
+	check = flag.String("check", "", "run operations in specified file and check output for equality")
+)
+
+func runMetaTest(ctx context.Context, t *testing.T, seed int64, checkFile io.Reader) {
+	for _, engineImpl := range mvccEngineImpls {
+		t.Run(engineImpl.name, func(t *testing.T) {
+			tempDir, cleanup := testutils.TempDir(t)
+			defer func() {
+				if !*keep {
+					cleanup()
+				}
+			}()
+
+			eng, err := engineImpl.create(path.Join(tempDir, engineImpl.name))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer eng.Close()
+
+			outputFilePath := path.Join(tempDir, fmt.Sprintf("%s.meta", engineImpl.name))
+			fmt.Printf("output file path: %s\n", outputFilePath)
+
+			outputFile, err := os.Create(outputFilePath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer outputFile.Close()
+
+			testRunner := metaTestRunner{
+				ctx:    ctx,
+				t:      t,
+				w:      outputFile,
+				seed:   seed,
+				engine: eng,
+			}
+
+			testRunner.init()
+			if checkFile != nil {
+				testRunner.parseFileAndRun(checkFile)
+			} else {
+				testRunner.generateAndRun(10000)
+			}
+		})
+	}
+}
+
+// TestMeta runs the MVCC Metamorphic test suite.
+func TestMeta(t *testing.T) {
+	defer leaktest.AfterTest(t)
+	ctx := context.Background()
+	seeds := []int64{123}
+
+	if *check != "" {
+		t.Run("check", func(t *testing.T) {
+			if _, err := os.Stat(*check); os.IsNotExist(err) {
+				t.Fatal(err)
+			}
+			checkFile, err := os.Open(*check)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer checkFile.Close()
+
+			runMetaTest(ctx, t, 0, checkFile)
+		})
+	}
+	for _, seed := range seeds {
+		t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) {
+			runMetaTest(ctx, t, seed, nil)
+		})
+	}
+}