Merge cockroachdb#37804

37804: workload/ycsb: support workload D, E and fix key selection for insertion and reading r=jeffrey-xiao a=jeffrey-xiao

This PR does the following items:

1. Support workload D (95% reads skewed to newer keys, 5% writes) by adding a latest generator
2. Support workload E (95% scans, 5% writes) by implementing scans
3. Fix key selection for insertion by having a separate index for next row to insert rather than using the `iMaxHead` in the zipfian generator
4. Fix key selection for reading by having a separate counter for number of rows instead of using the number of initial rows. I didn't fully understand the method proposed in the ycsb paper in section 5.3, so if this isn't the best way, I'm open to changes.
5. Improve key selection by hashing before modding. This follows the logic proposed in the ycsb paper. Originally we were hashing after applying a modulo, but that always causes the first inserted key to be the hotspot rather than distribute the hotspot around.
6. Change API for generators to be more consistent. All generators now take in a lower bound and upper bound and generate an inclusive range between the lower and upper bound (I.E. [lower, upper]).

Co-authored-by: Jeffrey Xiao <[email protected]>

pkg/cmd/roachtest/ycsb.go

@@ -45,11 +45,6 @@ func registerYCSB(r *registry) {
 	}
 
 	for _, wl := range []string{"A", "B", "C", "D", "E", "F"} {
-		if wl == "D" || wl == "E" {
-			// These workloads are currently unsupported by workload.
-			// See TODOs in workload/ycsb/ycsb.go.
-			continue
-		}
 		for _, cpus := range []int{8, 32} {
 			var name string
 			if cpus == 8 { // support legacy test name which didn't include cpu

pkg/workload/ycsb/acknowledged_counter.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+package ycsb
+
+import (
+	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+	"github.com/pkg/errors"
+)
+
+const (
+	// windowSize is the size of the window of pending acknowledgements.
+	windowSize = 1 << 20
+	// windowMask is the mask the apply to obtain an index in the window.
+	windowMask = windowSize - 1
+)
+
+// AcknowledgedCounter keeps track of the largest value v such that all values
+// in [initialCount, v) are acknowledged.
+type AcknowledgedCounter struct {
+	mu struct {
+		syncutil.Mutex
+		count  uint64
+		window []bool
+	}
+}
+
+// NewAcknowledgedCounter constructs a new AcknowledgedCounter with the given
+// parameters.
+func NewAcknowledgedCounter(initialCount uint64) *AcknowledgedCounter {
+	c := &AcknowledgedCounter{}
+	c.mu.count = initialCount
+	c.mu.window = make([]bool, windowSize)
+	return c
+}
+
+// Last returns the largest value v such that all values in [initialCount, v) are ackowledged.
+func (c *AcknowledgedCounter) Last() uint64 {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.mu.count
+}
+
+// Acknowledge marks v as being acknowledged.
+func (c *AcknowledgedCounter) Acknowledge(v uint64) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.mu.window[v&windowMask] {
+		return errors.Errorf("Number of pending acknowledgements exceeded window size: %d has been acknowledged, but %d is not acknowledged", v, c.mu.count)
+	}
+
+	c.mu.window[v&windowMask] = true
+	for c.mu.window[c.mu.count&windowMask] {
+		c.mu.window[c.mu.count&windowMask] = false
+		c.mu.count++
+	}
+	return nil
+}

pkg/workload/ycsb/skewed_latest_generator.go

@@ -0,0 +1,67 @@
+// Copyright 2019 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+package ycsb
+
+import (
+	"math/rand"
+
+	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+)
+
+// SkewedLatestGenerator is a random number generator that generates numbers in
+// the range [iMin, iMax], but skews it towards iMax using a zipfian
+// distribution.
+type SkewedLatestGenerator struct {
+	mu struct {
+		syncutil.Mutex
+		iMax    uint64
+		zipfGen *ZipfGenerator
+	}
+}
+
+// NewSkewedLatestGenerator constructs a new SkewedLatestGenerator with the
+// given parameters. It returns an error if the parameters are outside the
+// accepted range.
+func NewSkewedLatestGenerator(
+	rng *rand.Rand, iMin, iMax uint64, theta float64, verbose bool,
+) (*SkewedLatestGenerator, error) {
+
+	z := SkewedLatestGenerator{}
+	z.mu.iMax = iMax
+	zipfGen, err := NewZipfGenerator(rng, 0, iMax-iMin, theta, verbose)
+	if err != nil {
+		return nil, err
+	}
+	z.mu.zipfGen = zipfGen
+
+	return &z, nil
+}
+
+// IncrementIMax increments iMax.
+func (z *SkewedLatestGenerator) IncrementIMax() error {
+	z.mu.Lock()
+	defer z.mu.Unlock()
+	z.mu.iMax++
+	return z.mu.zipfGen.IncrementIMax()
+}
+
+// Uint64 returns a random Uint64 between iMin and iMax, where keys near iMax
+// are most likely to be drawn.
+func (z *SkewedLatestGenerator) Uint64() uint64 {
+	z.mu.Lock()
+	defer z.mu.Unlock()
+	return z.mu.iMax - z.mu.zipfGen.Uint64()
+}

pkg/workload/ycsb/uniform_generator.go

@@ -24,45 +24,38 @@ import (
 // UniformGenerator is a random number generator that generates draws from a
 // uniform distribution.
 type UniformGenerator struct {
-	mu struct {
+	iMin uint64
+	mu   struct {
 		syncutil.Mutex
-		r        *rand.Rand
-		sequence uint64
+		r    *rand.Rand
+		iMax uint64
 	}
 }
 
 // NewUniformGenerator constructs a new UniformGenerator with the given parameters.
 // It returns an error if the parameters are outside the accepted range.
-func NewUniformGenerator(rng *rand.Rand, minInsertRow uint64) (*UniformGenerator, error) {
+func NewUniformGenerator(rng *rand.Rand, iMin, iMax uint64) (*UniformGenerator, error) {
 
 	z := UniformGenerator{}
+	z.iMin = iMin
 	z.mu.r = rng
-	z.mu.sequence = minInsertRow
+	z.mu.iMax = iMax
 
 	return &z, nil
 }
 
-// IMaxHead returns the current value of IMaxHead, without incrementing.
-func (z *UniformGenerator) IMaxHead() uint64 {
-	z.mu.Lock()
-	max := z.mu.sequence
-	z.mu.Unlock()
-	return max
-}
-
-// IncrementIMax increments the sequence number.
+// IncrementIMax increments iMax.
 func (z *UniformGenerator) IncrementIMax() error {
 	z.mu.Lock()
-	z.mu.sequence++
-	z.mu.Unlock()
+	defer z.mu.Unlock()
+	z.mu.iMax++
 	return nil
 }
 
-// Uint64 returns a random Uint64 between min and sequence, drawn from a uniform
+// Uint64 returns a random Uint64 between iMin and iMax, drawn from a uniform
 // distribution.
 func (z *UniformGenerator) Uint64() uint64 {
 	z.mu.Lock()
-	result := rand.Uint64() % z.mu.sequence
-	z.mu.Unlock()
-	return result
+	defer z.mu.Unlock()
+	return (uint64)(z.mu.r.Int63n((int64)(z.mu.iMax-z.iMin+1))) + z.iMin
 }