Skip to content

Commit

Permalink
Merge pull request #711 from ambaxter/bench_random
Browse files Browse the repository at this point in the history 
Added rnd read capabilities to bbolt bench
  • Loading branch information
@ahrtr
ahrtr authored Apr 11, 2024
2 parents 6716581 + d3370d3 commit ee11a09
Showing 1 changed file with 119 additions and 25 deletions.
144 changes: 119 additions & 25 deletions cmd/bbolt/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1103,17 +1103,27 @@ func (cmd *benchCommand) Run(args ...string) error {
db.NoSync = options.NoSync
defer db.Close()

r := rand.New(rand.NewSource(time.Now().UnixNano()))

// Write to the database.
var writeResults BenchResults

fmt.Fprintf(cmd.Stderr, "starting write benchmark.\n")
if err := cmd.runWrites(db, options, &writeResults); err != nil {
keys, err := cmd.runWrites(db, options, &writeResults, r)
if err != nil {
return fmt.Errorf("write: %v", err)
}

if keys != nil {
r.Shuffle(len(keys), func(i, j int) {
keys[i], keys[j] = keys[j], keys[i]
})
}

var readResults BenchResults
fmt.Fprintf(cmd.Stderr, "starting read benchmark.\n")
// Read from the database.
if err := cmd.runReads(db, options, &readResults); err != nil {
if err := cmd.runReads(db, options, &readResults, keys); err != nil {
return fmt.Errorf("bench: read: %s", err)
}

Expand Down Expand Up @@ -1172,7 +1182,7 @@ func (cmd *benchCommand) ParseFlags(args []string) (*BenchOptions, error) {
}

// Writes to the database.
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
// Start profiling for writes.
if options.ProfileMode == "rw" || options.ProfileMode == "w" {
cmd.startProfiling(options)
Expand All @@ -1184,18 +1194,19 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *

t := time.Now()

var keys []nestedKey
var err error
switch options.WriteMode {
case "seq":
err = cmd.runWritesSequential(db, options, results)
keys, err = cmd.runWritesSequential(db, options, results)
case "rnd":
err = cmd.runWritesRandom(db, options, results)
keys, err = cmd.runWritesRandom(db, options, results, r)
case "seq-nest":
err = cmd.runWritesSequentialNested(db, options, results)
keys, err = cmd.runWritesSequentialNested(db, options, results)
case "rnd-nest":
err = cmd.runWritesRandomNested(db, options, results)
keys, err = cmd.runWritesRandomNested(db, options, results, r)
default:
return fmt.Errorf("invalid write mode: %s", options.WriteMode)
return nil, fmt.Errorf("invalid write mode: %s", options.WriteMode)
}

// Save time to write.
Expand All @@ -1206,30 +1217,33 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *
cmd.stopProfiling()
}

return err
return keys, err
}

func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) ([]nestedKey, error) {
var i = uint32(0)
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
}

func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
}

func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) ([]nestedKey, error) {
var i = uint32(0)
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { i++; return i })
}

func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
}

func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) ([]nestedKey, error) {
var keys []nestedKey
if options.ReadMode == "rnd" {
keys = make([]nestedKey, 0, options.Iterations)
}

for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
Expand All @@ -1247,20 +1261,27 @@ func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions,
if err := b.Put(key, value); err != nil {
return err
}

if keys != nil {
keys = append(keys, nestedKey{nil, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)

return nil
}); err != nil {
return err
return nil, err
}
}
return nil
return keys, nil
}

func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) ([]nestedKey, error) {
var keys []nestedKey
if options.ReadMode == "rnd" {
keys = make([]nestedKey, 0, options.Iterations)
}

for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
top, err := tx.CreateBucketIfNotExists(benchBucketName)
Expand Down Expand Up @@ -1292,21 +1313,23 @@ func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOp
if err := b.Put(key, value); err != nil {
return err
}

if keys != nil {
keys = append(keys, nestedKey{name, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)

return nil
}); err != nil {
return err
return nil, err
}
}
return nil
return keys, nil
}

// Reads from the database.
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults, keys []nestedKey) error {
// Start profiling for reads.
if options.ProfileMode == "r" {
cmd.startProfiling(options)
Expand All @@ -1327,6 +1350,13 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
default:
err = cmd.runReadsSequential(db, options, results)
}
case "rnd":
switch options.WriteMode {
case "seq-nest", "rnd-nest":
err = cmd.runReadsRandomNested(db, options, keys, results)
default:
err = cmd.runReadsRandom(db, options, keys, results)
}
default:
return fmt.Errorf("invalid read mode: %s", options.ReadMode)
}
Expand All @@ -1342,6 +1372,8 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
return err
}

type nestedKey struct{ bucket, key []byte }

func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
Expand All @@ -1353,7 +1385,37 @@ func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions,
numReads++
results.AddCompletedOps(1)
if v == nil {
return errors.New("invalid value")
return ErrInvalidValue
}
}

if options.WriteMode == "seq" && numReads != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func (cmd *benchCommand) runReadsRandom(db *bolt.DB, options *BenchOptions, keys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
b := tx.Bucket(benchBucketName)
for _, key := range keys {
v := b.Get(key.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
}
}

Expand Down Expand Up @@ -1408,6 +1470,38 @@ func (cmd *benchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOpt
})
}

func (cmd *benchCommand) runReadsRandomNested(db *bolt.DB, options *BenchOptions, nestedKeys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
var top = tx.Bucket(benchBucketName)
for _, nestedKey := range nestedKeys {
if b := top.Bucket(nestedKey.bucket); b != nil {
v := b.Get(nestedKey.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
}
}
}

if options.WriteMode == "seq-nest" && numReads != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func checkProgress(results *BenchResults, finishChan chan interface{}, stderr io.Writer) {
ticker := time.Tick(time.Second)
lastCompleted, lastTime := int64(0), time.Now()
Expand Down

0 comments on commit ee11a09

Please sign in to comment.