Skip to content

Commit

Permalink
Add rnd read-mode to bbolt bench command
Browse files Browse the repository at this point in the history 
Signed-off-by: Adam Baxter <[email protected]>
  • Loading branch information
@ambaxter
ambaxter committed Apr 9, 2024
1 parent c57b353 commit 3561017
Showing 1 changed file with 112 additions and 23 deletions.
135 changes: 112 additions & 23 deletions cmd/bbolt/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1103,17 +1103,33 @@ func (cmd *benchCommand) Run(args ...string) error {
db.NoSync = options.NoSync
defer db.Close()

r := rand.New(rand.NewSource(time.Now().UnixNano()))

// Write to the database.
var writeResults BenchResults
var keys []nestedKey

if options.ReadMode == "rnd" {
keys = make([]nestedKey, 0, options.Iterations)
} else {
keys = nil
}

fmt.Fprintf(cmd.Stderr, "starting write benchmark.\n")
if err := cmd.runWrites(db, options, &writeResults); err != nil {
if err := cmd.runWrites(db, options, &writeResults, r, &keys); err != nil {
return fmt.Errorf("write: %v", err)
}

if keys != nil {
r.Shuffle(len(keys), func(i, j int) {
keys[i], keys[j] = keys[j], keys[i]
})
}

var readResults BenchResults
fmt.Fprintf(cmd.Stderr, "starting read benchmark.\n")
// Read from the database.
if err := cmd.runReads(db, options, &readResults); err != nil {
if err := cmd.runReads(db, options, &readResults, keys); err != nil {
return fmt.Errorf("bench: read: %s", err)
}

Expand Down Expand Up @@ -1172,7 +1188,7 @@ func (cmd *benchCommand) ParseFlags(args []string) (*BenchOptions, error) {
}

// Writes to the database.
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand, keys *[]nestedKey) error {
// Start profiling for writes.
if options.ProfileMode == "rw" || options.ProfileMode == "w" {
cmd.startProfiling(options)
Expand All @@ -1187,13 +1203,13 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *
var err error
switch options.WriteMode {
case "seq":
err = cmd.runWritesSequential(db, options, results)
err = cmd.runWritesSequential(db, options, results, keys)
case "rnd":
err = cmd.runWritesRandom(db, options, results)
err = cmd.runWritesRandom(db, options, results, r, keys)
case "seq-nest":
err = cmd.runWritesSequentialNested(db, options, results)
err = cmd.runWritesSequentialNested(db, options, results, keys)
case "rnd-nest":
err = cmd.runWritesRandomNested(db, options, results)
err = cmd.runWritesRandomNested(db, options, results, r, keys)
default:
return fmt.Errorf("invalid write mode: %s", options.WriteMode)
}
Expand All @@ -1209,27 +1225,25 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *
return err
}

func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults, keys *[]nestedKey) error {
var i = uint32(0)
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }, keys)
}

func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand, keys *[]nestedKey) error {
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }, keys)
}

func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults, keys *[]nestedKey) error {
var i = uint32(0)
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { i++; return i })
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { i++; return i }, keys)
}

func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand, keys *[]nestedKey) error {
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { return r.Uint32() }, keys)
}

func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32, keys *[]nestedKey) error {
for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
Expand All @@ -1247,7 +1261,9 @@ func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions,
if err := b.Put(key, value); err != nil {
return err
}

if *keys != nil {
*keys = append(*keys, nestedKey{nil, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)
Expand All @@ -1260,7 +1276,7 @@ func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions,
return nil
}

func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32, keys *[]nestedKey) error {
for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
top, err := tx.CreateBucketIfNotExists(benchBucketName)
Expand Down Expand Up @@ -1292,7 +1308,9 @@ func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOp
if err := b.Put(key, value); err != nil {
return err
}

if *keys != nil {
*keys = append(*keys, nestedKey{nil, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)
Expand All @@ -1306,7 +1324,7 @@ func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOp
}

// Reads from the database.
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults, keys []nestedKey) error {
// Start profiling for reads.
if options.ProfileMode == "r" {
cmd.startProfiling(options)
Expand All @@ -1327,6 +1345,13 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
default:
err = cmd.runReadsSequential(db, options, results)
}
case "rnd":
switch options.WriteMode {
case "seq-nest", "rnd-nest":
err = cmd.runReadsRandomNested(db, options, keys, results)
default:
err = cmd.runReadsRandom(db, options, keys, results)
}
default:
return fmt.Errorf("invalid read mode: %s", options.ReadMode)
}
Expand All @@ -1342,6 +1367,8 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
return err
}

type nestedKey struct{ bucket, key []byte }

func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
Expand All @@ -1353,7 +1380,37 @@ func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions,
numReads++
results.AddCompletedOps(1)
if v == nil {
return errors.New("invalid value")
return ErrInvalidValue
}
}

if options.WriteMode == "seq" && numReads != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func (cmd *benchCommand) runReadsRandom(db *bolt.DB, options *BenchOptions, keys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
b := tx.Bucket(benchBucketName)
for _, key := range keys {
v := b.Get(key.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
}
}

Expand Down Expand Up @@ -1408,6 +1465,38 @@ func (cmd *benchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOpt
})
}

func (cmd *benchCommand) runReadsRandomNested(db *bolt.DB, options *BenchOptions, nestedKeys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
var top = tx.Bucket(benchBucketName)
for _, nestedKey := range nestedKeys {
if b := top.Bucket(nestedKey.bucket); b != nil {
v := b.Get(nestedKey.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
}
}
}

if options.WriteMode == "seq-nest" && numReads != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func checkProgress(results *BenchResults, finishChan chan interface{}, stderr io.Writer) {
ticker := time.Tick(time.Second)
lastCompleted, lastTime := int64(0), time.Now()
Expand Down

0 comments on commit 3561017

Please sign in to comment.