Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added rnd read capabilities to bbolt bench #711

Merged
merged 1 commit into from
Apr 11, 2024
Merged

Added rnd read capabilities to bbolt bench #711

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 119 additions & 25 deletions cmd/bbolt/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1103,17 +1103,27 @@ func (cmd *benchCommand) Run(args ...string) error {
db.NoSync = options.NoSync
defer db.Close()

r := rand.New(rand.NewSource(time.Now().UnixNano()))

// Write to the database.
var writeResults BenchResults

fmt.Fprintf(cmd.Stderr, "starting write benchmark.\n")
if err := cmd.runWrites(db, options, &writeResults); err != nil {
keys, err := cmd.runWrites(db, options, &writeResults, r)
if err != nil {
return fmt.Errorf("write: %v", err)
}

if keys != nil {
r.Shuffle(len(keys), func(i, j int) {
keys[i], keys[j] = keys[j], keys[i]
})
}

var readResults BenchResults
fmt.Fprintf(cmd.Stderr, "starting read benchmark.\n")
// Read from the database.
if err := cmd.runReads(db, options, &readResults); err != nil {
if err := cmd.runReads(db, options, &readResults, keys); err != nil {
return fmt.Errorf("bench: read: %s", err)
}

Expand Down Expand Up @@ -1172,7 +1182,7 @@ func (cmd *benchCommand) ParseFlags(args []string) (*BenchOptions, error) {
}

// Writes to the database.
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
// Start profiling for writes.
if options.ProfileMode == "rw" || options.ProfileMode == "w" {
cmd.startProfiling(options)
Expand All @@ -1184,18 +1194,19 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *

t := time.Now()

var keys []nestedKey
var err error
switch options.WriteMode {
case "seq":
err = cmd.runWritesSequential(db, options, results)
keys, err = cmd.runWritesSequential(db, options, results)
case "rnd":
err = cmd.runWritesRandom(db, options, results)
keys, err = cmd.runWritesRandom(db, options, results, r)
case "seq-nest":
err = cmd.runWritesSequentialNested(db, options, results)
keys, err = cmd.runWritesSequentialNested(db, options, results)
case "rnd-nest":
err = cmd.runWritesRandomNested(db, options, results)
keys, err = cmd.runWritesRandomNested(db, options, results, r)
default:
return fmt.Errorf("invalid write mode: %s", options.WriteMode)
return nil, fmt.Errorf("invalid write mode: %s", options.WriteMode)
}

// Save time to write.
Expand All @@ -1206,30 +1217,33 @@ func (cmd *benchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *
cmd.stopProfiling()
}

return err
return keys, err
}

func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) ([]nestedKey, error) {
var i = uint32(0)
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
}

func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
func (cmd *benchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
}

func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) ([]nestedKey, error) {
var i = uint32(0)
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { i++; return i })
}

func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
func (cmd *benchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults, r *rand.Rand) ([]nestedKey, error) {
return cmd.runWritesNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
}

func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) ([]nestedKey, error) {
var keys []nestedKey
if options.ReadMode == "rnd" {
keys = make([]nestedKey, 0, options.Iterations)
}

for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
Expand All @@ -1247,20 +1261,27 @@ func (cmd *benchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions,
if err := b.Put(key, value); err != nil {
return err
}

if keys != nil {
keys = append(keys, nestedKey{nil, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)

return nil
}); err != nil {
return err
return nil, err
}
}
return nil
return keys, nil
}

func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) ([]nestedKey, error) {
var keys []nestedKey
if options.ReadMode == "rnd" {
keys = make([]nestedKey, 0, options.Iterations)
}

for i := int64(0); i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
top, err := tx.CreateBucketIfNotExists(benchBucketName)
Expand Down Expand Up @@ -1292,21 +1313,23 @@ func (cmd *benchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOp
if err := b.Put(key, value); err != nil {
return err
}

if keys != nil {
keys = append(keys, nestedKey{name, key})
}
results.AddCompletedOps(1)
}
fmt.Fprintf(cmd.Stderr, "Finished write iteration %d\n", i)

return nil
}); err != nil {
return err
return nil, err
}
}
return nil
return keys, nil
}

// Reads from the database.
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults, keys []nestedKey) error {
// Start profiling for reads.
if options.ProfileMode == "r" {
cmd.startProfiling(options)
Expand All @@ -1327,6 +1350,13 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
default:
err = cmd.runReadsSequential(db, options, results)
}
case "rnd":
switch options.WriteMode {
case "seq-nest", "rnd-nest":
err = cmd.runReadsRandomNested(db, options, keys, results)
default:
err = cmd.runReadsRandom(db, options, keys, results)
}
default:
return fmt.Errorf("invalid read mode: %s", options.ReadMode)
}
Expand All @@ -1342,6 +1372,8 @@ func (cmd *benchCommand) runReads(db *bolt.DB, options *BenchOptions, results *B
return err
}

type nestedKey struct{ bucket, key []byte }

func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
Expand All @@ -1353,7 +1385,37 @@ func (cmd *benchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions,
numReads++
results.AddCompletedOps(1)
if v == nil {
return errors.New("invalid value")
return ErrInvalidValue
}
}

if options.WriteMode == "seq" && numReads != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func (cmd *benchCommand) runReadsRandom(db *bolt.DB, options *BenchOptions, keys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
b := tx.Bucket(benchBucketName)
for _, key := range keys {
v := b.Get(key.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
ahrtr marked this conversation as resolved.
Show resolved Hide resolved
}
}

Expand Down Expand Up @@ -1408,6 +1470,38 @@ func (cmd *benchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOpt
})
}

func (cmd *benchCommand) runReadsRandomNested(db *bolt.DB, options *BenchOptions, nestedKeys []nestedKey, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()

for {
numReads := int64(0)
var top = tx.Bucket(benchBucketName)
for _, nestedKey := range nestedKeys {
if b := top.Bucket(nestedKey.bucket); b != nil {
v := b.Get(nestedKey.key)
numReads++
results.AddCompletedOps(1)
if v == nil {
return ErrInvalidValue
}
}
}

if options.WriteMode == "seq-nest" && numReads != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, numReads)
}

// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}

return nil
})
}

func checkProgress(results *BenchResults, finishChan chan interface{}, stderr io.Writer) {
ticker := time.Tick(time.Second)
lastCompleted, lastTime := int64(0), time.Now()
Expand Down
Loading