Skip to content

Commit 1a0c61c

Browse files
jchiu255jchiu255
committed
Benchmark skiplist against map
1 parent 04490f9 commit 1a0c61c

File tree

5 files changed

+95
-185
lines changed

5 files changed

+95
-185
lines changed

‎skiplist/README.md‎

Lines changed: 12 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,15 @@
1-
Ran on macbook.
1+
Ran on Macbook.
22

33
```
4-
BenchmarkWrite/branch_2-4 1000000 44162 ns/op
5-
BenchmarkWrite/branch_3-4 1000000 6306 ns/op
6-
BenchmarkWrite/branch_4-4 1000000 6922 ns/op
7-
BenchmarkWrite/branch_5-4 1000000 9222 ns/op
8-
BenchmarkWrite/branch_6-4 1000000 7646 ns/op
9-
BenchmarkWrite/branch_7-4 500000 6604 ns/op
10-
BenchmarkWriteParallel/branch_2-4 500000 24437 ns/op
11-
BenchmarkWriteParallel/branch_3-4 1000000 7390 ns/op
12-
BenchmarkWriteParallel/branch_4-4 1000000 8112 ns/op
13-
BenchmarkWriteParallel/branch_5-4 500000 7885 ns/op
14-
BenchmarkWriteParallel/branch_6-4 500000 8459 ns/op
15-
BenchmarkWriteParallel/branch_7-4 500000 8593 ns/op
16-
BenchmarkRead/branch_2-4 200000 8491 ns/op
17-
BenchmarkRead/branch_3-4 500000 3108 ns/op
18-
BenchmarkRead/branch_4-4 500000 3223 ns/op
19-
BenchmarkRead/branch_5-4 500000 3586 ns/op
20-
BenchmarkRead/branch_6-4 500000 4118 ns/op
21-
BenchmarkRead/branch_7-4 300000 4729 ns/op
22-
BenchmarkReadParallel/branch_2-4 500000 2680 ns/op
23-
BenchmarkReadParallel/branch_3-4 1000000 1909 ns/op
24-
BenchmarkReadParallel/branch_4-4 1000000 1928 ns/op
25-
BenchmarkReadParallel/branch_5-4 1000000 1663 ns/op
26-
BenchmarkReadParallel/branch_6-4 1000000 1400 ns/op
27-
BenchmarkReadParallel/branch_7-4 1000000 1733 ns/op
28-
BenchmarkReadWrite/frac_0-4 1000000 6719 ns/op
29-
BenchmarkReadWrite/frac_1-4 1000000 6177 ns/op
30-
BenchmarkReadWrite/frac_2-4 1000000 6239 ns/op
31-
BenchmarkReadWrite/frac_3-4 1000000 5616 ns/op
32-
BenchmarkReadWrite/frac_4-4 1000000 5103 ns/op
33-
BenchmarkReadWrite/frac_5-4 1000000 4962 ns/op
34-
BenchmarkReadWrite/frac_6-4 1000000 4172 ns/op
35-
BenchmarkReadWrite/frac_7-4 1000000 3175 ns/op
36-
BenchmarkReadWrite/frac_8-4 1000000 2070 ns/op
37-
BenchmarkReadWrite/frac_9-4 1000000 1201 ns/op
38-
BenchmarkReadWrite/frac_10-4 5000000 456 ns/o
39-
```
40-
41-
For comparison with lock-free skiplist
42-
43-
```
44-
BenchmarkReadParallel/branch_4-4 1000000 1358 ns/op
45-
BenchmarkWriteParallelAlt-4 100 19644625 ns/op
4+
BenchmarkReadWrite/frac_0-4 500000 7073 ns/op
5+
BenchmarkReadWrite/frac_1-4 1000000 8509 ns/op
6+
BenchmarkReadWrite/frac_2-4 1000000 7978 ns/op
7+
BenchmarkReadWrite/frac_3-4 500000 5832 ns/op
8+
BenchmarkReadWrite/frac_4-4 1000000 5714 ns/op
9+
BenchmarkReadWrite/frac_5-4 1000000 5011 ns/op
10+
BenchmarkReadWrite/frac_6-4 1000000 4417 ns/op
11+
BenchmarkReadWrite/frac_7-4 1000000 3434 ns/op
12+
BenchmarkReadWrite/frac_8-4 1000000 2393 ns/op
13+
BenchmarkReadWrite/frac_9-4 1000000 1437 ns/op
14+
BenchmarkReadWrite/frac_10-4 3000000 483 ns/op
4615
```

‎skiplist/skiplist_test.go‎

Lines changed: 0 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -148,85 +148,6 @@ func randomKey() []byte {
148148
return bs
149149
}
150150

151-
func BenchmarkWrite(b *testing.B) {
152-
maxDepth := 10
153-
for branch := 2; branch <= 7; branch++ {
154-
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
155-
list := NewSkiplist(maxDepth, branch, DefaultComparator)
156-
for i := 0; i < b.N; i++ {
157-
list.Insert(randomKey())
158-
}
159-
})
160-
}
161-
}
162-
163-
func BenchmarkWriteParallel(b *testing.B) {
164-
maxDepth := 10
165-
for branch := 2; branch <= 7; branch++ {
166-
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
167-
list := NewSkiplist(maxDepth, branch, DefaultComparator)
168-
b.RunParallel(func(pb *testing.PB) {
169-
for pb.Next() {
170-
list.InsertConcurrently(randomKey())
171-
}
172-
})
173-
})
174-
}
175-
}
176-
177-
// Alternate version to WriteParallel that fixes the number of writes to be decently large. This
178-
// is needed for the lock-free skiplist benchmark and we need this here for a fairer comparison.
179-
// If we do intend to keep multiple skiplist implementations in the codebase, we can refactor
180-
// this benchmarking code.
181-
func BenchmarkWriteParallelAlt(b *testing.B) {
182-
maxDepth := 10
183-
branch := 3
184-
b.ResetTimer()
185-
for i := 0; i < b.N; i++ {
186-
list := NewSkiplist(maxDepth, branch, DefaultComparator)
187-
for i := 0; i < 10000; i++ {
188-
list.InsertConcurrently(randomKey())
189-
}
190-
}
191-
}
192-
193-
func BenchmarkRead(b *testing.B) {
194-
maxDepth := 10
195-
for branch := 2; branch <= 7; branch++ {
196-
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
197-
list := NewSkiplist(maxDepth, branch, DefaultComparator)
198-
for i := 0; i < 100000; i++ {
199-
list.Insert(randomKey())
200-
}
201-
b.ResetTimer()
202-
for i := 0; i < b.N; i++ {
203-
it := list.Iterator()
204-
it.Seek(randomKey())
205-
}
206-
})
207-
}
208-
}
209-
210-
// For comparison with lock-free skiplist.
211-
func BenchmarkReadParallel(b *testing.B) {
212-
maxDepth := 10
213-
for branch := 2; branch <= 7; branch++ {
214-
b.Run(fmt.Sprintf("branch_%d", branch), func(b *testing.B) {
215-
list := NewSkiplist(maxDepth, branch, DefaultComparator)
216-
for i := 0; i < 100000; i++ {
217-
list.Insert(randomKey())
218-
}
219-
b.ResetTimer()
220-
b.RunParallel(func(pb *testing.PB) {
221-
for pb.Next() {
222-
it := list.Iterator()
223-
it.Seek(randomKey())
224-
}
225-
})
226-
})
227-
}
228-
}
229-
230151
// Standard test. Some fraction is read. Some fraction is write. Writes have
231152
// to go through mutex lock.
232153
func BenchmarkReadWrite(b *testing.B) {

‎skl/README.md‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,31 @@
1-
This is better than `skiplist` and `slist`.
1+
This is much better than `skiplist` and `slist`.
22

3-
WriteParallel is about 3X faster and ReadParallel is 2X faster than `skiplist` and 20X faster
4-
than `slist`.
3+
```
4+
BenchmarkReadWrite/frac_0-4 1000000 1516 ns/op
5+
BenchmarkReadWrite/frac_1-4 1000000 1456 ns/op
6+
BenchmarkReadWrite/frac_2-4 1000000 1354 ns/op
7+
BenchmarkReadWrite/frac_3-4 1000000 1295 ns/op
8+
BenchmarkReadWrite/frac_4-4 1000000 1142 ns/op
9+
BenchmarkReadWrite/frac_5-4 1000000 1077 ns/op
10+
BenchmarkReadWrite/frac_6-4 1000000 1003 ns/op
11+
BenchmarkReadWrite/frac_7-4 2000000 1054 ns/op
12+
BenchmarkReadWrite/frac_8-4 2000000 929 ns/op
13+
BenchmarkReadWrite/frac_9-4 3000000 815 ns/op
14+
BenchmarkReadWrite/frac_10-4 5000000 472 ns/op
15+
```
16+
17+
But compared to a simple map with read-write lock, it is still slower.
518

619
```
7-
BenchmarkWriteParallelAlt-4 200 7700008 ns/op
8-
BenchmarkReadParallel-4 2000000 787 ns/op
20+
BenchmarkReadWriteMap/frac_0-4 2000000 883 ns/op
21+
BenchmarkReadWriteMap/frac_1-4 2000000 830 ns/op
22+
BenchmarkReadWriteMap/frac_2-4 2000000 658 ns/op
23+
BenchmarkReadWriteMap/frac_3-4 2000000 662 ns/op
24+
BenchmarkReadWriteMap/frac_4-4 2000000 657 ns/op
25+
BenchmarkReadWriteMap/frac_5-4 2000000 650 ns/op
26+
BenchmarkReadWriteMap/frac_6-4 3000000 592 ns/op
27+
BenchmarkReadWriteMap/frac_7-4 3000000 573 ns/op
28+
BenchmarkReadWriteMap/frac_8-4 3000000 539 ns/op
29+
BenchmarkReadWriteMap/frac_9-4 3000000 521 ns/op
30+
BenchmarkReadWriteMap/frac_10-4 3000000 479 ns/op
931
```

‎skl/skl_test.go‎

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -262,28 +262,58 @@ func randomKey() []byte {
262262
return b
263263
}
264264

265-
// Alternate version to WriteParallel that fixes the number of writes to be decently large.
266-
func BenchmarkWriteParallelAlt(b *testing.B) {
265+
// Standard test. Some fraction is read. Some fraction is write. Writes have
266+
// to go through mutex lock.
267+
func BenchmarkReadWrite(b *testing.B) {
267268
value := newValue(123)
268-
b.ResetTimer()
269-
for i := 0; i < b.N; i++ {
270-
list := NewSkiplist()
271-
for i := 0; i < 10000; i++ {
272-
list.Put(randomKey(), value, true)
273-
}
269+
for i := 0; i <= 10; i++ {
270+
readFrac := float32(i) / 10.0
271+
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
272+
list := NewSkiplist()
273+
b.ResetTimer()
274+
var count int
275+
b.RunParallel(func(pb *testing.PB) {
276+
for pb.Next() {
277+
if rand.Float32() < readFrac {
278+
if list.Get(randomKey()) != nil {
279+
count++
280+
}
281+
} else {
282+
list.Put(randomKey(), value, true)
283+
}
284+
}
285+
})
286+
})
274287
}
275288
}
276289

277-
func BenchmarkReadParallel(b *testing.B) {
290+
// Standard test. Some fraction is read. Some fraction is write. Writes have
291+
// to go through mutex lock.
292+
func BenchmarkReadWriteMap(b *testing.B) {
278293
value := newValue(123)
279-
list := NewSkiplist()
280-
for i := 0; i < 100000; i++ {
281-
list.Put(randomKey(), value, true)
294+
for i := 0; i <= 10; i++ {
295+
readFrac := float32(i) / 10.0
296+
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
297+
m := make(map[string]unsafe.Pointer)
298+
var mutex sync.RWMutex
299+
b.ResetTimer()
300+
var count int
301+
b.RunParallel(func(pb *testing.PB) {
302+
for pb.Next() {
303+
if rand.Float32() < readFrac {
304+
mutex.RLock()
305+
_, ok := m[string(randomKey())]
306+
mutex.RUnlock()
307+
if ok {
308+
count++
309+
}
310+
} else {
311+
mutex.Lock()
312+
m[string(randomKey())] = value
313+
mutex.Unlock()
314+
}
315+
}
316+
})
317+
})
282318
}
283-
b.ResetTimer()
284-
b.RunParallel(func(pb *testing.PB) {
285-
for pb.Next() {
286-
list.Get(randomKey())
287-
}
288-
})
289319
}

‎slist/slist_test.go‎

Lines changed: 8 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -223,34 +223,26 @@ func randomKey() []byte {
223223
return b
224224
}
225225

226-
func BenchmarkReadParallel(b *testing.B) {
227-
value := newValue(123)
228-
list := NewSkiplist()
229-
for i := 0; i < 100000; i++ {
230-
list.Put(randomKey(), value, true)
231-
}
232-
b.ResetTimer()
233-
b.RunParallel(func(pb *testing.PB) {
234-
for pb.Next() {
235-
list.Get(randomKey())
236-
}
237-
})
238-
}
239-
240226
// Standard test. Some fraction is read. Some fraction is write. Writes have
241227
// to go through mutex lock.
228+
// NOTE: For this implementation, this test doesn't seem to terminate sometimes.
229+
// I think the reason is that this skiplist likes to restart and there is high variance in
230+
// the running time. As a result, the benchmark framework is unsure and keeps retrying.
231+
// We used to have a read-parallel and write-parallel test. But let's just delete this lib soon.
242232
func BenchmarkReadWrite(b *testing.B) {
243233
value := newValue(123)
244234
for i := 0; i <= 10; i++ {
245235
readFrac := float32(i) / 10.0
246236
b.Run(fmt.Sprintf("frac_%d", i), func(b *testing.B) {
247237
list := NewSkiplist()
248238
b.ResetTimer()
239+
var count int
249240
b.RunParallel(func(pb *testing.PB) {
250241
for pb.Next() {
251242
if rand.Float32() < readFrac {
252-
it := list.NewIterator()
253-
it.Seek(randomKey())
243+
if list.Get(randomKey()) != nil {
244+
count++
245+
}
254246
} else {
255247
list.Put(randomKey(), value, true)
256248
}
@@ -259,27 +251,3 @@ func BenchmarkReadWrite(b *testing.B) {
259251
})
260252
}
261253
}
262-
263-
// Test takes a long time because early parallel writes tend to lead to a lot of starts and
264-
// variance in the running time.
265-
func BenchmarkWriteParallel(b *testing.B) {
266-
value := newValue(123)
267-
list := NewSkiplist()
268-
b.RunParallel(func(pb *testing.PB) {
269-
for pb.Next() {
270-
list.Put(randomKey(), value, true)
271-
}
272-
})
273-
}
274-
275-
// Alternate version to WriteParallel that fixes the number of writes to be decently large.
276-
func BenchmarkWriteParallelAlt(b *testing.B) {
277-
value := newValue(123)
278-
b.ResetTimer()
279-
for i := 0; i < b.N; i++ {
280-
list := NewSkiplist()
281-
for i := 0; i < 10000; i++ {
282-
list.Put(randomKey(), value, true)
283-
}
284-
}
285-
}

0 commit comments

Comments
 (0)