1
2
3
4
5
6
7 package leveldb
8
9 import (
10 "bytes"
11 "fmt"
12 "math/rand"
13 "os"
14 "path/filepath"
15 "runtime"
16 "sync/atomic"
17 "testing"
18
19 "github.com/syndtr/goleveldb/leveldb/iterator"
20 "github.com/syndtr/goleveldb/leveldb/opt"
21 "github.com/syndtr/goleveldb/leveldb/storage"
22 )
23
24 func randomString(r *rand.Rand, n int) []byte {
25 b := new(bytes.Buffer)
26 for i := 0; i < n; i++ {
27 b.WriteByte(' ' + byte(r.Intn(95)))
28 }
29 return b.Bytes()
30 }
31
32 func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
33 nn := int(float32(n) * frac)
34 rb := randomString(r, nn)
35 b := make([]byte, 0, n+nn)
36 for len(b) < n {
37 b = append(b, rb...)
38 }
39 return b[:n]
40 }
41
42 type valueGen struct {
43 src []byte
44 pos int
45 }
46
47 func newValueGen(frac float32) *valueGen {
48 v := new(valueGen)
49 r := rand.New(rand.NewSource(301))
50 v.src = make([]byte, 0, 1048576+100)
51 for len(v.src) < 1048576 {
52 v.src = append(v.src, compressibleStr(r, frac, 100)...)
53 }
54 return v
55 }
56
57 func (v *valueGen) get(n int) []byte {
58 if v.pos+n > len(v.src) {
59 v.pos = 0
60 }
61 v.pos += n
62 return v.src[v.pos-n : v.pos]
63 }
64
65 var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
66
67 type dbBench struct {
68 b *testing.B
69 stor storage.Storage
70 db *DB
71
72 o *opt.Options
73 ro *opt.ReadOptions
74 wo *opt.WriteOptions
75
76 keys, values [][]byte
77 }
78
79 func openDBBench(b *testing.B, noCompress bool) *dbBench {
80 _, err := os.Stat(benchDB)
81 if err == nil {
82 err = os.RemoveAll(benchDB)
83 if err != nil {
84 b.Fatal("cannot remove old db: ", err)
85 }
86 }
87
88 p := &dbBench{
89 b: b,
90 o: &opt.Options{},
91 ro: &opt.ReadOptions{},
92 wo: &opt.WriteOptions{},
93 }
94 p.stor, err = storage.OpenFile(benchDB, false)
95 if err != nil {
96 b.Fatal("cannot open stor: ", err)
97 }
98 if noCompress {
99 p.o.Compression = opt.NoCompression
100 }
101
102 p.db, err = Open(p.stor, p.o)
103 if err != nil {
104 b.Fatal("cannot open db: ", err)
105 }
106
107 return p
108 }
109
110 func (p *dbBench) reopen() {
111 p.db.Close()
112 var err error
113 p.db, err = Open(p.stor, p.o)
114 if err != nil {
115 p.b.Fatal("Reopen: got error: ", err)
116 }
117 }
118
119 func (p *dbBench) populate(n int) {
120 p.keys, p.values = make([][]byte, n), make([][]byte, n)
121 v := newValueGen(0.5)
122 for i := range p.keys {
123 p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
124 }
125 }
126
127 func (p *dbBench) randomize() {
128 m := len(p.keys)
129 times := m * 2
130 r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
131 for n := 0; n < times; n++ {
132 i, j := r1.Int()%m, r2.Int()%m
133 if i == j {
134 continue
135 }
136 p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
137 p.values[i], p.values[j] = p.values[j], p.values[i]
138 }
139 }
140
141 func (p *dbBench) writes(perBatch int) {
142 b := p.b
143 db := p.db
144
145 n := len(p.keys)
146 m := n / perBatch
147 if n%perBatch > 0 {
148 m++
149 }
150 batches := make([]Batch, m)
151 j := 0
152 for i := range batches {
153 first := true
154 for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
155 first = false
156 batches[i].Put(p.keys[j], p.values[j])
157 }
158 }
159 runtime.GC()
160
161 b.ResetTimer()
162 b.StartTimer()
163 for i := range batches {
164 err := db.Write(&(batches[i]), p.wo)
165 if err != nil {
166 b.Fatal("write failed: ", err)
167 }
168 }
169 b.StopTimer()
170 b.SetBytes(116)
171 }
172
173 func (p *dbBench) gc() {
174 p.keys, p.values = nil, nil
175 runtime.GC()
176 }
177
178 func (p *dbBench) puts() {
179 b := p.b
180 db := p.db
181
182 b.ResetTimer()
183 b.StartTimer()
184 for i := range p.keys {
185 err := db.Put(p.keys[i], p.values[i], p.wo)
186 if err != nil {
187 b.Fatal("put failed: ", err)
188 }
189 }
190 b.StopTimer()
191 b.SetBytes(116)
192 }
193
194 func (p *dbBench) fill() {
195 b := p.b
196 db := p.db
197
198 perBatch := 10000
199 batch := new(Batch)
200 for i, n := 0, len(p.keys); i < n; {
201 first := true
202 for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
203 first = false
204 batch.Put(p.keys[i], p.values[i])
205 }
206 err := db.Write(batch, p.wo)
207 if err != nil {
208 b.Fatal("write failed: ", err)
209 }
210 batch.Reset()
211 }
212 }
213
214 func (p *dbBench) gets() {
215 b := p.b
216 db := p.db
217
218 b.ResetTimer()
219 for i := range p.keys {
220 _, err := db.Get(p.keys[i], p.ro)
221 if err != nil {
222 b.Error("got error: ", err)
223 }
224 }
225 b.StopTimer()
226 }
227
228 func (p *dbBench) seeks() {
229 b := p.b
230
231 iter := p.newIter()
232 defer iter.Release()
233 b.ResetTimer()
234 for i := range p.keys {
235 if !iter.Seek(p.keys[i]) {
236 b.Error("value not found for: ", string(p.keys[i]))
237 }
238 }
239 b.StopTimer()
240 }
241
242 func (p *dbBench) newIter() iterator.Iterator {
243 iter := p.db.NewIterator(nil, p.ro)
244 err := iter.Error()
245 if err != nil {
246 p.b.Fatal("cannot create iterator: ", err)
247 }
248 return iter
249 }
250
251 func (p *dbBench) close() {
252 if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
253 p.b.Log("Block pool stats: ", bp)
254 }
255 p.db.Close()
256 p.stor.Close()
257 os.RemoveAll(benchDB)
258 p.db = nil
259 p.keys = nil
260 p.values = nil
261 runtime.GC()
262 }
263
264 func BenchmarkDBWrite(b *testing.B) {
265 p := openDBBench(b, false)
266 p.populate(b.N)
267 p.writes(1)
268 p.close()
269 }
270
271 func BenchmarkDBWriteBatch(b *testing.B) {
272 p := openDBBench(b, false)
273 p.populate(b.N)
274 p.writes(1000)
275 p.close()
276 }
277
278 func BenchmarkDBWriteUncompressed(b *testing.B) {
279 p := openDBBench(b, true)
280 p.populate(b.N)
281 p.writes(1)
282 p.close()
283 }
284
285 func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
286 p := openDBBench(b, true)
287 p.populate(b.N)
288 p.writes(1000)
289 p.close()
290 }
291
292 func BenchmarkDBWriteRandom(b *testing.B) {
293 p := openDBBench(b, false)
294 p.populate(b.N)
295 p.randomize()
296 p.writes(1)
297 p.close()
298 }
299
300 func BenchmarkDBWriteRandomSync(b *testing.B) {
301 p := openDBBench(b, false)
302 p.wo.Sync = true
303 p.populate(b.N)
304 p.writes(1)
305 p.close()
306 }
307
308 func BenchmarkDBOverwrite(b *testing.B) {
309 p := openDBBench(b, false)
310 p.populate(b.N)
311 p.writes(1)
312 p.writes(1)
313 p.close()
314 }
315
316 func BenchmarkDBOverwriteRandom(b *testing.B) {
317 p := openDBBench(b, false)
318 p.populate(b.N)
319 p.writes(1)
320 p.randomize()
321 p.writes(1)
322 p.close()
323 }
324
325 func BenchmarkDBPut(b *testing.B) {
326 p := openDBBench(b, false)
327 p.populate(b.N)
328 p.puts()
329 p.close()
330 }
331
332 func BenchmarkDBRead(b *testing.B) {
333 p := openDBBench(b, false)
334 p.populate(b.N)
335 p.fill()
336 p.gc()
337
338 iter := p.newIter()
339 b.ResetTimer()
340 for iter.Next() {
341 }
342 iter.Release()
343 b.StopTimer()
344 b.SetBytes(116)
345 p.close()
346 }
347
348 func BenchmarkDBReadGC(b *testing.B) {
349 p := openDBBench(b, false)
350 p.populate(b.N)
351 p.fill()
352
353 iter := p.newIter()
354 b.ResetTimer()
355 for iter.Next() {
356 }
357 iter.Release()
358 b.StopTimer()
359 b.SetBytes(116)
360 p.close()
361 }
362
363 func BenchmarkDBReadUncompressed(b *testing.B) {
364 p := openDBBench(b, true)
365 p.populate(b.N)
366 p.fill()
367 p.gc()
368
369 iter := p.newIter()
370 b.ResetTimer()
371 for iter.Next() {
372 }
373 iter.Release()
374 b.StopTimer()
375 b.SetBytes(116)
376 p.close()
377 }
378
379 func BenchmarkDBReadTable(b *testing.B) {
380 p := openDBBench(b, false)
381 p.populate(b.N)
382 p.fill()
383 p.reopen()
384 p.gc()
385
386 iter := p.newIter()
387 b.ResetTimer()
388 for iter.Next() {
389 }
390 iter.Release()
391 b.StopTimer()
392 b.SetBytes(116)
393 p.close()
394 }
395
396 func BenchmarkDBReadReverse(b *testing.B) {
397 p := openDBBench(b, false)
398 p.populate(b.N)
399 p.fill()
400 p.gc()
401
402 iter := p.newIter()
403 b.ResetTimer()
404 iter.Last()
405 for iter.Prev() {
406 }
407 iter.Release()
408 b.StopTimer()
409 b.SetBytes(116)
410 p.close()
411 }
412
413 func BenchmarkDBReadReverseTable(b *testing.B) {
414 p := openDBBench(b, false)
415 p.populate(b.N)
416 p.fill()
417 p.reopen()
418 p.gc()
419
420 iter := p.newIter()
421 b.ResetTimer()
422 iter.Last()
423 for iter.Prev() {
424 }
425 iter.Release()
426 b.StopTimer()
427 b.SetBytes(116)
428 p.close()
429 }
430
431 func BenchmarkDBSeek(b *testing.B) {
432 p := openDBBench(b, false)
433 p.populate(b.N)
434 p.fill()
435 p.seeks()
436 p.close()
437 }
438
439 func BenchmarkDBSeekRandom(b *testing.B) {
440 p := openDBBench(b, false)
441 p.populate(b.N)
442 p.fill()
443 p.randomize()
444 p.seeks()
445 p.close()
446 }
447
448 func BenchmarkDBGet(b *testing.B) {
449 p := openDBBench(b, false)
450 p.populate(b.N)
451 p.fill()
452 p.gets()
453 p.close()
454 }
455
456 func BenchmarkDBGetRandom(b *testing.B) {
457 p := openDBBench(b, false)
458 p.populate(b.N)
459 p.fill()
460 p.randomize()
461 p.gets()
462 p.close()
463 }
464
465 func BenchmarkDBReadConcurrent(b *testing.B) {
466 p := openDBBench(b, false)
467 p.populate(b.N)
468 p.fill()
469 p.gc()
470 defer p.close()
471
472 b.ResetTimer()
473 b.SetBytes(116)
474
475 b.RunParallel(func(pb *testing.PB) {
476 iter := p.newIter()
477 defer iter.Release()
478 for pb.Next() && iter.Next() {
479 }
480 })
481 }
482
483 func BenchmarkDBReadConcurrent2(b *testing.B) {
484 p := openDBBench(b, false)
485 p.populate(b.N)
486 p.fill()
487 p.gc()
488 defer p.close()
489
490 b.ResetTimer()
491 b.SetBytes(116)
492
493 var dir uint32
494 b.RunParallel(func(pb *testing.PB) {
495 iter := p.newIter()
496 defer iter.Release()
497 if atomic.AddUint32(&dir, 1)%2 == 0 {
498 for pb.Next() && iter.Next() {
499 }
500 } else {
501 if pb.Next() && iter.Last() {
502 for pb.Next() && iter.Prev() {
503 }
504 }
505 }
506 })
507 }
508
View as plain text