1 package diskv
2
3 import (
4 "bytes"
5 "errors"
6 "math/rand"
7 "regexp"
8 "strings"
9 "testing"
10 "time"
11 )
12
13 func cmpBytes(a, b []byte) bool {
14 if len(a) != len(b) {
15 return false
16 }
17 for i := 0; i < len(a); i++ {
18 if a[i] != b[i] {
19 return false
20 }
21 }
22 return true
23 }
24
25 func (d *Diskv) isCached(key string) bool {
26 d.mu.RLock()
27 defer d.mu.RUnlock()
28 _, ok := d.cache[key]
29 return ok
30 }
31
32 func TestWriteReadErase(t *testing.T) {
33 d := New(Options{
34 BasePath: "test-data",
35 CacheSizeMax: 1024,
36 })
37 defer d.EraseAll()
38 k, v := "a", []byte{'b'}
39 if err := d.Write(k, v); err != nil {
40 t.Fatalf("write: %s", err)
41 }
42 if readVal, err := d.Read(k); err != nil {
43 t.Fatalf("read: %s", err)
44 } else if bytes.Compare(v, readVal) != 0 {
45 t.Fatalf("read: expected %s, got %s", v, readVal)
46 }
47 if err := d.Erase(k); err != nil {
48 t.Fatalf("erase: %s", err)
49 }
50 }
51
52 func TestWRECache(t *testing.T) {
53 d := New(Options{
54 BasePath: "test-data",
55 CacheSizeMax: 1024,
56 })
57 defer d.EraseAll()
58 k, v := "xxx", []byte{' ', ' ', ' '}
59 if d.isCached(k) {
60 t.Fatalf("key cached before Write and Read")
61 }
62 if err := d.Write(k, v); err != nil {
63 t.Fatalf("write: %s", err)
64 }
65 if d.isCached(k) {
66 t.Fatalf("key cached before Read")
67 }
68 if readVal, err := d.Read(k); err != nil {
69 t.Fatalf("read: %s", err)
70 } else if bytes.Compare(v, readVal) != 0 {
71 t.Fatalf("read: expected %s, got %s", v, readVal)
72 }
73 for i := 0; i < 10 && !d.isCached(k); i++ {
74 time.Sleep(10 * time.Millisecond)
75 }
76 if !d.isCached(k) {
77 t.Fatalf("key not cached after Read")
78 }
79 if err := d.Erase(k); err != nil {
80 t.Fatalf("erase: %s", err)
81 }
82 if d.isCached(k) {
83 t.Fatalf("key cached after Erase")
84 }
85 }
86
87 func TestStrings(t *testing.T) {
88 d := New(Options{
89 BasePath: "test-data",
90 CacheSizeMax: 1024,
91 })
92 defer d.EraseAll()
93
94 keys := map[string]bool{"a": false, "b": false, "c": false, "d": false}
95 v := []byte{'1'}
96 for k := range keys {
97 if err := d.Write(k, v); err != nil {
98 t.Fatalf("write: %s: %s", k, err)
99 }
100 }
101
102 for k := range d.Keys(nil) {
103 if _, present := keys[k]; present {
104 t.Logf("got: %s", k)
105 keys[k] = true
106 } else {
107 t.Fatalf("strings() returns unknown key: %s", k)
108 }
109 }
110
111 for k, found := range keys {
112 if !found {
113 t.Errorf("never got %s", k)
114 }
115 }
116 }
117
118 func TestZeroByteCache(t *testing.T) {
119 d := New(Options{
120 BasePath: "test-data",
121 CacheSizeMax: 0,
122 })
123 defer d.EraseAll()
124
125 k, v := "a", []byte{'1', '2', '3'}
126 if err := d.Write(k, v); err != nil {
127 t.Fatalf("Write: %s", err)
128 }
129
130 if d.isCached(k) {
131 t.Fatalf("key cached, expected not-cached")
132 }
133
134 if _, err := d.Read(k); err != nil {
135 t.Fatalf("Read: %s", err)
136 }
137
138 if d.isCached(k) {
139 t.Fatalf("key cached, expected not-cached")
140 }
141 }
142
143 func TestOneByteCache(t *testing.T) {
144 d := New(Options{
145 BasePath: "test-data",
146 CacheSizeMax: 1,
147 })
148 defer d.EraseAll()
149
150 k1, k2, v1, v2 := "a", "b", []byte{'1'}, []byte{'1', '2'}
151 if err := d.Write(k1, v1); err != nil {
152 t.Fatal(err)
153 }
154
155 if v, err := d.Read(k1); err != nil {
156 t.Fatal(err)
157 } else if !cmpBytes(v, v1) {
158 t.Fatalf("Read: expected %s, got %s", string(v1), string(v))
159 }
160
161 for i := 0; i < 10 && !d.isCached(k1); i++ {
162 time.Sleep(10 * time.Millisecond)
163 }
164 if !d.isCached(k1) {
165 t.Fatalf("expected 1-byte value to be cached, but it wasn't")
166 }
167
168 if err := d.Write(k2, v2); err != nil {
169 t.Fatal(err)
170 }
171 if _, err := d.Read(k2); err != nil {
172 t.Fatalf("--> %s", err)
173 }
174
175 for i := 0; i < 10 && (!d.isCached(k1) || d.isCached(k2)); i++ {
176 time.Sleep(10 * time.Millisecond)
177 }
178 if !d.isCached(k1) {
179 t.Fatalf("1-byte value was uncached for no reason")
180 }
181
182 if d.isCached(k2) {
183 t.Fatalf("2-byte value was cached, but cache max size is 1")
184 }
185 }
186
187 func TestStaleCache(t *testing.T) {
188 d := New(Options{
189 BasePath: "test-data",
190 CacheSizeMax: 1,
191 })
192 defer d.EraseAll()
193
194 k, first, second := "a", "first", "second"
195 if err := d.Write(k, []byte(first)); err != nil {
196 t.Fatal(err)
197 }
198
199 v, err := d.Read(k)
200 if err != nil {
201 t.Fatal(err)
202 }
203 if string(v) != first {
204 t.Errorf("expected '%s', got '%s'", first, v)
205 }
206
207 if err := d.Write(k, []byte(second)); err != nil {
208 t.Fatal(err)
209 }
210
211 v, err = d.Read(k)
212 if err != nil {
213 t.Fatal(err)
214 }
215
216 if string(v) != second {
217 t.Errorf("expected '%s', got '%s'", second, v)
218 }
219 }
220
221 func TestHas(t *testing.T) {
222 d := New(Options{
223 BasePath: "test-data",
224 CacheSizeMax: 1024,
225 })
226 defer d.EraseAll()
227
228 for k, v := range map[string]string{
229 "a": "1",
230 "foo": "2",
231 "012345": "3",
232 } {
233 d.Write(k, []byte(v))
234 }
235
236 d.Read("foo")
237 if !d.isCached("foo") {
238 t.Errorf("'foo' didn't get cached")
239 }
240
241 for _, tuple := range []struct {
242 key string
243 expected bool
244 }{
245 {"a", true},
246 {"b", false},
247 {"foo", true},
248 {"bar", false},
249 {"01234", false},
250 {"012345", true},
251 {"0123456", false},
252 } {
253 if expected, got := tuple.expected, d.Has(tuple.key); expected != got {
254 t.Errorf("Has(%s): expected %v, got %v", tuple.key, expected, got)
255 }
256 }
257 }
258
259 type BrokenReader struct{}
260
261 func (BrokenReader) Read(p []byte) (n int, err error) {
262 return 0, errors.New("failed to read")
263 }
264
265 func TestRemovesIncompleteFiles(t *testing.T) {
266 opts := Options{
267 BasePath: "test-data",
268 CacheSizeMax: 1024,
269 }
270 d := New(opts)
271 defer d.EraseAll()
272
273 key, stream, sync := "key", BrokenReader{}, false
274
275 if err := d.WriteStream(key, stream, sync); err == nil {
276 t.Fatalf("Expected i/o copy error, none received.")
277 }
278
279 if _, err := d.Read(key); err == nil {
280 t.Fatal("Could read the key, but it shouldn't exist")
281 }
282 }
283
284 func TestTempDir(t *testing.T) {
285 opts := Options{
286 BasePath: "test-data",
287 TempDir: "test-data-temp",
288 CacheSizeMax: 1024,
289 }
290 d := New(opts)
291 defer d.EraseAll()
292
293 k, v := "a", []byte{'b'}
294 if err := d.Write(k, v); err != nil {
295 t.Fatalf("write: %s", err)
296 }
297 if readVal, err := d.Read(k); err != nil {
298 t.Fatalf("read: %s", err)
299 } else if bytes.Compare(v, readVal) != 0 {
300 t.Fatalf("read: expected %s, got %s", v, readVal)
301 }
302 if err := d.Erase(k); err != nil {
303 t.Fatalf("erase: %s", err)
304 }
305 }
306
307 type CrashingReader struct{}
308
309 func (CrashingReader) Read(p []byte) (n int, err error) {
310 panic("System has crashed while reading the stream")
311 }
312
313 func TestAtomicWrite(t *testing.T) {
314 opts := Options{
315 BasePath: "test-data",
316
317 TempDir: "test-data-temp",
318 CacheSizeMax: 1024,
319 }
320 d := New(opts)
321 defer d.EraseAll()
322
323 key := "key"
324 func() {
325 defer func() {
326 recover()
327 }()
328
329 stream := CrashingReader{}
330 d.WriteStream(key, stream, false)
331 }()
332
333 if d.Has(key) {
334 t.Fatal("Has key, but it shouldn't exist")
335 }
336 if _, ok := <-d.Keys(nil); ok {
337 t.Fatal("Store isn't empty")
338 }
339 }
340
341 const letterBytes = "abcdef0123456789"
342
343 func randStringBytes(n int) string {
344 b := make([]byte, n)
345 for i := range b {
346 b[i] = letterBytes[rand.Intn(len(letterBytes))]
347 }
348 return string(b)
349 }
350
351 func TestHybridStore(t *testing.T) {
352 regex := regexp.MustCompile("[0-9a-fA-F]{64}")
353
354 transformFunc := func(s string) *PathKey {
355
356 if regex.MatchString(s) {
357 return &PathKey{Path: []string{"objects", s[0:2]},
358 FileName: s,
359 }
360 }
361
362 folders := strings.Split(s, "/")
363 lfolders := len(folders)
364 if lfolders > 1 {
365 return &PathKey{Path: folders[:lfolders-1],
366 FileName: folders[lfolders-1],
367 }
368 }
369
370 return &PathKey{Path: []string{},
371 FileName: s,
372 }
373 }
374
375 inverseTransformFunc := func(pathKey *PathKey) string {
376
377 if regex.MatchString(pathKey.FileName) {
378 return pathKey.FileName
379
380 }
381
382 if len(pathKey.Path) == 0 {
383 return pathKey.FileName
384 }
385
386 return strings.Join(pathKey.Path, "/") + "/" + pathKey.FileName
387
388 }
389 opts := Options{
390 BasePath: "test-data",
391 CacheSizeMax: 1024,
392 AdvancedTransform: transformFunc,
393 InverseTransform: inverseTransformFunc,
394 }
395 d := New(opts)
396 defer d.EraseAll()
397
398 testData := map[string]string{}
399
400 for i := 0; i < 100; i++ {
401 testData[randStringBytes(64)] = randStringBytes(100)
402 }
403
404 for i := 0; i < 100; i++ {
405 testData[randStringBytes(20)] = randStringBytes(100)
406 }
407
408 for i := 0; i < 100; i++ {
409 numsep := rand.Intn(10) + 1
410 key := ""
411 for j := 0; j < numsep; j++ {
412 key += randStringBytes(10) + "/"
413 }
414 key += randStringBytes(40)
415 testData[key] = randStringBytes(100)
416 }
417
418 for k, v := range testData {
419 d.WriteString(k, v)
420 }
421
422 for k, v := range testData {
423 readVal := d.ReadString(k)
424
425 if v != readVal {
426 t.Fatalf("read: expected %s, got %s", v, readVal)
427 }
428 }
429
430 }
431
View as plain text