1 package huff0
2
3 import (
4 "bytes"
5 "fmt"
6 "io"
7 "os"
8 "testing"
9
10 "github.com/klauspost/compress/zip"
11 )
12
13 func TestDecompress1X(t *testing.T) {
14 for _, test := range testfiles {
15 t.Run(test.name, func(t *testing.T) {
16 var s = &Scratch{}
17 buf0, err := test.fn()
18 if err != nil {
19 t.Fatal(err)
20 }
21 if len(buf0) > BlockSizeMax {
22 buf0 = buf0[:BlockSizeMax]
23 }
24 b, re, err := Compress1X(buf0, s)
25 if err != test.err1X {
26 t.Errorf("want error %v (%T), got %v (%T)", test.err1X, test.err1X, err, err)
27 }
28 if err != nil {
29 t.Log(test.name, err.Error())
30 return
31 }
32 if b == nil {
33 t.Error("got no output")
34 return
35 }
36 if len(s.OutTable) == 0 {
37 t.Error("got no table definition")
38 }
39 if re {
40 t.Error("claimed to have re-used.")
41 }
42 if len(s.OutData) == 0 {
43 t.Error("got no data output")
44 }
45
46 wantRemain := len(s.OutData)
47 t.Logf("%s: %d -> %d bytes (%.2f:1) %t (table: %d bytes)", test.name, len(buf0), len(b), float64(len(buf0))/float64(len(b)), re, len(s.OutTable))
48
49 s.Out = nil
50 var remain []byte
51 s, remain, err = ReadTable(b, s)
52 if err != nil {
53 t.Error(err)
54 return
55 }
56 var buf bytes.Buffer
57 if s.matches(s.prevTable, &buf); buf.Len() > 0 {
58 t.Error(buf.String())
59 }
60 if len(remain) != wantRemain {
61 t.Fatalf("remain mismatch, want %d, got %d bytes", wantRemain, len(remain))
62 }
63 t.Logf("remain: %d bytes, ok", len(remain))
64 dc, err := s.Decompress1X(remain)
65 if err != nil {
66 t.Error(err)
67 return
68 }
69 if len(buf0) != len(dc) {
70 t.Errorf(test.name+"decompressed, want size: %d, got %d", len(buf0), len(dc))
71 if len(buf0) > len(dc) {
72 buf0 = buf0[:len(dc)]
73 } else {
74 dc = dc[:len(buf0)]
75 }
76 if !bytes.Equal(buf0, dc) {
77 if len(dc) > 1024 {
78 t.Log(string(dc[:1024]))
79 t.Errorf(test.name+"decompressed, got delta: \n(in)\t%02x !=\n(out)\t%02x\n", buf0[:1024], dc[:1024])
80 } else {
81 t.Log(string(dc))
82 t.Errorf(test.name+"decompressed, got delta: (in) %v != (out) %v\n", buf0, dc)
83 }
84 }
85 return
86 }
87 if !bytes.Equal(buf0, dc) {
88 if len(buf0) > 1024 {
89 t.Log(string(dc[:1024]))
90 } else {
91 t.Log(string(dc))
92 }
93
94 t.Errorf(test.name + ": decompressed, got delta")
95 }
96 if !t.Failed() {
97 t.Log("... roundtrip ok!")
98 }
99 })
100 }
101 }
102
103 func TestDecompress1XRegression(t *testing.T) {
104 data, err := os.ReadFile("testdata/decompress1x_regression.zip")
105 if err != nil {
106 t.Fatal(err)
107 }
108 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
109 if err != nil {
110 t.Fatal(err)
111 }
112 for _, tt := range zr.File {
113 if tt.UncompressedSize64 == 0 {
114 continue
115 }
116 rc, err := tt.Open()
117 if err != nil {
118 t.Fatal(err)
119 }
120 data, err := io.ReadAll(rc)
121 if err != nil {
122 t.Fatal(err)
123 }
124
125 t.Run(tt.Name, func(t *testing.T) {
126 s, rem, err := ReadTable(data, nil)
127 if err != nil {
128 t.Fatal(err)
129 }
130 _, err = s.Decompress1X(rem)
131 if err == nil {
132 t.Fatal("expected error to be returned")
133 }
134
135 t.Logf("returned error: %s", err)
136 })
137 }
138 }
139
140 func TestDecompress4X(t *testing.T) {
141 for _, test := range testfiles {
142 t.Run(test.name, func(t *testing.T) {
143 for _, tl := range []uint8{0, 5, 6, 7, 8, 9, 10, 11} {
144 t.Run(fmt.Sprintf("tablelog-%d", tl), func(t *testing.T) {
145 var s = &Scratch{}
146 s.TableLog = tl
147 buf0, err := test.fn()
148 if err != nil {
149 t.Fatal(err)
150 }
151 if len(buf0) > BlockSizeMax {
152 buf0 = buf0[:BlockSizeMax]
153 }
154 b, re, err := Compress4X(buf0, s)
155 if err != test.err4X {
156 t.Errorf("want error %v (%T), got %v (%T)", test.err1X, test.err1X, err, err)
157 }
158 if err != nil {
159 t.Log(test.name, err.Error())
160 return
161 }
162 if b == nil {
163 t.Error("got no output")
164 return
165 }
166 if len(s.OutTable) == 0 {
167 t.Error("got no table definition")
168 }
169 if re {
170 t.Error("claimed to have re-used.")
171 }
172 if len(s.OutData) == 0 {
173 t.Error("got no data output")
174 }
175
176 wantRemain := len(s.OutData)
177 t.Logf("%s: %d -> %d bytes (%.2f:1) %t (table: %d bytes)", test.name, len(buf0), len(b), float64(len(buf0))/float64(len(b)), re, len(s.OutTable))
178
179 s.Out = nil
180 var remain []byte
181 s, remain, err = ReadTable(b, s)
182 if err != nil {
183 t.Error(err)
184 return
185 }
186 var buf bytes.Buffer
187 if s.matches(s.prevTable, &buf); buf.Len() > 0 {
188 t.Error(buf.String())
189 }
190 if len(remain) != wantRemain {
191 t.Fatalf("remain mismatch, want %d, got %d bytes", wantRemain, len(remain))
192 }
193 t.Logf("remain: %d bytes, ok", len(remain))
194 dc, err := s.Decompress4X(remain, len(buf0))
195 if err != nil {
196 t.Error(err)
197 return
198 }
199 if len(buf0) != len(dc) {
200 t.Errorf(test.name+"decompressed, want size: %d, got %d", len(buf0), len(dc))
201 if len(buf0) > len(dc) {
202 buf0 = buf0[:len(dc)]
203 } else {
204 dc = dc[:len(buf0)]
205 }
206 if !bytes.Equal(buf0, dc) {
207 if len(dc) > 1024 {
208 t.Log(string(dc[:1024]))
209 t.Errorf(test.name+"decompressed, got delta: \n(in)\t%02x !=\n(out)\t%02x\n", buf0[:1024], dc[:1024])
210 } else {
211 t.Log(string(dc))
212 t.Errorf(test.name+"decompressed, got delta: (in) %v != (out) %v\n", buf0, dc)
213 }
214 }
215 return
216 }
217 if !bytes.Equal(buf0, dc) {
218 if len(buf0) > 1024 {
219 t.Log(string(dc[:1024]))
220 } else {
221 t.Log(string(dc))
222 }
223
224 t.Errorf(test.name + ": decompressed, got delta")
225 }
226 if !t.Failed() {
227 t.Log("... roundtrip ok!")
228 }
229
230 })
231 }
232 })
233 }
234 }
235
236 func TestRoundtrip1XFuzz(t *testing.T) {
237 for _, test := range testfilesExtended {
238 t.Run(test.name, func(t *testing.T) {
239 var s = &Scratch{}
240 buf0, err := test.fn()
241 if err != nil {
242 t.Fatal(err)
243 }
244 if len(buf0) > BlockSizeMax {
245 buf0 = buf0[:BlockSizeMax]
246 }
247 b, re, err := Compress1X(buf0, s)
248 if err != nil {
249 if err == ErrIncompressible || err == ErrUseRLE || err == ErrTooBig {
250 t.Log(test.name, err.Error())
251 return
252 }
253 t.Error(test.name, err.Error())
254 return
255 }
256 if b == nil {
257 t.Error("got no output")
258 return
259 }
260 if len(s.OutTable) == 0 {
261 t.Error("got no table definition")
262 }
263 if re {
264 t.Error("claimed to have re-used.")
265 }
266 if len(s.OutData) == 0 {
267 t.Error("got no data output")
268 }
269
270 wantRemain := len(s.OutData)
271 t.Logf("%s: %d -> %d bytes (%.2f:1) %t (table: %d bytes)", test.name, len(buf0), len(b), float64(len(buf0))/float64(len(b)), re, len(s.OutTable))
272
273 s.Out = nil
274 var remain []byte
275 s, remain, err = ReadTable(b, s)
276 if err != nil {
277 t.Error(err)
278 return
279 }
280 var buf bytes.Buffer
281 if s.matches(s.prevTable, &buf); buf.Len() > 0 {
282 t.Error(buf.String())
283 }
284 if len(remain) != wantRemain {
285 t.Fatalf("remain mismatch, want %d, got %d bytes", wantRemain, len(remain))
286 }
287 t.Logf("remain: %d bytes, ok", len(remain))
288 dc, err := s.Decompress1X(remain)
289 if err != nil {
290 t.Error(err)
291 return
292 }
293 if len(buf0) != len(dc) {
294 t.Errorf(test.name+"decompressed, want size: %d, got %d", len(buf0), len(dc))
295 if len(buf0) > len(dc) {
296 buf0 = buf0[:len(dc)]
297 } else {
298 dc = dc[:len(buf0)]
299 }
300 if !bytes.Equal(buf0, dc) {
301 if len(dc) > 1024 {
302 t.Log(string(dc[:1024]))
303 t.Errorf(test.name+"decompressed, got delta: \n(in)\t%02x !=\n(out)\t%02x\n", buf0[:1024], dc[:1024])
304 } else {
305 t.Log(string(dc))
306 t.Errorf(test.name+"decompressed, got delta: (in) %v != (out) %v\n", buf0, dc)
307 }
308 }
309 return
310 }
311 if !bytes.Equal(buf0, dc) {
312 if len(buf0) > 1024 {
313 t.Log(string(dc[:1024]))
314 } else {
315 t.Log(string(dc))
316 }
317
318 t.Errorf(test.name + ": decompressed, got delta")
319 }
320 if !t.Failed() {
321 t.Log("... roundtrip ok!")
322 }
323 })
324 }
325 }
326
327 func TestRoundtrip4XFuzz(t *testing.T) {
328 for _, test := range testfilesExtended {
329 t.Run(test.name, func(t *testing.T) {
330 var s = &Scratch{}
331 buf0, err := test.fn()
332 if err != nil {
333 t.Fatal(err)
334 }
335 if len(buf0) > BlockSizeMax {
336 buf0 = buf0[:BlockSizeMax]
337 }
338 b, re, err := Compress4X(buf0, s)
339 if err != nil {
340 if err == ErrIncompressible || err == ErrUseRLE || err == ErrTooBig {
341 t.Log(test.name, err.Error())
342 return
343 }
344 t.Error(test.name, err.Error())
345 return
346 }
347 if b == nil {
348 t.Error("got no output")
349 return
350 }
351 if len(s.OutTable) == 0 {
352 t.Error("got no table definition")
353 }
354 if re {
355 t.Error("claimed to have re-used.")
356 }
357 if len(s.OutData) == 0 {
358 t.Error("got no data output")
359 }
360
361 wantRemain := len(s.OutData)
362 t.Logf("%s: %d -> %d bytes (%.2f:1) %t (table: %d bytes)", test.name, len(buf0), len(b), float64(len(buf0))/float64(len(b)), re, len(s.OutTable))
363
364 s.Out = nil
365 var remain []byte
366 s, remain, err = ReadTable(b, s)
367 if err != nil {
368 t.Error(err)
369 return
370 }
371 var buf bytes.Buffer
372 if s.matches(s.prevTable, &buf); buf.Len() > 0 {
373 t.Error(buf.String())
374 }
375 if len(remain) != wantRemain {
376 t.Fatalf("remain mismatch, want %d, got %d bytes", wantRemain, len(remain))
377 }
378 t.Logf("remain: %d bytes, ok", len(remain))
379 dc, err := s.Decompress4X(remain, len(buf0))
380 if err != nil {
381 t.Error(err)
382 return
383 }
384 if len(buf0) != len(dc) {
385 t.Errorf(test.name+"decompressed, want size: %d, got %d", len(buf0), len(dc))
386 if len(buf0) > len(dc) {
387 buf0 = buf0[:len(dc)]
388 } else {
389 dc = dc[:len(buf0)]
390 }
391 if !bytes.Equal(buf0, dc) {
392 if len(dc) > 1024 {
393 t.Log(string(dc[:1024]))
394 t.Errorf(test.name+"decompressed, got delta: \n(in)\t%02x !=\n(out)\t%02x\n", buf0[:1024], dc[:1024])
395 } else {
396 t.Log(string(dc))
397 t.Errorf(test.name+"decompressed, got delta: (in) %v != (out) %v\n", buf0, dc)
398 }
399 }
400 return
401 }
402 if !bytes.Equal(buf0, dc) {
403 if len(buf0) > 1024 {
404 t.Log(string(dc[:1024]))
405 } else {
406 t.Log(string(dc))
407 }
408
409 t.Errorf(test.name + ": decompressed, got delta")
410 }
411 if !t.Failed() {
412 t.Log("... roundtrip ok!")
413 }
414 })
415 }
416 }
417
418 func BenchmarkDecompress1XTable(b *testing.B) {
419 for _, tt := range testfiles {
420 test := tt
421 if test.err1X != nil {
422 continue
423 }
424 b.Run(test.name, func(b *testing.B) {
425 var s = &Scratch{}
426 s.Reuse = ReusePolicyNone
427 buf0, err := test.fn()
428 if err != nil {
429 b.Fatal(err)
430 }
431 if len(buf0) > BlockSizeMax {
432 buf0 = buf0[:BlockSizeMax]
433 }
434 compressed, _, err := Compress1X(buf0, s)
435 if err != test.err1X {
436 b.Fatal("unexpected error:", err)
437 }
438 s.Out = nil
439 s, remain, _ := ReadTable(compressed, s)
440 s.Decompress1X(remain)
441 b.ResetTimer()
442 b.ReportAllocs()
443 b.SetBytes(int64(len(buf0)))
444 for i := 0; i < b.N; i++ {
445 s, remain, err := ReadTable(compressed, s)
446 if err != nil {
447 b.Fatal(err)
448 }
449 _, err = s.Decompress1X(remain)
450 if err != nil {
451 b.Fatal(err)
452 }
453 }
454 })
455 }
456 }
457
458 func BenchmarkDecompress1XNoTable(b *testing.B) {
459 for _, tt := range testfiles {
460 test := tt
461 if test.err1X != nil {
462 continue
463 }
464 b.Run(test.name, func(b *testing.B) {
465 for _, sz := range []int{1e2, 1e4, BlockSizeMax} {
466 b.Run(fmt.Sprintf("%d", sz), func(b *testing.B) {
467 var s = &Scratch{}
468 s.Reuse = ReusePolicyNone
469 buf0, err := test.fn()
470 if err != nil {
471 b.Fatal(err)
472 }
473 for len(buf0) < sz {
474 buf0 = append(buf0, buf0...)
475 }
476 if len(buf0) > sz {
477 buf0 = buf0[:sz]
478 }
479 compressed, _, err := Compress1X(buf0, s)
480 if err != test.err1X {
481 if err == ErrUseRLE {
482 b.Skip("RLE")
483 return
484 }
485 b.Fatal("unexpected error:", err)
486 }
487 s.Out = nil
488 s, remain, _ := ReadTable(compressed, s)
489 s.Decompress1X(remain)
490 b.ResetTimer()
491 b.ReportAllocs()
492 b.SetBytes(int64(len(buf0)))
493 for i := 0; i < b.N; i++ {
494 _, err = s.Decompress1X(remain)
495 if err != nil {
496 b.Fatal(err)
497 }
498 }
499 b.ReportMetric(float64(s.actualTableLog), "log")
500 b.ReportMetric(100*float64(len(compressed))/float64(len(buf0)), "pct")
501 })
502 }
503 })
504 }
505 }
506
507 func BenchmarkDecompress4XNoTable(b *testing.B) {
508 for _, tt := range testfiles {
509 test := tt
510 if test.err4X != nil {
511 continue
512 }
513 b.Run(test.name, func(b *testing.B) {
514 for _, sz := range []int{1e2, 1e4, BlockSizeMax} {
515 b.Run(fmt.Sprintf("%d", sz), func(b *testing.B) {
516 var s = &Scratch{}
517 s.Reuse = ReusePolicyNone
518 buf0, err := test.fn()
519 if err != nil {
520 b.Fatal(err)
521 }
522 for len(buf0) < sz {
523 buf0 = append(buf0, buf0...)
524 }
525 if len(buf0) > sz {
526 buf0 = buf0[:sz]
527 }
528 compressed, _, err := Compress4X(buf0, s)
529 if err != test.err4X {
530 if err == ErrUseRLE {
531 b.Skip("RLE")
532 return
533 }
534 b.Fatal("unexpected error:", err)
535 }
536 s.Out = nil
537 s, remain, _ := ReadTable(compressed, s)
538 s.Decompress4X(remain, len(buf0))
539 b.ResetTimer()
540 b.ReportAllocs()
541 b.SetBytes(int64(len(buf0)))
542 for i := 0; i < b.N; i++ {
543 _, err = s.Decompress4X(remain, len(buf0))
544 if err != nil {
545 b.Fatal(err)
546 }
547 }
548 b.ReportMetric(float64(s.actualTableLog), "log")
549 b.ReportMetric(100*float64(len(compressed))/float64(len(buf0)), "pct")
550
551 })
552 }
553 })
554 }
555 }
556
557 func BenchmarkDecompress4XNoTableTableLog8(b *testing.B) {
558 for _, tt := range testfiles[:1] {
559 test := tt
560 if test.err4X != nil {
561 continue
562 }
563 b.Run(test.name, func(b *testing.B) {
564 var s = &Scratch{}
565 s.Reuse = ReusePolicyNone
566 buf0, err := test.fn()
567 if err != nil {
568 b.Fatal(err)
569 }
570 if len(buf0) > BlockSizeMax {
571 buf0 = buf0[:BlockSizeMax]
572 }
573 s.TableLog = 8
574 compressed, _, err := Compress4X(buf0, s)
575 if err != test.err1X {
576 b.Fatal("unexpected error:", err)
577 }
578 s.Out = nil
579 s, remain, _ := ReadTable(compressed, s)
580 s.Decompress4X(remain, len(buf0))
581 b.ResetTimer()
582 b.ReportAllocs()
583 b.SetBytes(int64(len(buf0)))
584 for i := 0; i < b.N; i++ {
585 _, err = s.Decompress4X(remain, len(buf0))
586 if err != nil {
587 b.Fatal(err)
588 }
589 }
590 })
591 }
592 }
593
594 func BenchmarkDecompress4XTable(b *testing.B) {
595 for _, tt := range testfiles {
596 test := tt
597 if test.err4X != nil {
598 continue
599 }
600 b.Run(test.name, func(b *testing.B) {
601 var s = &Scratch{}
602 s.Reuse = ReusePolicyNone
603 buf0, err := test.fn()
604 if err != nil {
605 b.Fatal(err)
606 }
607 if len(buf0) > BlockSizeMax {
608 buf0 = buf0[:BlockSizeMax]
609 }
610 compressed, _, err := Compress4X(buf0, s)
611 if err != test.err1X {
612 b.Fatal("unexpected error:", err)
613 }
614 s.Out = nil
615 b.ResetTimer()
616 b.ReportAllocs()
617 b.SetBytes(int64(len(buf0)))
618 for i := 0; i < b.N; i++ {
619 s, remain, err := ReadTable(compressed, s)
620 if err != nil {
621 b.Fatal(err)
622 }
623 _, err = s.Decompress4X(remain, len(buf0))
624 if err != nil {
625 b.Fatal(err)
626 }
627 }
628 })
629 }
630 }
631
View as plain text