1
2
3
4
5 package flate
6
7 import (
8 "archive/zip"
9 "bytes"
10 "compress/flate"
11 "fmt"
12 "io"
13 "math"
14 "math/rand"
15 "os"
16 "runtime"
17 "strconv"
18 "strings"
19 "testing"
20 )
21
22 func TestWriterMemUsage(t *testing.T) {
23 testMem := func(t *testing.T, fn func()) {
24 var before, after runtime.MemStats
25 runtime.GC()
26 runtime.ReadMemStats(&before)
27 fn()
28 runtime.GC()
29 runtime.ReadMemStats(&after)
30 t.Logf("%s: Memory Used: %dKB, %d allocs", t.Name(), (after.HeapInuse-before.HeapInuse)/1024, after.HeapObjects-before.HeapObjects)
31 }
32 data := make([]byte, 100000)
33 t.Run("stateless", func(t *testing.T) {
34 testMem(t, func() {
35 StatelessDeflate(io.Discard, data, false, nil)
36 })
37 })
38 for level := HuffmanOnly; level <= BestCompression; level++ {
39 t.Run(fmt.Sprint("level-", level), func(t *testing.T) {
40 var zr *Writer
41 var err error
42 testMem(t, func() {
43 zr, err = NewWriter(io.Discard, level)
44 if err != nil {
45 t.Fatal(err)
46 }
47 zr.Write(data)
48 })
49 zr.Close()
50 })
51 }
52 for level := HuffmanOnly; level <= BestCompression; level++ {
53 t.Run(fmt.Sprint("stdlib-", level), func(t *testing.T) {
54 var zr *flate.Writer
55 var err error
56 testMem(t, func() {
57 zr, err = flate.NewWriter(io.Discard, level)
58 if err != nil {
59 t.Fatal(err)
60 }
61 zr.Write(data)
62 })
63 zr.Close()
64 })
65 }
66 }
67
68 func TestWriterRegression(t *testing.T) {
69 data, err := os.ReadFile("testdata/regression.zip")
70 if err != nil {
71 t.Fatal(err)
72 }
73 for level := HuffmanOnly; level <= BestCompression; level++ {
74 t.Run(fmt.Sprint("level_", level), func(t *testing.T) {
75 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
76 if err != nil {
77 t.Fatal(err)
78 }
79
80 for _, tt := range zr.File {
81 if !strings.HasSuffix(t.Name(), "") {
82 continue
83 }
84
85 t.Run(tt.Name, func(t *testing.T) {
86 if testing.Short() && tt.FileInfo().Size() > 10000 {
87 t.SkipNow()
88 }
89 r, err := tt.Open()
90 if err != nil {
91 t.Error(err)
92 return
93 }
94 in, err := io.ReadAll(r)
95 if err != nil {
96 t.Error(err)
97 }
98 msg := "level " + strconv.Itoa(level) + ":"
99 buf := new(bytes.Buffer)
100 fw, err := NewWriter(buf, level)
101 if err != nil {
102 t.Fatal(msg + err.Error())
103 }
104 n, err := fw.Write(in)
105 if n != len(in) {
106 t.Fatal(msg + "short write")
107 }
108 if err != nil {
109 t.Fatal(msg + err.Error())
110 }
111 err = fw.Close()
112 if err != nil {
113 t.Fatal(msg + err.Error())
114 }
115 fr1 := NewReader(buf)
116 data2, err := io.ReadAll(fr1)
117 if err != nil {
118 t.Fatal(msg + err.Error())
119 }
120 if !bytes.Equal(in, data2) {
121 t.Fatal(msg + "not equal")
122 }
123
124 msg = "level " + strconv.Itoa(level) + " (reset):"
125 buf.Reset()
126 fw.Reset(buf)
127 n, err = fw.Write(in)
128 if n != len(in) {
129 t.Fatal(msg + "short write")
130 }
131 if err != nil {
132 t.Fatal(msg + err.Error())
133 }
134 err = fw.Close()
135 if err != nil {
136 t.Fatal(msg + err.Error())
137 }
138 fr1 = NewReader(buf)
139 data2, err = io.ReadAll(fr1)
140 if err != nil {
141 t.Fatal(msg + err.Error())
142 }
143 if !bytes.Equal(in, data2) {
144 t.Fatal(msg + "not equal")
145 }
146 })
147 }
148 })
149 }
150 }
151
152 func benchmarkEncoder(b *testing.B, testfile, level, n int) {
153 b.SetBytes(int64(n))
154 buf0, err := os.ReadFile(testfiles[testfile])
155 if err != nil {
156 b.Fatal(err)
157 }
158 if len(buf0) == 0 {
159 b.Fatalf("test file %q has no data", testfiles[testfile])
160 }
161 buf1 := make([]byte, n)
162 for i := 0; i < n; i += len(buf0) {
163 if len(buf0) > n-i {
164 buf0 = buf0[:n-i]
165 }
166 copy(buf1[i:], buf0)
167 }
168 buf0 = nil
169 runtime.GC()
170 w, err := NewWriter(io.Discard, level)
171 if err != nil {
172 b.Fatal(err)
173 }
174 b.ResetTimer()
175 b.ReportAllocs()
176 for i := 0; i < b.N; i++ {
177 w.Reset(io.Discard)
178 _, err = w.Write(buf1)
179 if err != nil {
180 b.Fatal(err)
181 }
182 err = w.Close()
183 if err != nil {
184 b.Fatal(err)
185 }
186 }
187 }
188
189 func BenchmarkEncodeDigitsConstant1e4(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e4) }
190 func BenchmarkEncodeDigitsConstant1e5(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e5) }
191 func BenchmarkEncodeDigitsConstant1e6(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e6) }
192 func BenchmarkEncodeDigitsSpeed1e4(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e4) }
193 func BenchmarkEncodeDigitsSpeed1e5(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e5) }
194 func BenchmarkEncodeDigitsSpeed1e6(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e6) }
195 func BenchmarkEncodeDigitsDefault1e4(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e4) }
196 func BenchmarkEncodeDigitsDefault1e5(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e5) }
197 func BenchmarkEncodeDigitsDefault1e6(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e6) }
198 func BenchmarkEncodeDigitsCompress1e4(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e4) }
199 func BenchmarkEncodeDigitsCompress1e5(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e5) }
200 func BenchmarkEncodeDigitsCompress1e6(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e6) }
201 func BenchmarkEncodeDigitsSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e4) }
202 func BenchmarkEncodeDigitsSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e5) }
203 func BenchmarkEncodeDigitsSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e6) }
204 func BenchmarkEncodeTwainConstant1e4(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e4) }
205 func BenchmarkEncodeTwainConstant1e5(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e5) }
206 func BenchmarkEncodeTwainConstant1e6(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e6) }
207 func BenchmarkEncodeTwainSpeed1e4(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e4) }
208 func BenchmarkEncodeTwainSpeed1e5(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e5) }
209 func BenchmarkEncodeTwainSpeed1e6(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e6) }
210 func BenchmarkEncodeTwainDefault1e4(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e4) }
211 func BenchmarkEncodeTwainDefault1e5(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e5) }
212 func BenchmarkEncodeTwainDefault1e6(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e6) }
213 func BenchmarkEncodeTwainCompress1e4(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e4) }
214 func BenchmarkEncodeTwainCompress1e5(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e5) }
215 func BenchmarkEncodeTwainCompress1e6(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e6) }
216 func BenchmarkEncodeTwainSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e4) }
217 func BenchmarkEncodeTwainSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e5) }
218 func BenchmarkEncodeTwainSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e6) }
219
220 func BenchmarkEncodeTwain1024Win1e4(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e4) }
221 func BenchmarkEncodeTwain1024Win1e5(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e5) }
222 func BenchmarkEncodeTwain1024Win1e6(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e6) }
223
224 func benchmarkStatelessEncoder(b *testing.B, testfile, n int) {
225 b.SetBytes(int64(n))
226 buf0, err := os.ReadFile(testfiles[testfile])
227 if err != nil {
228 b.Fatal(err)
229 }
230 if len(buf0) == 0 {
231 b.Fatalf("test file %q has no data", testfiles[testfile])
232 }
233 buf1 := make([]byte, n)
234 for i := 0; i < n; i += len(buf0) {
235 if len(buf0) > n-i {
236 buf0 = buf0[:n-i]
237 }
238 copy(buf1[i:], buf0)
239 }
240 buf0 = nil
241 runtime.GC()
242 b.ResetTimer()
243 b.ReportAllocs()
244 for i := 0; i < b.N; i++ {
245 w := NewStatelessWriter(io.Discard)
246 _, err = w.Write(buf1)
247 if err != nil {
248 b.Fatal(err)
249 }
250 err = w.Close()
251 if err != nil {
252 b.Fatal(err)
253 }
254 }
255 }
256
257
258 type errorWriter struct {
259 N int
260 }
261
262 func (e *errorWriter) Write(b []byte) (int, error) {
263 if e.N <= 0 {
264 return 0, io.ErrClosedPipe
265 }
266 e.N--
267 return len(b), nil
268 }
269
270
271 func TestWriteError(t *testing.T) {
272 buf := new(bytes.Buffer)
273 n := 65536
274 if !testing.Short() {
275 n *= 4
276 }
277 for i := 0; i < n; i++ {
278 fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i)
279 }
280 in := buf.Bytes()
281
282 copyBuf := make([]byte, 128)
283 for l := 0; l < 10; l++ {
284 for fail := 1; fail <= 256; fail *= 2 {
285
286 ew := &errorWriter{N: fail}
287 w, err := NewWriter(ew, l)
288 if err != nil {
289 t.Fatalf("NewWriter: level %d: %v", l, err)
290 }
291 n, err := copyBuffer(w, bytes.NewBuffer(in), copyBuf)
292 if err == nil {
293 t.Fatalf("Level %d: Expected an error, writer was %#v", l, ew)
294 }
295 n2, err := w.Write([]byte{1, 2, 2, 3, 4, 5})
296 if n2 != 0 {
297 t.Fatal("Level", l, "Expected 0 length write, got", n)
298 }
299 if err == nil {
300 t.Fatal("Level", l, "Expected an error")
301 }
302 err = w.Flush()
303 if err == nil {
304 t.Fatal("Level", l, "Expected an error on flush")
305 }
306 err = w.Close()
307 if err == nil {
308 t.Fatal("Level", l, "Expected an error on close")
309 }
310
311 w.Reset(io.Discard)
312 n2, err = w.Write([]byte{1, 2, 3, 4, 5, 6})
313 if err != nil {
314 t.Fatal("Level", l, "Got unexpected error after reset:", err)
315 }
316 if n2 == 0 {
317 t.Fatal("Level", l, "Got 0 length write, expected > 0")
318 }
319 if testing.Short() {
320 return
321 }
322 }
323 }
324 }
325
326
327 func TestWriter_Reset(t *testing.T) {
328 buf := new(bytes.Buffer)
329 n := 65536
330 if !testing.Short() {
331 n *= 4
332 }
333 for i := 0; i < n; i++ {
334 fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i)
335 }
336 in := buf.Bytes()
337 for l := 0; l < 10; l++ {
338 l := l
339 if testing.Short() && l > 1 {
340 continue
341 }
342 t.Run(fmt.Sprintf("level-%d", l), func(t *testing.T) {
343 t.Parallel()
344 offset := 1
345 if testing.Short() {
346 offset = 256
347 }
348 for ; offset <= 256; offset *= 2 {
349
350 w, err := NewWriter(io.Discard, l)
351 if err != nil {
352 t.Fatalf("NewWriter: level %d: %v", l, err)
353 }
354 if w.d.fast == nil {
355 t.Skip("Not Fast...")
356 return
357 }
358 for i := 0; i < (bufferReset-len(in)-offset-maxMatchOffset)/maxMatchOffset; i++ {
359
360 w.d.fast.Reset()
361 }
362 w.d.fast.Reset()
363 _, err = w.Write(in)
364 if err != nil {
365 t.Fatal(err)
366 }
367 for i := 0; i < 50; i++ {
368
369 w.d.fast.Reset()
370 }
371 w.d.fast.Reset()
372
373 _, err = w.Write(in)
374 if err != nil {
375 t.Fatal(err)
376 }
377 for i := 0; i < (math.MaxUint32-bufferReset)/maxMatchOffset; i++ {
378
379 w.d.fast.Reset()
380 }
381
382 _, err = w.Write(in)
383 if err != nil {
384 t.Fatal(err)
385 }
386 err = w.Close()
387 if err != nil {
388 t.Fatal(err)
389 }
390 }
391 })
392 }
393 }
394
395 func TestDeterministicL1(t *testing.T) { testDeterministic(1, t) }
396 func TestDeterministicL2(t *testing.T) { testDeterministic(2, t) }
397 func TestDeterministicL3(t *testing.T) { testDeterministic(3, t) }
398 func TestDeterministicL4(t *testing.T) { testDeterministic(4, t) }
399 func TestDeterministicL5(t *testing.T) { testDeterministic(5, t) }
400 func TestDeterministicL6(t *testing.T) { testDeterministic(6, t) }
401 func TestDeterministicL7(t *testing.T) { testDeterministic(7, t) }
402 func TestDeterministicL8(t *testing.T) { testDeterministic(8, t) }
403 func TestDeterministicL9(t *testing.T) { testDeterministic(9, t) }
404 func TestDeterministicL0(t *testing.T) { testDeterministic(0, t) }
405 func TestDeterministicLM2(t *testing.T) { testDeterministic(-2, t) }
406
407 func testDeterministic(i int, t *testing.T) {
408
409 var length = maxStoreBlockSize*30 + 500
410 if testing.Short() {
411 length /= 10
412 }
413
414
415 rng := rand.New(rand.NewSource(1))
416 t1 := make([]byte, length)
417 for i := range t1 {
418 t1[i] = byte(rng.Int63() & 7)
419 }
420
421
422 var b1 bytes.Buffer
423 br := bytes.NewBuffer(t1)
424 w, err := NewWriter(&b1, i)
425 if err != nil {
426 t.Fatal(err)
427 }
428
429 cbuf := make([]byte, 787)
430 _, err = copyBuffer(w, br, cbuf)
431 if err != nil {
432 t.Fatal(err)
433 }
434 w.Close()
435
436
437
438 var b2 bytes.Buffer
439 cbuf = make([]byte, 81761)
440 br2 := bytes.NewBuffer(t1)
441 w2, err := NewWriter(&b2, i)
442 if err != nil {
443 t.Fatal(err)
444 }
445 _, err = copyBuffer(w2, br2, cbuf)
446 if err != nil {
447 t.Fatal(err)
448 }
449 w2.Close()
450
451 b1b := b1.Bytes()
452 b2b := b2.Bytes()
453
454 if !bytes.Equal(b1b, b2b) {
455 t.Errorf("level %d did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b))
456 }
457
458
459 var b3 bytes.Buffer
460 br = bytes.NewBuffer(t1)
461 w, err = NewWriter(&b3, i)
462 if err != nil {
463 t.Fatal(err)
464 }
465 _, err = br.WriteTo(w)
466 if err != nil {
467 t.Fatal(err)
468 }
469 w.Close()
470
471 b3b := b3.Bytes()
472 if !bytes.Equal(b1b, b3b) {
473 t.Errorf("level %d (io.WriterTo) did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b3b))
474 }
475 }
476
477
478
479 func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
480 if buf == nil {
481 buf = make([]byte, 32*1024)
482 }
483 for {
484 nr, er := src.Read(buf)
485 if nr > 0 {
486 nw, ew := dst.Write(buf[0:nr])
487 if nw > 0 {
488 written += int64(nw)
489 }
490 if ew != nil {
491 err = ew
492 break
493 }
494 if nr != nw {
495 err = io.ErrShortWrite
496 break
497 }
498 }
499 if er == io.EOF {
500 break
501 }
502 if er != nil {
503 err = er
504 break
505 }
506 }
507 return written, err
508 }
509
510 func BenchmarkCompressAllocations(b *testing.B) {
511 payload := []byte(strings.Repeat("Tiny payload", 20))
512 for j := -2; j <= 9; j++ {
513 b.Run("level("+strconv.Itoa(j)+")", func(b *testing.B) {
514 b.Run("flate", func(b *testing.B) {
515 b.ReportAllocs()
516
517 for i := 0; i < b.N; i++ {
518 w, err := NewWriter(io.Discard, j)
519 if err != nil {
520 b.Fatal(err)
521 }
522 w.Write(payload)
523 w.Close()
524 }
525 })
526 })
527 }
528 }
529
530 func BenchmarkCompressAllocationsSingle(b *testing.B) {
531 payload := []byte(strings.Repeat("Tiny payload", 20))
532 const level = 2
533 b.Run("flate", func(b *testing.B) {
534 b.ReportAllocs()
535
536 for i := 0; i < b.N; i++ {
537 w, err := NewWriter(io.Discard, level)
538 if err != nil {
539 b.Fatal(err)
540 }
541 w.Write(payload)
542 w.Close()
543 }
544 })
545 }
546
View as plain text