1 package zstd
2
3 import (
4 "bytes"
5 "io"
6 "os"
7 "testing"
8
9 snappy "github.com/klauspost/compress/internal/snapref"
10 )
11
12 func TestSnappy_ConvertSimple(t *testing.T) {
13 in, err := os.ReadFile("testdata/z000028")
14 if err != nil {
15 t.Fatal(err)
16 }
17
18 var comp bytes.Buffer
19 w := snappy.NewBufferedWriter(&comp)
20 _, err = io.Copy(w, bytes.NewBuffer(in))
21 if err != nil {
22 t.Fatal(err)
23 }
24 err = w.Close()
25 if err != nil {
26 t.Fatal(err)
27 }
28 snapLen := comp.Len()
29 s := SnappyConverter{}
30 var dst bytes.Buffer
31 n, err := s.Convert(&comp, &dst)
32 if err != io.EOF {
33 t.Fatal(err)
34 }
35 if n != int64(dst.Len()) {
36 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
37 }
38 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
39
40 dec, err := NewReader(nil)
41 if err != nil {
42 t.Fatal(err)
43 }
44 defer dec.Close()
45 decoded, err := dec.DecodeAll(dst.Bytes(), nil)
46 if err != nil {
47 t.Error(err, len(decoded))
48 }
49 if !bytes.Equal(decoded, in) {
50 os.WriteFile("testdata/z000028.got", decoded, os.ModePerm)
51 t.Fatal("Decoded does not match")
52 }
53 t.Log("Encoded content matched")
54 }
55
56 func TestSnappy_ConvertXML(t *testing.T) {
57 f, err := os.Open("testdata/xml.zst")
58 if err != nil {
59 t.Fatal(err)
60 }
61 defer f.Close()
62
63 dec, err := NewReader(f)
64 if err != nil {
65 t.Fatal(err)
66 }
67 defer dec.Close()
68 in, err := io.ReadAll(dec)
69 if err != nil {
70 t.Fatal(err)
71 }
72 var comp bytes.Buffer
73 w := snappy.NewBufferedWriter(&comp)
74 _, err = io.Copy(w, bytes.NewBuffer(in))
75 if err != nil {
76 t.Fatal(err)
77 }
78 err = w.Close()
79 if err != nil {
80 t.Fatal(err)
81 }
82 snapLen := comp.Len()
83 s := SnappyConverter{}
84 var dst bytes.Buffer
85 n, err := s.Convert(&comp, &dst)
86 if err != io.EOF {
87 t.Fatal(err)
88 }
89 if n != int64(dst.Len()) {
90 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
91 }
92 t.Log("Snappy len", snapLen, "-> zstd len", dst.Len())
93
94 decoded, err := dec.DecodeAll(dst.Bytes(), nil)
95 if err != nil {
96 t.Error(err, len(decoded))
97 }
98 if !bytes.Equal(decoded, in) {
99 os.WriteFile("testdata/xml.got", decoded, os.ModePerm)
100 t.Fatal("Decoded does not match")
101 }
102 t.Log("Encoded content matched")
103 }
104
105 func TestSnappy_ConvertSilesia(t *testing.T) {
106 if testing.Short() {
107 t.SkipNow()
108 }
109 in, err := os.ReadFile("testdata/silesia.tar")
110 if err != nil {
111 if os.IsNotExist(err) {
112 t.Skip("Missing testdata/silesia.tar")
113 return
114 }
115 t.Fatal(err)
116 }
117
118 var comp bytes.Buffer
119 w := snappy.NewBufferedWriter(&comp)
120 _, err = io.Copy(w, bytes.NewBuffer(in))
121 if err != nil {
122 t.Fatal(err)
123 }
124 err = w.Close()
125 if err != nil {
126 t.Fatal(err)
127 }
128 snapLen := comp.Len()
129 s := SnappyConverter{}
130 var dst bytes.Buffer
131 n, err := s.Convert(&comp, &dst)
132 if err != io.EOF {
133 t.Fatal(err)
134 }
135 if n != int64(dst.Len()) {
136 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
137 }
138 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
139
140 dec, err := NewReader(nil)
141 if err != nil {
142 t.Fatal(err)
143 }
144 defer dec.Close()
145 decoded, err := dec.DecodeAll(dst.Bytes(), nil)
146 if err != nil {
147 t.Error(err, len(decoded))
148 }
149 if !bytes.Equal(decoded, in) {
150 os.WriteFile("testdata/silesia.tar.got", decoded, os.ModePerm)
151 t.Fatal("Decoded does not match")
152 }
153 t.Log("Encoded content matched")
154 }
155
156 func TestSnappy_ConvertEnwik9(t *testing.T) {
157 if testing.Short() {
158 t.SkipNow()
159 }
160 file := "testdata/enwik9.zst"
161 f, err := os.Open(file)
162 if err != nil {
163 if os.IsNotExist(err) {
164 t.Skip("To run extended tests, download http://mattmahoney.net/dc/enwik9.zip unzip it \n" +
165 "compress it with 'zstd -15 -T0 enwik9' and place it in " + file)
166 }
167 }
168 defer f.Close()
169
170 dec, err := NewReader(f)
171 if err != nil {
172 t.Fatal(err)
173 }
174 defer dec.Close()
175 in, err := io.ReadAll(dec)
176 if err != nil {
177 t.Fatal(err)
178 }
179
180 var comp bytes.Buffer
181 w := snappy.NewBufferedWriter(&comp)
182 _, err = io.Copy(w, bytes.NewBuffer(in))
183 if err != nil {
184 t.Fatal(err)
185 }
186 err = w.Close()
187 if err != nil {
188 t.Fatal(err)
189 }
190 snapLen := comp.Len()
191 s := SnappyConverter{}
192 var dst bytes.Buffer
193 n, err := s.Convert(&comp, &dst)
194 if err != io.EOF {
195 t.Fatal(err)
196 }
197 if n != int64(dst.Len()) {
198 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
199 }
200 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
201
202 decoded, err := dec.DecodeAll(dst.Bytes(), nil)
203 if err != nil {
204 t.Error(err, len(decoded))
205 }
206 if !bytes.Equal(decoded, in) {
207 os.WriteFile("testdata/enwik9.got", decoded, os.ModePerm)
208 t.Fatal("Decoded does not match")
209 }
210 t.Log("Encoded content matched")
211 }
212
213 func BenchmarkSnappy_ConvertXML(b *testing.B) {
214 f, err := os.Open("testdata/xml.zst")
215 if err != nil {
216 b.Fatal(err)
217 }
218 defer f.Close()
219
220 dec, err := NewReader(f)
221 if err != nil {
222 b.Fatal(err)
223 }
224 defer dec.Close()
225 in, err := io.ReadAll(dec)
226 if err != nil {
227 b.Fatal(err)
228 }
229
230 var comp bytes.Buffer
231 w := snappy.NewBufferedWriter(&comp)
232 _, err = io.Copy(w, bytes.NewBuffer(in))
233 if err != nil {
234 b.Fatal(err)
235 }
236 err = w.Close()
237 if err != nil {
238 b.Fatal(err)
239 }
240 s := SnappyConverter{}
241 compBytes := comp.Bytes()
242 _, err = s.Convert(&comp, io.Discard)
243 if err != io.EOF {
244 b.Fatal(err)
245 }
246 b.ResetTimer()
247 b.ReportAllocs()
248 b.SetBytes(int64(len(in)))
249 for i := 0; i < b.N; i++ {
250 _, err := s.Convert(bytes.NewBuffer(compBytes), io.Discard)
251 if err != io.EOF {
252 b.Fatal(err)
253 }
254 }
255 }
256
257 func BenchmarkSnappy_Enwik9(b *testing.B) {
258 f, err := os.Open("testdata/enwik9.zst")
259 if err != nil {
260 b.Skip(err)
261 return
262 }
263 defer f.Close()
264
265 dec, err := NewReader(f)
266 if err != nil {
267 b.Fatal(err)
268 }
269 defer dec.Close()
270 in, err := io.ReadAll(dec)
271 if err != nil {
272 b.Fatal(err)
273 }
274 defer dec.Close()
275
276 var comp bytes.Buffer
277 w := snappy.NewBufferedWriter(&comp)
278 _, err = io.Copy(w, bytes.NewBuffer(in))
279 if err != nil {
280 b.Fatal(err)
281 }
282 err = w.Close()
283 if err != nil {
284 b.Fatal(err)
285 }
286 s := SnappyConverter{}
287 compBytes := comp.Bytes()
288 _, err = s.Convert(&comp, io.Discard)
289 if err != io.EOF {
290 b.Fatal(err)
291 }
292 b.ResetTimer()
293 b.ReportAllocs()
294 b.SetBytes(int64(len(in)))
295 for i := 0; i < b.N; i++ {
296 _, err := s.Convert(bytes.NewBuffer(compBytes), io.Discard)
297 if err != io.EOF {
298 b.Fatal(err)
299 }
300 }
301 }
302
303 func BenchmarkSnappy_ConvertSilesia(b *testing.B) {
304 in, err := os.ReadFile("testdata/silesia.tar")
305 if err != nil {
306 if os.IsNotExist(err) {
307 b.Skip("Missing testdata/silesia.tar")
308 return
309 }
310 b.Fatal(err)
311 }
312
313 var comp bytes.Buffer
314 w := snappy.NewBufferedWriter(&comp)
315 _, err = io.Copy(w, bytes.NewBuffer(in))
316 if err != nil {
317 b.Fatal(err)
318 }
319 err = w.Close()
320 if err != nil {
321 b.Fatal(err)
322 }
323 s := SnappyConverter{}
324 compBytes := comp.Bytes()
325 _, err = s.Convert(&comp, io.Discard)
326 if err != io.EOF {
327 b.Fatal(err)
328 }
329 b.ResetTimer()
330 b.ReportAllocs()
331 b.SetBytes(int64(len(in)))
332 for i := 0; i < b.N; i++ {
333 _, err := s.Convert(bytes.NewBuffer(compBytes), io.Discard)
334 if err != io.EOF {
335 b.Fatal(err)
336 }
337 }
338 }
339
View as plain text