1 package xerial
2
3 import (
4 "bytes"
5 "math/rand"
6 "testing"
7
8 "github.com/klauspost/compress/s2"
9 )
10
11 const largeString = `Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias except`
12
13 var snappyStreamTestCases = map[string][]byte{
14 "PLAINDATA": {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 11, 9, 32, 80, 76, 65, 73, 78, 68, 65, 84, 65},
15 `{"a":"UtaitILHMDAAAAfU","b":"日本"}`: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 39, 37, 144, 123, 34, 97, 34, 58, 34, 85, 116, 97, 105, 116, 73, 76, 72, 77, 68, 65, 65, 65, 65, 102, 85, 34, 44, 34, 98, 34, 58, 34, 230, 151, 165, 230, 156, 172, 34, 125},
16 largeString: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 89, 128, 8, 240, 90, 83, 101, 100, 32, 117, 116, 32, 112, 101, 114, 115, 112, 105, 99, 105, 97, 116, 105, 115, 32, 117, 110, 100, 101, 32, 111, 109, 110, 105, 115, 32, 105, 115, 116, 101, 32, 110, 97, 116, 117, 115, 32, 101, 114, 114, 111, 114, 32, 115, 105, 116, 32, 118, 111, 108, 117, 112, 116, 97, 116, 101, 109, 32, 97, 99, 99, 117, 115, 97, 110, 116, 105, 117, 109, 32, 100, 111, 108, 111, 114, 101, 109, 113, 117, 101, 32, 108, 97, 117, 100, 97, 5, 22, 240, 60, 44, 32, 116, 111, 116, 97, 109, 32, 114, 101, 109, 32, 97, 112, 101, 114, 105, 97, 109, 44, 32, 101, 97, 113, 117, 101, 32, 105, 112, 115, 97, 32, 113, 117, 97, 101, 32, 97, 98, 32, 105, 108, 108, 111, 32, 105, 110, 118, 101, 110, 116, 111, 114, 101, 32, 118, 101, 114, 105, 116, 97, 1, 141, 4, 101, 116, 1, 36, 88, 115, 105, 32, 97, 114, 99, 104, 105, 116, 101, 99, 116, 111, 32, 98, 101, 97, 116, 97, 101, 32, 118, 105, 1, 6, 120, 100, 105, 99, 116, 97, 32, 115, 117, 110, 116, 32, 101, 120, 112, 108, 105, 99, 97, 98, 111, 46, 32, 78, 101, 109, 111, 32, 101, 110, 105, 109, 5, 103, 0, 109, 46, 180, 0, 12, 113, 117, 105, 97, 17, 16, 0, 115, 5, 209, 72, 97, 115, 112, 101, 114, 110, 97, 116, 117, 114, 32, 97, 117, 116, 32, 111, 100, 105, 116, 5, 9, 36, 102, 117, 103, 105, 116, 44, 32, 115, 101, 100, 9, 53, 32, 99, 111, 110, 115, 101, 113, 117, 117, 110, 1, 42, 20, 109, 97, 103, 110, 105, 32, 9, 245, 16, 115, 32, 101, 111, 115, 1, 36, 28, 32, 114, 97, 116, 105, 111, 110, 101, 17, 96, 33, 36, 1, 51, 36, 105, 32, 110, 101, 115, 99, 105, 117, 110, 116, 1, 155, 1, 254, 16, 112, 111, 114, 114, 111, 1, 51, 36, 115, 113, 117, 97, 109, 32, 101, 115, 116, 44, 1, 14, 13, 81, 5, 183, 4, 117, 109, 1, 18, 0, 97, 9, 19, 4, 32, 115, 1, 149, 12, 109, 101, 116, 44, 9, 135, 76, 99, 116, 101, 116, 117, 114, 44, 32, 97, 100, 105, 112, 105, 115, 99, 105, 32, 118, 101, 108, 50, 173, 0, 24, 110, 111, 110, 32, 110, 117, 109, 9, 94, 84, 105, 117, 115, 32, 109, 111, 100, 105, 32, 116, 101, 109, 112, 111, 114, 97, 32, 105, 110, 99, 105, 100, 33, 52, 20, 117, 116, 32, 108, 97, 98, 33, 116, 4, 101, 116, 9, 106, 0, 101, 5, 219, 20, 97, 109, 32, 97, 108, 105, 5, 62, 33, 164, 8, 114, 97, 116, 29, 212, 12, 46, 32, 85, 116, 41, 94, 52, 97, 100, 32, 109, 105, 110, 105, 109, 97, 32, 118, 101, 110, 105, 33, 221, 72, 113, 117, 105, 115, 32, 110, 111, 115, 116, 114, 117, 109, 32, 101, 120, 101, 114, 99, 105, 33, 202, 104, 111, 110, 101, 109, 32, 117, 108, 108, 97, 109, 32, 99, 111, 114, 112, 111, 114, 105, 115, 32, 115, 117, 115, 99, 105, 112, 105, 13, 130, 8, 105, 111, 115, 1, 64, 12, 110, 105, 115, 105, 1, 150, 5, 126, 44, 105, 100, 32, 101, 120, 32, 101, 97, 32, 99, 111, 109, 5, 192, 0, 99, 41, 131, 33, 172, 8, 63, 32, 81, 1, 107, 4, 97, 117, 33, 101, 96, 118, 101, 108, 32, 101, 117, 109, 32, 105, 117, 114, 101, 32, 114, 101, 112, 114, 101, 104, 101, 110, 100, 101, 114, 105, 65, 63, 12, 105, 32, 105, 110, 1, 69, 16, 118, 111, 108, 117, 112, 65, 185, 1, 47, 24, 105, 116, 32, 101, 115, 115, 101, 1, 222, 64, 109, 32, 110, 105, 104, 105, 108, 32, 109, 111, 108, 101, 115, 116, 105, 97, 101, 46, 103, 0, 0, 44, 1, 45, 16, 32, 105, 108, 108, 117, 37, 143, 45, 36, 0, 109, 5, 110, 65, 33, 20, 97, 116, 32, 113, 117, 111, 17, 92, 44, 115, 32, 110, 117, 108, 108, 97, 32, 112, 97, 114, 105, 9, 165, 24, 65, 116, 32, 118, 101, 114, 111, 69, 34, 44, 101, 116, 32, 97, 99, 99, 117, 115, 97, 109, 117, 115, 1, 13, 104, 105, 117, 115, 116, 111, 32, 111, 100, 105, 111, 32, 100, 105, 103, 110, 105, 115, 115, 105, 109, 111, 115, 32, 100, 117, 99, 105, 1, 34, 80, 113, 117, 105, 32, 98, 108, 97, 110, 100, 105, 116, 105, 105, 115, 32, 112, 114, 97, 101, 115, 101, 101, 87, 17, 111, 56, 116, 117, 109, 32, 100, 101, 108, 101, 110, 105, 116, 105, 32, 97, 116, 65, 89, 28, 99, 111, 114, 114, 117, 112, 116, 105, 1, 150, 0, 115, 13, 174, 5, 109, 8, 113, 117, 97, 65, 5, 52, 108, 101, 115, 116, 105, 97, 115, 32, 101, 120, 99, 101, 112, 116, 0, 0, 0, 1, 0},
17 }
18
19 func makeMassive(input string, numCopies int) string {
20 outBuff := make([]byte, len(input)*numCopies)
21
22 for i := 0; i < numCopies; i++ {
23 copy(outBuff[len(outBuff):], input)
24 }
25
26 return string(outBuff)
27 }
28
29 func TestSnappyEncodeStream(t *testing.T) {
30 for src := range snappyStreamTestCases {
31 dst := Encode(nil, []byte(src))
32
33
34 dec, err := Decode(dst)
35 if err != nil {
36 t.Error(err)
37 }
38 if src != string(dec) {
39 t.Errorf("Expected decode to match encode orig = %s, decoded = %s", src, string(dec))
40 }
41 }
42 }
43
44 func TestSnappyLargeStringEncodeStream(t *testing.T) {
45 massiveString := makeMassive(largeString, 10000)
46 dst := Encode(nil, []byte(massiveString))
47 dec, err := Decode(dst)
48 if err != nil {
49 t.Error(err)
50 }
51 if massiveString != string(dec) {
52 t.Errorf("Decoded string didn't match original input (not printing due to size)")
53 }
54 }
55
56 func TestSnappyDecodeStreams(t *testing.T) {
57 for exp, src := range snappyStreamTestCases {
58 dst, err := Decode(src)
59 if err != nil {
60 t.Error("Encoding error: ", err)
61 } else if !bytes.Equal(dst, []byte(exp)) {
62 t.Errorf("Expected %s to be generated from [%d]byte, but was %s", exp, len(src), string(dst))
63 }
64 }
65 }
66
67 func TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) {
68
69 for i := 0; i < len(xerialHeader); i++ {
70 buf := make([]byte, i)
71 copy(buf, xerialHeader[:i])
72 if _, err := Decode(buf); err != ErrMalformed {
73 t.Errorf("expected ErrMalformed got %v", err)
74 }
75 }
76 }
77
78 func TestSnappyDecodeMalformedTruncatedSize(t *testing.T) {
79
80 sizes := []int{sizeOffset + 1, sizeOffset + 2, sizeOffset + 3}
81 for _, size := range sizes {
82 buf := make([]byte, size)
83 copy(buf, xerialHeader)
84 if _, err := Decode(buf); err != ErrMalformed {
85 t.Errorf("expected ErrMalformed got %v", err)
86 }
87 }
88 }
89
90 func TestSnappyDecodeMalformedBNoData(t *testing.T) {
91
92 buf := make([]byte, 20)
93 copy(buf, xerialHeader)
94
95 buf[len(buf)-1] = 1
96 if _, err := Decode(buf); err != ErrMalformed {
97 t.Errorf("expected ErrMalformed got %v", err)
98 }
99 }
100
101 func TestSnappyMasterDecodeFailed(t *testing.T) {
102 buf := make([]byte, 21)
103 copy(buf, xerialHeader)
104
105 buf[len(buf)-2] = 1
106
107 buf[len(buf)-1] = 1
108 if _, err := Decode(buf); err == ErrMalformed || err == nil {
109 t.Errorf("unexpected err: %v", err)
110 }
111 }
112
113 func BenchmarkSnappyStreamDecode(b *testing.B) {
114 b.ReportAllocs()
115 b.ResetTimer()
116
117 for n := 0; n < b.N; n++ {
118 bytes := 0
119 for _, test := range snappyStreamTestCases {
120 dst, err := Decode(test)
121 if err != nil {
122 b.Error("Decoding error: ", err)
123 }
124 bytes += len(dst)
125 }
126 b.SetBytes(int64(bytes))
127 }
128 }
129
130 func BenchmarkSnappyStreamDecodeInto(b *testing.B) {
131 b.ReportAllocs()
132 b.ResetTimer()
133
134 var (
135 dst = make([]byte, 1024, 1024)
136 err error
137 )
138
139 for n := 0; n < b.N; n++ {
140 bytes := 0
141 for _, test := range snappyStreamTestCases {
142 dst, err = DecodeInto(dst, test)
143 if err != nil {
144 b.Error("Decoding error: ", err)
145 }
146 bytes += len(dst)
147 }
148 b.SetBytes(int64(bytes))
149 }
150 }
151 func BenchmarkSnappyStreamDecodeMassive(b *testing.B) {
152 massiveString := makeMassive(largeString, 10000)
153 enc := Encode(nil, []byte(massiveString))
154
155 b.ReportAllocs()
156 b.ResetTimer()
157 b.SetBytes(int64(len(massiveString)))
158
159 for n := 0; n < b.N; n++ {
160 _, err := Decode(enc)
161 if err != nil {
162 b.Error("Decoding error: ", err)
163 }
164 }
165 }
166
167 func BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) {
168 massiveString := makeMassive(largeString, 10000)
169 enc := Encode(nil, []byte(massiveString))
170
171 var (
172 dst = make([]byte, 1024, len(massiveString))
173 err error
174 )
175
176 b.ReportAllocs()
177 b.ResetTimer()
178 b.SetBytes(int64(len(massiveString)))
179
180 for n := 0; n < b.N; n++ {
181 dst, err = DecodeInto(dst, enc)
182 if err != nil {
183 b.Error("Decoding error: ", err)
184 }
185 }
186 }
187
188 func BenchmarkSnappyStreamEncode(b *testing.B) {
189 test := []byte(largeString)
190
191 var (
192 dst = make([]byte, 0, 20+s2.MaxEncodedLen(len(test)))
193 err error
194 )
195
196 b.ReportAllocs()
197 b.ResetTimer()
198 b.SetBytes(int64(len(test)))
199
200 for n := 0; n < b.N; n++ {
201 dst = Encode(dst[:0], test)
202 if err != nil {
203 b.Error("Encoding error: ", err)
204 }
205 }
206 }
207
208 func BenchmarkSnappyStreamEncodeBetter(b *testing.B) {
209 test := []byte(largeString)
210
211 var (
212 dst = make([]byte, 0, 20+s2.MaxEncodedLen(len(test)))
213 err error
214 )
215
216 b.ReportAllocs()
217 b.ResetTimer()
218 b.SetBytes(int64(len(test)))
219
220 for n := 0; n < b.N; n++ {
221 dst = EncodeBetter(dst[:0], test)
222 if err != nil {
223 b.Error("Encoding error: ", err)
224 }
225 }
226 }
227
228 func BenchmarkSnappyStreamEncodeMassive(b *testing.B) {
229 massiveString := []byte(makeMassive(largeString, 10000))
230
231
232 rng := rand.New(rand.NewSource(0))
233 for i := 0; i < len(massiveString)/10; i++ {
234 massiveString[rng.Intn(len(massiveString))]++
235 }
236 var (
237 dst = make([]byte, 0, s2.MaxEncodedLen(len(massiveString)))
238 err error
239 )
240
241 b.ReportAllocs()
242 b.ResetTimer()
243 b.SetBytes(int64(len(massiveString)))
244
245 for n := 0; n < b.N; n++ {
246 dst = Encode(dst[:0], massiveString)
247 if err != nil {
248 b.Error("Encoding error: ", err)
249 }
250 }
251 }
252
253 func BenchmarkSnappyStreamEncodeBetterMassive(b *testing.B) {
254 massiveString := []byte(makeMassive(largeString, 10000))
255
256
257 rng := rand.New(rand.NewSource(0))
258 for i := 0; i < len(massiveString)/10; i++ {
259 massiveString[rng.Intn(len(massiveString))]++
260 }
261 var (
262 dst = make([]byte, 0, s2.MaxEncodedLen(len(massiveString)))
263 err error
264 )
265
266 b.ReportAllocs()
267 b.ResetTimer()
268 b.SetBytes(int64(len(massiveString)))
269
270 for n := 0; n < b.N; n++ {
271 dst = EncodeBetter(dst[:0], massiveString)
272 if err != nil {
273 b.Error("Encoding error: ", err)
274 }
275 }
276 }
277
View as plain text