1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 package packed
30
31 import (
32 "bytes"
33 "fmt"
34 math_rand "math/rand"
35 "runtime"
36 "testing"
37 "time"
38 "unsafe"
39
40 "github.com/gogo/protobuf/proto"
41 )
42
43 func BenchmarkVarintIssue436withCount(b *testing.B) {
44 var arraySizes = []struct {
45 name string
46 value int64
47 }{
48 {"2^0_ints", 1 << 0},
49 {"2^1_ints", 1 << 1},
50 {"2^2_ints", 1 << 2},
51 {"2^3_ints", 1 << 3},
52 {"2^4_ints", 1 << 4},
53 {"2^8_ints", 1 << 8},
54 {"2^16_ints", 1 << 16},
55 {"2^20_ints", 1 << 20},
56 {"2^24_ints", 1 << 24},
57 }
58
59 var varintSizes = []struct {
60 name string
61 value int64
62 }{
63 {"max_int 2^7-4", 1<<7 - 4},
64 {"max_int 2^15-4", 1<<15 - 4},
65 {"max_int 2^31-4", 1<<31 - 4},
66 {"max_int 2^63-4", 1<<63 - 4},
67 }
68
69 for _, arraySize := range arraySizes {
70 for _, varintSize := range varintSizes {
71 seed := time.Now().UnixNano()
72 rng := math_rand.New(math_rand.NewSource(seed))
73 buf := make([]int64, arraySize.value)
74 for j := range buf {
75 buf[j] = rng.Int63n(varintSize.value)
76 }
77
78 b.Run(arraySize.name+", "+varintSize.name, func(b *testing.B) {
79 msg := &NinRepNative{
80 Field8: buf,
81 }
82
83 data, err := proto.Marshal(msg)
84 if err != nil {
85 b.Fatal(err)
86 }
87
88 normalmsg := &NinRepNative{}
89
90 for i := 0; i < b.N; i++ {
91 err = proto.Unmarshal(data, normalmsg)
92 if err != nil {
93 b.Fatal(err)
94 }
95 }
96 })
97 }
98 }
99 }
100
101 func TestVarintIssue436(t *testing.T) {
102 n := 1 << 22
103
104 m := &runtime.MemStats{}
105
106 msgNormal := &NinRepNative{
107 Field8: make([]int64, n),
108 }
109 dataNormal, err := proto.Marshal(msgNormal)
110 if err != nil {
111 t.Fatal(err)
112 }
113
114 normalmsg := &NinRepNative{}
115 runtime.ReadMemStats(m)
116 beforeNormal := m.TotalAlloc
117 err = proto.Unmarshal(dataNormal, normalmsg)
118 runtime.ReadMemStats(m)
119 afterNormal := m.TotalAlloc
120 if err != nil {
121 t.Fatal(err)
122 }
123
124 msgPacked := &NinRepPackedNative{
125 Field8: make([]int64, n),
126 }
127 dataPacked, err := proto.Marshal(msgPacked)
128 if err != nil {
129 t.Fatal(err)
130 }
131
132 packedmsg := &NinRepPackedNative{}
133 runtime.ReadMemStats(m)
134 beforePacked := m.TotalAlloc
135 err = proto.Unmarshal(dataPacked, packedmsg)
136 runtime.ReadMemStats(m)
137 afterPacked := m.TotalAlloc
138 if err != nil {
139 t.Fatal(err)
140 }
141
142 totalNormal := afterNormal - beforeNormal
143 totalPacked := afterPacked - beforePacked
144 usedRatio := float64(totalPacked) / float64(totalNormal)
145 if usedRatio > 0.5 {
146 t.Fatalf("unmarshaling packed msg allocated too much memory:\nnormal:\t\t%d bytes\npacked:\t\t%d bytes\nused ratio:\t%.2f%%", totalNormal, totalPacked, usedRatio*100)
147 }
148 }
149
150 func TestIssue436(t *testing.T) {
151 n := 1 << 22
152
153 m := &runtime.MemStats{}
154
155 msgNormal := &NinRepNative{
156 Field1: make([]float64, n),
157 }
158 dataNormal, err := proto.Marshal(msgNormal)
159 if err != nil {
160 t.Fatal(err)
161 }
162
163 normalmsg := &NinRepNative{}
164 runtime.ReadMemStats(m)
165 beforeNormal := m.TotalAlloc
166 err = proto.Unmarshal(dataNormal, normalmsg)
167 runtime.ReadMemStats(m)
168 afterNormal := m.TotalAlloc
169 if err != nil {
170 t.Fatal(err)
171 }
172
173 msgPacked := &NinRepPackedNative{
174 Field1: make([]float64, n),
175 }
176 dataPacked, err := proto.Marshal(msgPacked)
177 if err != nil {
178 t.Fatal(err)
179 }
180
181 packedmsg := &NinRepPackedNative{}
182 runtime.ReadMemStats(m)
183 beforePacked := m.TotalAlloc
184 err = proto.Unmarshal(dataPacked, packedmsg)
185 runtime.ReadMemStats(m)
186 afterPacked := m.TotalAlloc
187 if err != nil {
188 t.Fatal(err)
189 }
190
191 totalNormal := afterNormal - beforeNormal
192 totalPacked := afterPacked - beforePacked
193 usedRatio := float64(totalPacked) / float64(totalNormal)
194 if usedRatio > 0.5 {
195 t.Fatalf("unmarshaling packed msg allocated too much memory:\nnormal:\t\t%d bytes\npacked:\t\t%d bytes\nused ratio:\t%.2f%%", totalNormal, totalPacked, usedRatio*100)
196 }
197 }
198
199
203 func TestTestPackedPreallocation(t *testing.T) {
204 n1 := 900
205 n2 := 700
206
207 msgPacked := &NinRepPackedNative{
208 Field3: make([]int32, n1),
209 Field4: make([]int64, n2),
210 }
211
212 dataPacked, err := proto.Marshal(msgPacked)
213 if err != nil {
214 t.Fatal(err)
215 }
216
217 packedmsg := &NinRepPackedNative{}
218 err = proto.Unmarshal(dataPacked, packedmsg)
219 if err != nil {
220 t.Fatal(err)
221 }
222
223 if v := len(packedmsg.Field3); v != n1 {
224 t.Errorf("Field3 incorrect len: %v != %v", v, n1)
225 }
226
227 if v := len(packedmsg.Field4); v != n2 {
228 t.Errorf("Field4 incorrect len: %v != %v", v, n2)
229 }
230
231 if v := cap(packedmsg.Field3); v != n1 {
232 t.Errorf("Field3 incorrect cap: %v != %v", v, n1)
233 }
234
235 if v := cap(packedmsg.Field4); v != n2 {
236 t.Errorf("Field4 incorrect cap: %v != %v", v, n2)
237 }
238 }
239
240
245 func TestSafeIssue21(t *testing.T) {
246 popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
247 msg1 := NewPopulatedNinRepNative(popr, true)
248 data1, err := proto.Marshal(msg1)
249 if err != nil {
250 t.Fatal(err)
251 }
252 packedmsg := &NinRepPackedNative{}
253 err = proto.Unmarshal(data1, packedmsg)
254 if err != nil {
255 t.Fatal(err)
256 }
257 if len(packedmsg.XXX_unrecognized) != 0 {
258 t.Fatalf("packed msg unmarshaled unrecognized fields, even though there aren't any")
259 }
260 if err := VerboseEqual(msg1, packedmsg); err != nil {
261 t.Fatalf("%v", err)
262 }
263 }
264
265 func TestUnsafeIssue21(t *testing.T) {
266 var bigendian uint32 = 0x01020304
267 if *(*byte)(unsafe.Pointer(&bigendian)) == 1 {
268 t.Skip("unsafe does not work on big endian architectures")
269 }
270 popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
271 msg1 := NewPopulatedNinRepNativeUnsafe(popr, true)
272 data1, err := proto.Marshal(msg1)
273 if err != nil {
274 t.Fatal(err)
275 }
276 packedmsg := &NinRepPackedNativeUnsafe{}
277 err = proto.Unmarshal(data1, packedmsg)
278 if err != nil {
279 t.Fatal(err)
280 }
281 if len(packedmsg.XXX_unrecognized) != 0 {
282 t.Fatalf("packed msg unmarshaled unrecognized fields, even though there aren't any")
283 }
284 if err := VerboseEqualUnsafe(msg1, packedmsg); err != nil {
285 t.Fatalf("%v", err)
286 }
287 }
288
289 func VerboseEqual(this *NinRepNative, that *NinRepPackedNative) error {
290 if that == nil {
291 if this == nil {
292 return nil
293 }
294 return fmt.Errorf("that == nil && this != nil")
295 } else if this == nil {
296 return fmt.Errorf("that != nil && this == nil")
297 }
298
299 if len(this.Field1) != len(that.Field1) {
300 return fmt.Errorf("Field1 this(%v) Not Equal that(%v)", len(this.Field1), len(that.Field1))
301 }
302 for i := range this.Field1 {
303 if this.Field1[i] != that.Field1[i] {
304 return fmt.Errorf("Field1 this[%v](%v) Not Equal that[%v](%v)", i, this.Field1[i], i, that.Field1[i])
305 }
306 }
307 if len(this.Field2) != len(that.Field2) {
308 return fmt.Errorf("Field2 this(%v) Not Equal that(%v)", len(this.Field2), len(that.Field2))
309 }
310 for i := range this.Field2 {
311 if this.Field2[i] != that.Field2[i] {
312 return fmt.Errorf("Field2 this[%v](%v) Not Equal that[%v](%v)", i, this.Field2[i], i, that.Field2[i])
313 }
314 }
315 if len(this.Field3) != len(that.Field3) {
316 return fmt.Errorf("Field3 this(%v) Not Equal that(%v)", len(this.Field3), len(that.Field3))
317 }
318 for i := range this.Field3 {
319 if this.Field3[i] != that.Field3[i] {
320 return fmt.Errorf("Field3 this[%v](%v) Not Equal that[%v](%v)", i, this.Field3[i], i, that.Field3[i])
321 }
322 }
323 if len(this.Field4) != len(that.Field4) {
324 return fmt.Errorf("Field4 this(%v) Not Equal that(%v)", len(this.Field4), len(that.Field4))
325 }
326 for i := range this.Field4 {
327 if this.Field4[i] != that.Field4[i] {
328 return fmt.Errorf("Field4 this[%v](%v) Not Equal that[%v](%v)", i, this.Field4[i], i, that.Field4[i])
329 }
330 }
331 if len(this.Field5) != len(that.Field5) {
332 return fmt.Errorf("Field5 this(%v) Not Equal that(%v)", len(this.Field5), len(that.Field5))
333 }
334 for i := range this.Field5 {
335 if this.Field5[i] != that.Field5[i] {
336 return fmt.Errorf("Field5 this[%v](%v) Not Equal that[%v](%v)", i, this.Field5[i], i, that.Field5[i])
337 }
338 }
339 if len(this.Field6) != len(that.Field6) {
340 return fmt.Errorf("Field6 this(%v) Not Equal that(%v)", len(this.Field6), len(that.Field6))
341 }
342 for i := range this.Field6 {
343 if this.Field6[i] != that.Field6[i] {
344 return fmt.Errorf("Field6 this[%v](%v) Not Equal that[%v](%v)", i, this.Field6[i], i, that.Field6[i])
345 }
346 }
347 if len(this.Field7) != len(that.Field7) {
348 return fmt.Errorf("Field7 this(%v) Not Equal that(%v)", len(this.Field7), len(that.Field7))
349 }
350 for i := range this.Field7 {
351 if this.Field7[i] != that.Field7[i] {
352 return fmt.Errorf("Field7 this[%v](%v) Not Equal that[%v](%v)", i, this.Field7[i], i, that.Field7[i])
353 }
354 }
355 if len(this.Field8) != len(that.Field8) {
356 return fmt.Errorf("Field8 this(%v) Not Equal that(%v)", len(this.Field8), len(that.Field8))
357 }
358 for i := range this.Field8 {
359 if this.Field8[i] != that.Field8[i] {
360 return fmt.Errorf("Field8 this[%v](%v) Not Equal that[%v](%v)", i, this.Field8[i], i, that.Field8[i])
361 }
362 }
363 if len(this.Field9) != len(that.Field9) {
364 return fmt.Errorf("Field9 this(%v) Not Equal that(%v)", len(this.Field9), len(that.Field9))
365 }
366 for i := range this.Field9 {
367 if this.Field9[i] != that.Field9[i] {
368 return fmt.Errorf("Field9 this[%v](%v) Not Equal that[%v](%v)", i, this.Field9[i], i, that.Field9[i])
369 }
370 }
371 if len(this.Field10) != len(that.Field10) {
372 return fmt.Errorf("Field10 this(%v) Not Equal that(%v)", len(this.Field10), len(that.Field10))
373 }
374 for i := range this.Field10 {
375 if this.Field10[i] != that.Field10[i] {
376 return fmt.Errorf("Field10 this[%v](%v) Not Equal that[%v](%v)", i, this.Field10[i], i, that.Field10[i])
377 }
378 }
379 if len(this.Field11) != len(that.Field11) {
380 return fmt.Errorf("Field11 this(%v) Not Equal that(%v)", len(this.Field11), len(that.Field11))
381 }
382 for i := range this.Field11 {
383 if this.Field11[i] != that.Field11[i] {
384 return fmt.Errorf("Field11 this[%v](%v) Not Equal that[%v](%v)", i, this.Field11[i], i, that.Field11[i])
385 }
386 }
387 if len(this.Field12) != len(that.Field12) {
388 return fmt.Errorf("Field12 this(%v) Not Equal that(%v)", len(this.Field12), len(that.Field12))
389 }
390 for i := range this.Field12 {
391 if this.Field12[i] != that.Field12[i] {
392 return fmt.Errorf("Field12 this[%v](%v) Not Equal that[%v](%v)", i, this.Field12[i], i, that.Field12[i])
393 }
394 }
395 if len(this.Field13) != len(that.Field13) {
396 return fmt.Errorf("Field13 this(%v) Not Equal that(%v)", len(this.Field13), len(that.Field13))
397 }
398 for i := range this.Field13 {
399 if this.Field13[i] != that.Field13[i] {
400 return fmt.Errorf("Field13 this[%v](%v) Not Equal that[%v](%v)", i, this.Field13[i], i, that.Field13[i])
401 }
402 }
403 if !bytes.Equal(this.XXX_unrecognized, that.XXX_unrecognized) {
404 return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that.XXX_unrecognized)
405 }
406 return nil
407 }
408
409 func VerboseEqualUnsafe(this *NinRepNativeUnsafe, that *NinRepPackedNativeUnsafe) error {
410 if that == nil {
411 if this == nil {
412 return nil
413 }
414 return fmt.Errorf("that == nil && this != nil")
415 } else if this == nil {
416 return fmt.Errorf("that != nil && this == nil")
417 }
418
419 if len(this.Field1) != len(that.Field1) {
420 return fmt.Errorf("Field1 this(%v) Not Equal that(%v)", len(this.Field1), len(that.Field1))
421 }
422 for i := range this.Field1 {
423 if this.Field1[i] != that.Field1[i] {
424 return fmt.Errorf("Field1 this[%v](%v) Not Equal that[%v](%v)", i, this.Field1[i], i, that.Field1[i])
425 }
426 }
427 if len(this.Field2) != len(that.Field2) {
428 return fmt.Errorf("Field2 this(%v) Not Equal that(%v)", len(this.Field2), len(that.Field2))
429 }
430 for i := range this.Field2 {
431 if this.Field2[i] != that.Field2[i] {
432 return fmt.Errorf("Field2 this[%v](%v) Not Equal that[%v](%v)", i, this.Field2[i], i, that.Field2[i])
433 }
434 }
435 if len(this.Field3) != len(that.Field3) {
436 return fmt.Errorf("Field3 this(%v) Not Equal that(%v)", len(this.Field3), len(that.Field3))
437 }
438 for i := range this.Field3 {
439 if this.Field3[i] != that.Field3[i] {
440 return fmt.Errorf("Field3 this[%v](%v) Not Equal that[%v](%v)", i, this.Field3[i], i, that.Field3[i])
441 }
442 }
443 if len(this.Field4) != len(that.Field4) {
444 return fmt.Errorf("Field4 this(%v) Not Equal that(%v)", len(this.Field4), len(that.Field4))
445 }
446 for i := range this.Field4 {
447 if this.Field4[i] != that.Field4[i] {
448 return fmt.Errorf("Field4 this[%v](%v) Not Equal that[%v](%v)", i, this.Field4[i], i, that.Field4[i])
449 }
450 }
451 if len(this.Field5) != len(that.Field5) {
452 return fmt.Errorf("Field5 this(%v) Not Equal that(%v)", len(this.Field5), len(that.Field5))
453 }
454 for i := range this.Field5 {
455 if this.Field5[i] != that.Field5[i] {
456 return fmt.Errorf("Field5 this[%v](%v) Not Equal that[%v](%v)", i, this.Field5[i], i, that.Field5[i])
457 }
458 }
459 if len(this.Field6) != len(that.Field6) {
460 return fmt.Errorf("Field6 this(%v) Not Equal that(%v)", len(this.Field6), len(that.Field6))
461 }
462 for i := range this.Field6 {
463 if this.Field6[i] != that.Field6[i] {
464 return fmt.Errorf("Field6 this[%v](%v) Not Equal that[%v](%v)", i, this.Field6[i], i, that.Field6[i])
465 }
466 }
467 if len(this.Field7) != len(that.Field7) {
468 return fmt.Errorf("Field7 this(%v) Not Equal that(%v)", len(this.Field7), len(that.Field7))
469 }
470 for i := range this.Field7 {
471 if this.Field7[i] != that.Field7[i] {
472 return fmt.Errorf("Field7 this[%v](%v) Not Equal that[%v](%v)", i, this.Field7[i], i, that.Field7[i])
473 }
474 }
475 if len(this.Field8) != len(that.Field8) {
476 return fmt.Errorf("Field8 this(%v) Not Equal that(%v)", len(this.Field8), len(that.Field8))
477 }
478 for i := range this.Field8 {
479 if this.Field8[i] != that.Field8[i] {
480 return fmt.Errorf("Field8 this[%v](%v) Not Equal that[%v](%v)", i, this.Field8[i], i, that.Field8[i])
481 }
482 }
483 if len(this.Field9) != len(that.Field9) {
484 return fmt.Errorf("Field9 this(%v) Not Equal that(%v)", len(this.Field9), len(that.Field9))
485 }
486 for i := range this.Field9 {
487 if this.Field9[i] != that.Field9[i] {
488 return fmt.Errorf("Field9 this[%v](%v) Not Equal that[%v](%v)", i, this.Field9[i], i, that.Field9[i])
489 }
490 }
491 if len(this.Field10) != len(that.Field10) {
492 return fmt.Errorf("Field10 this(%v) Not Equal that(%v)", len(this.Field10), len(that.Field10))
493 }
494 for i := range this.Field10 {
495 if this.Field10[i] != that.Field10[i] {
496 return fmt.Errorf("Field10 this[%v](%v) Not Equal that[%v](%v)", i, this.Field10[i], i, that.Field10[i])
497 }
498 }
499 if len(this.Field11) != len(that.Field11) {
500 return fmt.Errorf("Field11 this(%v) Not Equal that(%v)", len(this.Field11), len(that.Field11))
501 }
502 for i := range this.Field11 {
503 if this.Field11[i] != that.Field11[i] {
504 return fmt.Errorf("Field11 this[%v](%v) Not Equal that[%v](%v)", i, this.Field11[i], i, that.Field11[i])
505 }
506 }
507 if len(this.Field12) != len(that.Field12) {
508 return fmt.Errorf("Field12 this(%v) Not Equal that(%v)", len(this.Field12), len(that.Field12))
509 }
510 for i := range this.Field12 {
511 if this.Field12[i] != that.Field12[i] {
512 return fmt.Errorf("Field12 this[%v](%v) Not Equal that[%v](%v)", i, this.Field12[i], i, that.Field12[i])
513 }
514 }
515 if len(this.Field13) != len(that.Field13) {
516 return fmt.Errorf("Field13 this(%v) Not Equal that(%v)", len(this.Field13), len(that.Field13))
517 }
518 for i := range this.Field13 {
519 if this.Field13[i] != that.Field13[i] {
520 return fmt.Errorf("Field13 this[%v](%v) Not Equal that[%v](%v)", i, this.Field13[i], i, that.Field13[i])
521 }
522 }
523 if !bytes.Equal(this.XXX_unrecognized, that.XXX_unrecognized) {
524 return fmt.Errorf("XXX_unrecognized this(%v) Not Equal that(%v)", this.XXX_unrecognized, that.XXX_unrecognized)
525 }
526 return nil
527 }
528
View as plain text