1
2
3
4
5 package json
6
7 import (
8 "bufio"
9 "bytes"
10 "compress/gzip"
11 "crypto/sha256"
12 "encoding/binary"
13 "encoding/hex"
14 "errors"
15 "flag"
16 "io"
17 "math"
18 "net/http"
19 "path"
20 "reflect"
21 "strconv"
22 "strings"
23 "testing"
24 "time"
25 "unicode"
26 )
27
28
29 func TestEncoder(t *testing.T) {
30 for _, td := range coderTestdata {
31 for _, formatName := range []string{"Compact", "Escaped", "Indented"} {
32 for _, typeName := range []string{"Token", "Value", "TokenDelims"} {
33 t.Run(path.Join(td.name.name, typeName, formatName), func(t *testing.T) {
34 testEncoder(t, td.name.where, formatName, typeName, td)
35 })
36 }
37 }
38 }
39 }
40 func testEncoder(t *testing.T, where pc, formatName, typeName string, td coderTestdataEntry) {
41 var want string
42 dst := new(bytes.Buffer)
43 enc := NewEncoder(dst)
44 enc.options.omitTopLevelNewline = true
45 want = td.outCompacted
46 switch formatName {
47 case "Escaped":
48 enc.options.EscapeRune = func(rune) bool { return true }
49 if td.outEscaped != "" {
50 want = td.outEscaped
51 }
52 case "Indented":
53 enc.options.multiline = true
54 enc.options.IndentPrefix = "\t"
55 enc.options.Indent = " "
56 if td.outIndented != "" {
57 want = td.outIndented
58 }
59 }
60
61 switch typeName {
62 case "Token":
63 var pointers []string
64 for _, tok := range td.tokens {
65 if err := enc.WriteToken(tok); err != nil {
66 t.Fatalf("%s: Encoder.WriteToken error: %v", where, err)
67 }
68 if td.pointers != nil {
69 pointers = append(pointers, enc.StackPointer())
70 }
71 }
72 if !reflect.DeepEqual(pointers, td.pointers) {
73 t.Fatalf("%s: pointers mismatch:\ngot %q\nwant %q", where, pointers, td.pointers)
74 }
75 case "Value":
76 if err := enc.WriteValue(RawValue(td.in)); err != nil {
77 t.Fatalf("%s: Encoder.WriteValue error: %v", where, err)
78 }
79 case "TokenDelims":
80
81 for _, tok := range td.tokens {
82 switch tok.Kind() {
83 case '{', '}', '[', ']':
84 if err := enc.WriteToken(tok); err != nil {
85 t.Fatalf("%s: Encoder.WriteToken error: %v", where, err)
86 }
87 default:
88 val := RawValue(tok.String())
89 if tok.Kind() == '"' {
90 val, _ = appendString(nil, tok.String(), false, nil)
91 }
92 if err := enc.WriteValue(val); err != nil {
93 t.Fatalf("%s: Encoder.WriteValue error: %v", where, err)
94 }
95 }
96 }
97 }
98
99 got := dst.String()
100 if got != want {
101 t.Errorf("%s: output mismatch:\ngot %q\nwant %q", where, got, want)
102 }
103 }
104
105
106 func TestFaultyEncoder(t *testing.T) {
107 for _, td := range coderTestdata {
108 for _, typeName := range []string{"Token", "Value"} {
109 t.Run(path.Join(td.name.name, typeName), func(t *testing.T) {
110 testFaultyEncoder(t, td.name.where, typeName, td)
111 })
112 }
113 }
114 }
115 func testFaultyEncoder(t *testing.T, where pc, typeName string, td coderTestdataEntry) {
116 b := &FaultyBuffer{
117 MaxBytes: 1,
118 MayError: io.ErrShortWrite,
119 }
120
121
122
123
124
125
126 enc := NewEncoder(b)
127 switch typeName {
128 case "Token":
129 for i, tok := range td.tokens {
130 err := enc.WriteToken(tok)
131 if err != nil && !errors.Is(err, io.ErrShortWrite) {
132 t.Fatalf("%s: %d: Encoder.WriteToken error: %v", where, i, err)
133 }
134 }
135 case "Value":
136 err := enc.WriteValue(RawValue(td.in))
137 if err != nil && !errors.Is(err, io.ErrShortWrite) {
138 t.Fatalf("%s: Encoder.WriteValue error: %v", where, err)
139 }
140 }
141 gotOutput := string(append(b.B, enc.unflushedBuffer()...))
142 wantOutput := td.outCompacted + "\n"
143 if gotOutput != wantOutput {
144 t.Fatalf("%s: output mismatch:\ngot %s\nwant %s", where, gotOutput, wantOutput)
145 }
146 }
147
148 type encoderMethodCall struct {
149 in tokOrVal
150 wantErr error
151 wantPointer string
152 }
153
154 var encoderErrorTestdata = []struct {
155 name testName
156 opts EncodeOptions
157 calls []encoderMethodCall
158 wantOut string
159 }{{
160 name: name("InvalidToken"),
161 calls: []encoderMethodCall{
162 {zeroToken, &SyntacticError{str: "invalid json.Token"}, ""},
163 },
164 }, {
165 name: name("InvalidValue"),
166 calls: []encoderMethodCall{
167 {RawValue(`#`), newInvalidCharacterError([]byte("#"), "at start of value"), ""},
168 },
169 }, {
170 name: name("InvalidValue/DoubleZero"),
171 calls: []encoderMethodCall{
172 {RawValue(`00`), newInvalidCharacterError([]byte("0"), "after top-level value"), ""},
173 },
174 }, {
175 name: name("TruncatedValue"),
176 calls: []encoderMethodCall{
177 {zeroValue, io.ErrUnexpectedEOF, ""},
178 },
179 }, {
180 name: name("TruncatedNull"),
181 calls: []encoderMethodCall{
182 {RawValue(`nul`), io.ErrUnexpectedEOF, ""},
183 },
184 }, {
185 name: name("InvalidNull"),
186 calls: []encoderMethodCall{
187 {RawValue(`nulL`), newInvalidCharacterError([]byte("L"), "within literal null (expecting 'l')"), ""},
188 },
189 }, {
190 name: name("TruncatedFalse"),
191 calls: []encoderMethodCall{
192 {RawValue(`fals`), io.ErrUnexpectedEOF, ""},
193 },
194 }, {
195 name: name("InvalidFalse"),
196 calls: []encoderMethodCall{
197 {RawValue(`falsE`), newInvalidCharacterError([]byte("E"), "within literal false (expecting 'e')"), ""},
198 },
199 }, {
200 name: name("TruncatedTrue"),
201 calls: []encoderMethodCall{
202 {RawValue(`tru`), io.ErrUnexpectedEOF, ""},
203 },
204 }, {
205 name: name("InvalidTrue"),
206 calls: []encoderMethodCall{
207 {RawValue(`truE`), newInvalidCharacterError([]byte("E"), "within literal true (expecting 'e')"), ""},
208 },
209 }, {
210 name: name("TruncatedString"),
211 calls: []encoderMethodCall{
212 {RawValue(`"star`), io.ErrUnexpectedEOF, ""},
213 },
214 }, {
215 name: name("InvalidString"),
216 calls: []encoderMethodCall{
217 {RawValue(`"ok` + "\x00"), newInvalidCharacterError([]byte("\x00"), `within string (expecting non-control character)`), ""},
218 },
219 }, {
220 name: name("ValidString/AllowInvalidUTF8/Token"),
221 opts: EncodeOptions{AllowInvalidUTF8: true},
222 calls: []encoderMethodCall{
223 {String("living\xde\xad\xbe\xef"), nil, ""},
224 },
225 wantOut: "\"living\xde\xad\ufffd\ufffd\"\n",
226 }, {
227 name: name("ValidString/AllowInvalidUTF8/Value"),
228 opts: EncodeOptions{AllowInvalidUTF8: true},
229 calls: []encoderMethodCall{
230 {RawValue("\"living\xde\xad\xbe\xef\""), nil, ""},
231 },
232 wantOut: "\"living\xde\xad\ufffd\ufffd\"\n",
233 }, {
234 name: name("InvalidString/RejectInvalidUTF8"),
235 opts: EncodeOptions{AllowInvalidUTF8: false},
236 calls: []encoderMethodCall{
237 {String("living\xde\xad\xbe\xef"), &SyntacticError{str: "invalid UTF-8 within string"}, ""},
238 {RawValue("\"living\xde\xad\xbe\xef\""), &SyntacticError{str: "invalid UTF-8 within string"}, ""},
239 },
240 }, {
241 name: name("TruncatedNumber"),
242 calls: []encoderMethodCall{
243 {RawValue(`0.`), io.ErrUnexpectedEOF, ""},
244 },
245 }, {
246 name: name("InvalidNumber"),
247 calls: []encoderMethodCall{
248 {RawValue(`0.e`), newInvalidCharacterError([]byte("e"), "within number (expecting digit)"), ""},
249 },
250 }, {
251 name: name("TruncatedObject/AfterStart"),
252 calls: []encoderMethodCall{
253 {RawValue(`{`), io.ErrUnexpectedEOF, ""},
254 },
255 }, {
256 name: name("TruncatedObject/AfterName"),
257 calls: []encoderMethodCall{
258 {RawValue(`{"0"`), io.ErrUnexpectedEOF, ""},
259 },
260 }, {
261 name: name("TruncatedObject/AfterColon"),
262 calls: []encoderMethodCall{
263 {RawValue(`{"0":`), io.ErrUnexpectedEOF, ""},
264 },
265 }, {
266 name: name("TruncatedObject/AfterValue"),
267 calls: []encoderMethodCall{
268 {RawValue(`{"0":0`), io.ErrUnexpectedEOF, ""},
269 },
270 }, {
271 name: name("TruncatedObject/AfterComma"),
272 calls: []encoderMethodCall{
273 {RawValue(`{"0":0,`), io.ErrUnexpectedEOF, ""},
274 },
275 }, {
276 name: name("InvalidObject/MissingColon"),
277 calls: []encoderMethodCall{
278 {RawValue(` { "fizz" "buzz" } `), newInvalidCharacterError([]byte("\""), "after object name (expecting ':')"), ""},
279 {RawValue(` { "fizz" , "buzz" } `), newInvalidCharacterError([]byte(","), "after object name (expecting ':')"), ""},
280 },
281 }, {
282 name: name("InvalidObject/MissingComma"),
283 calls: []encoderMethodCall{
284 {RawValue(` { "fizz" : "buzz" "gazz" } `), newInvalidCharacterError([]byte("\""), "after object value (expecting ',' or '}')"), ""},
285 {RawValue(` { "fizz" : "buzz" : "gazz" } `), newInvalidCharacterError([]byte(":"), "after object value (expecting ',' or '}')"), ""},
286 },
287 }, {
288 name: name("InvalidObject/ExtraComma"),
289 calls: []encoderMethodCall{
290 {RawValue(` { , } `), newInvalidCharacterError([]byte(","), `at start of string (expecting '"')`), ""},
291 {RawValue(` { "fizz" : "buzz" , } `), newInvalidCharacterError([]byte("}"), `at start of string (expecting '"')`), ""},
292 },
293 }, {
294 name: name("InvalidObject/InvalidName"),
295 calls: []encoderMethodCall{
296 {RawValue(`{ null }`), newInvalidCharacterError([]byte("n"), `at start of string (expecting '"')`), ""},
297 {RawValue(`{ false }`), newInvalidCharacterError([]byte("f"), `at start of string (expecting '"')`), ""},
298 {RawValue(`{ true }`), newInvalidCharacterError([]byte("t"), `at start of string (expecting '"')`), ""},
299 {RawValue(`{ 0 }`), newInvalidCharacterError([]byte("0"), `at start of string (expecting '"')`), ""},
300 {RawValue(`{ {} }`), newInvalidCharacterError([]byte("{"), `at start of string (expecting '"')`), ""},
301 {RawValue(`{ [] }`), newInvalidCharacterError([]byte("["), `at start of string (expecting '"')`), ""},
302 {ObjectStart, nil, ""},
303 {Null, errMissingName, ""},
304 {RawValue(`null`), errMissingName, ""},
305 {False, errMissingName, ""},
306 {RawValue(`false`), errMissingName, ""},
307 {True, errMissingName, ""},
308 {RawValue(`true`), errMissingName, ""},
309 {Uint(0), errMissingName, ""},
310 {RawValue(`0`), errMissingName, ""},
311 {ObjectStart, errMissingName, ""},
312 {RawValue(`{}`), errMissingName, ""},
313 {ArrayStart, errMissingName, ""},
314 {RawValue(`[]`), errMissingName, ""},
315 {ObjectEnd, nil, ""},
316 },
317 wantOut: "{}\n",
318 }, {
319 name: name("InvalidObject/InvalidValue"),
320 calls: []encoderMethodCall{
321 {RawValue(`{ "0": x }`), newInvalidCharacterError([]byte("x"), `at start of value`), ""},
322 },
323 }, {
324 name: name("InvalidObject/MismatchingDelim"),
325 calls: []encoderMethodCall{
326 {RawValue(` { ] `), newInvalidCharacterError([]byte("]"), `at start of string (expecting '"')`), ""},
327 {RawValue(` { "0":0 ] `), newInvalidCharacterError([]byte("]"), `after object value (expecting ',' or '}')`), ""},
328 {ObjectStart, nil, ""},
329 {ArrayEnd, errMismatchDelim, ""},
330 {RawValue(`]`), newInvalidCharacterError([]byte("]"), "at start of value"), ""},
331 {ObjectEnd, nil, ""},
332 },
333 wantOut: "{}\n",
334 }, {
335 name: name("ValidObject/UniqueNames"),
336 calls: []encoderMethodCall{
337 {ObjectStart, nil, ""},
338 {String("0"), nil, ""},
339 {Uint(0), nil, ""},
340 {String("1"), nil, ""},
341 {Uint(1), nil, ""},
342 {ObjectEnd, nil, ""},
343 {RawValue(` { "0" : 0 , "1" : 1 } `), nil, ""},
344 },
345 wantOut: `{"0":0,"1":1}` + "\n" + `{"0":0,"1":1}` + "\n",
346 }, {
347 name: name("ValidObject/DuplicateNames"),
348 opts: EncodeOptions{AllowDuplicateNames: true},
349 calls: []encoderMethodCall{
350 {ObjectStart, nil, ""},
351 {String("0"), nil, ""},
352 {Uint(0), nil, ""},
353 {String("0"), nil, ""},
354 {Uint(0), nil, ""},
355 {ObjectEnd, nil, ""},
356 {RawValue(` { "0" : 0 , "0" : 0 } `), nil, ""},
357 },
358 wantOut: `{"0":0,"0":0}` + "\n" + `{"0":0,"0":0}` + "\n",
359 }, {
360 name: name("InvalidObject/DuplicateNames"),
361 calls: []encoderMethodCall{
362 {ObjectStart, nil, ""},
363 {String("0"), nil, ""},
364 {ObjectStart, nil, ""},
365 {ObjectEnd, nil, ""},
366 {String("0"), &SyntacticError{str: `duplicate name "0" in object`}, "/0"},
367 {RawValue(`"0"`), &SyntacticError{str: `duplicate name "0" in object`}, "/0"},
368 {String("1"), nil, ""},
369 {ObjectStart, nil, ""},
370 {ObjectEnd, nil, ""},
371 {String("0"), &SyntacticError{str: `duplicate name "0" in object`}, "/1"},
372 {RawValue(`"0"`), &SyntacticError{str: `duplicate name "0" in object`}, "/1"},
373 {String("1"), &SyntacticError{str: `duplicate name "1" in object`}, "/1"},
374 {RawValue(`"1"`), &SyntacticError{str: `duplicate name "1" in object`}, "/1"},
375 {ObjectEnd, nil, ""},
376 {RawValue(` { "0" : 0 , "1" : 1 , "0" : 0 } `), &SyntacticError{str: `duplicate name "0" in object`}, ""},
377 },
378 wantOut: `{"0":{},"1":{}}` + "\n",
379 }, {
380 name: name("TruncatedArray/AfterStart"),
381 calls: []encoderMethodCall{
382 {RawValue(`[`), io.ErrUnexpectedEOF, ""},
383 },
384 }, {
385 name: name("TruncatedArray/AfterValue"),
386 calls: []encoderMethodCall{
387 {RawValue(`[0`), io.ErrUnexpectedEOF, ""},
388 },
389 }, {
390 name: name("TruncatedArray/AfterComma"),
391 calls: []encoderMethodCall{
392 {RawValue(`[0,`), io.ErrUnexpectedEOF, ""},
393 },
394 }, {
395 name: name("TruncatedArray/MissingComma"),
396 calls: []encoderMethodCall{
397 {RawValue(` [ "fizz" "buzz" ] `), newInvalidCharacterError([]byte("\""), "after array value (expecting ',' or ']')"), ""},
398 },
399 }, {
400 name: name("InvalidArray/MismatchingDelim"),
401 calls: []encoderMethodCall{
402 {RawValue(` [ } `), newInvalidCharacterError([]byte("}"), `at start of value`), ""},
403 {ArrayStart, nil, ""},
404 {ObjectEnd, errMismatchDelim, ""},
405 {RawValue(`}`), newInvalidCharacterError([]byte("}"), "at start of value"), ""},
406 {ArrayEnd, nil, ""},
407 },
408 wantOut: "[]\n",
409 }}
410
411
412
413 func TestEncoderErrors(t *testing.T) {
414 for _, td := range encoderErrorTestdata {
415 t.Run(path.Join(td.name.name), func(t *testing.T) {
416 testEncoderErrors(t, td.name.where, td.opts, td.calls, td.wantOut)
417 })
418 }
419 }
420 func testEncoderErrors(t *testing.T, where pc, opts EncodeOptions, calls []encoderMethodCall, wantOut string) {
421 dst := new(bytes.Buffer)
422 enc := opts.NewEncoder(dst)
423 for i, call := range calls {
424 var gotErr error
425 switch tokVal := call.in.(type) {
426 case Token:
427 gotErr = enc.WriteToken(tokVal)
428 case RawValue:
429 gotErr = enc.WriteValue(tokVal)
430 }
431 if !reflect.DeepEqual(gotErr, call.wantErr) {
432 t.Fatalf("%s: %d: error mismatch: got %#v, want %#v", where, i, gotErr, call.wantErr)
433 }
434 if call.wantPointer != "" {
435 gotPointer := enc.StackPointer()
436 if gotPointer != call.wantPointer {
437 t.Fatalf("%s: %d: Encoder.StackPointer = %s, want %s", where, i, gotPointer, call.wantPointer)
438 }
439 }
440 }
441 gotOut := dst.String() + string(enc.unflushedBuffer())
442 if gotOut != wantOut {
443 t.Fatalf("%s: output mismatch:\ngot %q\nwant %q", where, gotOut, wantOut)
444 }
445 gotOffset := int(enc.OutputOffset())
446 wantOffset := len(wantOut)
447 if gotOffset != wantOffset {
448 t.Fatalf("%s: Encoder.OutputOffset = %v, want %v", where, gotOffset, wantOffset)
449 }
450 }
451
452 func TestAppendString(t *testing.T) {
453 var (
454 escapeNothing = func(r rune) bool { return false }
455 escapeHTML = func(r rune) bool { return r == '<' || r == '>' || r == '&' || r == '\u2028' || r == '\u2029' }
456 escapeNonASCII = func(r rune) bool { return r > unicode.MaxASCII }
457 escapeEverything = func(r rune) bool { return true }
458 )
459
460 tests := []struct {
461 in string
462 escapeRune func(rune) bool
463 want string
464 wantErr error
465 wantErrUTF8 error
466 }{
467 {"", nil, `""`, nil, nil},
468 {"hello", nil, `"hello"`, nil, nil},
469 {"\x00", nil, `"\u0000"`, nil, nil},
470 {"\x1f", nil, `"\u001f"`, nil, nil},
471 {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", nil, `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, nil, nil},
472 {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", nil, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil},
473 {"x\x80\ufffd", nil, "\"x\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
474 {"x\xff\ufffd", nil, "\"x\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
475 {"x\x80\ufffd", escapeNonASCII, "\"x\\ufffd\\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
476 {"x\xff\ufffd", escapeNonASCII, "\"x\\ufffd\\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
477 {"x\xc0", nil, "\"x\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
478 {"x\xc0\x80", nil, "\"x\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
479 {"x\xe0", nil, "\"x\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
480 {"x\xe0\x80", nil, "\"x\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
481 {"x\xe0\x80\x80", nil, "\"x\ufffd\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
482 {"x\xf0", nil, "\"x\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
483 {"x\xf0\x80", nil, "\"x\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
484 {"x\xf0\x80\x80", nil, "\"x\ufffd\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
485 {"x\xf0\x80\x80\x80", nil, "\"x\ufffd\ufffd\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
486 {"x\xed\xba\xad", nil, "\"x\ufffd\ufffd\ufffd\"", nil, &SyntacticError{str: "invalid UTF-8 within string"}},
487 {"\"\\/\b\f\n\r\t", nil, `"\"\\/\b\f\n\r\t"`, nil, nil},
488 {"\"\\/\b\f\n\r\t", escapeEverything, `"\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009"`, nil, nil},
489 {"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", nil, `"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃)."`, nil, nil},
490 {"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", escapeNonASCII, `"\u0669(-\u032e\u032e\u0303-\u0303)\u06f6 \u0669(\u25cf\u032e\u032e\u0303\u2022\u0303)\u06f6 \u0669(\u0361\u0e4f\u032f\u0361\u0e4f)\u06f6 \u0669(-\u032e\u032e\u0303\u2022\u0303)."`, nil, nil},
491 {"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", escapeEverything, `"\u0669\u0028\u002d\u032e\u032e\u0303\u002d\u0303\u0029\u06f6\u0020\u0669\u0028\u25cf\u032e\u032e\u0303\u2022\u0303\u0029\u06f6\u0020\u0669\u0028\u0361\u0e4f\u032f\u0361\u0e4f\u0029\u06f6\u0020\u0669\u0028\u002d\u032e\u032e\u0303\u2022\u0303\u0029\u002e"`, nil, nil},
492 {"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", nil, "\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", nil, nil},
493 {"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", escapeEverything, `"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\ud83d\ude02"`, nil, nil},
494 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", nil, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\u0080\u2028\u2029\ufffd\U0001f602\"", nil, nil},
495 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", escapeNothing, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\u0080\u2028\u2029\ufffd\U0001f602\"", nil, nil},
496 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", escapeHTML, "\"\\u0000\\u001f\u0020\\\"\\u0026\\u003c\\u003e\\\\\u007f\u0080\\u2028\\u2029\ufffd\U0001f602\"", nil, nil},
497 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", escapeNonASCII, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\\u0080\\u2028\\u2029\\ufffd\\ud83d\\ude02\"", nil, nil},
498 {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", escapeEverything, "\"\\u0000\\u001f\\u0020\\u0022\\u0026\\u003c\\u003e\\u005c\\u007f\\u0080\\u2028\\u2029\\ufffd\\ud83d\\ude02\"", nil, nil},
499 }
500
501 for _, tt := range tests {
502 t.Run("", func(t *testing.T) {
503 got, gotErr := appendString(nil, tt.in, false, tt.escapeRune)
504 if string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) {
505 t.Errorf("appendString(nil, %q, false, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
506 }
507 switch got, gotErr := appendString(nil, tt.in, true, tt.escapeRune); {
508 case tt.wantErrUTF8 == nil && (string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr)):
509 t.Errorf("appendString(nil, %q, true, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr)
510 case tt.wantErrUTF8 != nil && (!strings.HasPrefix(tt.want, string(got)) || !reflect.DeepEqual(gotErr, tt.wantErrUTF8)):
511 t.Errorf("appendString(nil, %q, true, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErrUTF8)
512 }
513 })
514 }
515 }
516
517 func TestAppendNumber(t *testing.T) {
518 tests := []struct {
519 in float64
520 want32 string
521 want64 string
522 }{
523 {math.E, "2.7182817", "2.718281828459045"},
524 {math.Pi, "3.1415927", "3.141592653589793"},
525 {math.SmallestNonzeroFloat32, "1e-45", "1.401298464324817e-45"},
526 {math.SmallestNonzeroFloat64, "0", "5e-324"},
527 {math.MaxFloat32, "3.4028235e+38", "3.4028234663852886e+38"},
528 {math.MaxFloat64, "", "1.7976931348623157e+308"},
529 {0.1111111111111111, "0.11111111", "0.1111111111111111"},
530 {0.2222222222222222, "0.22222222", "0.2222222222222222"},
531 {0.3333333333333333, "0.33333334", "0.3333333333333333"},
532 {0.4444444444444444, "0.44444445", "0.4444444444444444"},
533 {0.5555555555555555, "0.5555556", "0.5555555555555555"},
534 {0.6666666666666666, "0.6666667", "0.6666666666666666"},
535 {0.7777777777777777, "0.7777778", "0.7777777777777777"},
536 {0.8888888888888888, "0.8888889", "0.8888888888888888"},
537 {0.9999999999999999, "1", "0.9999999999999999"},
538
539
540
541 {math.Float64frombits(0x0000000000000000), "0", "0"},
542 {math.Float64frombits(0x8000000000000000), "-0", "-0"},
543 {math.Float64frombits(0x0000000000000001), "0", "5e-324"},
544 {math.Float64frombits(0x8000000000000001), "-0", "-5e-324"},
545 {math.Float64frombits(0x7fefffffffffffff), "", "1.7976931348623157e+308"},
546 {math.Float64frombits(0xffefffffffffffff), "", "-1.7976931348623157e+308"},
547 {math.Float64frombits(0x4340000000000000), "9007199000000000", "9007199254740992"},
548 {math.Float64frombits(0xc340000000000000), "-9007199000000000", "-9007199254740992"},
549 {math.Float64frombits(0x4430000000000000), "295147900000000000000", "295147905179352830000"},
550 {math.Float64frombits(0x44b52d02c7e14af5), "1e+23", "9.999999999999997e+22"},
551 {math.Float64frombits(0x44b52d02c7e14af6), "1e+23", "1e+23"},
552 {math.Float64frombits(0x44b52d02c7e14af7), "1e+23", "1.0000000000000001e+23"},
553 {math.Float64frombits(0x444b1ae4d6e2ef4e), "1e+21", "999999999999999700000"},
554 {math.Float64frombits(0x444b1ae4d6e2ef4f), "1e+21", "999999999999999900000"},
555 {math.Float64frombits(0x444b1ae4d6e2ef50), "1e+21", "1e+21"},
556 {math.Float64frombits(0x3eb0c6f7a0b5ed8c), "0.000001", "9.999999999999997e-7"},
557 {math.Float64frombits(0x3eb0c6f7a0b5ed8d), "0.000001", "0.000001"},
558 {math.Float64frombits(0x41b3de4355555553), "333333340", "333333333.3333332"},
559 {math.Float64frombits(0x41b3de4355555554), "333333340", "333333333.33333325"},
560 {math.Float64frombits(0x41b3de4355555555), "333333340", "333333333.3333333"},
561 {math.Float64frombits(0x41b3de4355555556), "333333340", "333333333.3333334"},
562 {math.Float64frombits(0x41b3de4355555557), "333333340", "333333333.33333343"},
563 {math.Float64frombits(0xbecbf647612f3696), "-0.0000033333333", "-0.0000033333333333333333"},
564 {math.Float64frombits(0x43143ff3c1cb0959), "1424953900000000", "1424953923781206.2"},
565
566
567
568 {float64(math.Float32frombits(0x65a96815)), "9.999999e+22", "9.999998877476383e+22"},
569 {float64(math.Float32frombits(0x65a96816)), "1e+23", "9.999999778196308e+22"},
570 {float64(math.Float32frombits(0x65a96817)), "1.0000001e+23", "1.0000000678916234e+23"},
571 {float64(math.Float32frombits(0x6258d725)), "999999900000000000000", "999999879303389000000"},
572 {float64(math.Float32frombits(0x6258d726)), "999999950000000000000", "999999949672133200000"},
573 {float64(math.Float32frombits(0x6258d727)), "1e+21", "1.0000000200408773e+21"},
574 {float64(math.Float32frombits(0x6258d728)), "1.0000001e+21", "1.0000000904096215e+21"},
575 {float64(math.Float32frombits(0x358637bc)), "9.999999e-7", "9.99999883788405e-7"},
576 {float64(math.Float32frombits(0x358637bd)), "0.000001", "9.999999974752427e-7"},
577 {float64(math.Float32frombits(0x358637be)), "0.0000010000001", "0.0000010000001111620804"},
578 }
579
580 for _, tt := range tests {
581 t.Run("", func(t *testing.T) {
582 if got32 := string(appendNumber(nil, tt.in, 32)); got32 != tt.want32 && tt.want32 != "" {
583 t.Errorf("appendNumber(nil, %v, 32) = %v, want %v", tt.in, got32, tt.want32)
584 }
585 if got64 := string(appendNumber(nil, tt.in, 64)); got64 != tt.want64 && tt.want64 != "" {
586 t.Errorf("appendNumber(nil, %v, 64) = %v, want %v", tt.in, got64, tt.want64)
587 }
588 })
589 }
590 }
591
592
593
594
595
596
597 var testCanonicalNumberLines = flag.Float64("canonical-number-lines", 1e4, "specify the number of lines to check from the canonical numbers testdata")
598
599
600
601
602 func TestCanonicalNumber(t *testing.T) {
603 const testfileURL = "https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz"
604 hashes := map[float64]string{
605 1e3: "be18b62b6f69cdab33a7e0dae0d9cfa869fda80ddc712221570f9f40a5878687",
606 1e4: "b9f7a8e75ef22a835685a52ccba7f7d6bdc99e34b010992cbc5864cd12be6892",
607 1e5: "22776e6d4b49fa294a0d0f349268e5c28808fe7e0cb2bcbe28f63894e494d4c7",
608 1e6: "49415fee2c56c77864931bd3624faad425c3c577d6d74e89a83bc725506dad16",
609 1e7: "b9f8a44a91d46813b21b9602e72f112613c91408db0b8341fb94603d9db135e0",
610 1e8: "0f7dda6b0837dde083c5d6b896f7d62340c8a2415b0c7121d83145e08a755272",
611 }
612 wantHash := hashes[*testCanonicalNumberLines]
613 if wantHash == "" {
614 t.Fatalf("canonical-number-lines must be one of the following values: 1e3, 1e4, 1e5, 1e6, 1e7, 1e8")
615 }
616 numLines := int(*testCanonicalNumberLines)
617
618
619
620 generator := func() func() float64 {
621 static := [...]uint64{
622 0x0000000000000000, 0x8000000000000000, 0x0000000000000001, 0x8000000000000001,
623 0xc46696695dbd1cc3, 0xc43211ede4974a35, 0xc3fce97ca0f21056, 0xc3c7213080c1a6ac,
624 0xc39280f39a348556, 0xc35d9b1f5d20d557, 0xc327af4c4a80aaac, 0xc2f2f2a36ecd5556,
625 0xc2be51057e155558, 0xc28840d131aaaaac, 0xc253670dc1555557, 0xc21f0b4935555557,
626 0xc1e8d5d42aaaaaac, 0xc1b3de4355555556, 0xc17fca0555555556, 0xc1496e6aaaaaaaab,
627 0xc114585555555555, 0xc0e046aaaaaaaaab, 0xc0aa0aaaaaaaaaaa, 0xc074d55555555555,
628 0xc040aaaaaaaaaaab, 0xc00aaaaaaaaaaaab, 0xbfd5555555555555, 0xbfa1111111111111,
629 0xbf6b4e81b4e81b4f, 0xbf35d867c3ece2a5, 0xbf0179ec9cbd821e, 0xbecbf647612f3696,
630 0xbe965e9f80f29212, 0xbe61e54c672874db, 0xbe2ca213d840baf8, 0xbdf6e80fe033c8c6,
631 0xbdc2533fe68fd3d2, 0xbd8d51ffd74c861c, 0xbd5774ccac3d3817, 0xbd22c3d6f030f9ac,
632 0xbcee0624b3818f79, 0xbcb804ea293472c7, 0xbc833721ba905bd3, 0xbc4ebe9c5db3c61e,
633 0xbc18987d17c304e5, 0xbbe3ad30dfcf371d, 0xbbaf7b816618582f, 0xbb792f9ab81379bf,
634 0xbb442615600f9499, 0xbb101e77800c76e1, 0xbad9ca58cce0be35, 0xbaa4a1e0a3e6fe90,
635 0xba708180831f320d, 0xba3a68cd9e985016, 0x446696695dbd1cc3, 0x443211ede4974a35,
636 0x43fce97ca0f21056, 0x43c7213080c1a6ac, 0x439280f39a348556, 0x435d9b1f5d20d557,
637 0x4327af4c4a80aaac, 0x42f2f2a36ecd5556, 0x42be51057e155558, 0x428840d131aaaaac,
638 0x4253670dc1555557, 0x421f0b4935555557, 0x41e8d5d42aaaaaac, 0x41b3de4355555556,
639 0x417fca0555555556, 0x41496e6aaaaaaaab, 0x4114585555555555, 0x40e046aaaaaaaaab,
640 0x40aa0aaaaaaaaaaa, 0x4074d55555555555, 0x4040aaaaaaaaaaab, 0x400aaaaaaaaaaaab,
641 0x3fd5555555555555, 0x3fa1111111111111, 0x3f6b4e81b4e81b4f, 0x3f35d867c3ece2a5,
642 0x3f0179ec9cbd821e, 0x3ecbf647612f3696, 0x3e965e9f80f29212, 0x3e61e54c672874db,
643 0x3e2ca213d840baf8, 0x3df6e80fe033c8c6, 0x3dc2533fe68fd3d2, 0x3d8d51ffd74c861c,
644 0x3d5774ccac3d3817, 0x3d22c3d6f030f9ac, 0x3cee0624b3818f79, 0x3cb804ea293472c7,
645 0x3c833721ba905bd3, 0x3c4ebe9c5db3c61e, 0x3c18987d17c304e5, 0x3be3ad30dfcf371d,
646 0x3baf7b816618582f, 0x3b792f9ab81379bf, 0x3b442615600f9499, 0x3b101e77800c76e1,
647 0x3ad9ca58cce0be35, 0x3aa4a1e0a3e6fe90, 0x3a708180831f320d, 0x3a3a68cd9e985016,
648 0x4024000000000000, 0x4014000000000000, 0x3fe0000000000000, 0x3fa999999999999a,
649 0x3f747ae147ae147b, 0x3f40624dd2f1a9fc, 0x3f0a36e2eb1c432d, 0x3ed4f8b588e368f1,
650 0x3ea0c6f7a0b5ed8d, 0x3e6ad7f29abcaf48, 0x3e35798ee2308c3a, 0x3ed539223589fa95,
651 0x3ed4ff26cd5a7781, 0x3ed4f95a762283ff, 0x3ed4f8c60703520c, 0x3ed4f8b72f19cd0d,
652 0x3ed4f8b5b31c0c8d, 0x3ed4f8b58d1c461a, 0x3ed4f8b5894f7f0e, 0x3ed4f8b588ee37f3,
653 0x3ed4f8b588e47da4, 0x3ed4f8b588e3849c, 0x3ed4f8b588e36bb5, 0x3ed4f8b588e36937,
654 0x3ed4f8b588e368f8, 0x3ed4f8b588e368f1, 0x3ff0000000000000, 0xbff0000000000000,
655 0xbfeffffffffffffa, 0xbfeffffffffffffb, 0x3feffffffffffffa, 0x3feffffffffffffb,
656 0x3feffffffffffffc, 0x3feffffffffffffe, 0xbfefffffffffffff, 0xbfefffffffffffff,
657 0x3fefffffffffffff, 0x3fefffffffffffff, 0x3fd3333333333332, 0x3fd3333333333333,
658 0x3fd3333333333334, 0x0010000000000000, 0x000ffffffffffffd, 0x000fffffffffffff,
659 0x7fefffffffffffff, 0xffefffffffffffff, 0x4340000000000000, 0xc340000000000000,
660 0x4430000000000000, 0x44b52d02c7e14af5, 0x44b52d02c7e14af6, 0x44b52d02c7e14af7,
661 0x444b1ae4d6e2ef4e, 0x444b1ae4d6e2ef4f, 0x444b1ae4d6e2ef50, 0x3eb0c6f7a0b5ed8c,
662 0x3eb0c6f7a0b5ed8d, 0x41b3de4355555553, 0x41b3de4355555554, 0x41b3de4355555555,
663 0x41b3de4355555556, 0x41b3de4355555557, 0xbecbf647612f3696, 0x43143ff3c1cb0959,
664 }
665 var state struct {
666 idx int
667 data []byte
668 block [sha256.Size]byte
669 }
670 return func() float64 {
671 const numSerial = 2000
672 var f float64
673 switch {
674 case state.idx < len(static):
675 f = math.Float64frombits(static[state.idx])
676 case state.idx < len(static)+numSerial:
677 f = math.Float64frombits(0x0010000000000000 + uint64(state.idx-len(static)))
678 default:
679 for f == 0 || math.IsNaN(f) || math.IsInf(f, 0) {
680 if len(state.data) == 0 {
681 state.block = sha256.Sum256(state.block[:])
682 state.data = state.block[:]
683 }
684 f = math.Float64frombits(binary.LittleEndian.Uint64(state.data))
685 state.data = state.data[8:]
686 }
687 }
688 state.idx++
689 return f
690 }
691 }
692
693
694
695
696 for _, checkGolden := range []bool{false, true} {
697 var br *bufio.Reader
698 if checkGolden {
699 resp, err := http.Get(testfileURL)
700 if err != nil {
701 t.Fatalf("http.Get error: %v", err)
702 }
703 defer resp.Body.Close()
704
705 zr, err := gzip.NewReader(resp.Body)
706 if err != nil {
707 t.Fatalf("gzip.NewReader error: %v", err)
708 }
709
710 br = bufio.NewReader(zr)
711 }
712
713
714 appendNumberJCS := func(b []byte, f float64) []byte {
715 if math.Signbit(f) && f == 0 {
716 return append(b, '0')
717 }
718 return appendNumber(b, f, 64)
719 }
720
721 var gotLine []byte
722 next := generator()
723 hash := sha256.New()
724 start := time.Now()
725 lastPrint := start
726 for n := 1; n <= numLines; n++ {
727
728 f := next()
729 gotLine = gotLine[:0]
730 gotLine = strconv.AppendUint(gotLine, math.Float64bits(f), 16)
731 gotLine = append(gotLine, ',')
732 gotLine = appendNumberJCS(gotLine, f)
733 gotLine = append(gotLine, '\n')
734 hash.Write(gotLine)
735
736
737 if checkGolden {
738 wantLine, err := br.ReadBytes('\n')
739 if err != nil {
740 t.Fatalf("bufio.Reader.ReadBytes error: %v", err)
741 }
742 if !bytes.Equal(gotLine, wantLine) {
743 t.Errorf("mismatch on line %d:\n\tgot %v\n\twant %v",
744 n, strings.TrimSpace(string(gotLine)), strings.TrimSpace(string(wantLine)))
745 }
746 }
747
748
749 if now := time.Now(); now.Sub(lastPrint) > time.Second || n == numLines {
750 remaining := float64(now.Sub(start)) * float64(numLines-n) / float64(n)
751 t.Logf("%0.3f%% (%v remaining)",
752 100.0*float64(n)/float64(numLines),
753 time.Duration(remaining).Round(time.Second))
754 lastPrint = now
755 }
756 }
757
758 gotHash := hex.EncodeToString(hash.Sum(nil))
759 if gotHash == wantHash {
760 return
761 }
762 }
763 }
764
View as plain text