1
16
17 package yaml
18
19 import (
20 "bufio"
21 "bytes"
22 "encoding/json"
23 "fmt"
24 "io"
25 "math/rand"
26 "reflect"
27 "strings"
28 "testing"
29 )
30
31 func TestYAMLDecoderReadBytesLength(t *testing.T) {
32 d := `---
33 stuff: 1
34 test-foo: 1
35 `
36 testCases := []struct {
37 bufLen int
38 expectLen int
39 expectErr error
40 }{
41 {len(d), len(d), nil},
42 {len(d) + 10, len(d), nil},
43 {len(d) - 10, len(d) - 10, io.ErrShortBuffer},
44 }
45
46 for i, testCase := range testCases {
47 r := NewDocumentDecoder(io.NopCloser(bytes.NewReader([]byte(d))))
48 b := make([]byte, testCase.bufLen)
49 n, err := r.Read(b)
50 if err != testCase.expectErr || n != testCase.expectLen {
51 t.Fatalf("%d: unexpected body: %d / %v", i, n, err)
52 }
53 }
54 }
55
56 func TestBigYAML(t *testing.T) {
57 d := `
58 stuff: 1
59 `
60 maxLen := 5 * 1024 * 1024
61 bufferLen := 4 * 1024
62
63 dd := strings.Repeat(d, 512*1024)
64 r := NewDocumentDecoder(io.NopCloser(bytes.NewReader([]byte(dd[:maxLen-1]))))
65 b := make([]byte, bufferLen)
66 n, err := r.Read(b)
67 if err != io.ErrShortBuffer {
68 t.Fatalf("expected ErrShortBuffer: %d / %v", n, err)
69 }
70 b = make([]byte, maxLen)
71 n, err = r.Read(b)
72 if err != nil {
73 t.Fatalf("expected nil: %d / %v", n, err)
74 }
75 r = NewDocumentDecoder(io.NopCloser(bytes.NewReader([]byte(dd))))
76 b = make([]byte, maxLen)
77 n, err = r.Read(b)
78 if err != bufio.ErrTooLong {
79 t.Fatalf("bufio.Scanner: token too long: %d / %v", n, err)
80 }
81 }
82
83 func TestYAMLDecoderCallsAfterErrShortBufferRestOfFrame(t *testing.T) {
84 d := `---
85 stuff: 1
86 test-foo: 1`
87 r := NewDocumentDecoder(io.NopCloser(bytes.NewReader([]byte(d))))
88 b := make([]byte, 12)
89 n, err := r.Read(b)
90 if err != io.ErrShortBuffer || n != 12 {
91 t.Fatalf("expected ErrShortBuffer: %d / %v", n, err)
92 }
93 expected := "---\nstuff: 1"
94 if string(b) != expected {
95 t.Fatalf("expected bytes read to be: %s got: %s", expected, string(b))
96 }
97 b = make([]byte, 13)
98 n, err = r.Read(b)
99 if err != nil || n != 13 {
100 t.Fatalf("expected nil: %d / %v", n, err)
101 }
102 expected = "\n\ttest-foo: 1"
103 if string(b) != expected {
104 t.Fatalf("expected bytes read to be: '%s' got: '%s'", expected, string(b))
105 }
106 b = make([]byte, 15)
107 n, err = r.Read(b)
108 if err != io.EOF || n != 0 {
109 t.Fatalf("expected EOF: %d / %v", n, err)
110 }
111 }
112
113 func TestSplitYAMLDocument(t *testing.T) {
114 testCases := []struct {
115 input string
116 atEOF bool
117 expect string
118 adv int
119 }{
120 {"foo", true, "foo", 3},
121 {"fo", false, "", 0},
122
123 {"---", true, "---", 3},
124 {"---\n", true, "---\n", 4},
125 {"---\n", false, "", 0},
126
127 {"\n---\n", false, "", 5},
128 {"\n---\n", true, "", 5},
129
130 {"abc\n---\ndef", true, "abc", 8},
131 {"def", true, "def", 3},
132 {"", true, "", 0},
133 }
134 for i, testCase := range testCases {
135 adv, token, err := splitYAMLDocument([]byte(testCase.input), testCase.atEOF)
136 if err != nil {
137 t.Errorf("%d: unexpected error: %v", i, err)
138 continue
139 }
140 if adv != testCase.adv {
141 t.Errorf("%d: advance did not match: %d %d", i, testCase.adv, adv)
142 }
143 if testCase.expect != string(token) {
144 t.Errorf("%d: token did not match: %q %q", i, testCase.expect, string(token))
145 }
146 }
147 }
148
149 func TestGuessJSON(t *testing.T) {
150 if r, _, isJSON := GuessJSONStream(bytes.NewReader([]byte(" \n{}")), 100); !isJSON {
151 t.Fatalf("expected stream to be JSON")
152 } else {
153 b := make([]byte, 30)
154 n, err := r.Read(b)
155 if err != nil || n != 4 {
156 t.Fatalf("unexpected body: %d / %v", n, err)
157 }
158 if string(b[:n]) != " \n{}" {
159 t.Fatalf("unexpected body: %q", string(b[:n]))
160 }
161 }
162 }
163
164 func TestScanYAML(t *testing.T) {
165 s := bufio.NewScanner(bytes.NewReader([]byte(`---
166 stuff: 1
167
168 ---
169 `)))
170 s.Split(splitYAMLDocument)
171 if !s.Scan() {
172 t.Fatalf("should have been able to scan")
173 }
174 t.Logf("scan: %s", s.Text())
175 if !s.Scan() {
176 t.Fatalf("should have been able to scan")
177 }
178 t.Logf("scan: %s", s.Text())
179 if s.Scan() {
180 t.Fatalf("scan should have been done")
181 }
182 if s.Err() != nil {
183 t.Fatalf("err should have been nil: %v", s.Err())
184 }
185 }
186
187 func TestDecodeYAML(t *testing.T) {
188 s := NewYAMLToJSONDecoder(bytes.NewReader([]byte(`---
189 stuff: 1
190
191 ---
192 `)))
193 obj := generic{}
194 if err := s.Decode(&obj); err != nil {
195 t.Fatalf("unexpected error: %v", err)
196 }
197 if fmt.Sprintf("%#v", obj) != `yaml.generic{"stuff":1}` {
198 t.Errorf("unexpected object: %#v", obj)
199 }
200 obj = generic{}
201 if err := s.Decode(&obj); err != nil {
202 t.Fatalf("unexpected error: %v", err)
203 }
204 if len(obj) != 0 {
205 t.Fatalf("unexpected object: %#v", obj)
206 }
207 obj = generic{}
208 if err := s.Decode(&obj); err != io.EOF {
209 t.Fatalf("unexpected error: %v", err)
210 }
211 }
212
213 func TestDecodeYAMLSeparatorValidation(t *testing.T) {
214 s := NewYAMLToJSONDecoder(bytes.NewReader([]byte(`---
215 stuff: 1
216 --- # Make sure termination happen with inline comment
217 stuff: 2
218 ---
219 stuff: 3
220 --- Make sure uncommented content results YAMLSyntaxError
221
222 `)))
223 obj := generic{}
224 if err := s.Decode(&obj); err != nil {
225 t.Fatalf("unexpected error: %v", err)
226 }
227 if fmt.Sprintf("%#v", obj) != `yaml.generic{"stuff":1}` {
228 t.Errorf("unexpected object: %#v", obj)
229 }
230 obj = generic{}
231 if err := s.Decode(&obj); err != nil {
232 t.Fatalf("unexpected error: %v", err)
233 }
234 if fmt.Sprintf("%#v", obj) != `yaml.generic{"stuff":2}` {
235 t.Errorf("unexpected object: %#v", obj)
236 }
237 obj = generic{}
238 err := s.Decode(&obj)
239 if err == nil {
240 t.Fatalf("expected YamlSyntaxError, got nil instead")
241 }
242 if _, ok := err.(YAMLSyntaxError); !ok {
243 t.Fatalf("unexpected error: %v", err)
244 }
245 }
246
247 func TestDecodeBrokenYAML(t *testing.T) {
248 s := NewYAMLOrJSONDecoder(bytes.NewReader([]byte(`---
249 stuff: 1
250 test-foo: 1
251
252 ---
253 `)), 100)
254 obj := generic{}
255 err := s.Decode(&obj)
256 if err == nil {
257 t.Fatal("expected error with yaml: violate, got no error")
258 }
259 fmt.Printf("err: %s\n", err.Error())
260 if !strings.Contains(err.Error(), "yaml: line 3:") {
261 t.Fatalf("expected %q to have 'yaml: line 3:' found a tab character", err.Error())
262 }
263 }
264
265 func TestDecodeBrokenJSON(t *testing.T) {
266 s := NewYAMLOrJSONDecoder(bytes.NewReader([]byte(`{
267 "foo": {
268 "stuff": 1
269 "otherStuff": 2
270 }
271 }
272 `)), 100)
273 obj := generic{}
274 err := s.Decode(&obj)
275 if err == nil {
276 t.Fatal("expected error with json: prefix, got no error")
277 }
278 const msg = `json: offset 28: invalid character '"' after object key:value pair`
279 if msg != err.Error() {
280 t.Fatalf("expected %q, got %q", msg, err.Error())
281 }
282 }
283
284 type generic map[string]interface{}
285
286 func TestYAMLOrJSONDecoder(t *testing.T) {
287 testCases := []struct {
288 input string
289 buffer int
290 isJSON bool
291 err bool
292 out []generic
293 }{
294 {` {"1":2}{"3":4}`, 2, true, false, []generic{
295 {"1": 2},
296 {"3": 4},
297 }},
298 {" \n{}", 3, true, false, []generic{
299 {},
300 }},
301 {" \na: b", 2, false, false, []generic{
302 {"a": "b"},
303 }},
304 {" \n{\"a\": \"b\"}", 2, false, true, []generic{
305 {"a": "b"},
306 }},
307 {" \n{\"a\": \"b\"}", 3, true, false, []generic{
308 {"a": "b"},
309 }},
310 {` {"a":"b"}`, 100, true, false, []generic{
311 {"a": "b"},
312 }},
313 {"", 1, false, false, []generic{}},
314 {"foo: bar\n---\nbaz: biz", 100, false, false, []generic{
315 {"foo": "bar"},
316 {"baz": "biz"},
317 }},
318 {"---\nfoo: bar\n--- # with Comment\nbaz: biz", 100, false, false, []generic{
319 {"foo": "bar"},
320 {"baz": "biz"},
321 }},
322 {"foo: bar\n---\n", 100, false, false, []generic{
323 {"foo": "bar"},
324 }},
325 {"foo: bar\n---", 100, false, false, []generic{
326 {"foo": "bar"},
327 }},
328 {"foo: bar\n--", 100, false, true, []generic{
329 {"foo": "bar"},
330 }},
331 {"foo: bar\n-", 100, false, true, []generic{
332 {"foo": "bar"},
333 }},
334 {"foo: bar\n", 100, false, false, []generic{
335 {"foo": "bar"},
336 }},
337 }
338 for i, testCase := range testCases {
339 decoder := NewYAMLOrJSONDecoder(bytes.NewReader([]byte(testCase.input)), testCase.buffer)
340 objs := []generic{}
341
342 var err error
343 for {
344 out := make(generic)
345 err = decoder.Decode(&out)
346 if err != nil {
347 break
348 }
349 objs = append(objs, out)
350 }
351 if err != io.EOF {
352 switch {
353 case testCase.err && err == nil:
354 t.Errorf("%d: unexpected non-error", i)
355 continue
356 case !testCase.err && err != nil:
357 t.Errorf("%d: unexpected error: %v", i, err)
358 continue
359 case err != nil:
360 continue
361 }
362 }
363 switch decoder.decoder.(type) {
364 case *YAMLToJSONDecoder:
365 if testCase.isJSON {
366 t.Errorf("%d: expected JSON decoder, got YAML", i)
367 }
368 case *json.Decoder:
369 if !testCase.isJSON {
370 t.Errorf("%d: expected YAML decoder, got JSON", i)
371 }
372 }
373 if fmt.Sprintf("%#v", testCase.out) != fmt.Sprintf("%#v", objs) {
374 t.Errorf("%d: objects were not equal: \n%#v\n%#v", i, testCase.out, objs)
375 }
376 }
377 }
378
379 func TestReadSingleLongLine(t *testing.T) {
380 testReadLines(t, []int{128 * 1024})
381 }
382
383 func TestReadRandomLineLengths(t *testing.T) {
384 minLength := 100
385 maxLength := 96 * 1024
386 maxLines := 100
387
388 lineLengths := make([]int, maxLines)
389 for i := 0; i < maxLines; i++ {
390 lineLengths[i] = rand.Intn(maxLength-minLength) + minLength
391 }
392
393 testReadLines(t, lineLengths)
394 }
395
396 func testReadLines(t *testing.T, lineLengths []int) {
397 var (
398 lines [][]byte
399 inputStream []byte
400 )
401 for _, lineLength := range lineLengths {
402 inputLine := make([]byte, lineLength+1)
403 for i := 0; i < lineLength; i++ {
404 char := rand.Intn('z'-'A') + 'A'
405 inputLine[i] = byte(char)
406 }
407 inputLine[len(inputLine)-1] = '\n'
408 lines = append(lines, inputLine)
409 }
410 for _, line := range lines {
411 inputStream = append(inputStream, line...)
412 }
413
414
415 reader := bufio.NewReader(bytes.NewReader(inputStream))
416 lineReader := &LineReader{reader: reader}
417
418
419 var readLines [][]byte
420 for range lines {
421 bytes, err := lineReader.Read()
422 if err != nil && err != io.EOF {
423 t.Fatalf("failed to read lines: %v", err)
424 }
425 readLines = append(readLines, bytes)
426 }
427
428
429 for i := range lines {
430 if len(lines[i]) != len(readLines[i]) {
431 t.Fatalf("expected line length: %d, but got %d", len(lines[i]), len(readLines[i]))
432 }
433 if !reflect.DeepEqual(lines[i], readLines[i]) {
434 t.Fatalf("expected line: %v, but got %v", lines[i], readLines[i])
435 }
436 }
437 }
438
439 func TestTypedJSONOrYamlErrors(t *testing.T) {
440 s := NewYAMLOrJSONDecoder(bytes.NewReader([]byte(`{
441 "foo": {
442 "stuff": 1
443 "otherStuff": 2
444 }
445 }
446 `)), 100)
447 obj := generic{}
448 err := s.Decode(&obj)
449 if err == nil {
450 t.Fatal("expected error with json: prefix, got no error")
451 }
452 if _, ok := err.(JSONSyntaxError); !ok {
453 t.Fatalf("expected %q to be of type JSONSyntaxError", err.Error())
454 }
455
456 s = NewYAMLOrJSONDecoder(bytes.NewReader([]byte(`---
457 stuff: 1
458 test-foo: 1
459
460 ---
461 `)), 100)
462 obj = generic{}
463 err = s.Decode(&obj)
464 if err == nil {
465 t.Fatal("expected error with yaml: prefix, got no error")
466 }
467 if _, ok := err.(YAMLSyntaxError); !ok {
468 t.Fatalf("expected %q to be of type YAMLSyntaxError", err.Error())
469 }
470 }
471
472 func TestUnmarshal(t *testing.T) {
473 mapWithIntegerBytes := []byte(`replicas: 1`)
474 mapWithInteger := make(map[string]interface{})
475 if err := Unmarshal(mapWithIntegerBytes, &mapWithInteger); err != nil {
476 t.Fatalf("unexpected error unmarshaling yaml: %v", err)
477 }
478 if _, ok := mapWithInteger["replicas"].(int64); !ok {
479 t.Fatalf(`Expected number in map to be int64 but got "%T"`, mapWithInteger["replicas"])
480 }
481
482 sliceWithIntegerBytes := []byte(`- 1`)
483 var sliceWithInteger []interface{}
484 if err := Unmarshal(sliceWithIntegerBytes, &sliceWithInteger); err != nil {
485 t.Fatalf("unexpected error unmarshaling yaml: %v", err)
486 }
487 if _, ok := sliceWithInteger[0].(int64); !ok {
488 t.Fatalf(`Expected number in slice to be int64 but got "%T"`, sliceWithInteger[0])
489 }
490
491 integerBytes := []byte(`1`)
492 var integer interface{}
493 if err := Unmarshal(integerBytes, &integer); err != nil {
494 t.Fatalf("unexpected error unmarshaling yaml: %v", err)
495 }
496 if _, ok := integer.(int64); !ok {
497 t.Fatalf(`Expected number to be int64 but got "%T"`, integer)
498 }
499
500 otherTypeBytes := []byte(`123: 2`)
501 otherType := make(map[int]interface{})
502 if err := Unmarshal(otherTypeBytes, &otherType); err != nil {
503 t.Fatalf("unexpected error unmarshaling yaml: %v", err)
504 }
505 if _, ok := otherType[123].(int64); ok {
506 t.Fatalf(`Expected number not to be converted to int64`)
507 }
508 if _, ok := otherType[123].(float64); !ok {
509 t.Fatalf(`Expected number to be float64 but got "%T"`, otherType[123])
510 }
511 }
512
View as plain text