1 package toml
2
3 import (
4 "bytes"
5 "fmt"
6 "reflect"
7 "testing"
8 "text/tabwriter"
9 )
10
11 func testFlow(t *testing.T, input string, expectedFlow []token) {
12 tokens := lexToml([]byte(input))
13
14 if !reflect.DeepEqual(tokens, expectedFlow) {
15 diffFlowsColumnsFatal(t, expectedFlow, tokens)
16 }
17 }
18
19 func diffFlowsColumnsFatal(t *testing.T, expectedFlow []token, actualFlow []token) {
20 max := len(expectedFlow)
21 if len(actualFlow) > max {
22 max = len(actualFlow)
23 }
24
25 b := &bytes.Buffer{}
26 w := tabwriter.NewWriter(b, 0, 0, 1, ' ', tabwriter.Debug)
27
28 fmt.Fprintln(w, "expected\tT\tP\tactual\tT\tP\tdiff")
29
30 for i := 0; i < max; i++ {
31 expected := ""
32 expectedType := ""
33 expectedPos := ""
34 if i < len(expectedFlow) {
35 expected = fmt.Sprintf("%s", expectedFlow[i])
36 expectedType = fmt.Sprintf("%s", expectedFlow[i].typ)
37 expectedPos = expectedFlow[i].Position.String()
38 }
39 actual := ""
40 actualType := ""
41 actualPos := ""
42 if i < len(actualFlow) {
43 actual = fmt.Sprintf("%s", actualFlow[i])
44 actualType = fmt.Sprintf("%s", actualFlow[i].typ)
45 actualPos = actualFlow[i].Position.String()
46 }
47 different := ""
48 if i >= len(expectedFlow) {
49 different = "+"
50 } else if i >= len(actualFlow) {
51 different = "-"
52 } else if !reflect.DeepEqual(expectedFlow[i], actualFlow[i]) {
53 different = "x"
54 }
55 fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", expected, expectedType, expectedPos, actual, actualType, actualPos, different)
56 }
57 w.Flush()
58 t.Errorf("Different flows:\n%s", b.String())
59 }
60
61 func TestValidKeyGroup(t *testing.T) {
62 testFlow(t, "[hello world]", []token{
63 {Position{1, 1}, tokenLeftBracket, "["},
64 {Position{1, 2}, tokenKeyGroup, "hello world"},
65 {Position{1, 13}, tokenRightBracket, "]"},
66 {Position{1, 14}, tokenEOF, ""},
67 })
68 }
69
70 func TestNestedQuotedUnicodeKeyGroup(t *testing.T) {
71 testFlow(t, `[ j . "ʞ" . l . 'ɯ' ]`, []token{
72 {Position{1, 1}, tokenLeftBracket, "["},
73 {Position{1, 2}, tokenKeyGroup, ` j . "ʞ" . l . 'ɯ' `},
74 {Position{1, 21}, tokenRightBracket, "]"},
75 {Position{1, 22}, tokenEOF, ""},
76 })
77 }
78
79 func TestNestedQuotedUnicodeKeyAssign(t *testing.T) {
80 testFlow(t, ` j . "ʞ" . l . 'ɯ' = 3`, []token{
81 {Position{1, 2}, tokenKey, `j . "ʞ" . l . 'ɯ'`},
82 {Position{1, 20}, tokenEqual, "="},
83 {Position{1, 22}, tokenInteger, "3"},
84 {Position{1, 23}, tokenEOF, ""},
85 })
86 }
87
88 func TestUnclosedKeyGroup(t *testing.T) {
89 testFlow(t, "[hello world", []token{
90 {Position{1, 1}, tokenLeftBracket, "["},
91 {Position{1, 2}, tokenError, "unclosed table key"},
92 })
93 }
94
95 func TestComment(t *testing.T) {
96 testFlow(t, "# blahblah", []token{
97 {Position{1, 11}, tokenEOF, ""},
98 })
99 }
100
101 func TestKeyGroupComment(t *testing.T) {
102 testFlow(t, "[hello world] # blahblah", []token{
103 {Position{1, 1}, tokenLeftBracket, "["},
104 {Position{1, 2}, tokenKeyGroup, "hello world"},
105 {Position{1, 13}, tokenRightBracket, "]"},
106 {Position{1, 25}, tokenEOF, ""},
107 })
108 }
109
110 func TestMultipleKeyGroupsComment(t *testing.T) {
111 testFlow(t, "[hello world] # blahblah\n[test]", []token{
112 {Position{1, 1}, tokenLeftBracket, "["},
113 {Position{1, 2}, tokenKeyGroup, "hello world"},
114 {Position{1, 13}, tokenRightBracket, "]"},
115 {Position{2, 1}, tokenLeftBracket, "["},
116 {Position{2, 2}, tokenKeyGroup, "test"},
117 {Position{2, 6}, tokenRightBracket, "]"},
118 {Position{2, 7}, tokenEOF, ""},
119 })
120 }
121
122 func TestSimpleWindowsCRLF(t *testing.T) {
123 testFlow(t, "a=4\r\nb=2", []token{
124 {Position{1, 1}, tokenKey, "a"},
125 {Position{1, 2}, tokenEqual, "="},
126 {Position{1, 3}, tokenInteger, "4"},
127 {Position{2, 1}, tokenKey, "b"},
128 {Position{2, 2}, tokenEqual, "="},
129 {Position{2, 3}, tokenInteger, "2"},
130 {Position{2, 4}, tokenEOF, ""},
131 })
132 }
133
134 func TestBasicKey(t *testing.T) {
135 testFlow(t, "hello", []token{
136 {Position{1, 1}, tokenKey, "hello"},
137 {Position{1, 6}, tokenEOF, ""},
138 })
139 }
140
141 func TestBasicKeyWithUnderscore(t *testing.T) {
142 testFlow(t, "hello_hello", []token{
143 {Position{1, 1}, tokenKey, "hello_hello"},
144 {Position{1, 12}, tokenEOF, ""},
145 })
146 }
147
148 func TestBasicKeyWithDash(t *testing.T) {
149 testFlow(t, "hello-world", []token{
150 {Position{1, 1}, tokenKey, "hello-world"},
151 {Position{1, 12}, tokenEOF, ""},
152 })
153 }
154
155 func TestBasicKeyWithUppercaseMix(t *testing.T) {
156 testFlow(t, "helloHELLOHello", []token{
157 {Position{1, 1}, tokenKey, "helloHELLOHello"},
158 {Position{1, 16}, tokenEOF, ""},
159 })
160 }
161
162 func TestBasicKeyWithInternationalCharacters(t *testing.T) {
163 testFlow(t, "'héllÖ'", []token{
164 {Position{1, 1}, tokenKey, "'héllÖ'"},
165 {Position{1, 8}, tokenEOF, ""},
166 })
167 }
168
169 func TestBasicKeyAndEqual(t *testing.T) {
170 testFlow(t, "hello =", []token{
171 {Position{1, 1}, tokenKey, "hello"},
172 {Position{1, 7}, tokenEqual, "="},
173 {Position{1, 8}, tokenEOF, ""},
174 })
175 }
176
177 func TestKeyWithSharpAndEqual(t *testing.T) {
178 testFlow(t, "key#name = 5", []token{
179 {Position{1, 1}, tokenError, "keys cannot contain # character"},
180 })
181 }
182
183 func TestKeyWithSymbolsAndEqual(t *testing.T) {
184 testFlow(t, "~!@$^&*()_+-`1234567890[]\\|/?><.,;:' = 5", []token{
185 {Position{1, 1}, tokenError, "keys cannot contain ~ character"},
186 })
187 }
188
189 func TestKeyEqualStringEscape(t *testing.T) {
190 testFlow(t, `foo = "hello\""`, []token{
191 {Position{1, 1}, tokenKey, "foo"},
192 {Position{1, 5}, tokenEqual, "="},
193 {Position{1, 8}, tokenString, "hello\""},
194 {Position{1, 16}, tokenEOF, ""},
195 })
196 }
197
198 func TestKeyEqualStringUnfinished(t *testing.T) {
199 testFlow(t, `foo = "bar`, []token{
200 {Position{1, 1}, tokenKey, "foo"},
201 {Position{1, 5}, tokenEqual, "="},
202 {Position{1, 8}, tokenError, "unclosed string"},
203 })
204 }
205
206 func TestKeyEqualString(t *testing.T) {
207 testFlow(t, `foo = "bar"`, []token{
208 {Position{1, 1}, tokenKey, "foo"},
209 {Position{1, 5}, tokenEqual, "="},
210 {Position{1, 8}, tokenString, "bar"},
211 {Position{1, 12}, tokenEOF, ""},
212 })
213 }
214
215 func TestKeyEqualTrue(t *testing.T) {
216 testFlow(t, "foo = true", []token{
217 {Position{1, 1}, tokenKey, "foo"},
218 {Position{1, 5}, tokenEqual, "="},
219 {Position{1, 7}, tokenTrue, "true"},
220 {Position{1, 11}, tokenEOF, ""},
221 })
222 }
223
224 func TestKeyEqualFalse(t *testing.T) {
225 testFlow(t, "foo = false", []token{
226 {Position{1, 1}, tokenKey, "foo"},
227 {Position{1, 5}, tokenEqual, "="},
228 {Position{1, 7}, tokenFalse, "false"},
229 {Position{1, 12}, tokenEOF, ""},
230 })
231 }
232
233 func TestArrayNestedString(t *testing.T) {
234 testFlow(t, `a = [ ["hello", "world"] ]`, []token{
235 {Position{1, 1}, tokenKey, "a"},
236 {Position{1, 3}, tokenEqual, "="},
237 {Position{1, 5}, tokenLeftBracket, "["},
238 {Position{1, 7}, tokenLeftBracket, "["},
239 {Position{1, 9}, tokenString, "hello"},
240 {Position{1, 15}, tokenComma, ","},
241 {Position{1, 18}, tokenString, "world"},
242 {Position{1, 24}, tokenRightBracket, "]"},
243 {Position{1, 26}, tokenRightBracket, "]"},
244 {Position{1, 27}, tokenEOF, ""},
245 })
246 }
247
248 func TestArrayNestedInts(t *testing.T) {
249 testFlow(t, "a = [ [42, 21], [10] ]", []token{
250 {Position{1, 1}, tokenKey, "a"},
251 {Position{1, 3}, tokenEqual, "="},
252 {Position{1, 5}, tokenLeftBracket, "["},
253 {Position{1, 7}, tokenLeftBracket, "["},
254 {Position{1, 8}, tokenInteger, "42"},
255 {Position{1, 10}, tokenComma, ","},
256 {Position{1, 12}, tokenInteger, "21"},
257 {Position{1, 14}, tokenRightBracket, "]"},
258 {Position{1, 15}, tokenComma, ","},
259 {Position{1, 17}, tokenLeftBracket, "["},
260 {Position{1, 18}, tokenInteger, "10"},
261 {Position{1, 20}, tokenRightBracket, "]"},
262 {Position{1, 22}, tokenRightBracket, "]"},
263 {Position{1, 23}, tokenEOF, ""},
264 })
265 }
266
267 func TestArrayInts(t *testing.T) {
268 testFlow(t, "a = [ 42, 21, 10, ]", []token{
269 {Position{1, 1}, tokenKey, "a"},
270 {Position{1, 3}, tokenEqual, "="},
271 {Position{1, 5}, tokenLeftBracket, "["},
272 {Position{1, 7}, tokenInteger, "42"},
273 {Position{1, 9}, tokenComma, ","},
274 {Position{1, 11}, tokenInteger, "21"},
275 {Position{1, 13}, tokenComma, ","},
276 {Position{1, 15}, tokenInteger, "10"},
277 {Position{1, 17}, tokenComma, ","},
278 {Position{1, 19}, tokenRightBracket, "]"},
279 {Position{1, 20}, tokenEOF, ""},
280 })
281 }
282
283 func TestMultilineArrayComments(t *testing.T) {
284 testFlow(t, "a = [1, # wow\n2, # such items\n3, # so array\n]", []token{
285 {Position{1, 1}, tokenKey, "a"},
286 {Position{1, 3}, tokenEqual, "="},
287 {Position{1, 5}, tokenLeftBracket, "["},
288 {Position{1, 6}, tokenInteger, "1"},
289 {Position{1, 7}, tokenComma, ","},
290 {Position{2, 1}, tokenInteger, "2"},
291 {Position{2, 2}, tokenComma, ","},
292 {Position{3, 1}, tokenInteger, "3"},
293 {Position{3, 2}, tokenComma, ","},
294 {Position{4, 1}, tokenRightBracket, "]"},
295 {Position{4, 2}, tokenEOF, ""},
296 })
297 }
298
299 func TestNestedArraysComment(t *testing.T) {
300 toml := `
301 someArray = [
302 # does not work
303 ["entry1"]
304 ]`
305 testFlow(t, toml, []token{
306 {Position{2, 1}, tokenKey, "someArray"},
307 {Position{2, 11}, tokenEqual, "="},
308 {Position{2, 13}, tokenLeftBracket, "["},
309 {Position{4, 1}, tokenLeftBracket, "["},
310 {Position{4, 3}, tokenString, "entry1"},
311 {Position{4, 10}, tokenRightBracket, "]"},
312 {Position{5, 1}, tokenRightBracket, "]"},
313 {Position{5, 2}, tokenEOF, ""},
314 })
315 }
316
317 func TestKeyEqualArrayBools(t *testing.T) {
318 testFlow(t, "foo = [true, false, true]", []token{
319 {Position{1, 1}, tokenKey, "foo"},
320 {Position{1, 5}, tokenEqual, "="},
321 {Position{1, 7}, tokenLeftBracket, "["},
322 {Position{1, 8}, tokenTrue, "true"},
323 {Position{1, 12}, tokenComma, ","},
324 {Position{1, 14}, tokenFalse, "false"},
325 {Position{1, 19}, tokenComma, ","},
326 {Position{1, 21}, tokenTrue, "true"},
327 {Position{1, 25}, tokenRightBracket, "]"},
328 {Position{1, 26}, tokenEOF, ""},
329 })
330 }
331
332 func TestKeyEqualArrayBoolsWithComments(t *testing.T) {
333 testFlow(t, "foo = [true, false, true] # YEAH", []token{
334 {Position{1, 1}, tokenKey, "foo"},
335 {Position{1, 5}, tokenEqual, "="},
336 {Position{1, 7}, tokenLeftBracket, "["},
337 {Position{1, 8}, tokenTrue, "true"},
338 {Position{1, 12}, tokenComma, ","},
339 {Position{1, 14}, tokenFalse, "false"},
340 {Position{1, 19}, tokenComma, ","},
341 {Position{1, 21}, tokenTrue, "true"},
342 {Position{1, 25}, tokenRightBracket, "]"},
343 {Position{1, 33}, tokenEOF, ""},
344 })
345 }
346
347 func TestKeyEqualDate(t *testing.T) {
348 t.Run("local date time", func(t *testing.T) {
349 testFlow(t, "foo = 1979-05-27T07:32:00", []token{
350 {Position{1, 1}, tokenKey, "foo"},
351 {Position{1, 5}, tokenEqual, "="},
352 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
353 {Position{1, 18}, tokenLocalTime, "07:32:00"},
354 {Position{1, 26}, tokenEOF, ""},
355 })
356 })
357
358 t.Run("local date time space", func(t *testing.T) {
359 testFlow(t, "foo = 1979-05-27 07:32:00", []token{
360 {Position{1, 1}, tokenKey, "foo"},
361 {Position{1, 5}, tokenEqual, "="},
362 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
363 {Position{1, 18}, tokenLocalTime, "07:32:00"},
364 {Position{1, 26}, tokenEOF, ""},
365 })
366 })
367
368 t.Run("local date time fraction", func(t *testing.T) {
369 testFlow(t, "foo = 1979-05-27T00:32:00.999999", []token{
370 {Position{1, 1}, tokenKey, "foo"},
371 {Position{1, 5}, tokenEqual, "="},
372 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
373 {Position{1, 18}, tokenLocalTime, "00:32:00.999999"},
374 {Position{1, 33}, tokenEOF, ""},
375 })
376 })
377
378 t.Run("local date time fraction space", func(t *testing.T) {
379 testFlow(t, "foo = 1979-05-27 00:32:00.999999", []token{
380 {Position{1, 1}, tokenKey, "foo"},
381 {Position{1, 5}, tokenEqual, "="},
382 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
383 {Position{1, 18}, tokenLocalTime, "00:32:00.999999"},
384 {Position{1, 33}, tokenEOF, ""},
385 })
386 })
387
388 t.Run("offset date-time utc", func(t *testing.T) {
389 testFlow(t, "foo = 1979-05-27T07:32:00Z", []token{
390 {Position{1, 1}, tokenKey, "foo"},
391 {Position{1, 5}, tokenEqual, "="},
392 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
393 {Position{1, 18}, tokenLocalTime, "07:32:00"},
394 {Position{1, 26}, tokenTimeOffset, "Z"},
395 {Position{1, 27}, tokenEOF, ""},
396 })
397 })
398
399 t.Run("offset date-time -07:00", func(t *testing.T) {
400 testFlow(t, "foo = 1979-05-27T00:32:00-07:00", []token{
401 {Position{1, 1}, tokenKey, "foo"},
402 {Position{1, 5}, tokenEqual, "="},
403 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
404 {Position{1, 18}, tokenLocalTime, "00:32:00"},
405 {Position{1, 26}, tokenTimeOffset, "-07:00"},
406 {Position{1, 32}, tokenEOF, ""},
407 })
408 })
409
410 t.Run("offset date-time fractions -07:00", func(t *testing.T) {
411 testFlow(t, "foo = 1979-05-27T00:32:00.999999-07:00", []token{
412 {Position{1, 1}, tokenKey, "foo"},
413 {Position{1, 5}, tokenEqual, "="},
414 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
415 {Position{1, 18}, tokenLocalTime, "00:32:00.999999"},
416 {Position{1, 33}, tokenTimeOffset, "-07:00"},
417 {Position{1, 39}, tokenEOF, ""},
418 })
419 })
420
421 t.Run("offset date-time space separated utc", func(t *testing.T) {
422 testFlow(t, "foo = 1979-05-27 07:32:00Z", []token{
423 {Position{1, 1}, tokenKey, "foo"},
424 {Position{1, 5}, tokenEqual, "="},
425 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
426 {Position{1, 18}, tokenLocalTime, "07:32:00"},
427 {Position{1, 26}, tokenTimeOffset, "Z"},
428 {Position{1, 27}, tokenEOF, ""},
429 })
430 })
431
432 t.Run("offset date-time space separated offset", func(t *testing.T) {
433 testFlow(t, "foo = 1979-05-27 00:32:00-07:00", []token{
434 {Position{1, 1}, tokenKey, "foo"},
435 {Position{1, 5}, tokenEqual, "="},
436 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
437 {Position{1, 18}, tokenLocalTime, "00:32:00"},
438 {Position{1, 26}, tokenTimeOffset, "-07:00"},
439 {Position{1, 32}, tokenEOF, ""},
440 })
441 })
442
443 t.Run("offset date-time space separated fraction offset", func(t *testing.T) {
444 testFlow(t, "foo = 1979-05-27 00:32:00.999999-07:00", []token{
445 {Position{1, 1}, tokenKey, "foo"},
446 {Position{1, 5}, tokenEqual, "="},
447 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
448 {Position{1, 18}, tokenLocalTime, "00:32:00.999999"},
449 {Position{1, 33}, tokenTimeOffset, "-07:00"},
450 {Position{1, 39}, tokenEOF, ""},
451 })
452 })
453
454 t.Run("local date", func(t *testing.T) {
455 testFlow(t, "foo = 1979-05-27", []token{
456 {Position{1, 1}, tokenKey, "foo"},
457 {Position{1, 5}, tokenEqual, "="},
458 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
459 {Position{1, 17}, tokenEOF, ""},
460 })
461 })
462
463 t.Run("local time", func(t *testing.T) {
464 testFlow(t, "foo = 07:32:00", []token{
465 {Position{1, 1}, tokenKey, "foo"},
466 {Position{1, 5}, tokenEqual, "="},
467 {Position{1, 7}, tokenLocalTime, "07:32:00"},
468 {Position{1, 15}, tokenEOF, ""},
469 })
470 })
471
472 t.Run("local time fraction", func(t *testing.T) {
473 testFlow(t, "foo = 00:32:00.999999", []token{
474 {Position{1, 1}, tokenKey, "foo"},
475 {Position{1, 5}, tokenEqual, "="},
476 {Position{1, 7}, tokenLocalTime, "00:32:00.999999"},
477 {Position{1, 22}, tokenEOF, ""},
478 })
479 })
480
481 t.Run("local time invalid minute digit", func(t *testing.T) {
482 testFlow(t, "foo = 00:3x:00.999999", []token{
483 {Position{1, 1}, tokenKey, "foo"},
484 {Position{1, 5}, tokenEqual, "="},
485 {Position{1, 7}, tokenError, "invalid minute digit in time: x"},
486 })
487 })
488
489 t.Run("local time invalid minute/second digit", func(t *testing.T) {
490 testFlow(t, "foo = 00:30x00.999999", []token{
491 {Position{1, 1}, tokenKey, "foo"},
492 {Position{1, 5}, tokenEqual, "="},
493 {Position{1, 7}, tokenError, "time minute/second separator should be :, not x"},
494 })
495 })
496
497 t.Run("local time invalid second digit", func(t *testing.T) {
498 testFlow(t, "foo = 00:30:x0.999999", []token{
499 {Position{1, 1}, tokenKey, "foo"},
500 {Position{1, 5}, tokenEqual, "="},
501 {Position{1, 7}, tokenError, "invalid second digit in time: x"},
502 })
503 })
504
505 t.Run("local time invalid second digit", func(t *testing.T) {
506 testFlow(t, "foo = 00:30:00.F", []token{
507 {Position{1, 1}, tokenKey, "foo"},
508 {Position{1, 5}, tokenEqual, "="},
509 {Position{1, 7}, tokenError, "expected at least one digit in time's fraction, not F"},
510 })
511 })
512
513 t.Run("local date-time invalid minute digit", func(t *testing.T) {
514 testFlow(t, "foo = 1979-05-27 00:3x:00.999999", []token{
515 {Position{1, 1}, tokenKey, "foo"},
516 {Position{1, 5}, tokenEqual, "="},
517 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
518 {Position{1, 18}, tokenError, "invalid minute digit in time: x"},
519 })
520 })
521
522 t.Run("local date-time invalid hour digit", func(t *testing.T) {
523 testFlow(t, "foo = 1979-05-27T0x:30:00.999999", []token{
524 {Position{1, 1}, tokenKey, "foo"},
525 {Position{1, 5}, tokenEqual, "="},
526 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
527 {Position{1, 18}, tokenError, "invalid hour digit in time: x"},
528 })
529 })
530
531 t.Run("local date-time invalid hour digit", func(t *testing.T) {
532 testFlow(t, "foo = 1979-05-27T00x30:00.999999", []token{
533 {Position{1, 1}, tokenKey, "foo"},
534 {Position{1, 5}, tokenEqual, "="},
535 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
536 {Position{1, 18}, tokenError, "time hour/minute separator should be :, not x"},
537 })
538 })
539
540 t.Run("local date-time invalid minute/second digit", func(t *testing.T) {
541 testFlow(t, "foo = 1979-05-27 00:30x00.999999", []token{
542 {Position{1, 1}, tokenKey, "foo"},
543 {Position{1, 5}, tokenEqual, "="},
544 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
545 {Position{1, 18}, tokenError, "time minute/second separator should be :, not x"},
546 })
547 })
548
549 t.Run("local date-time invalid second digit", func(t *testing.T) {
550 testFlow(t, "foo = 1979-05-27 00:30:x0.999999", []token{
551 {Position{1, 1}, tokenKey, "foo"},
552 {Position{1, 5}, tokenEqual, "="},
553 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
554 {Position{1, 18}, tokenError, "invalid second digit in time: x"},
555 })
556 })
557
558 t.Run("local date-time invalid fraction", func(t *testing.T) {
559 testFlow(t, "foo = 1979-05-27 00:30:00.F", []token{
560 {Position{1, 1}, tokenKey, "foo"},
561 {Position{1, 5}, tokenEqual, "="},
562 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
563 {Position{1, 18}, tokenError, "expected at least one digit in time's fraction, not F"},
564 })
565 })
566
567 t.Run("local date-time invalid month-date separator", func(t *testing.T) {
568 testFlow(t, "foo = 1979-05X27 00:30:00.F", []token{
569 {Position{1, 1}, tokenKey, "foo"},
570 {Position{1, 5}, tokenEqual, "="},
571 {Position{1, 7}, tokenError, "expected - to separate month of a date, not X"},
572 })
573 })
574
575 t.Run("local date-time extra whitespace", func(t *testing.T) {
576 testFlow(t, "foo = 1979-05-27 ", []token{
577 {Position{1, 1}, tokenKey, "foo"},
578 {Position{1, 5}, tokenEqual, "="},
579 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
580 {Position{1, 19}, tokenEOF, ""},
581 })
582 })
583
584 t.Run("local date-time extra whitespace", func(t *testing.T) {
585 testFlow(t, "foo = 1979-05-27 ", []token{
586 {Position{1, 1}, tokenKey, "foo"},
587 {Position{1, 5}, tokenEqual, "="},
588 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
589 {Position{1, 22}, tokenEOF, ""},
590 })
591 })
592
593 t.Run("offset date-time space separated offset", func(t *testing.T) {
594 testFlow(t, "foo = 1979-05-27 00:32:00-0x:00", []token{
595 {Position{1, 1}, tokenKey, "foo"},
596 {Position{1, 5}, tokenEqual, "="},
597 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
598 {Position{1, 18}, tokenLocalTime, "00:32:00"},
599 {Position{1, 26}, tokenError, "invalid hour digit in time offset: x"},
600 })
601 })
602
603 t.Run("offset date-time space separated offset", func(t *testing.T) {
604 testFlow(t, "foo = 1979-05-27 00:32:00-07x00", []token{
605 {Position{1, 1}, tokenKey, "foo"},
606 {Position{1, 5}, tokenEqual, "="},
607 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
608 {Position{1, 18}, tokenLocalTime, "00:32:00"},
609 {Position{1, 26}, tokenError, "time offset hour/minute separator should be :, not x"},
610 })
611 })
612
613 t.Run("offset date-time space separated offset", func(t *testing.T) {
614 testFlow(t, "foo = 1979-05-27 00:32:00-07:x0", []token{
615 {Position{1, 1}, tokenKey, "foo"},
616 {Position{1, 5}, tokenEqual, "="},
617 {Position{1, 7}, tokenLocalDate, "1979-05-27"},
618 {Position{1, 18}, tokenLocalTime, "00:32:00"},
619 {Position{1, 26}, tokenError, "invalid minute digit in time offset: x"},
620 })
621 })
622 }
623
624 func TestFloatEndingWithDot(t *testing.T) {
625 testFlow(t, "foo = 42.", []token{
626 {Position{1, 1}, tokenKey, "foo"},
627 {Position{1, 5}, tokenEqual, "="},
628 {Position{1, 7}, tokenError, "float cannot end with a dot"},
629 })
630 }
631
632 func TestFloatWithTwoDots(t *testing.T) {
633 testFlow(t, "foo = 4.2.", []token{
634 {Position{1, 1}, tokenKey, "foo"},
635 {Position{1, 5}, tokenEqual, "="},
636 {Position{1, 7}, tokenError, "cannot have two dots in one float"},
637 })
638 }
639
640 func TestFloatWithExponent1(t *testing.T) {
641 testFlow(t, "a = 5e+22", []token{
642 {Position{1, 1}, tokenKey, "a"},
643 {Position{1, 3}, tokenEqual, "="},
644 {Position{1, 5}, tokenFloat, "5e+22"},
645 {Position{1, 10}, tokenEOF, ""},
646 })
647 }
648
649 func TestFloatWithExponent2(t *testing.T) {
650 testFlow(t, "a = 5E+22", []token{
651 {Position{1, 1}, tokenKey, "a"},
652 {Position{1, 3}, tokenEqual, "="},
653 {Position{1, 5}, tokenFloat, "5E+22"},
654 {Position{1, 10}, tokenEOF, ""},
655 })
656 }
657
658 func TestFloatWithExponent3(t *testing.T) {
659 testFlow(t, "a = -5e+22", []token{
660 {Position{1, 1}, tokenKey, "a"},
661 {Position{1, 3}, tokenEqual, "="},
662 {Position{1, 5}, tokenFloat, "-5e+22"},
663 {Position{1, 11}, tokenEOF, ""},
664 })
665 }
666
667 func TestFloatWithExponent4(t *testing.T) {
668 testFlow(t, "a = -5e-22", []token{
669 {Position{1, 1}, tokenKey, "a"},
670 {Position{1, 3}, tokenEqual, "="},
671 {Position{1, 5}, tokenFloat, "-5e-22"},
672 {Position{1, 11}, tokenEOF, ""},
673 })
674 }
675
676 func TestFloatWithExponent5(t *testing.T) {
677 testFlow(t, "a = 6.626e-34", []token{
678 {Position{1, 1}, tokenKey, "a"},
679 {Position{1, 3}, tokenEqual, "="},
680 {Position{1, 5}, tokenFloat, "6.626e-34"},
681 {Position{1, 14}, tokenEOF, ""},
682 })
683 }
684
685 func TestInvalidEsquapeSequence(t *testing.T) {
686 testFlow(t, `foo = "\x"`, []token{
687 {Position{1, 1}, tokenKey, "foo"},
688 {Position{1, 5}, tokenEqual, "="},
689 {Position{1, 8}, tokenError, "invalid escape sequence: \\x"},
690 })
691 }
692
693 func TestNestedArrays(t *testing.T) {
694 testFlow(t, "foo = [[[]]]", []token{
695 {Position{1, 1}, tokenKey, "foo"},
696 {Position{1, 5}, tokenEqual, "="},
697 {Position{1, 7}, tokenLeftBracket, "["},
698 {Position{1, 8}, tokenLeftBracket, "["},
699 {Position{1, 9}, tokenLeftBracket, "["},
700 {Position{1, 10}, tokenRightBracket, "]"},
701 {Position{1, 11}, tokenRightBracket, "]"},
702 {Position{1, 12}, tokenRightBracket, "]"},
703 {Position{1, 13}, tokenEOF, ""},
704 })
705 }
706
707 func TestKeyEqualNumber(t *testing.T) {
708 testFlow(t, "foo = 42", []token{
709 {Position{1, 1}, tokenKey, "foo"},
710 {Position{1, 5}, tokenEqual, "="},
711 {Position{1, 7}, tokenInteger, "42"},
712 {Position{1, 9}, tokenEOF, ""},
713 })
714
715 testFlow(t, "foo = +42", []token{
716 {Position{1, 1}, tokenKey, "foo"},
717 {Position{1, 5}, tokenEqual, "="},
718 {Position{1, 7}, tokenInteger, "+42"},
719 {Position{1, 10}, tokenEOF, ""},
720 })
721
722 testFlow(t, "foo = -42", []token{
723 {Position{1, 1}, tokenKey, "foo"},
724 {Position{1, 5}, tokenEqual, "="},
725 {Position{1, 7}, tokenInteger, "-42"},
726 {Position{1, 10}, tokenEOF, ""},
727 })
728
729 testFlow(t, "foo = 4.2", []token{
730 {Position{1, 1}, tokenKey, "foo"},
731 {Position{1, 5}, tokenEqual, "="},
732 {Position{1, 7}, tokenFloat, "4.2"},
733 {Position{1, 10}, tokenEOF, ""},
734 })
735
736 testFlow(t, "foo = +4.2", []token{
737 {Position{1, 1}, tokenKey, "foo"},
738 {Position{1, 5}, tokenEqual, "="},
739 {Position{1, 7}, tokenFloat, "+4.2"},
740 {Position{1, 11}, tokenEOF, ""},
741 })
742
743 testFlow(t, "foo = -4.2", []token{
744 {Position{1, 1}, tokenKey, "foo"},
745 {Position{1, 5}, tokenEqual, "="},
746 {Position{1, 7}, tokenFloat, "-4.2"},
747 {Position{1, 11}, tokenEOF, ""},
748 })
749
750 testFlow(t, "foo = 1_000", []token{
751 {Position{1, 1}, tokenKey, "foo"},
752 {Position{1, 5}, tokenEqual, "="},
753 {Position{1, 7}, tokenInteger, "1_000"},
754 {Position{1, 12}, tokenEOF, ""},
755 })
756
757 testFlow(t, "foo = 5_349_221", []token{
758 {Position{1, 1}, tokenKey, "foo"},
759 {Position{1, 5}, tokenEqual, "="},
760 {Position{1, 7}, tokenInteger, "5_349_221"},
761 {Position{1, 16}, tokenEOF, ""},
762 })
763
764 testFlow(t, "foo = 1_2_3_4_5", []token{
765 {Position{1, 1}, tokenKey, "foo"},
766 {Position{1, 5}, tokenEqual, "="},
767 {Position{1, 7}, tokenInteger, "1_2_3_4_5"},
768 {Position{1, 16}, tokenEOF, ""},
769 })
770
771 testFlow(t, "flt8 = 9_224_617.445_991_228_313", []token{
772 {Position{1, 1}, tokenKey, "flt8"},
773 {Position{1, 6}, tokenEqual, "="},
774 {Position{1, 8}, tokenFloat, "9_224_617.445_991_228_313"},
775 {Position{1, 33}, tokenEOF, ""},
776 })
777
778 testFlow(t, "foo = +", []token{
779 {Position{1, 1}, tokenKey, "foo"},
780 {Position{1, 5}, tokenEqual, "="},
781 {Position{1, 7}, tokenError, "no digit in that number"},
782 })
783 }
784
785 func TestMultiline(t *testing.T) {
786 testFlow(t, "foo = 42\nbar=21", []token{
787 {Position{1, 1}, tokenKey, "foo"},
788 {Position{1, 5}, tokenEqual, "="},
789 {Position{1, 7}, tokenInteger, "42"},
790 {Position{2, 1}, tokenKey, "bar"},
791 {Position{2, 4}, tokenEqual, "="},
792 {Position{2, 5}, tokenInteger, "21"},
793 {Position{2, 7}, tokenEOF, ""},
794 })
795 }
796
797 func TestKeyEqualStringUnicodeEscape(t *testing.T) {
798 testFlow(t, `foo = "hello \u2665"`, []token{
799 {Position{1, 1}, tokenKey, "foo"},
800 {Position{1, 5}, tokenEqual, "="},
801 {Position{1, 8}, tokenString, "hello ♥"},
802 {Position{1, 21}, tokenEOF, ""},
803 })
804 testFlow(t, `foo = "hello \U000003B4"`, []token{
805 {Position{1, 1}, tokenKey, "foo"},
806 {Position{1, 5}, tokenEqual, "="},
807 {Position{1, 8}, tokenString, "hello δ"},
808 {Position{1, 25}, tokenEOF, ""},
809 })
810 testFlow(t, `foo = "\uabcd"`, []token{
811 {Position{1, 1}, tokenKey, "foo"},
812 {Position{1, 5}, tokenEqual, "="},
813 {Position{1, 8}, tokenString, "\uabcd"},
814 {Position{1, 15}, tokenEOF, ""},
815 })
816 testFlow(t, `foo = "\uABCD"`, []token{
817 {Position{1, 1}, tokenKey, "foo"},
818 {Position{1, 5}, tokenEqual, "="},
819 {Position{1, 8}, tokenString, "\uABCD"},
820 {Position{1, 15}, tokenEOF, ""},
821 })
822 testFlow(t, `foo = "\U000bcdef"`, []token{
823 {Position{1, 1}, tokenKey, "foo"},
824 {Position{1, 5}, tokenEqual, "="},
825 {Position{1, 8}, tokenString, "\U000bcdef"},
826 {Position{1, 19}, tokenEOF, ""},
827 })
828 testFlow(t, `foo = "\U000BCDEF"`, []token{
829 {Position{1, 1}, tokenKey, "foo"},
830 {Position{1, 5}, tokenEqual, "="},
831 {Position{1, 8}, tokenString, "\U000BCDEF"},
832 {Position{1, 19}, tokenEOF, ""},
833 })
834 testFlow(t, `foo = "\u2"`, []token{
835 {Position{1, 1}, tokenKey, "foo"},
836 {Position{1, 5}, tokenEqual, "="},
837 {Position{1, 8}, tokenError, "unfinished unicode escape"},
838 })
839 testFlow(t, `foo = "\U2"`, []token{
840 {Position{1, 1}, tokenKey, "foo"},
841 {Position{1, 5}, tokenEqual, "="},
842 {Position{1, 8}, tokenError, "unfinished unicode escape"},
843 })
844 }
845
846 func TestKeyEqualStringNoEscape(t *testing.T) {
847 testFlow(t, "foo = \"hello \u0002\"", []token{
848 {Position{1, 1}, tokenKey, "foo"},
849 {Position{1, 5}, tokenEqual, "="},
850 {Position{1, 8}, tokenError, "unescaped control character U+0002"},
851 })
852 testFlow(t, "foo = \"hello \u001F\"", []token{
853 {Position{1, 1}, tokenKey, "foo"},
854 {Position{1, 5}, tokenEqual, "="},
855 {Position{1, 8}, tokenError, "unescaped control character U+001F"},
856 })
857 }
858
859 func TestLiteralString(t *testing.T) {
860 testFlow(t, `foo = 'C:\Users\nodejs\templates'`, []token{
861 {Position{1, 1}, tokenKey, "foo"},
862 {Position{1, 5}, tokenEqual, "="},
863 {Position{1, 8}, tokenString, `C:\Users\nodejs\templates`},
864 {Position{1, 34}, tokenEOF, ""},
865 })
866 testFlow(t, `foo = '\\ServerX\admin$\system32\'`, []token{
867 {Position{1, 1}, tokenKey, "foo"},
868 {Position{1, 5}, tokenEqual, "="},
869 {Position{1, 8}, tokenString, `\\ServerX\admin$\system32\`},
870 {Position{1, 35}, tokenEOF, ""},
871 })
872 testFlow(t, `foo = 'Tom "Dubs" Preston-Werner'`, []token{
873 {Position{1, 1}, tokenKey, "foo"},
874 {Position{1, 5}, tokenEqual, "="},
875 {Position{1, 8}, tokenString, `Tom "Dubs" Preston-Werner`},
876 {Position{1, 34}, tokenEOF, ""},
877 })
878 testFlow(t, `foo = '<\i\c*\s*>'`, []token{
879 {Position{1, 1}, tokenKey, "foo"},
880 {Position{1, 5}, tokenEqual, "="},
881 {Position{1, 8}, tokenString, `<\i\c*\s*>`},
882 {Position{1, 19}, tokenEOF, ""},
883 })
884 testFlow(t, `foo = 'C:\Users\nodejs\unfinis`, []token{
885 {Position{1, 1}, tokenKey, "foo"},
886 {Position{1, 5}, tokenEqual, "="},
887 {Position{1, 8}, tokenError, "unclosed string"},
888 })
889 }
890
891 func TestMultilineLiteralString(t *testing.T) {
892 testFlow(t, `foo = '''hello 'literal' world'''`, []token{
893 {Position{1, 1}, tokenKey, "foo"},
894 {Position{1, 5}, tokenEqual, "="},
895 {Position{1, 10}, tokenString, `hello 'literal' world`},
896 {Position{1, 34}, tokenEOF, ""},
897 })
898
899 testFlow(t, "foo = '''\nhello\n'literal'\nworld'''", []token{
900 {Position{1, 1}, tokenKey, "foo"},
901 {Position{1, 5}, tokenEqual, "="},
902 {Position{2, 1}, tokenString, "hello\n'literal'\nworld"},
903 {Position{4, 9}, tokenEOF, ""},
904 })
905 testFlow(t, "foo = '''\r\nhello\r\n'literal'\r\nworld'''", []token{
906 {Position{1, 1}, tokenKey, "foo"},
907 {Position{1, 5}, tokenEqual, "="},
908 {Position{2, 1}, tokenString, "hello\r\n'literal'\r\nworld"},
909 {Position{4, 9}, tokenEOF, ""},
910 })
911 }
912
913 func TestMultilineString(t *testing.T) {
914 testFlow(t, `foo = """hello "literal" world"""`, []token{
915 {Position{1, 1}, tokenKey, "foo"},
916 {Position{1, 5}, tokenEqual, "="},
917 {Position{1, 10}, tokenString, `hello "literal" world`},
918 {Position{1, 34}, tokenEOF, ""},
919 })
920
921 testFlow(t, "foo = \"\"\"\r\nhello\\\r\n\"literal\"\\\nworld\"\"\"", []token{
922 {Position{1, 1}, tokenKey, "foo"},
923 {Position{1, 5}, tokenEqual, "="},
924 {Position{2, 1}, tokenString, "hello\"literal\"world"},
925 {Position{4, 9}, tokenEOF, ""},
926 })
927
928 testFlow(t, "foo = \"\"\"\\\n \\\n \\\n hello\\\nmultiline\\\nworld\"\"\"", []token{
929 {Position{1, 1}, tokenKey, "foo"},
930 {Position{1, 5}, tokenEqual, "="},
931 {Position{1, 10}, tokenString, "hellomultilineworld"},
932 {Position{6, 9}, tokenEOF, ""},
933 })
934
935 testFlow(t, `foo = """hello world"""`, []token{
936 {Position{1, 1}, tokenKey, "foo"},
937 {Position{1, 5}, tokenEqual, "="},
938 {Position{1, 10}, tokenString, "hello\tworld"},
939 {Position{1, 24}, tokenEOF, ""},
940 })
941
942 testFlow(t, "key2 = \"\"\"\nThe quick brown \\\n\n\n fox jumps over \\\n the lazy dog.\"\"\"", []token{
943 {Position{1, 1}, tokenKey, "key2"},
944 {Position{1, 6}, tokenEqual, "="},
945 {Position{2, 1}, tokenString, "The quick brown fox jumps over the lazy dog."},
946 {Position{6, 21}, tokenEOF, ""},
947 })
948
949 testFlow(t, "key2 = \"\"\"\\\n The quick brown \\\n fox jumps over \\\n the lazy dog.\\\n \"\"\"", []token{
950 {Position{1, 1}, tokenKey, "key2"},
951 {Position{1, 6}, tokenEqual, "="},
952 {Position{1, 11}, tokenString, "The quick brown fox jumps over the lazy dog."},
953 {Position{5, 11}, tokenEOF, ""},
954 })
955
956 testFlow(t, `key2 = "Roses are red\nViolets are blue"`, []token{
957 {Position{1, 1}, tokenKey, "key2"},
958 {Position{1, 6}, tokenEqual, "="},
959 {Position{1, 9}, tokenString, "Roses are red\nViolets are blue"},
960 {Position{1, 41}, tokenEOF, ""},
961 })
962
963 testFlow(t, "key2 = \"\"\"\nRoses are red\nViolets are blue\"\"\"", []token{
964 {Position{1, 1}, tokenKey, "key2"},
965 {Position{1, 6}, tokenEqual, "="},
966 {Position{2, 1}, tokenString, "Roses are red\nViolets are blue"},
967 {Position{3, 20}, tokenEOF, ""},
968 })
969 }
970
971 func TestUnicodeString(t *testing.T) {
972 testFlow(t, `foo = "hello ♥ world"`, []token{
973 {Position{1, 1}, tokenKey, "foo"},
974 {Position{1, 5}, tokenEqual, "="},
975 {Position{1, 8}, tokenString, "hello ♥ world"},
976 {Position{1, 22}, tokenEOF, ""},
977 })
978 }
979
980 func TestEscapeInString(t *testing.T) {
981 testFlow(t, `foo = "\b\f\/"`, []token{
982 {Position{1, 1}, tokenKey, "foo"},
983 {Position{1, 5}, tokenEqual, "="},
984 {Position{1, 8}, tokenString, "\b\f/"},
985 {Position{1, 15}, tokenEOF, ""},
986 })
987 }
988
989 func TestTabInString(t *testing.T) {
990 testFlow(t, `foo = "hello world"`, []token{
991 {Position{1, 1}, tokenKey, "foo"},
992 {Position{1, 5}, tokenEqual, "="},
993 {Position{1, 8}, tokenString, "hello\tworld"},
994 {Position{1, 20}, tokenEOF, ""},
995 })
996 }
997
998 func TestKeyGroupArray(t *testing.T) {
999 testFlow(t, "[[foo]]", []token{
1000 {Position{1, 1}, tokenDoubleLeftBracket, "[["},
1001 {Position{1, 3}, tokenKeyGroupArray, "foo"},
1002 {Position{1, 6}, tokenDoubleRightBracket, "]]"},
1003 {Position{1, 8}, tokenEOF, ""},
1004 })
1005 }
1006
1007 func TestQuotedKey(t *testing.T) {
1008 testFlow(t, "\"a b\" = 42", []token{
1009 {Position{1, 1}, tokenKey, "\"a b\""},
1010 {Position{1, 7}, tokenEqual, "="},
1011 {Position{1, 9}, tokenInteger, "42"},
1012 {Position{1, 11}, tokenEOF, ""},
1013 })
1014 }
1015
1016 func TestQuotedKeyTab(t *testing.T) {
1017 testFlow(t, "\"num\tber\" = 123", []token{
1018 {Position{1, 1}, tokenKey, "\"num\tber\""},
1019 {Position{1, 11}, tokenEqual, "="},
1020 {Position{1, 13}, tokenInteger, "123"},
1021 {Position{1, 16}, tokenEOF, ""},
1022 })
1023 }
1024
1025 func TestKeyNewline(t *testing.T) {
1026 testFlow(t, "a\n= 4", []token{
1027 {Position{1, 1}, tokenError, "keys cannot contain new lines"},
1028 })
1029 }
1030
1031 func TestInvalidFloat(t *testing.T) {
1032 testFlow(t, "a=7e1_", []token{
1033 {Position{1, 1}, tokenKey, "a"},
1034 {Position{1, 2}, tokenEqual, "="},
1035 {Position{1, 3}, tokenFloat, "7e1_"},
1036 {Position{1, 7}, tokenEOF, ""},
1037 })
1038 }
1039
1040 func TestLexUnknownRvalue(t *testing.T) {
1041 testFlow(t, `a = !b`, []token{
1042 {Position{1, 1}, tokenKey, "a"},
1043 {Position{1, 3}, tokenEqual, "="},
1044 {Position{1, 5}, tokenError, "no value can start with !"},
1045 })
1046
1047 testFlow(t, `a = \b`, []token{
1048 {Position{1, 1}, tokenKey, "a"},
1049 {Position{1, 3}, tokenEqual, "="},
1050 {Position{1, 5}, tokenError, `no value can start with \`},
1051 })
1052 }
1053
1054 func TestLexInlineTableEmpty(t *testing.T) {
1055 testFlow(t, `foo = {}`, []token{
1056 {Position{1, 1}, tokenKey, "foo"},
1057 {Position{1, 5}, tokenEqual, "="},
1058 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1059 {Position{1, 8}, tokenRightCurlyBrace, "}"},
1060 {Position{1, 9}, tokenEOF, ""},
1061 })
1062 }
1063
1064 func TestLexInlineTableBareKey(t *testing.T) {
1065 testFlow(t, `foo = { bar = "baz" }`, []token{
1066 {Position{1, 1}, tokenKey, "foo"},
1067 {Position{1, 5}, tokenEqual, "="},
1068 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1069 {Position{1, 9}, tokenKey, "bar"},
1070 {Position{1, 13}, tokenEqual, "="},
1071 {Position{1, 16}, tokenString, "baz"},
1072 {Position{1, 21}, tokenRightCurlyBrace, "}"},
1073 {Position{1, 22}, tokenEOF, ""},
1074 })
1075 }
1076
1077 func TestLexInlineTableBareKeyDash(t *testing.T) {
1078 testFlow(t, `foo = { -bar = "baz" }`, []token{
1079 {Position{1, 1}, tokenKey, "foo"},
1080 {Position{1, 5}, tokenEqual, "="},
1081 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1082 {Position{1, 9}, tokenKey, "-bar"},
1083 {Position{1, 14}, tokenEqual, "="},
1084 {Position{1, 17}, tokenString, "baz"},
1085 {Position{1, 22}, tokenRightCurlyBrace, "}"},
1086 {Position{1, 23}, tokenEOF, ""},
1087 })
1088 }
1089
1090 func TestLexInlineTableBareKeyInArray(t *testing.T) {
1091 testFlow(t, `foo = [{ -bar_ = "baz" }]`, []token{
1092 {Position{1, 1}, tokenKey, "foo"},
1093 {Position{1, 5}, tokenEqual, "="},
1094 {Position{1, 7}, tokenLeftBracket, "["},
1095 {Position{1, 8}, tokenLeftCurlyBrace, "{"},
1096 {Position{1, 10}, tokenKey, "-bar_"},
1097 {Position{1, 16}, tokenEqual, "="},
1098 {Position{1, 19}, tokenString, "baz"},
1099 {Position{1, 24}, tokenRightCurlyBrace, "}"},
1100 {Position{1, 25}, tokenRightBracket, "]"},
1101 {Position{1, 26}, tokenEOF, ""},
1102 })
1103 }
1104
1105 func TestLexInlineTableError1(t *testing.T) {
1106 testFlow(t, `foo = { 123 = 0 ]`, []token{
1107 {Position{1, 1}, tokenKey, "foo"},
1108 {Position{1, 5}, tokenEqual, "="},
1109 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1110 {Position{1, 9}, tokenKey, "123"},
1111 {Position{1, 13}, tokenEqual, "="},
1112 {Position{1, 15}, tokenInteger, "0"},
1113 {Position{1, 17}, tokenRightBracket, "]"},
1114 {Position{1, 18}, tokenError, "cannot have ']' here"},
1115 })
1116 }
1117
1118 func TestLexInlineTableError2(t *testing.T) {
1119 testFlow(t, `foo = { 123 = 0 }}`, []token{
1120 {Position{1, 1}, tokenKey, "foo"},
1121 {Position{1, 5}, tokenEqual, "="},
1122 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1123 {Position{1, 9}, tokenKey, "123"},
1124 {Position{1, 13}, tokenEqual, "="},
1125 {Position{1, 15}, tokenInteger, "0"},
1126 {Position{1, 17}, tokenRightCurlyBrace, "}"},
1127 {Position{1, 18}, tokenRightCurlyBrace, "}"},
1128 {Position{1, 19}, tokenError, "cannot have '}' here"},
1129 })
1130 }
1131
1132 func TestLexInlineTableDottedKey1(t *testing.T) {
1133 testFlow(t, `foo = { a = 0, 123.45abc = 0 }`, []token{
1134 {Position{1, 1}, tokenKey, "foo"},
1135 {Position{1, 5}, tokenEqual, "="},
1136 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1137 {Position{1, 9}, tokenKey, "a"},
1138 {Position{1, 11}, tokenEqual, "="},
1139 {Position{1, 13}, tokenInteger, "0"},
1140 {Position{1, 14}, tokenComma, ","},
1141 {Position{1, 16}, tokenKey, "123.45abc"},
1142 {Position{1, 26}, tokenEqual, "="},
1143 {Position{1, 28}, tokenInteger, "0"},
1144 {Position{1, 30}, tokenRightCurlyBrace, "}"},
1145 {Position{1, 31}, tokenEOF, ""},
1146 })
1147 }
1148
1149 func TestLexInlineTableDottedKey2(t *testing.T) {
1150 testFlow(t, `foo = { a = 0, '123'.'45abc' = 0 }`, []token{
1151 {Position{1, 1}, tokenKey, "foo"},
1152 {Position{1, 5}, tokenEqual, "="},
1153 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1154 {Position{1, 9}, tokenKey, "a"},
1155 {Position{1, 11}, tokenEqual, "="},
1156 {Position{1, 13}, tokenInteger, "0"},
1157 {Position{1, 14}, tokenComma, ","},
1158 {Position{1, 16}, tokenKey, "'123'.'45abc'"},
1159 {Position{1, 30}, tokenEqual, "="},
1160 {Position{1, 32}, tokenInteger, "0"},
1161 {Position{1, 34}, tokenRightCurlyBrace, "}"},
1162 {Position{1, 35}, tokenEOF, ""},
1163 })
1164 }
1165
1166 func TestLexInlineTableDottedKey3(t *testing.T) {
1167 testFlow(t, `foo = { a = 0, "123"."45ʎǝʞ" = 0 }`, []token{
1168 {Position{1, 1}, tokenKey, "foo"},
1169 {Position{1, 5}, tokenEqual, "="},
1170 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1171 {Position{1, 9}, tokenKey, "a"},
1172 {Position{1, 11}, tokenEqual, "="},
1173 {Position{1, 13}, tokenInteger, "0"},
1174 {Position{1, 14}, tokenComma, ","},
1175 {Position{1, 16}, tokenKey, `"123"."45ʎǝʞ"`},
1176 {Position{1, 30}, tokenEqual, "="},
1177 {Position{1, 32}, tokenInteger, "0"},
1178 {Position{1, 34}, tokenRightCurlyBrace, "}"},
1179 {Position{1, 35}, tokenEOF, ""},
1180 })
1181 }
1182
1183 func TestLexInlineTableBareKeyWithComma(t *testing.T) {
1184 testFlow(t, `foo = { -bar1 = "baz", -bar_ = "baz" }`, []token{
1185 {Position{1, 1}, tokenKey, "foo"},
1186 {Position{1, 5}, tokenEqual, "="},
1187 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1188 {Position{1, 9}, tokenKey, "-bar1"},
1189 {Position{1, 15}, tokenEqual, "="},
1190 {Position{1, 18}, tokenString, "baz"},
1191 {Position{1, 22}, tokenComma, ","},
1192 {Position{1, 24}, tokenKey, "-bar_"},
1193 {Position{1, 30}, tokenEqual, "="},
1194 {Position{1, 33}, tokenString, "baz"},
1195 {Position{1, 38}, tokenRightCurlyBrace, "}"},
1196 {Position{1, 39}, tokenEOF, ""},
1197 })
1198 }
1199
1200 func TestLexInlineTableBareKeyUnderscore(t *testing.T) {
1201 testFlow(t, `foo = { _bar = "baz" }`, []token{
1202 {Position{1, 1}, tokenKey, "foo"},
1203 {Position{1, 5}, tokenEqual, "="},
1204 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1205 {Position{1, 9}, tokenKey, "_bar"},
1206 {Position{1, 14}, tokenEqual, "="},
1207 {Position{1, 17}, tokenString, "baz"},
1208 {Position{1, 22}, tokenRightCurlyBrace, "}"},
1209 {Position{1, 23}, tokenEOF, ""},
1210 })
1211 }
1212
1213 func TestLexInlineTableQuotedKey(t *testing.T) {
1214 testFlow(t, `foo = { "bar" = "baz" }`, []token{
1215 {Position{1, 1}, tokenKey, "foo"},
1216 {Position{1, 5}, tokenEqual, "="},
1217 {Position{1, 7}, tokenLeftCurlyBrace, "{"},
1218 {Position{1, 9}, tokenKey, "\"bar\""},
1219 {Position{1, 15}, tokenEqual, "="},
1220 {Position{1, 18}, tokenString, "baz"},
1221 {Position{1, 23}, tokenRightCurlyBrace, "}"},
1222 {Position{1, 24}, tokenEOF, ""},
1223 })
1224 }
1225
1226 func BenchmarkLexer(b *testing.B) {
1227 sample := `title = "Hugo: A Fast and Flexible Website Generator"
1228 baseurl = "http://gohugo.io/"
1229 MetaDataFormat = "yaml"
1230 pluralizeListTitles = false
1231
1232 [params]
1233 description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go"
1234 author = "Steve Francia (spf13) and friends"
1235 release = "0.22-DEV"
1236
1237 [[menu.main]]
1238 name = "Download Hugo"
1239 pre = "<i class='fa fa-download'></i>"
1240 url = "https://github.com/spf13/hugo/releases"
1241 weight = -200
1242 `
1243 b.ResetTimer()
1244 for i := 0; i < b.N; i++ {
1245 lexToml([]byte(sample))
1246 }
1247 }
1248
View as plain text