1 package scanner
2
3 import (
4 "bytes"
5 "fmt"
6 "testing"
7
8 "github.com/hashicorp/hcl/json/token"
9 )
10
11 var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
12
13 type tokenPair struct {
14 tok token.Type
15 text string
16 }
17
18 var tokenLists = map[string][]tokenPair{
19 "operator": []tokenPair{
20 {token.LBRACK, "["},
21 {token.LBRACE, "{"},
22 {token.COMMA, ","},
23 {token.PERIOD, "."},
24 {token.RBRACK, "]"},
25 {token.RBRACE, "}"},
26 },
27 "bool": []tokenPair{
28 {token.BOOL, "true"},
29 {token.BOOL, "false"},
30 },
31 "string": []tokenPair{
32 {token.STRING, `" "`},
33 {token.STRING, `"a"`},
34 {token.STRING, `"本"`},
35 {token.STRING, `"${file(\"foo\")}"`},
36 {token.STRING, `"\a"`},
37 {token.STRING, `"\b"`},
38 {token.STRING, `"\f"`},
39 {token.STRING, `"\n"`},
40 {token.STRING, `"\r"`},
41 {token.STRING, `"\t"`},
42 {token.STRING, `"\v"`},
43 {token.STRING, `"\""`},
44 {token.STRING, `"\000"`},
45 {token.STRING, `"\777"`},
46 {token.STRING, `"\x00"`},
47 {token.STRING, `"\xff"`},
48 {token.STRING, `"\u0000"`},
49 {token.STRING, `"\ufA16"`},
50 {token.STRING, `"\U00000000"`},
51 {token.STRING, `"\U0000ffAB"`},
52 {token.STRING, `"` + f100 + `"`},
53 },
54 "number": []tokenPair{
55 {token.NUMBER, "0"},
56 {token.NUMBER, "1"},
57 {token.NUMBER, "9"},
58 {token.NUMBER, "42"},
59 {token.NUMBER, "1234567890"},
60 {token.NUMBER, "-0"},
61 {token.NUMBER, "-1"},
62 {token.NUMBER, "-9"},
63 {token.NUMBER, "-42"},
64 {token.NUMBER, "-1234567890"},
65 },
66 "float": []tokenPair{
67 {token.FLOAT, "0."},
68 {token.FLOAT, "1."},
69 {token.FLOAT, "42."},
70 {token.FLOAT, "01234567890."},
71 {token.FLOAT, ".0"},
72 {token.FLOAT, ".1"},
73 {token.FLOAT, ".42"},
74 {token.FLOAT, ".0123456789"},
75 {token.FLOAT, "0.0"},
76 {token.FLOAT, "1.0"},
77 {token.FLOAT, "42.0"},
78 {token.FLOAT, "01234567890.0"},
79 {token.FLOAT, "0e0"},
80 {token.FLOAT, "1e0"},
81 {token.FLOAT, "42e0"},
82 {token.FLOAT, "01234567890e0"},
83 {token.FLOAT, "0E0"},
84 {token.FLOAT, "1E0"},
85 {token.FLOAT, "42E0"},
86 {token.FLOAT, "01234567890E0"},
87 {token.FLOAT, "0e+10"},
88 {token.FLOAT, "1e-10"},
89 {token.FLOAT, "42e+10"},
90 {token.FLOAT, "01234567890e-10"},
91 {token.FLOAT, "0E+10"},
92 {token.FLOAT, "1E-10"},
93 {token.FLOAT, "42E+10"},
94 {token.FLOAT, "01234567890E-10"},
95 {token.FLOAT, "01.8e0"},
96 {token.FLOAT, "1.4e0"},
97 {token.FLOAT, "42.2e0"},
98 {token.FLOAT, "01234567890.12e0"},
99 {token.FLOAT, "0.E0"},
100 {token.FLOAT, "1.12E0"},
101 {token.FLOAT, "42.123E0"},
102 {token.FLOAT, "01234567890.213E0"},
103 {token.FLOAT, "0.2e+10"},
104 {token.FLOAT, "1.2e-10"},
105 {token.FLOAT, "42.54e+10"},
106 {token.FLOAT, "01234567890.98e-10"},
107 {token.FLOAT, "0.1E+10"},
108 {token.FLOAT, "1.1E-10"},
109 {token.FLOAT, "42.1E+10"},
110 {token.FLOAT, "01234567890.1E-10"},
111 {token.FLOAT, "-0.0"},
112 {token.FLOAT, "-1.0"},
113 {token.FLOAT, "-42.0"},
114 {token.FLOAT, "-01234567890.0"},
115 {token.FLOAT, "-0e0"},
116 {token.FLOAT, "-1e0"},
117 {token.FLOAT, "-42e0"},
118 {token.FLOAT, "-01234567890e0"},
119 {token.FLOAT, "-0E0"},
120 {token.FLOAT, "-1E0"},
121 {token.FLOAT, "-42E0"},
122 {token.FLOAT, "-01234567890E0"},
123 {token.FLOAT, "-0e+10"},
124 {token.FLOAT, "-1e-10"},
125 {token.FLOAT, "-42e+10"},
126 {token.FLOAT, "-01234567890e-10"},
127 {token.FLOAT, "-0E+10"},
128 {token.FLOAT, "-1E-10"},
129 {token.FLOAT, "-42E+10"},
130 {token.FLOAT, "-01234567890E-10"},
131 {token.FLOAT, "-01.8e0"},
132 {token.FLOAT, "-1.4e0"},
133 {token.FLOAT, "-42.2e0"},
134 {token.FLOAT, "-01234567890.12e0"},
135 {token.FLOAT, "-0.E0"},
136 {token.FLOAT, "-1.12E0"},
137 {token.FLOAT, "-42.123E0"},
138 {token.FLOAT, "-01234567890.213E0"},
139 {token.FLOAT, "-0.2e+10"},
140 {token.FLOAT, "-1.2e-10"},
141 {token.FLOAT, "-42.54e+10"},
142 {token.FLOAT, "-01234567890.98e-10"},
143 {token.FLOAT, "-0.1E+10"},
144 {token.FLOAT, "-1.1E-10"},
145 {token.FLOAT, "-42.1E+10"},
146 {token.FLOAT, "-01234567890.1E-10"},
147 },
148 }
149
150 var orderedTokenLists = []string{
151 "comment",
152 "operator",
153 "bool",
154 "string",
155 "number",
156 "float",
157 }
158
159 func TestPosition(t *testing.T) {
160
161 buf := new(bytes.Buffer)
162
163 for _, listName := range orderedTokenLists {
164 for _, ident := range tokenLists[listName] {
165 fmt.Fprintf(buf, "\t\t\t\t%s\n", ident.text)
166 }
167 }
168
169 s := New(buf.Bytes())
170
171 pos := token.Pos{"", 4, 1, 5}
172 s.Scan()
173 for _, listName := range orderedTokenLists {
174
175 for _, k := range tokenLists[listName] {
176 curPos := s.tokPos
177
178
179 if curPos.Offset != pos.Offset {
180 t.Fatalf("offset = %d, want %d for %q", curPos.Offset, pos.Offset, k.text)
181 }
182 if curPos.Line != pos.Line {
183 t.Fatalf("line = %d, want %d for %q", curPos.Line, pos.Line, k.text)
184 }
185 if curPos.Column != pos.Column {
186 t.Fatalf("column = %d, want %d for %q", curPos.Column, pos.Column, k.text)
187 }
188 pos.Offset += 4 + len(k.text) + 1
189 pos.Line += countNewlines(k.text) + 1
190
191 s.Error = func(pos token.Pos, msg string) {
192 t.Errorf("error %q for %q", msg, k.text)
193 }
194
195 s.Scan()
196 }
197 }
198
199 if s.ErrorCount != 0 {
200 t.Errorf("%d errors", s.ErrorCount)
201 }
202 }
203
204 func TestComment(t *testing.T) {
205 testTokenList(t, tokenLists["comment"])
206 }
207
208 func TestOperator(t *testing.T) {
209 testTokenList(t, tokenLists["operator"])
210 }
211
212 func TestBool(t *testing.T) {
213 testTokenList(t, tokenLists["bool"])
214 }
215
216 func TestIdent(t *testing.T) {
217 testTokenList(t, tokenLists["ident"])
218 }
219
220 func TestString(t *testing.T) {
221 testTokenList(t, tokenLists["string"])
222 }
223
224 func TestNumber(t *testing.T) {
225 testTokenList(t, tokenLists["number"])
226 }
227
228 func TestFloat(t *testing.T) {
229 testTokenList(t, tokenLists["float"])
230 }
231
232 func TestRealExample(t *testing.T) {
233 complexReal := `
234 {
235 "variable": {
236 "foo": {
237 "default": "bar",
238 "description": "bar",
239 "depends_on": ["something"]
240 }
241 }
242 }`
243
244 literals := []struct {
245 tokenType token.Type
246 literal string
247 }{
248 {token.LBRACE, `{`},
249 {token.STRING, `"variable"`},
250 {token.COLON, `:`},
251 {token.LBRACE, `{`},
252 {token.STRING, `"foo"`},
253 {token.COLON, `:`},
254 {token.LBRACE, `{`},
255 {token.STRING, `"default"`},
256 {token.COLON, `:`},
257 {token.STRING, `"bar"`},
258 {token.COMMA, `,`},
259 {token.STRING, `"description"`},
260 {token.COLON, `:`},
261 {token.STRING, `"bar"`},
262 {token.COMMA, `,`},
263 {token.STRING, `"depends_on"`},
264 {token.COLON, `:`},
265 {token.LBRACK, `[`},
266 {token.STRING, `"something"`},
267 {token.RBRACK, `]`},
268 {token.RBRACE, `}`},
269 {token.RBRACE, `}`},
270 {token.RBRACE, `}`},
271 {token.EOF, ``},
272 }
273
274 s := New([]byte(complexReal))
275 for _, l := range literals {
276 tok := s.Scan()
277 if l.tokenType != tok.Type {
278 t.Errorf("got: %s want %s for %s\n", tok, l.tokenType, tok.String())
279 }
280
281 if l.literal != tok.Text {
282 t.Errorf("got: %s want %s\n", tok, l.literal)
283 }
284 }
285
286 }
287
288 func TestError(t *testing.T) {
289 testError(t, "\x80", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
290 testError(t, "\xff", "1:1", "illegal UTF-8 encoding", token.ILLEGAL)
291
292 testError(t, `"ab`+"\x80", "1:4", "illegal UTF-8 encoding", token.STRING)
293 testError(t, `"abc`+"\xff", "1:5", "illegal UTF-8 encoding", token.STRING)
294
295 testError(t, `01238`, "1:7", "numbers cannot start with 0", token.NUMBER)
296 testError(t, `01238123`, "1:10", "numbers cannot start with 0", token.NUMBER)
297 testError(t, `'aa'`, "1:1", "illegal char: '", token.ILLEGAL)
298
299 testError(t, `"`, "1:2", "literal not terminated", token.STRING)
300 testError(t, `"abc`, "1:5", "literal not terminated", token.STRING)
301 testError(t, `"abc`+"\n", "1:5", "literal not terminated", token.STRING)
302 }
303
304 func testError(t *testing.T, src, pos, msg string, tok token.Type) {
305 s := New([]byte(src))
306
307 errorCalled := false
308 s.Error = func(p token.Pos, m string) {
309 if !errorCalled {
310 if pos != p.String() {
311 t.Errorf("pos = %q, want %q for %q", p, pos, src)
312 }
313
314 if m != msg {
315 t.Errorf("msg = %q, want %q for %q", m, msg, src)
316 }
317 errorCalled = true
318 }
319 }
320
321 tk := s.Scan()
322 if tk.Type != tok {
323 t.Errorf("tok = %s, want %s for %q", tk, tok, src)
324 }
325 if !errorCalled {
326 t.Errorf("error handler not called for %q", src)
327 }
328 if s.ErrorCount == 0 {
329 t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src)
330 }
331 }
332
333 func testTokenList(t *testing.T, tokenList []tokenPair) {
334
335 buf := new(bytes.Buffer)
336 for _, ident := range tokenList {
337 fmt.Fprintf(buf, "%s\n", ident.text)
338 }
339
340 s := New(buf.Bytes())
341 for _, ident := range tokenList {
342 tok := s.Scan()
343 if tok.Type != ident.tok {
344 t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text)
345 }
346
347 if tok.Text != ident.text {
348 t.Errorf("text = %q want %q", tok.String(), ident.text)
349 }
350
351 }
352 }
353
354 func countNewlines(s string) int {
355 n := 0
356 for _, ch := range s {
357 if ch == '\n' {
358 n++
359 }
360 }
361 return n
362 }
363
View as plain text