1 package parser
2
3 import (
4 "testing"
5
6 "github.com/dop251/goja/file"
7 "github.com/dop251/goja/token"
8 "github.com/dop251/goja/unistring"
9 )
10
11 func TestLexer(t *testing.T) {
12 tt(t, func() {
13 setup := func(src string) *_parser {
14 parser := newParser("", src)
15 return parser
16 }
17
18 test := func(src string, test ...interface{}) {
19 parser := setup(src)
20 for len(test) > 0 {
21 tkn, literal, _, idx := parser.scan()
22 if len(test) > 0 {
23 is(tkn, test[0].(token.Token))
24 test = test[1:]
25 }
26 if len(test) > 0 {
27 is(literal, unistring.String(test[0].(string)))
28 test = test[1:]
29 }
30 if len(test) > 0 {
31
32 is(idx, file.Idx(test[0].(int)))
33 test = test[1:]
34 }
35 }
36 }
37
38 test("",
39 token.EOF, "", 1,
40 )
41
42 test("#!",
43 token.EOF, "", 3,
44 )
45
46 test("#!\n1",
47 token.NUMBER, "1", 4,
48 token.EOF, "", 5,
49 )
50
51 test("1",
52 token.NUMBER, "1", 1,
53 token.EOF, "", 2,
54 )
55
56 test(".0",
57 token.NUMBER, ".0", 1,
58 token.EOF, "", 3,
59 )
60
61 test("abc",
62 token.IDENTIFIER, "abc", 1,
63 token.EOF, "", 4,
64 )
65
66 test("abc(1)",
67 token.IDENTIFIER, "abc", 1,
68 token.LEFT_PARENTHESIS, "", 4,
69 token.NUMBER, "1", 5,
70 token.RIGHT_PARENTHESIS, "", 6,
71 token.EOF, "", 7,
72 )
73
74 test(".",
75 token.PERIOD, "", 1,
76 token.EOF, "", 2,
77 )
78
79 test("===.",
80 token.STRICT_EQUAL, "", 1,
81 token.PERIOD, "", 4,
82 token.EOF, "", 5,
83 )
84
85 test(">>>=.0",
86 token.UNSIGNED_SHIFT_RIGHT_ASSIGN, "", 1,
87 token.NUMBER, ".0", 5,
88 token.EOF, "", 7,
89 )
90
91 test(">>>=0.0.",
92 token.UNSIGNED_SHIFT_RIGHT_ASSIGN, "", 1,
93 token.NUMBER, "0.0", 5,
94 token.PERIOD, "", 8,
95 token.EOF, "", 9,
96 )
97
98 test("\"abc\"",
99 token.STRING, "\"abc\"", 1,
100 token.EOF, "", 6,
101 )
102
103 test("abc = //",
104 token.IDENTIFIER, "abc", 1,
105 token.ASSIGN, "", 5,
106 token.EOF, "", 9,
107 )
108
109 test("abc = 1 / 2",
110 token.IDENTIFIER, "abc", 1,
111 token.ASSIGN, "", 5,
112 token.NUMBER, "1", 7,
113 token.SLASH, "", 9,
114 token.NUMBER, "2", 11,
115 token.EOF, "", 12,
116 )
117
118 test("xyzzy = 'Nothing happens.'",
119 token.IDENTIFIER, "xyzzy", 1,
120 token.ASSIGN, "", 7,
121 token.STRING, "'Nothing happens.'", 9,
122 token.EOF, "", 27,
123 )
124
125 test("abc = !false",
126 token.IDENTIFIER, "abc", 1,
127 token.ASSIGN, "", 5,
128 token.NOT, "", 7,
129 token.BOOLEAN, "false", 8,
130 token.EOF, "", 13,
131 )
132
133 test("abc = !!true",
134 token.IDENTIFIER, "abc", 1,
135 token.ASSIGN, "", 5,
136 token.NOT, "", 7,
137 token.NOT, "", 8,
138 token.BOOLEAN, "true", 9,
139 token.EOF, "", 13,
140 )
141
142 test("abc *= 1",
143 token.IDENTIFIER, "abc", 1,
144 token.MULTIPLY_ASSIGN, "", 5,
145 token.NUMBER, "1", 8,
146 token.EOF, "", 9,
147 )
148
149 test("if 1 else",
150 token.IF, "if", 1,
151 token.NUMBER, "1", 4,
152 token.ELSE, "else", 6,
153 token.EOF, "", 10,
154 )
155
156 test("null",
157 token.NULL, "null", 1,
158 token.EOF, "", 5,
159 )
160
161 test(`"\u007a\x79\u000a\x78"`,
162 token.STRING, "\"\\u007a\\x79\\u000a\\x78\"", 1,
163 token.EOF, "", 23,
164 )
165
166 test(`"[First line \
167 Second line \
168 Third line\
169 . ]"
170 `,
171 token.STRING, "\"[First line \\\nSecond line \\\n Third line\\\n. ]\"", 1,
172 token.EOF, "", 53,
173 )
174
175 test("/",
176 token.SLASH, "", 1,
177 token.EOF, "", 2,
178 )
179
180 test("var abc = \"abc\uFFFFabc\"",
181 token.VAR, "var", 1,
182 token.IDENTIFIER, "abc", 5,
183 token.ASSIGN, "", 9,
184 token.STRING, "\"abc\uFFFFabc\"", 11,
185 token.EOF, "", 22,
186 )
187
188 test(`'\t' === '\r'`,
189 token.STRING, "'\\t'", 1,
190 token.STRICT_EQUAL, "", 6,
191 token.STRING, "'\\r'", 10,
192 token.EOF, "", 14,
193 )
194
195 test(`var \u0024 = 1`,
196 token.VAR, "var", 1,
197 token.IDENTIFIER, "\\u0024", 5,
198 token.ASSIGN, "", 12,
199 token.NUMBER, "1", 14,
200 token.EOF, "", 15,
201 )
202
203 test("10e10000",
204 token.NUMBER, "10e10000", 1,
205 token.EOF, "", 9,
206 )
207
208 test(`var if var class`,
209 token.VAR, "var", 1,
210 token.IF, "if", 5,
211 token.VAR, "var", 8,
212 token.CLASS, "class", 12,
213 token.EOF, "", 17,
214 )
215
216 test(`-0`,
217 token.MINUS, "", 1,
218 token.NUMBER, "0", 2,
219 token.EOF, "", 3,
220 )
221
222 test(`.01`,
223 token.NUMBER, ".01", 1,
224 token.EOF, "", 4,
225 )
226
227 test(`.01e+2`,
228 token.NUMBER, ".01e+2", 1,
229 token.EOF, "", 7,
230 )
231
232 test(";",
233 token.SEMICOLON, "", 1,
234 token.EOF, "", 2,
235 )
236
237 test(";;",
238 token.SEMICOLON, "", 1,
239 token.SEMICOLON, "", 2,
240 token.EOF, "", 3,
241 )
242
243 test("//",
244 token.EOF, "", 3,
245 )
246
247 test(";;//",
248 token.SEMICOLON, "", 1,
249 token.SEMICOLON, "", 2,
250 token.EOF, "", 5,
251 )
252
253 test("1",
254 token.NUMBER, "1", 1,
255 )
256
257 test("12 123",
258 token.NUMBER, "12", 1,
259 token.NUMBER, "123", 4,
260 )
261
262 test("1.2 12.3",
263 token.NUMBER, "1.2", 1,
264 token.NUMBER, "12.3", 5,
265 )
266
267 test("/ /=",
268 token.SLASH, "", 1,
269 token.QUOTIENT_ASSIGN, "", 3,
270 )
271
272 test(`"abc"`,
273 token.STRING, `"abc"`, 1,
274 )
275
276 test(`'abc'`,
277 token.STRING, `'abc'`, 1,
278 )
279
280 test("++",
281 token.INCREMENT, "", 1,
282 )
283
284 test(">",
285 token.GREATER, "", 1,
286 )
287
288 test(">=",
289 token.GREATER_OR_EQUAL, "", 1,
290 )
291
292 test(">>",
293 token.SHIFT_RIGHT, "", 1,
294 )
295
296 test(">>=",
297 token.SHIFT_RIGHT_ASSIGN, "", 1,
298 )
299
300 test(">>>",
301 token.UNSIGNED_SHIFT_RIGHT, "", 1,
302 )
303
304 test(">>>=",
305 token.UNSIGNED_SHIFT_RIGHT_ASSIGN, "", 1,
306 )
307
308 test("1 \"abc\"",
309 token.NUMBER, "1", 1,
310 token.STRING, "\"abc\"", 3,
311 )
312
313 test(",",
314 token.COMMA, "", 1,
315 )
316
317 test("1, \"abc\"",
318 token.NUMBER, "1", 1,
319 token.COMMA, "", 2,
320 token.STRING, "\"abc\"", 4,
321 )
322
323 test("new abc(1, 3.14159);",
324 token.NEW, "new", 1,
325 token.IDENTIFIER, "abc", 5,
326 token.LEFT_PARENTHESIS, "", 8,
327 token.NUMBER, "1", 9,
328 token.COMMA, "", 10,
329 token.NUMBER, "3.14159", 12,
330 token.RIGHT_PARENTHESIS, "", 19,
331 token.SEMICOLON, "", 20,
332 )
333
334 test("1 == \"1\"",
335 token.NUMBER, "1", 1,
336 token.EQUAL, "", 3,
337 token.STRING, "\"1\"", 6,
338 )
339
340 test("1\n[]\n",
341 token.NUMBER, "1", 1,
342 token.LEFT_BRACKET, "", 3,
343 token.RIGHT_BRACKET, "", 4,
344 )
345
346 test("1\ufeff[]\ufeff",
347 token.NUMBER, "1", 1,
348 token.LEFT_BRACKET, "", 5,
349 token.RIGHT_BRACKET, "", 6,
350 )
351
352 test("x ?.30 : false",
353 token.IDENTIFIER, "x", 1,
354 token.QUESTION_MARK, "", 3,
355 token.NUMBER, ".30", 4,
356 token.COLON, "", 8,
357 token.BOOLEAN, "false", 10,
358 )
359
360 test("a\n?.b",
361 token.IDENTIFIER, "a", 1,
362 token.QUESTION_DOT, "", 3,
363 token.IDENTIFIER, "b", 5,
364 )
365
366
367
368 test(`3ea`,
369 token.ILLEGAL, "3e", 1,
370 token.IDENTIFIER, "a", 3,
371 token.EOF, "", 4,
372 )
373
374 test(`3in`,
375 token.ILLEGAL, "3", 1,
376 token.IN, "in", 2,
377 token.EOF, "", 4,
378 )
379
380 test("\"Hello\nWorld\"",
381 token.ILLEGAL, "", 1,
382 token.IDENTIFIER, "World", 8,
383 token.ILLEGAL, "", 13,
384 token.EOF, "", 14,
385 )
386
387 test("\u203f = 10",
388 token.ILLEGAL, "", 1,
389 token.ASSIGN, "", 5,
390 token.NUMBER, "10", 7,
391 token.EOF, "", 9,
392 )
393
394 test(`"\x0G"`,
395 token.ILLEGAL, "\"\\x0G\"", 1,
396
397 token.EOF, "", 7,
398 )
399
400 })
401 }
402
View as plain text