1 package chroma
2
3 import (
4 "testing"
5
6 assert "github.com/alecthomas/assert/v2"
7 )
8
9 func mustNewLexer(t *testing.T, config *Config, rules Rules) *RegexLexer {
10 lexer, err := NewLexer(config, func() Rules {
11 return rules
12 })
13 assert.NoError(t, err)
14 return lexer
15 }
16
17 func TestNewlineAtEndOfFile(t *testing.T) {
18 l := Coalesce(mustNewLexer(t, &Config{EnsureNL: true}, Rules{
19 "root": {
20 {`(\w+)(\n)`, ByGroups(Keyword, Whitespace), nil},
21 },
22 }))
23 it, err := l.Tokenise(nil, `hello`)
24 assert.NoError(t, err)
25 assert.Equal(t, []Token{{Keyword, "hello"}, {Whitespace, "\n"}}, it.Tokens())
26
27 l = Coalesce(mustNewLexer(t, nil, Rules{
28 "root": {
29 {`(\w+)(\n)`, ByGroups(Keyword, Whitespace), nil},
30 },
31 }))
32 it, err = l.Tokenise(nil, `hello`)
33 assert.NoError(t, err)
34 assert.Equal(t, []Token{{Error, "hello"}}, it.Tokens())
35 }
36
37 func TestMatchingAtStart(t *testing.T) {
38 l := Coalesce(mustNewLexer(t, &Config{}, Rules{
39 "root": {
40 {`\s+`, Whitespace, nil},
41 {`^-`, Punctuation, Push("directive")},
42 {`->`, Operator, nil},
43 },
44 "directive": {
45 {"module", NameEntity, Pop(1)},
46 },
47 }))
48 it, err := l.Tokenise(nil, `-module ->`)
49 assert.NoError(t, err)
50 assert.Equal(t,
51 []Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
52 it.Tokens())
53 }
54
55 func TestEnsureLFOption(t *testing.T) {
56 l := Coalesce(mustNewLexer(t, &Config{}, Rules{
57 "root": {
58 {`(\w+)(\r?\n|\r)`, ByGroups(Keyword, Whitespace), nil},
59 },
60 }))
61 it, err := l.Tokenise(&TokeniseOptions{
62 State: "root",
63 EnsureLF: true,
64 }, "hello\r\nworld\r")
65 assert.NoError(t, err)
66 assert.Equal(t, []Token{
67 {Keyword, "hello"},
68 {Whitespace, "\n"},
69 {Keyword, "world"},
70 {Whitespace, "\n"},
71 }, it.Tokens())
72
73 l = Coalesce(mustNewLexer(t, nil, Rules{
74 "root": {
75 {`(\w+)(\r?\n|\r)`, ByGroups(Keyword, Whitespace), nil},
76 },
77 }))
78 it, err = l.Tokenise(&TokeniseOptions{
79 State: "root",
80 EnsureLF: false,
81 }, "hello\r\nworld\r")
82 assert.NoError(t, err)
83 assert.Equal(t, []Token{
84 {Keyword, "hello"},
85 {Whitespace, "\r\n"},
86 {Keyword, "world"},
87 {Whitespace, "\r"},
88 }, it.Tokens())
89 }
90
91 func TestEnsureLFFunc(t *testing.T) {
92 tests := []struct{ in, out string }{
93 {in: "", out: ""},
94 {in: "abc", out: "abc"},
95 {in: "\r", out: "\n"},
96 {in: "a\r", out: "a\n"},
97 {in: "\rb", out: "\nb"},
98 {in: "a\rb", out: "a\nb"},
99 {in: "\r\n", out: "\n"},
100 {in: "a\r\n", out: "a\n"},
101 {in: "\r\nb", out: "\nb"},
102 {in: "a\r\nb", out: "a\nb"},
103 {in: "\r\r\r\n\r", out: "\n\n\n\n"},
104 }
105 for _, test := range tests {
106 out := ensureLF(test.in)
107 assert.Equal(t, out, test.out)
108 }
109 }
110
111 func TestByGroupNames(t *testing.T) {
112 l := Coalesce(mustNewLexer(t, nil, Rules{
113 "root": {
114 {
115 `(?<key>\w+)(?<operator>=)(?<value>\w+)`,
116 ByGroupNames(map[string]Emitter{
117 `key`: String,
118 `operator`: Operator,
119 `value`: String,
120 }),
121 nil,
122 },
123 },
124 }))
125 it, err := l.Tokenise(nil, `abc=123`)
126 assert.NoError(t, err)
127 assert.Equal(t, []Token{{String, `abc`}, {Operator, `=`}, {String, `123`}}, it.Tokens())
128
129 l = Coalesce(mustNewLexer(t, nil, Rules{
130 "root": {
131 {
132 `(?<key>\w+)(?<operator>=)(?<value>\w+)`,
133 ByGroupNames(map[string]Emitter{
134 `key`: String,
135 `value`: String,
136 }),
137 nil,
138 },
139 },
140 }))
141 it, err = l.Tokenise(nil, `abc=123`)
142 assert.NoError(t, err)
143 assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, it.Tokens())
144
145 l = Coalesce(mustNewLexer(t, nil, Rules{
146 "root": {
147 {
148 `(?<key>\w+)=(?<value>\w+)`,
149 ByGroupNames(map[string]Emitter{
150 `key`: String,
151 `value`: String,
152 }),
153 nil,
154 },
155 },
156 }))
157 it, err = l.Tokenise(nil, `abc=123`)
158 assert.NoError(t, err)
159 assert.Equal(t, []Token{{String, `abc123`}}, it.Tokens())
160
161 l = Coalesce(mustNewLexer(t, nil, Rules{
162 "root": {
163 {
164 `(?<key>\w+)(?<op>=)(?<value>\w+)`,
165 ByGroupNames(map[string]Emitter{
166 `key`: String,
167 `operator`: Operator,
168 `value`: String,
169 }),
170 nil,
171 },
172 },
173 }))
174 it, err = l.Tokenise(nil, `abc=123`)
175 assert.NoError(t, err)
176 assert.Equal(t, []Token{{String, `abc`}, {Error, `=`}, {String, `123`}}, it.Tokens())
177
178 l = Coalesce(mustNewLexer(t, nil, Rules{
179 "root": {
180 {
181 `\w+=\w+`,
182 ByGroupNames(map[string]Emitter{
183 `key`: String,
184 `operator`: Operator,
185 `value`: String,
186 }),
187 nil,
188 },
189 },
190 }))
191 it, err = l.Tokenise(nil, `abc=123`)
192 assert.NoError(t, err)
193 assert.Equal(t, []Token{{Error, `abc=123`}}, it.Tokens())
194 }
195
View as plain text