1 package lexers_test
2
3 import (
4 "testing"
5
6 assert "github.com/alecthomas/assert/v2"
7
8 "github.com/alecthomas/chroma/v2"
9 "github.com/alecthomas/chroma/v2/lexers"
10 )
11
12 const lexerBenchSource = `/*
13 * Licensed to the Apache Software Foundation (ASF) under one or more
14 * contributor license agreements. See the NOTICE file distributed with
15 * this work for additional information regarding copyright ownership.
16 * The ASF licenses this file to You under the Apache License, Version 2.0
17 * (the "License"); you may not use this file except in compliance with
18 * the License. You may obtain a copy of the License at
19 *
20 * http://www.apache.org/licenses/LICENSE-2.0
21 *
22 * Unless required by applicable law or agreed to in writing, software
23 * distributed under the License is distributed on an "AS IS" BASIS,
24 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 * See the License for the specific language governing permissions and
26 * limitations under the License.
27 */
28 package org.apache.kafka.server.common;
29
30 import org.apache.kafka.common.utils.Utils;
31
32 import java.io.BufferedReader;
33 import java.io.BufferedWriter;
34 import java.io.File;
35 import java.io.FileOutputStream;
36 import java.io.IOException;
37 import java.io.OutputStreamWriter;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.FileAlreadyExistsException;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.Paths;
43 import java.util.ArrayList;
44 import java.util.Collection;
45 import java.util.Collections;
46 import java.util.List;
47 import java.util.Optional;
48
49 /**
50 * This class represents a utility to capture a checkpoint in a file. It writes down to the file in the below format.
51 *
52 * ========= File beginning =========
53 * version: int
54 * entries-count: int
55 * entry-as-string-on-each-line
56 * ========= File end ===============
57 *
58 * Each entry is represented as a string on each line in the checkpoint file. {@link EntryFormatter} is used
59 * to convert the entry into a string and vice versa.
60 *
61 * @param <T> entry type.
62 */
63 public class CheckpointFile<T> {
64
65 private final int version;
66 private final EntryFormatter<T> formatter;
67 private final Object lock = new Object();
68 private final Path absolutePath;
69 private final Path tempPath;
70
71 public CheckpointFile(File file,
72 int version,
73 EntryFormatter<T> formatter) throws IOException {
74 this.version = version;
75 this.formatter = formatter;
76 try {
77 // Create the file if it does not exist.
78 Files.createFile(file.toPath());
79 } catch (FileAlreadyExistsException ex) {
80 // Ignore if file already exists.
81 }
82 absolutePath = file.toPath().toAbsolutePath();
83 tempPath = Paths.get(absolutePath.toString() + ".tmp");
84 }
85
86 public void write(Collection<T> entries) throws IOException {
87 synchronized (lock) {
88 // write to temp file and then swap with the existing file
89 try (FileOutputStream fileOutputStream = new FileOutputStream(tempPath.toFile());
90 BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8))) {
91 // Write the version
92 writer.write(Integer.toString(version));
93 writer.newLine();
94
95 // Write the entries count
96 writer.write(Integer.toString(entries.size()));
97 writer.newLine();
98
99 // Write each entry on a new line.
100 for (T entry : entries) {
101 writer.write(formatter.toString(entry));
102 writer.newLine();
103 }
104
105 writer.flush();
106 fileOutputStream.getFD().sync();
107 }
108
109 Utils.atomicMoveWithFallback(tempPath, absolutePath);
110 }
111 }
112
113 public List<T> read() throws IOException {
114 synchronized (lock) {
115 try (BufferedReader reader = Files.newBufferedReader(absolutePath)) {
116 CheckpointReadBuffer<T> checkpointBuffer = new CheckpointReadBuffer<>(absolutePath.toString(), reader, version, formatter);
117 return checkpointBuffer.read();
118 }
119 }
120 }
121
122 private static class CheckpointReadBuffer<T> {
123
124 private final String location;
125 private final BufferedReader reader;
126 private final int version;
127 private final EntryFormatter<T> formatter;
128
129 CheckpointReadBuffer(String location,
130 BufferedReader reader,
131 int version,
132 EntryFormatter<T> formatter) {
133 this.location = location;
134 this.reader = reader;
135 this.version = version;
136 this.formatter = formatter;
137 }
138
139 List<T> read() throws IOException {
140 String line = reader.readLine();
141 if (line == null)
142 return Collections.emptyList();
143
144 int readVersion = toInt(line);
145 if (readVersion != version) {
146 throw new IOException("Unrecognised version:" + readVersion + ", expected version: " + version
147 + " in checkpoint file at: " + location);
148 }
149
150 line = reader.readLine();
151 if (line == null) {
152 return Collections.emptyList();
153 }
154 int expectedSize = toInt(line);
155 List<T> entries = new ArrayList<>(expectedSize);
156 line = reader.readLine();
157 while (line != null) {
158 Optional<T> maybeEntry = formatter.fromString(line);
159 if (!maybeEntry.isPresent()) {
160 throw buildMalformedLineException(line);
161 }
162 entries.add(maybeEntry.get());
163 line = reader.readLine();
164 }
165
166 if (entries.size() != expectedSize) {
167 throw new IOException("Expected [" + expectedSize + "] entries in checkpoint file ["
168 + location + "], but found only [" + entries.size() + "]");
169 }
170
171 return entries;
172 }
173
174 private int toInt(String line) throws IOException {
175 try {
176 return Integer.parseInt(line);
177 } catch (NumberFormatException e) {
178 throw buildMalformedLineException(line);
179 }
180 }
181
182 private IOException buildMalformedLineException(String line) {
183 return new IOException(String.format("Malformed line in checkpoint file [%s]: %s", location, line));
184 }
185 }
186
187 /**
188 * This is used to convert the given entry of type {@code T} into a string and vice versa.
189 *
190 * @param <T> entry type
191 */
192 public interface EntryFormatter<T> {
193
194 /**
195 * @param entry entry to be converted into string.
196 * @return String representation of the given entry.
197 */
198 String toString(T entry);
199
200 /**
201 * @param value string representation of an entry.
202 * @return entry converted from the given string representation if possible. {@link Optional#empty()} represents
203 * that the given string representation could not be converted into an entry.
204 */
205 Optional<T> fromString(String value);
206 }
207 }
208 `
209
210 func Benchmark(b *testing.B) {
211 b.ReportAllocs()
212 for i := 0; i < b.N; i++ {
213 it, err := lexers.GlobalLexerRegistry.Get("Java").Tokenise(nil, lexerBenchSource)
214 assert.NoError(b, err)
215 for t := it(); t != chroma.EOF; t = it() {
216 }
217 }
218 }
219
View as plain text