1#!/usr/bin/env python3
2#
3# Copyright 2021 Google Inc. All rights reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import filecmp
18import glob
19import shutil
20import subprocess
21import generate_grpc_examples
22from pathlib import Path
23from util import flatc, root_path, tests_path, args, flatc_path
24
25# Specify the other paths that will be referenced
26swift_code_gen = Path(root_path, "tests/swift/tests/CodeGenerationTests")
27ts_code_gen = Path(root_path, "tests/ts")
28samples_path = Path(root_path, "samples")
29reflection_path = Path(root_path, "reflection")
30
31
32# Generate the code for flatbuffers reflection schema
33def flatc_reflection(options, location, target):
34 full_options = ["--no-prefix"] + options
35 temp_dir = ".tmp"
36 flatc(
37 full_options,
38 prefix=temp_dir,
39 schema="reflection.fbs",
40 cwd=reflection_path,
41 )
42 new_reflection_path = Path(reflection_path, temp_dir, target)
43 original_reflection_path = Path(root_path, location, target)
44 if not filecmp.cmp(str(new_reflection_path), str(original_reflection_path)):
45 shutil.rmtree(str(original_reflection_path), ignore_errors=True)
46 shutil.move(str(new_reflection_path), str(original_reflection_path))
47 shutil.rmtree(str(Path(reflection_path, temp_dir)))
48
49
50def flatc_annotate(schema, file, include=None, cwd=tests_path):
51 cmd = [str(flatc_path)]
52 if include:
53 cmd += ["-I"] + [include]
54 cmd += ["--annotate", schema, file]
55 result = subprocess.run(cmd, cwd=str(cwd), check=True)
56
57
58# Glob a pattern relative to file path
59def glob(path, pattern):
60 return [str(p) for p in path.glob(pattern)]
61
62
63# flatc options that are shared
64BASE_OPTS = ["--reflect-names", "--gen-mutable", "--gen-object-api"]
65NO_INCL_OPTS = BASE_OPTS + ["--no-includes"]
66
67# Language specific options
68CS_OPTS = ["--csharp", "--cs-gen-json-serializer"]
69CPP_OPTS = [
70 "--cpp",
71 "--gen-compare",
72] + (["--cpp-std", "c++0x"] if args.cpp_0x else [])
73
74CPP_17_OPTS = NO_INCL_OPTS + [
75 "--cpp",
76 "--cpp-std",
77 "c++17",
78 "--cpp-static-reflection",
79 "--gen-object-api",
80]
81RUST_OPTS = BASE_OPTS + [
82 "--rust",
83 "--gen-all",
84 "--gen-name-strings",
85 "--rust-module-root-file",
86]
87RUST_SERIALIZE_OPTS = BASE_OPTS + [
88 "--rust",
89 "--gen-all",
90 "--gen-name-strings",
91 "--rust-serialize",
92 "--rust-module-root-file",
93]
94TS_OPTS = ["--ts", "--gen-name-strings"]
95LOBSTER_OPTS = ["--lobster"]
96SWIFT_OPTS = ["--swift", "--gen-json-emit", "--bfbs-filenames", str(tests_path)]
97SWIFT_OPTS_CODE_GEN = [
98 "--swift",
99 "--gen-json-emit",
100 "--bfbs-filenames",
101 str(swift_code_gen),
102]
103JAVA_OPTS = ["--java"]
104KOTLIN_OPTS = ["--kotlin"]
105PHP_OPTS = ["--php"]
106DART_OPTS = ["--dart"]
107PYTHON_OPTS = ["--python", "--python-typing"]
108BINARY_OPTS = ["-b", "--schema", "--bfbs-comments", "--bfbs-builtins"]
109PROTO_OPTS = ["--proto"]
110
111# Basic Usage
112
113flatc(
114 NO_INCL_OPTS
115 + CPP_OPTS
116 + CS_OPTS
117 + [
118 "--binary",
119 "--java",
120 "--kotlin",
121 "--dart",
122 "--go",
123 "--lobster",
124 "--php",
125 ],
126 schema="monster_test.fbs",
127 include="include_test",
128 data="monsterdata_test.json",
129)
130
131flatc(
132 NO_INCL_OPTS + DART_OPTS,
133 schema="include_test/include_test1.fbs",
134 include="include_test/sub",
135)
136
137flatc(
138 NO_INCL_OPTS + DART_OPTS,
139 schema="include_test/sub/include_test2.fbs",
140 include="include_test",
141)
142
143flatc(
144 NO_INCL_OPTS + TS_OPTS,
145 cwd=ts_code_gen,
146 schema="../monster_test.fbs",
147 include="../include_test",
148 data="../monsterdata_test.json",
149)
150
151flatc(
152 ["--lua", "--bfbs-filenames", str(tests_path)],
153 schema="monster_test.fbs",
154 include="include_test",
155)
156
157flatc(
158 NO_INCL_OPTS + CPP_OPTS + ["--grpc"],
159 schema="monster_test.fbs",
160 include="include_test",
161 data="monsterdata_test.json",
162)
163
164flatc(
165 RUST_OPTS,
166 schema="monster_test.fbs",
167 include="include_test",
168 prefix="monster_test",
169 data="monsterdata_test.json",
170)
171
172flatc(
173 RUST_SERIALIZE_OPTS,
174 schema="monster_test.fbs",
175 include="include_test",
176 prefix="monster_test_serialize",
177 data="monsterdata_test.json",
178)
179
180flatc(
181 options=BASE_OPTS + ["--python"],
182 schema="monster_test.fbs",
183 include="include_test",
184 data="monsterdata_test.json",
185)
186
187flatc(
188 options=BASE_OPTS + ["--python", "--gen-onefile"],
189 schema="monster_test.fbs",
190 include="include_test",
191 data="monsterdata_test.json",
192)
193
194flatc(
195 PROTO_OPTS,
196 schema="prototest/test.proto",
197)
198
199# For Rust we currently generate two independent schemas, with namespace_test2
200# duplicating the types in namespace_test1
201flatc(
202 RUST_OPTS + CS_OPTS,
203 prefix="namespace_test",
204 schema=[
205 "namespace_test/namespace_test1.fbs",
206 "namespace_test/namespace_test2.fbs",
207 ],
208)
209
210flatc(
211 [
212 "--cpp",
213 "--reflect-names",
214 "--no-includes",
215 "--gen-mutable",
216 "--gen-object-api",
217 "--gen-compare",
218 "--gen-name-strings",
219 ],
220 prefix="namespace_test",
221 schema=[
222 "namespace_test/namespace_test1.fbs",
223 "namespace_test/namespace_test2.fbs",
224 ],
225)
226
227flatc(
228 BASE_OPTS + CPP_OPTS + CS_OPTS + JAVA_OPTS + KOTLIN_OPTS + PHP_OPTS,
229 prefix="union_vector",
230 schema="union_vector/union_vector.fbs",
231)
232
233flatc(
234 BASE_OPTS + TS_OPTS,
235 cwd=ts_code_gen,
236 prefix="union_vector",
237 schema="../union_vector/union_vector.fbs",
238)
239
240flatc(
241 BASE_OPTS + TS_OPTS + ["--gen-name-strings", "--gen-mutable"],
242 cwd=ts_code_gen,
243 include="../include_test",
244 schema="../monster_test.fbs",
245)
246
247flatc(
248 BASE_OPTS + TS_OPTS + ["-b"],
249 cwd=ts_code_gen,
250 include="../include_test",
251 schema="../monster_test.fbs",
252 data="../unicode_test.json",
253)
254
255flatc(
256 BASE_OPTS + TS_OPTS + ["--gen-name-strings"],
257 cwd=ts_code_gen,
258 prefix="union_vector",
259 schema="../union_vector/union_vector.fbs",
260)
261
262flatc(
263 RUST_OPTS,
264 prefix="include_test1",
265 include="include_test",
266 schema="include_test/include_test1.fbs",
267)
268
269flatc(
270 RUST_OPTS,
271 prefix="include_test2",
272 include="include_test",
273 schema="include_test/sub/include_test2.fbs",
274)
275
276flatc(
277 BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
278 include="include_test",
279 schema="monster_test.fbs",
280)
281
282# Generate the annotated binary of the monster_test binary schema.
283flatc_annotate(
284 schema="../reflection/reflection.fbs",
285 file="monster_test.bfbs",
286 include="include_test",
287)
288
289flatc_annotate(
290 schema="monster_test.fbs", file="monsterdata_test.mon", include="include_test"
291)
292
293flatc(
294 CPP_OPTS
295 + NO_INCL_OPTS
296 + [
297 "--bfbs-comments",
298 "--bfbs-builtins",
299 "--bfbs-gen-embed",
300 "--bfbs-filenames",
301 str(tests_path),
302 ],
303 include="include_test",
304 schema="monster_test.fbs",
305)
306
307flatc(
308 BINARY_OPTS + ["--bfbs-filenames", str(tests_path)],
309 include="include_test",
310 schema="arrays_test.fbs",
311)
312
313flatc(
314 ["--jsonschema", "--schema"],
315 include="include_test",
316 schema="monster_test.fbs",
317)
318
319if not args.skip_monster_extra:
320 flatc(
321 CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + KOTLIN_OPTS + PYTHON_OPTS,
322 schema="monster_extra.fbs",
323 data="monsterdata_extra.json",
324 )
325
326 flatc(
327 DART_OPTS + ["--gen-object-api"],
328 schema="monster_extra.fbs",
329 )
330
331flatc(
332 CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + ["--jsonschema", "--scoped-enums"],
333 schema="arrays_test.fbs",
334)
335
336flatc(
337 RUST_OPTS,
338 prefix="arrays_test",
339 schema="arrays_test.fbs",
340)
341
342flatc(
343 RUST_OPTS,
344 prefix="rust_namer_test",
345 schema="rust_namer_test.fbs",
346)
347
348flatc(
349 BASE_OPTS + PYTHON_OPTS,
350 schema="arrays_test.fbs",
351)
352
353
354flatc(
355 BASE_OPTS + PYTHON_OPTS,
356 schema="nested_union_test.fbs",
357)
358
359
360# Optional Scalars
361optional_scalars_schema = "optional_scalars.fbs"
362flatc(["--java", "--kotlin", "--lobster"], schema=optional_scalars_schema)
363flatc(TS_OPTS, cwd=ts_code_gen, schema="../optional_scalars.fbs")
364
365flatc(["--csharp", "--python", "--gen-object-api"], schema=optional_scalars_schema)
366
367flatc(RUST_OPTS, prefix="optional_scalars", schema=optional_scalars_schema)
368
369flatc(NO_INCL_OPTS + CPP_OPTS, schema=optional_scalars_schema)
370
371# Type / field collsion
372type_field_collsion_schema = "type_field_collsion.fbs"
373
374flatc(["--csharp", "--gen-object-api"], schema=type_field_collsion_schema)
375
376# Union / value collision
377flatc(
378 CS_OPTS + ["--gen-object-api", "--gen-onefile"],
379 prefix="union_value_collsion",
380 schema="union_value_collision.fbs",
381)
382
383# Generate string/vector default code for tests
384flatc(RUST_OPTS, prefix="more_defaults", schema="more_defaults.fbs")
385
386# Generate the schema evolution tests
387flatc(
388 CPP_OPTS + ["--scoped-enums"],
389 prefix="evolution_test",
390 schema=glob(tests_path, "evolution_test/evolution_v*.fbs"),
391)
392
393# Generate the keywords tests
394flatc(BASE_OPTS + CS_OPTS, schema="keyword_test.fbs")
395flatc(RUST_OPTS, prefix="keyword_test", schema="keyword_test.fbs")
396flatc(
397 BASE_OPTS + CS_OPTS + ["--cs-global-alias", "--gen-onefile"],
398 prefix="nested_namespace_test",
399 schema=glob(tests_path, "nested_namespace_test/nested_namespace_test*.fbs"),
400)
401flatc(BASE_OPTS + DART_OPTS, prefix="../dart/test/", schema="keyword_test.fbs")
402
403# Field key lookup with default value test
404dictionary_lookup_schema = "dictionary_lookup.fbs"
405flatc(["--java", "--kotlin"], schema=dictionary_lookup_schema)
406
407# Swift Tests
408swift_prefix = "swift/tests/Tests/FlatBuffers.Test.SwiftTests"
409flatc(
410 SWIFT_OPTS + BASE_OPTS + ["--grpc"],
411 schema="monster_test.fbs",
412 include="include_test",
413 prefix=swift_prefix,
414)
415flatc(
416 SWIFT_OPTS + BASE_OPTS,
417 schema="union_vector/union_vector.fbs",
418 prefix=swift_prefix,
419)
420flatc(SWIFT_OPTS, schema="optional_scalars.fbs", prefix=swift_prefix)
421flatc(SWIFT_OPTS, schema="vector_has_test.fbs", prefix=swift_prefix)
422flatc(SWIFT_OPTS, schema="nan_inf_test.fbs", prefix=swift_prefix)
423flatc(
424 SWIFT_OPTS + ["--gen-object-api"],
425 schema="more_defaults.fbs",
426 prefix=swift_prefix,
427)
428flatc(
429 SWIFT_OPTS + BASE_OPTS,
430 schema="MutatingBool.fbs",
431 prefix=swift_prefix,
432)
433
434flatc(
435 SWIFT_OPTS_CODE_GEN + BASE_OPTS + ["--grpc", "--swift-implementation-only"],
436 schema="test_import.fbs",
437 cwd=swift_code_gen,
438)
439
440flatc(
441 SWIFT_OPTS_CODE_GEN + NO_INCL_OPTS + ["--grpc"],
442 schema="test_no_include.fbs",
443 cwd=swift_code_gen,
444)
445
446# Swift Wasm Tests
447swift_Wasm_prefix = "swift/Wasm.tests/Tests/FlatBuffers.Test.Swift.WasmTests"
448flatc(
449 SWIFT_OPTS + BASE_OPTS,
450 schema="monster_test.fbs",
451 include="include_test",
452 prefix=swift_Wasm_prefix,
453)
454
455# Nim Tests
456NIM_OPTS = BASE_OPTS + ["--nim"]
457flatc(NIM_OPTS, schema="monster_test.fbs", include="include_test")
458flatc(NIM_OPTS, schema="optional_scalars.fbs")
459flatc(NIM_OPTS, schema="more_defaults.fbs")
460flatc(NIM_OPTS, schema="MutatingBool.fbs")
461
462# --filename-suffix and --filename-ext tests
463flatc(
464 CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-ext", "hpp"],
465 include="include_test",
466 prefix="monster_test_suffix/ext_only",
467 schema="monster_test.fbs",
468)
469flatc(
470 CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-suffix", "_suffix"],
471 include="include_test",
472 prefix="monster_test_suffix/filesuffix_only",
473 schema="monster_test.fbs",
474)
475flatc(
476 CPP_OPTS
477 + NO_INCL_OPTS
478 + ["--grpc", "--filename-suffix", "_suffix", "--filename-ext", "hpp"],
479 include="include_test",
480 prefix="monster_test_suffix",
481 schema="monster_test.fbs",
482)
483
484# Flag c++17 requires Clang6, GCC7, MSVC2017 (_MSC_VER >= 1914) or higher.
485cpp_17_prefix = "cpp17/generated_cpp17"
486flatc(
487 CPP_17_OPTS,
488 schema="monster_test.fbs",
489 include="include_test",
490 prefix=cpp_17_prefix,
491)
492flatc(
493 CPP_17_OPTS,
494 schema="optional_scalars.fbs",
495 prefix=cpp_17_prefix,
496)
497flatc(
498 CPP_17_OPTS,
499 schema="union_vector/union_vector.fbs",
500 prefix=cpp_17_prefix,
501)
502
503# Private annotations
504annotations_test_schema = "private_annotation_test.fbs"
505
506flatc(
507 RUST_OPTS + ["--no-leak-private-annotation", "--gen-object-api"],
508 prefix="private_annotation_test",
509 schema=annotations_test_schema,
510)
511
512# Sample files
513samples_schema = "monster.fbs"
514flatc(BASE_OPTS + CPP_OPTS + LOBSTER_OPTS, schema=samples_schema, cwd=samples_path)
515flatc(RUST_OPTS, prefix="rust_generated", schema=samples_schema, cwd=samples_path)
516flatc(
517 BINARY_OPTS + ["--bfbs-filenames", str(samples_path)],
518 schema=samples_schema,
519 cwd=samples_path,
520)
521
522# Reflection
523
524# Skip generating the reflection if told too, as we run this script after
525# building flatc which uses the reflection_generated.h itself.
526if not args.skip_gen_reflection:
527 # C++ Reflection
528 flatc_reflection(
529 ["-c", "--cpp-std", "c++0x"], "include/flatbuffers", "reflection_generated.h"
530 )
531
532# Python Reflection
533flatc_reflection(["-p"], "python/flatbuffers", "reflection")
534
535# Java Reflection
536flatc_reflection(
537 ["-j", "--java-package-prefix", "com.google.flatbuffers"],
538 "java/src/main/java",
539 "com/google/flatbuffers/reflection",
540)
541
542# Annotation
543
544
545def flatc_annotate(schema, include=None, data=None, cwd=tests_path):
546 cmd = [str(flatc_path)]
547 if include:
548 cmd += ["-I"] + [include]
549 cmd += ["--annotate", schema]
550 if data:
551 cmd += [data] if isinstance(data, str) else data
552 subprocess.run(cmd, cwd=str(cwd), check=True)
553
554
555flatc_annotate(
556 schema="monster_test.fbs", include="include_test", data="monsterdata_test.mon"
557)
558
559# Run the generate_grpc_examples script
560generate_grpc_examples.GenerateGRPCExamples()
View as plain text