1#include "flexbuffers_test.h"
2
3#include <limits>
4
5#include "flatbuffers/flexbuffers.h"
6#include "flatbuffers/idl.h"
7#include "is_quiet_nan.h"
8#include "test_assert.h"
9
10namespace flatbuffers {
11namespace tests {
12
13// Shortcuts for the infinity.
14static const auto infinity_d = std::numeric_limits<double>::infinity();
15
16void FlexBuffersTest() {
17 flexbuffers::Builder slb(512,
18 flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
19
20 // Write the equivalent of:
21 // { vec: [ -100, "Fred", 4.0, false ], bar: [ 1, 2, 3 ], bar3: [ 1, 2, 3 ],
22 // foo: 100, bool: true, mymap: { foo: "Fred" } }
23
24 // It's possible to do this without std::function support as well.
25 slb.Map([&]() {
26 slb.Vector("vec", [&]() {
27 slb += -100; // Equivalent to slb.Add(-100) or slb.Int(-100);
28 slb += "Fred";
29 slb.IndirectFloat(4.0f);
30 auto i_f = slb.LastValue();
31 uint8_t blob[] = { 77 };
32 slb.Blob(blob, 1);
33 slb += false;
34 slb.ReuseValue(i_f);
35 });
36 int ints[] = { 1, 2, 3 };
37 slb.Vector("bar", ints, 3);
38 slb.FixedTypedVector("bar3", ints, 3);
39 bool bools[] = { true, false, true, false };
40 slb.Vector("bools", bools, 4);
41 slb.Bool("bool", true);
42 slb.Double("foo", 100);
43 slb.Map("mymap", [&]() {
44 slb.String("foo", "Fred"); // Testing key and string reuse.
45 });
46 });
47 slb.Finish();
48
49 // clang-format off
50 #ifdef FLATBUFFERS_TEST_VERBOSE
51 for (size_t i = 0; i < slb.GetBuffer().size(); i++)
52 printf("%d ", slb.GetBuffer().data()[i]);
53 printf("\n");
54 #endif
55 // clang-format on
56
57 std::vector<uint8_t> reuse_tracker;
58 TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
59 slb.GetBuffer().size(), &reuse_tracker),
60 true);
61
62 auto map = flexbuffers::GetRoot(slb.GetBuffer()).AsMap();
63 TEST_EQ(map.size(), 7);
64 auto vec = map["vec"].AsVector();
65 TEST_EQ(vec.size(), 6);
66 TEST_EQ(vec[0].AsInt64(), -100);
67 TEST_EQ_STR(vec[1].AsString().c_str(), "Fred");
68 TEST_EQ(vec[1].AsInt64(), 0); // Number parsing failed.
69 TEST_EQ(vec[2].AsDouble(), 4.0);
70 TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type.
71 TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though.
72 TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted.
73 // Few tests for templated version of As.
74 TEST_EQ(vec[0].As<int64_t>(), -100);
75 TEST_EQ_STR(vec[1].As<std::string>().c_str(), "Fred");
76 TEST_EQ(vec[1].As<int64_t>(), 0); // Number parsing failed.
77 TEST_EQ(vec[2].As<double>(), 4.0);
78 // Test that the blob can be accessed.
79 TEST_EQ(vec[3].IsBlob(), true);
80 auto blob = vec[3].AsBlob();
81 TEST_EQ(blob.size(), 1);
82 TEST_EQ(blob.data()[0], 77);
83 TEST_EQ(vec[4].IsBool(), true); // Check if type is a bool
84 TEST_EQ(vec[4].AsBool(), false); // Check if value is false
85 TEST_EQ(vec[5].AsDouble(), 4.0); // This is shared with vec[2] !
86 auto tvec = map["bar"].AsTypedVector();
87 TEST_EQ(tvec.size(), 3);
88 TEST_EQ(tvec[2].AsInt8(), 3);
89 auto tvec3 = map["bar3"].AsFixedTypedVector();
90 TEST_EQ(tvec3.size(), 3);
91 TEST_EQ(tvec3[2].AsInt8(), 3);
92 TEST_EQ(map["bool"].AsBool(), true);
93 auto tvecb = map["bools"].AsTypedVector();
94 TEST_EQ(tvecb.ElementType(), flexbuffers::FBT_BOOL);
95 TEST_EQ(map["foo"].AsUInt8(), 100);
96 TEST_EQ(map["unknown"].IsNull(), true);
97 auto mymap = map["mymap"].AsMap();
98 // These should be equal by pointer equality, since key and value are shared.
99 TEST_EQ(mymap.Keys()[0].AsKey(), map.Keys()[4].AsKey());
100 TEST_EQ(mymap.Values()[0].AsString().c_str(), vec[1].AsString().c_str());
101 // We can mutate values in the buffer.
102 TEST_EQ(vec[0].MutateInt(-99), true);
103 TEST_EQ(vec[0].AsInt64(), -99);
104 TEST_EQ(vec[1].MutateString("John"), true); // Size must match.
105 TEST_EQ_STR(vec[1].AsString().c_str(), "John");
106 TEST_EQ(vec[1].MutateString("Alfred"), false); // Too long.
107 TEST_EQ(vec[2].MutateFloat(2.0f), true);
108 TEST_EQ(vec[2].AsFloat(), 2.0f);
109 TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float.
110 TEST_EQ(vec[4].AsBool(), false); // Is false before change
111 TEST_EQ(vec[4].MutateBool(true), true); // Can change a bool
112 TEST_EQ(vec[4].AsBool(), true); // Changed bool is now true
113
114 // Parse from JSON:
115 flatbuffers::Parser parser;
116 slb.Clear();
117 auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\", c: true, d: false }";
118 TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
119 TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
120 slb.GetBuffer().size(), &reuse_tracker),
121 true);
122 auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
123 auto jmap = jroot.AsMap();
124 auto jvec = jmap["a"].AsVector();
125 TEST_EQ(jvec[0].AsInt64(), 123);
126 TEST_EQ(jvec[1].AsDouble(), 456.0);
127 TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
128 TEST_EQ(jmap["c"].IsBool(), true); // Parsed correctly to a bool
129 TEST_EQ(jmap["c"].AsBool(), true); // Parsed correctly to true
130 TEST_EQ(jmap["d"].IsBool(), true); // Parsed correctly to a bool
131 TEST_EQ(jmap["d"].AsBool(), false); // Parsed correctly to false
132 // And from FlexBuffer back to JSON:
133 auto jsonback = jroot.ToString();
134 TEST_EQ_STR(jsontest, jsonback.c_str());
135
136 slb.Clear();
137 slb.Vector([&]() {
138 for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
139 slb.Vector([&]() {
140 for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
141 slb.Vector([] {});
142 });
143 });
144 slb.Finish();
145 TEST_EQ(slb.GetSize(), 664);
146}
147
148void FlexBuffersReuseBugTest() {
149 flexbuffers::Builder slb;
150 slb.Map([&]() {
151 slb.Vector("vec", [&]() {});
152 slb.Bool("bool", true);
153 });
154 slb.Finish();
155 std::vector<uint8_t> reuse_tracker;
156 // This would fail before, since the reuse_tracker would use the address of
157 // the vector reference to check for reuse, but in this case we have an empty
158 // vector, and since the size field is before the pointer, its address is the
159 // same as thing after it, the key "bool".
160 // We fix this by using the address of the size field for tracking reuse.
161 TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
162 slb.GetBuffer().size(), &reuse_tracker),
163 true);
164}
165
166void FlexBuffersFloatingPointTest() {
167#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
168 flexbuffers::Builder slb(512,
169 flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
170 // Parse floating-point values from JSON:
171 flatbuffers::Parser parser;
172 slb.Clear();
173 auto jsontest =
174 "{ a: [1.0, nan, inf, infinity, -inf, +inf, -infinity, 8.0] }";
175 TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
176 auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
177 TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
178 slb.GetBuffer().size(), nullptr),
179 true);
180 auto jmap = jroot.AsMap();
181 auto jvec = jmap["a"].AsVector();
182 TEST_EQ(8, jvec.size());
183 TEST_EQ(1.0, jvec[0].AsDouble());
184 TEST_ASSERT(is_quiet_nan(jvec[1].AsDouble()));
185 TEST_EQ(infinity_d, jvec[2].AsDouble());
186 TEST_EQ(infinity_d, jvec[3].AsDouble());
187 TEST_EQ(-infinity_d, jvec[4].AsDouble());
188 TEST_EQ(+infinity_d, jvec[5].AsDouble());
189 TEST_EQ(-infinity_d, jvec[6].AsDouble());
190 TEST_EQ(8.0, jvec[7].AsDouble());
191#endif
192}
193
194void FlexBuffersDeprecatedTest() {
195 // FlexBuffers as originally designed had a flaw involving the
196 // FBT_VECTOR_STRING datatype, and this test documents/tests the fix for it.
197 // Discussion: https://github.com/google/flatbuffers/issues/5627
198 flexbuffers::Builder slb;
199 // FBT_VECTOR_* are "typed vectors" where all elements are of the same type.
200 // Problem is, when storing FBT_STRING elements, it relies on that type to
201 // get the bit-width for the size field of the string, which in this case
202 // isn't present, and instead defaults to 8-bit. This means that any strings
203 // stored inside such a vector, when accessed thru the old API that returns
204 // a String reference, will appear to be truncated if the string stored is
205 // actually >=256 bytes.
206 std::string test_data(300, 'A');
207 auto start = slb.StartVector();
208 // This one will have a 16-bit size field.
209 slb.String(test_data);
210 // This one will have an 8-bit size field.
211 slb.String("hello");
212 // We're asking this to be serialized as a typed vector (true), but not
213 // fixed size (false). The type will be FBT_VECTOR_STRING with a bit-width
214 // of whatever the offsets in the vector need, the bit-widths of the strings
215 // are not stored(!) <- the actual design flaw.
216 // Note that even in the fixed code, we continue to serialize the elements of
217 // FBT_VECTOR_STRING as FBT_STRING, since there may be old code out there
218 // reading new data that we want to continue to function.
219 // Thus, FBT_VECTOR_STRING, while deprecated, will always be represented the
220 // same way, the fix lies on the reading side.
221 slb.EndVector(start, true, false);
222 slb.Finish();
223 // Verify because why not.
224 TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
225 slb.GetBuffer().size(), nullptr),
226 true);
227 // So now lets read this data back.
228 // For existing data, since we have no way of knowing what the actual
229 // bit-width of the size field of the string is, we are going to ignore this
230 // field, and instead treat these strings as FBT_KEY (null-terminated), so we
231 // can deal with strings of arbitrary length. This of course truncates strings
232 // with embedded nulls, but we think that that is preferrable over truncating
233 // strings >= 256 bytes.
234 auto vec = flexbuffers::GetRoot(slb.GetBuffer()).AsTypedVector();
235 // Even though this was serialized as FBT_VECTOR_STRING, it is read as
236 // FBT_VECTOR_KEY:
237 TEST_EQ(vec.ElementType(), flexbuffers::FBT_KEY);
238 // Access the long string. Previously, this would return a string of size 1,
239 // since it would read the high-byte of the 16-bit length.
240 // This should now correctly test the full 300 bytes, using AsKey():
241 TEST_EQ_STR(vec[0].AsKey(), test_data.c_str());
242 // Old code that called AsString will continue to work, as the String
243 // accessor objects now use a cached size that can come from a key as well.
244 TEST_EQ_STR(vec[0].AsString().c_str(), test_data.c_str());
245 // Short strings work as before:
246 TEST_EQ_STR(vec[1].AsKey(), "hello");
247 TEST_EQ_STR(vec[1].AsString().c_str(), "hello");
248 // So, while existing code and data mostly "just work" with the fixes applied
249 // to AsTypedVector and AsString, what do you do going forward?
250 // Code accessing existing data doesn't necessarily need to change, though
251 // you could consider using AsKey instead of AsString for a) documenting
252 // that you are accessing keys, or b) a speedup if you don't actually use
253 // the string size.
254 // For new data, or data that doesn't need to be backwards compatible,
255 // instead serialize as FBT_VECTOR (call EndVector with typed = false, then
256 // read elements with AsString), or, for maximum compactness, use
257 // FBT_VECTOR_KEY (call slb.Key above instead, read with AsKey or AsString).
258}
259
260void ParseFlexbuffersFromJsonWithNullTest() {
261 // Test nulls are handled appropriately through flexbuffers to exercise other
262 // code paths of ParseSingleValue in the optional scalars change.
263 // TODO(cneo): Json -> Flatbuffers test once some language can generate code
264 // with optional scalars.
265 {
266 char json[] = "{\"opt_field\": 123 }";
267 flatbuffers::Parser parser;
268 flexbuffers::Builder flexbuild;
269 parser.ParseFlexBuffer(json, nullptr, &flexbuild);
270 auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
271 TEST_EQ(root.AsMap()["opt_field"].AsInt64(), 123);
272 }
273 {
274 char json[] = "{\"opt_field\": 123.4 }";
275 flatbuffers::Parser parser;
276 flexbuffers::Builder flexbuild;
277 parser.ParseFlexBuffer(json, nullptr, &flexbuild);
278 auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
279 TEST_EQ(root.AsMap()["opt_field"].AsDouble(), 123.4);
280 }
281 {
282 char json[] = "{\"opt_field\": null }";
283 flatbuffers::Parser parser;
284 flexbuffers::Builder flexbuild;
285 parser.ParseFlexBuffer(json, nullptr, &flexbuild);
286 auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
287 TEST_ASSERT(!root.AsMap().IsTheEmptyMap());
288 TEST_ASSERT(root.AsMap()["opt_field"].IsNull());
289 TEST_EQ(root.ToString(), std::string("{ opt_field: null }"));
290 }
291}
292
293} // namespace tests
294} // namespace flatbuffers
View as plain text