1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package bigquery
16
17 import (
18 "encoding/json"
19 "fmt"
20 "math/big"
21 "reflect"
22 "testing"
23 "time"
24
25 "cloud.google.com/go/civil"
26 "cloud.google.com/go/internal/pretty"
27 "cloud.google.com/go/internal/testutil"
28 "github.com/google/go-cmp/cmp"
29 bq "google.golang.org/api/bigquery/v2"
30 )
31
32 func (fs *FieldSchema) GoString() string {
33 if fs == nil {
34 return "<nil>"
35 }
36
37 return fmt.Sprintf("{Name:%s Description:%s Repeated:%t Required:%t Type:%s Schema:%s}",
38 fs.Name,
39 fs.Description,
40 fs.Repeated,
41 fs.Required,
42 fs.Type,
43 fmt.Sprintf("%#v", fs.Schema),
44 )
45 }
46
47 func bqTableFieldSchema(desc, name, typ, mode string, tags []string) *bq.TableFieldSchema {
48 var policy *bq.TableFieldSchemaPolicyTags
49 if tags != nil {
50 policy = &bq.TableFieldSchemaPolicyTags{
51 Names: tags,
52 }
53 }
54 return &bq.TableFieldSchema{
55 Description: desc,
56 Name: name,
57 Mode: mode,
58 Type: typ,
59 PolicyTags: policy,
60 }
61 }
62
63 func fieldSchema(desc, name, typ string, repeated, required bool, tags []string) *FieldSchema {
64 var policy *PolicyTagList
65 if tags != nil {
66 policy = &PolicyTagList{
67 Names: tags,
68 }
69 }
70 return &FieldSchema{
71 Description: desc,
72 Name: name,
73 Repeated: repeated,
74 Required: required,
75 Type: FieldType(typ),
76 PolicyTags: policy,
77 }
78 }
79
80 func TestRelaxSchema(t *testing.T) {
81 testCases := []struct {
82 in Schema
83 expected Schema
84 }{
85 {
86 Schema{
87 &FieldSchema{
88 Description: "a relaxed schema",
89 Required: false,
90 Type: StringFieldType,
91 },
92 },
93 Schema{
94 &FieldSchema{
95 Description: "a relaxed schema",
96 Required: false,
97 Type: StringFieldType,
98 },
99 },
100 },
101 {
102 Schema{
103 &FieldSchema{
104 Description: "a required string",
105 Required: true,
106 Type: StringFieldType,
107 },
108 &FieldSchema{
109 Description: "a required integer",
110 Required: true,
111 Type: IntegerFieldType,
112 },
113 },
114 Schema{
115 &FieldSchema{
116 Description: "a required string",
117 Required: false,
118 Type: StringFieldType,
119 },
120 &FieldSchema{
121 Description: "a required integer",
122 Required: false,
123 Type: IntegerFieldType,
124 },
125 },
126 },
127 {
128 Schema{
129 &FieldSchema{
130 Description: "An outer schema wrapping a nested schema",
131 Name: "outer",
132 Required: true,
133 Type: RecordFieldType,
134 Schema: Schema{
135 {
136 Description: "inner field",
137 Name: "inner",
138 Type: StringFieldType,
139 Required: true,
140 },
141 },
142 },
143 },
144 Schema{
145 &FieldSchema{
146 Description: "An outer schema wrapping a nested schema",
147 Name: "outer",
148 Required: false,
149 Type: "RECORD",
150 Schema: Schema{
151 {
152 Description: "inner field",
153 Name: "inner",
154 Type: "STRING",
155 Required: false,
156 },
157 },
158 },
159 },
160 },
161 }
162 for _, tc := range testCases {
163 converted := tc.in.Relax()
164 if !testutil.Equal(converted, tc.expected) {
165 t.Errorf("relaxing schema: got:\n%v\nwant:\n%v",
166 pretty.Value(converted), pretty.Value(tc.expected))
167 }
168 }
169 }
170 func TestSchemaConversion(t *testing.T) {
171 testCases := []struct {
172 schema Schema
173 bqSchema *bq.TableSchema
174 }{
175 {
176
177 bqSchema: &bq.TableSchema{
178 Fields: []*bq.TableFieldSchema{
179 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
180 },
181 },
182 schema: Schema{
183 fieldSchema("desc", "name", "STRING", false, true, nil),
184 },
185 },
186 {
187
188 bqSchema: &bq.TableSchema{
189 Fields: []*bq.TableFieldSchema{
190 bqTableFieldSchema("desc", "name", "STRING", "REPEATED", nil),
191 },
192 },
193 schema: Schema{
194 fieldSchema("desc", "name", "STRING", true, false, nil),
195 },
196 },
197 {
198
199 bqSchema: &bq.TableSchema{
200 Fields: []*bq.TableFieldSchema{
201 bqTableFieldSchema("desc", "name", "STRING", "", nil),
202 },
203 },
204 schema: Schema{
205 fieldSchema("desc", "name", "STRING", false, false, nil),
206 },
207 },
208 {
209
210 bqSchema: &bq.TableSchema{
211 Fields: []*bq.TableFieldSchema{
212 bqTableFieldSchema("desc", "name", "INTEGER", "", nil),
213 },
214 },
215 schema: Schema{
216 fieldSchema("desc", "name", "INTEGER", false, false, nil),
217 },
218 },
219 {
220
221 bqSchema: &bq.TableSchema{
222 Fields: []*bq.TableFieldSchema{
223 bqTableFieldSchema("desc", "name", "FLOAT", "", nil),
224 },
225 },
226 schema: Schema{
227 fieldSchema("desc", "name", "FLOAT", false, false, nil),
228 },
229 },
230 {
231
232 bqSchema: &bq.TableSchema{
233 Fields: []*bq.TableFieldSchema{
234 bqTableFieldSchema("desc", "name", "BOOLEAN", "", nil),
235 },
236 },
237 schema: Schema{
238 fieldSchema("desc", "name", "BOOLEAN", false, false, nil),
239 },
240 },
241 {
242
243 bqSchema: &bq.TableSchema{
244 Fields: []*bq.TableFieldSchema{
245 bqTableFieldSchema("desc", "name", "TIMESTAMP", "", nil),
246 },
247 },
248 schema: Schema{
249 fieldSchema("desc", "name", "TIMESTAMP", false, false, nil),
250 },
251 },
252 {
253
254 bqSchema: &bq.TableSchema{
255 Fields: []*bq.TableFieldSchema{
256 bqTableFieldSchema("desc", "f1", "TIME", "", nil),
257 bqTableFieldSchema("desc", "f2", "DATE", "", nil),
258 bqTableFieldSchema("desc", "f3", "DATETIME", "", nil),
259 },
260 },
261 schema: Schema{
262 fieldSchema("desc", "f1", "TIME", false, false, nil),
263 fieldSchema("desc", "f2", "DATE", false, false, nil),
264 fieldSchema("desc", "f3", "DATETIME", false, false, nil),
265 },
266 },
267 {
268
269 bqSchema: &bq.TableSchema{
270 Fields: []*bq.TableFieldSchema{
271 bqTableFieldSchema("desc", "n", "NUMERIC", "", nil),
272 },
273 },
274 schema: Schema{
275 fieldSchema("desc", "n", "NUMERIC", false, false, nil),
276 },
277 },
278 {
279
280 bqSchema: &bq.TableSchema{
281 Fields: []*bq.TableFieldSchema{
282 bqTableFieldSchema("geo", "g", "GEOGRAPHY", "", nil),
283 },
284 },
285 schema: Schema{
286 fieldSchema("geo", "g", "GEOGRAPHY", false, false, nil),
287 },
288 },
289 {
290
291 bqSchema: &bq.TableSchema{
292 Fields: []*bq.TableFieldSchema{
293 {
294 Name: "foo",
295 Type: "STRING",
296 MaxLength: 0,
297 Precision: 0,
298 Scale: 0,
299 },
300 {
301 Name: "bar",
302 Type: "STRING",
303 MaxLength: 1,
304 Precision: 2,
305 Scale: 3,
306 },
307 }},
308 schema: Schema{
309 {Name: "foo",
310 Type: StringFieldType,
311 MaxLength: 0,
312 Precision: 0,
313 Scale: 0,
314 },
315 {Name: "bar",
316 Type: StringFieldType,
317 MaxLength: 1,
318 Precision: 2,
319 Scale: 3,
320 },
321 },
322 },
323 {
324
325 bqSchema: &bq.TableSchema{
326 Fields: []*bq.TableFieldSchema{
327 {
328 Name: "foo",
329 Type: "STRING",
330 DefaultValueExpression: "I_LOVE_FOO",
331 },
332 {
333 Name: "bar",
334 Type: "TIMESTAMP",
335 DefaultValueExpression: "CURRENT_TIMESTAMP()",
336 },
337 }},
338 schema: Schema{
339 {
340 Name: "foo",
341 Type: StringFieldType,
342 DefaultValueExpression: "I_LOVE_FOO",
343 },
344 {
345 Name: "bar",
346 Type: TimestampFieldType,
347 DefaultValueExpression: "CURRENT_TIMESTAMP()",
348 },
349 },
350 },
351 {
352
353 bqSchema: &bq.TableSchema{
354 Fields: []*bq.TableFieldSchema{
355 {
356 Name: "name",
357 Type: "STRING",
358 Collation: "und:ci",
359 },
360 {
361 Name: "another_name",
362 Type: "STRING",
363 Collation: "",
364 },
365 }},
366 schema: Schema{
367 {
368 Name: "name",
369 Type: StringFieldType,
370 Collation: "und:ci",
371 },
372 {
373 Name: "another_name",
374 Type: StringFieldType,
375 Collation: "",
376 },
377 },
378 },
379 {
380
381 bqSchema: &bq.TableSchema{
382 Fields: []*bq.TableFieldSchema{
383 bqTableFieldSchema("some pii", "restrictedfield", "STRING", "", []string{"tag1"}),
384 },
385 },
386 schema: Schema{
387 fieldSchema("some pii", "restrictedfield", "STRING", false, false, []string{"tag1"}),
388 },
389 },
390 {
391
392 bqSchema: &bq.TableSchema{
393 Fields: []*bq.TableFieldSchema{
394 {
395 Description: "An outer schema wrapping a nested schema",
396 Name: "outer",
397 Mode: "REQUIRED",
398 Type: "RECORD",
399 Fields: []*bq.TableFieldSchema{
400 bqTableFieldSchema("inner field", "inner", "STRING", "", nil),
401 },
402 },
403 },
404 },
405 schema: Schema{
406 &FieldSchema{
407 Description: "An outer schema wrapping a nested schema",
408 Name: "outer",
409 Required: true,
410 Type: "RECORD",
411 Schema: Schema{
412 {
413 Description: "inner field",
414 Name: "inner",
415 Type: "STRING",
416 },
417 },
418 },
419 },
420 },
421 {
422
423 bqSchema: &bq.TableSchema{
424 Fields: []*bq.TableFieldSchema{
425 func() *bq.TableFieldSchema {
426 f := bqTableFieldSchema("desc", "rt", "RANGE", "", nil)
427 f.RangeElementType = &bq.TableFieldSchemaRangeElementType{
428 Type: "DATE",
429 }
430 return f
431 }(),
432 },
433 },
434 schema: Schema{
435 func() *FieldSchema {
436 f := fieldSchema("desc", "rt", "RANGE", false, false, nil)
437 f.RangeElementType = &RangeElementType{
438 Type: DateFieldType,
439 }
440 return f
441 }(),
442 },
443 },
444 }
445 for _, tc := range testCases {
446 bqSchema := tc.schema.toBQ()
447 if !testutil.Equal(bqSchema, tc.bqSchema) {
448 t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v",
449 pretty.Value(bqSchema), pretty.Value(tc.bqSchema))
450 }
451 schema := bqToSchema(tc.bqSchema)
452 if !testutil.Equal(schema, tc.schema) {
453 t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema)
454 }
455 }
456 }
457
458 type allStrings struct {
459 String string
460 ByteSlice []byte
461 }
462
463 type allSignedIntegers struct {
464 Int64 int64
465 Int32 int32
466 Int16 int16
467 Int8 int8
468 Int int
469 }
470
471 type allUnsignedIntegers struct {
472 Uint32 uint32
473 Uint16 uint16
474 Uint8 uint8
475 }
476
477 type allFloat struct {
478 Float64 float64
479 Float32 float32
480
481 }
482
483 type allBoolean struct {
484 Bool bool
485 }
486
487 type allTime struct {
488 Timestamp time.Time
489 Time civil.Time
490 Date civil.Date
491 DateTime civil.DateTime
492 Interval *IntervalValue
493 RangeGeneric *RangeValue
494 }
495
496 type allNumeric struct {
497 Numeric *big.Rat
498 }
499
500 func reqField(name, typ string) *FieldSchema {
501 return &FieldSchema{
502 Name: name,
503 Type: FieldType(typ),
504 Required: true,
505 }
506 }
507
508 func optField(name, typ string) *FieldSchema {
509 return &FieldSchema{
510 Name: name,
511 Type: FieldType(typ),
512 Required: false,
513 }
514 }
515
516 type jsonFields struct {
517 IntMap map[string]int
518 StructMap map[string]allTime
519 SliceOfMaps []map[string]int
520 }
521
522 func TestSimpleInference(t *testing.T) {
523 testCases := []struct {
524 in interface{}
525 want Schema
526 }{
527 {
528 in: allSignedIntegers{},
529 want: Schema{
530 reqField("Int64", "INTEGER"),
531 reqField("Int32", "INTEGER"),
532 reqField("Int16", "INTEGER"),
533 reqField("Int8", "INTEGER"),
534 reqField("Int", "INTEGER"),
535 },
536 },
537 {
538 in: allUnsignedIntegers{},
539 want: Schema{
540 reqField("Uint32", "INTEGER"),
541 reqField("Uint16", "INTEGER"),
542 reqField("Uint8", "INTEGER"),
543 },
544 },
545 {
546 in: allFloat{},
547 want: Schema{
548 reqField("Float64", "FLOAT"),
549 reqField("Float32", "FLOAT"),
550 },
551 },
552 {
553 in: allBoolean{},
554 want: Schema{
555 reqField("Bool", "BOOLEAN"),
556 },
557 },
558 {
559 in: &allBoolean{},
560 want: Schema{
561 reqField("Bool", "BOOLEAN"),
562 },
563 },
564 {
565 in: allTime{},
566 want: Schema{
567 reqField("Timestamp", "TIMESTAMP"),
568 reqField("Time", "TIME"),
569 reqField("Date", "DATE"),
570 reqField("DateTime", "DATETIME"),
571 reqField("Interval", "INTERVAL"),
572 reqField("RangeGeneric", "RANGE"),
573 },
574 },
575 {
576 in: &allNumeric{},
577 want: Schema{
578 reqField("Numeric", "NUMERIC"),
579 },
580 },
581 {
582 in: allStrings{},
583 want: Schema{
584 reqField("String", "STRING"),
585 reqField("ByteSlice", "BYTES"),
586 },
587 },
588 {
589 in: jsonFields{},
590 want: Schema{
591 reqField("IntMap", "JSON"),
592 reqField("StructMap", "JSON"),
593 repField("SliceOfMaps", "JSON"),
594 },
595 },
596 }
597 for _, tc := range testCases {
598 got, err := InferSchema(tc.in)
599 if err != nil {
600 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
601 }
602 if d := testutil.Diff(got, tc.want); d != "" {
603 t.Errorf("%T: inferring TableSchema: %s", tc.in, d)
604 }
605 }
606 }
607
608 type containsNested struct {
609 NotNested int
610 Nested struct {
611 Inside int
612 }
613 }
614
615 type containsDoubleNested struct {
616 NotNested int
617 Nested struct {
618 InsideNested struct {
619 Inside int
620 }
621 }
622 }
623
624 type ptrNested struct {
625 Ptr *struct{ Inside int }
626 }
627
628 type dup struct {
629 A, B allBoolean
630 }
631
632 func TestNestedInference(t *testing.T) {
633 testCases := []struct {
634 in interface{}
635 want Schema
636 }{
637 {
638 in: containsNested{},
639 want: Schema{
640 reqField("NotNested", "INTEGER"),
641 &FieldSchema{
642 Name: "Nested",
643 Required: true,
644 Type: "RECORD",
645 Schema: Schema{reqField("Inside", "INTEGER")},
646 },
647 },
648 },
649 {
650 in: containsDoubleNested{},
651 want: Schema{
652 reqField("NotNested", "INTEGER"),
653 &FieldSchema{
654 Name: "Nested",
655 Required: true,
656 Type: "RECORD",
657 Schema: Schema{
658 {
659 Name: "InsideNested",
660 Required: true,
661 Type: "RECORD",
662 Schema: Schema{reqField("Inside", "INTEGER")},
663 },
664 },
665 },
666 },
667 },
668 {
669 in: ptrNested{},
670 want: Schema{
671 &FieldSchema{
672 Name: "Ptr",
673 Required: true,
674 Type: "RECORD",
675 Schema: Schema{reqField("Inside", "INTEGER")},
676 },
677 },
678 },
679 {
680 in: dup{},
681 want: Schema{
682 &FieldSchema{
683 Name: "A",
684 Required: true,
685 Type: "RECORD",
686 Schema: Schema{reqField("Bool", "BOOLEAN")},
687 },
688 &FieldSchema{
689 Name: "B",
690 Required: true,
691 Type: "RECORD",
692 Schema: Schema{reqField("Bool", "BOOLEAN")},
693 },
694 },
695 },
696 }
697
698 for _, tc := range testCases {
699 got, err := InferSchema(tc.in)
700 if err != nil {
701 t.Fatalf("%T: error inferring TableSchema: %v", tc.in, err)
702 }
703 if !testutil.Equal(got, tc.want) {
704 t.Errorf("%T: inferring TableSchema: got:\n%#v\nwant:\n%#v", tc.in,
705 pretty.Value(got), pretty.Value(tc.want))
706 }
707 }
708 }
709
710 type repeated struct {
711 NotRepeated []byte
712 RepeatedByteSlice [][]byte
713 Slice []int
714 Array [5]bool
715 }
716
717 type nestedRepeated struct {
718 NotRepeated int
719 Repeated []struct {
720 Inside int
721 }
722 RepeatedPtr []*struct{ Inside int }
723 }
724
725 func repField(name, typ string) *FieldSchema {
726 return &FieldSchema{
727 Name: name,
728 Type: FieldType(typ),
729 Repeated: true,
730 }
731 }
732
733 func TestRepeatedInference(t *testing.T) {
734 testCases := []struct {
735 in interface{}
736 want Schema
737 }{
738 {
739 in: repeated{},
740 want: Schema{
741 reqField("NotRepeated", "BYTES"),
742 repField("RepeatedByteSlice", "BYTES"),
743 repField("Slice", "INTEGER"),
744 repField("Array", "BOOLEAN"),
745 },
746 },
747 {
748 in: nestedRepeated{},
749 want: Schema{
750 reqField("NotRepeated", "INTEGER"),
751 {
752 Name: "Repeated",
753 Repeated: true,
754 Type: "RECORD",
755 Schema: Schema{reqField("Inside", "INTEGER")},
756 },
757 {
758 Name: "RepeatedPtr",
759 Repeated: true,
760 Type: "RECORD",
761 Schema: Schema{reqField("Inside", "INTEGER")},
762 },
763 },
764 },
765 }
766
767 for i, tc := range testCases {
768 got, err := InferSchema(tc.in)
769 if err != nil {
770 t.Fatalf("%d: error inferring TableSchema: %v", i, err)
771 }
772 if !testutil.Equal(got, tc.want) {
773 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
774 pretty.Value(got), pretty.Value(tc.want))
775 }
776 }
777 }
778
779 type allNulls struct {
780 A NullInt64
781 B NullFloat64
782 C NullBool
783 D NullString
784 E NullTimestamp
785 F NullTime
786 G NullDate
787 H NullDateTime
788 I NullGeography
789 }
790
791 func TestNullInference(t *testing.T) {
792 got, err := InferSchema(allNulls{})
793 if err != nil {
794 t.Fatal(err)
795 }
796 want := Schema{
797 optField("A", "INTEGER"),
798 optField("B", "FLOAT"),
799 optField("C", "BOOLEAN"),
800 optField("D", "STRING"),
801 optField("E", "TIMESTAMP"),
802 optField("F", "TIME"),
803 optField("G", "DATE"),
804 optField("H", "DATETIME"),
805 optField("I", "GEOGRAPHY"),
806 }
807 if diff := testutil.Diff(got, want); diff != "" {
808 t.Error(diff)
809 }
810 }
811
812 type Embedded struct {
813 Embedded int
814 }
815
816 type embedded struct {
817 Embedded2 int
818 }
819
820 type nestedEmbedded struct {
821 Embedded
822 embedded
823 }
824
825 func TestEmbeddedInference(t *testing.T) {
826 got, err := InferSchema(nestedEmbedded{})
827 if err != nil {
828 t.Fatal(err)
829 }
830 want := Schema{
831 reqField("Embedded", "INTEGER"),
832 reqField("Embedded2", "INTEGER"),
833 }
834 if !testutil.Equal(got, want) {
835 t.Errorf("got %v, want %v", pretty.Value(got), pretty.Value(want))
836 }
837 }
838
839 func TestRecursiveInference(t *testing.T) {
840 type List struct {
841 Val int
842 Next *List
843 }
844
845 _, err := InferSchema(List{})
846 if err == nil {
847 t.Fatal("got nil, want error")
848 }
849 }
850
851 type withTags struct {
852 NoTag int
853 ExcludeTag int `bigquery:"-"`
854 SimpleTag int `bigquery:"simple_tag"`
855 UnderscoreTag int `bigquery:"_id"`
856 MixedCase int `bigquery:"MIXEDcase"`
857 Nullable []byte `bigquery:",nullable"`
858 NullNumeric *big.Rat `bigquery:",nullable"`
859 JSON struct{ X int } `bigquery:",json"`
860 JSONPtr *struct{ X int } `bigquery:",json"`
861 }
862
863 type withTagsNested struct {
864 Nested withTags `bigquery:"nested"`
865 NestedAnonymous struct {
866 ExcludeTag int `bigquery:"-"`
867 Inside int `bigquery:"inside"`
868 } `bigquery:"anon"`
869 PNested *struct{ X int }
870 PNestedNullable *struct{ X int } `bigquery:",nullable"`
871 }
872
873 type withTagsRepeated struct {
874 Repeated []withTags `bigquery:"repeated"`
875 RepeatedAnonymous []struct {
876 ExcludeTag int `bigquery:"-"`
877 Inside int `bigquery:"inside"`
878 } `bigquery:"anon"`
879 }
880
881 type withTagsEmbedded struct {
882 withTags
883 }
884
885 var withTagsSchema = Schema{
886 reqField("NoTag", "INTEGER"),
887 reqField("simple_tag", "INTEGER"),
888 reqField("_id", "INTEGER"),
889 reqField("MIXEDcase", "INTEGER"),
890 optField("Nullable", "BYTES"),
891 optField("NullNumeric", "NUMERIC"),
892 reqField("JSON", "JSON"),
893 reqField("JSONPtr", "JSON"),
894 }
895
896 func TestTagInference(t *testing.T) {
897 testCases := []struct {
898 in interface{}
899 want Schema
900 }{
901 {
902 in: withTags{},
903 want: withTagsSchema,
904 },
905 {
906 in: withTagsNested{},
907 want: Schema{
908 &FieldSchema{
909 Name: "nested",
910 Required: true,
911 Type: "RECORD",
912 Schema: withTagsSchema,
913 },
914 &FieldSchema{
915 Name: "anon",
916 Required: true,
917 Type: "RECORD",
918 Schema: Schema{reqField("inside", "INTEGER")},
919 },
920 &FieldSchema{
921 Name: "PNested",
922 Required: true,
923 Type: "RECORD",
924 Schema: Schema{reqField("X", "INTEGER")},
925 },
926 &FieldSchema{
927 Name: "PNestedNullable",
928 Required: false,
929 Type: "RECORD",
930 Schema: Schema{reqField("X", "INTEGER")},
931 },
932 },
933 },
934 {
935 in: withTagsRepeated{},
936 want: Schema{
937 &FieldSchema{
938 Name: "repeated",
939 Repeated: true,
940 Type: "RECORD",
941 Schema: withTagsSchema,
942 },
943 &FieldSchema{
944 Name: "anon",
945 Repeated: true,
946 Type: "RECORD",
947 Schema: Schema{reqField("inside", "INTEGER")},
948 },
949 },
950 },
951 {
952 in: withTagsEmbedded{},
953 want: withTagsSchema,
954 },
955 }
956 for i, tc := range testCases {
957 got, err := InferSchema(tc.in)
958 if err != nil {
959 t.Fatalf("%d: error inferring TableSchema: %v", i, err)
960 }
961 if !testutil.Equal(got, tc.want) {
962 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant:\n%#v", i,
963 pretty.Value(got), pretty.Value(tc.want))
964 }
965 }
966 }
967
968 func TestTagInferenceErrors(t *testing.T) {
969 testCases := []interface{}{
970 struct {
971 LongTag int `bigquery:"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy"`
972 }{},
973 struct {
974 UnsupporedStartChar int `bigquery:"øab"`
975 }{},
976 struct {
977 UnsupportedEndChar int `bigquery:"abø"`
978 }{},
979 struct {
980 UnsupportedMiddleChar int `bigquery:"aøb"`
981 }{},
982 struct {
983 StartInt int `bigquery:"1abc"`
984 }{},
985 struct {
986 Hyphens int `bigquery:"a-b"`
987 }{},
988 }
989 for i, tc := range testCases {
990
991 _, got := InferSchema(tc)
992 if _, ok := got.(invalidFieldNameError); !ok {
993 t.Errorf("%d: inferring TableSchema: got:\n%#v\nwant invalidFieldNameError", i, got)
994 }
995 }
996
997 _, err := InferSchema(struct {
998 X int `bigquery:",optional"`
999 }{})
1000 if err == nil {
1001 t.Error("got nil, want error")
1002 }
1003 }
1004
1005 func TestSchemaErrors(t *testing.T) {
1006 testCases := []struct {
1007 in interface{}
1008 want interface{}
1009 }{
1010 {
1011 in: []byte{},
1012 want: noStructError{},
1013 },
1014 {
1015 in: new(int),
1016 want: noStructError{},
1017 },
1018 {
1019 in: struct{ Uint uint }{},
1020 want: unsupportedFieldTypeError{},
1021 },
1022 {
1023 in: struct{ Uint64 uint64 }{},
1024 want: unsupportedFieldTypeError{},
1025 },
1026 {
1027 in: struct{ Uintptr uintptr }{},
1028 want: unsupportedFieldTypeError{},
1029 },
1030 {
1031 in: struct{ Complex complex64 }{},
1032 want: unsupportedFieldTypeError{},
1033 },
1034 {
1035 in: struct{ Chan chan bool }{},
1036 want: unsupportedFieldTypeError{},
1037 },
1038 {
1039 in: struct{ Ptr *int }{},
1040 want: unsupportedFieldTypeError{},
1041 },
1042 {
1043 in: struct{ Interface interface{} }{},
1044 want: unsupportedFieldTypeError{},
1045 },
1046 {
1047 in: struct{ MultiDimensional [][]int }{},
1048 want: unsupportedFieldTypeError{},
1049 },
1050 {
1051 in: struct{ MultiDimensional [][][]byte }{},
1052 want: unsupportedFieldTypeError{},
1053 },
1054 {
1055 in: struct{ SliceOfPointer []*int }{},
1056 want: unsupportedFieldTypeError{},
1057 },
1058 {
1059 in: struct{ SliceOfNull []NullInt64 }{},
1060 want: unsupportedFieldTypeError{},
1061 },
1062 {
1063 in: struct{ ChanSlice []chan bool }{},
1064 want: unsupportedFieldTypeError{},
1065 },
1066 {
1067 in: struct{ NestedChan struct{ Chan []chan bool } }{},
1068 want: unsupportedFieldTypeError{},
1069 },
1070 {
1071 in: struct {
1072 X int `bigquery:",nullable"`
1073 }{},
1074 want: badNullableError{},
1075 },
1076 {
1077 in: struct {
1078 X bool `bigquery:",nullable"`
1079 }{},
1080 want: badNullableError{},
1081 },
1082 {
1083 in: struct {
1084 X struct{ N int } `bigquery:",nullable"`
1085 }{},
1086 want: badNullableError{},
1087 },
1088 {
1089 in: struct {
1090 X []int `bigquery:",nullable"`
1091 }{},
1092 want: badNullableError{},
1093 },
1094 {
1095 in: struct {
1096 X int `bigquery:",json"`
1097 }{},
1098 want: badJSONError{},
1099 },
1100 {
1101 in: struct{ X *[]byte }{},
1102 want: unsupportedFieldTypeError{},
1103 },
1104 {
1105 in: struct{ X *[]int }{},
1106 want: unsupportedFieldTypeError{},
1107 },
1108 {
1109 in: struct{ X *int }{},
1110 want: unsupportedFieldTypeError{},
1111 },
1112 {
1113 in: struct{ X map[struct{}]interface{} }{},
1114 want: unsupportedFieldTypeError{},
1115 },
1116 }
1117 for _, tc := range testCases {
1118 _, got := InferSchema(tc.in)
1119 if reflect.TypeOf(got) != reflect.TypeOf(tc.want) {
1120 t.Errorf("%#v: got:\n%#v\nwant type %T", tc.in, got, tc.want)
1121 }
1122 }
1123 }
1124
1125 func TestHasRecursiveType(t *testing.T) {
1126 type (
1127 nonStruct int
1128 nonRec struct{ A string }
1129 dup struct{ A, B nonRec }
1130 rec struct {
1131 A int
1132 B *rec
1133 }
1134 recUnexported struct {
1135 A int
1136 }
1137 hasRec struct {
1138 A int
1139 R *rec
1140 }
1141 recSlicePointer struct {
1142 A []*recSlicePointer
1143 }
1144 )
1145 for _, test := range []struct {
1146 in interface{}
1147 want bool
1148 }{
1149 {nonStruct(0), false},
1150 {nonRec{}, false},
1151 {dup{}, false},
1152 {rec{}, true},
1153 {recUnexported{}, false},
1154 {hasRec{}, true},
1155 {&recSlicePointer{}, true},
1156 } {
1157 got, err := hasRecursiveType(reflect.TypeOf(test.in), nil)
1158 if err != nil {
1159 t.Fatal(err)
1160 }
1161 if got != test.want {
1162 t.Errorf("%T: got %t, want %t", test.in, got, test.want)
1163 }
1164 }
1165 }
1166
1167 func TestSchemaFromJSON(t *testing.T) {
1168 testCasesExpectingSuccess := []struct {
1169 bqSchemaJSON []byte
1170 description string
1171 expectedSchema Schema
1172 }{
1173 {
1174 description: "Flat table with a mixture of NULLABLE and REQUIRED fields",
1175 bqSchemaJSON: []byte(`
1176 [
1177 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"},
1178 {"name":"flat_bytes","type":"BYTES","mode":"REQUIRED","description":"Flat required BYTES"},
1179 {"name":"flat_integer","type":"INTEGER","mode":"NULLABLE","description":"Flat nullable INTEGER"},
1180 {"name":"flat_float","type":"FLOAT","mode":"REQUIRED","description":"Flat required FLOAT"},
1181 {"name":"flat_boolean","type":"BOOLEAN","mode":"NULLABLE","description":"Flat nullable BOOLEAN"},
1182 {"name":"flat_timestamp","type":"TIMESTAMP","mode":"REQUIRED","description":"Flat required TIMESTAMP"},
1183 {"name":"flat_date","type":"DATE","mode":"NULLABLE","description":"Flat required DATE"},
1184 {"name":"flat_time","type":"TIME","mode":"REQUIRED","description":"Flat nullable TIME"},
1185 {"name":"flat_datetime","type":"DATETIME","mode":"NULLABLE","description":"Flat required DATETIME"},
1186 {"name":"flat_numeric","type":"NUMERIC","mode":"REQUIRED","description":"Flat required NUMERIC"},
1187 {"name":"flat_bignumeric","type":"BIGNUMERIC","mode":"NULLABLE","description":"Flat nullable BIGNUMERIC"},
1188 {"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"},
1189 {"name":"aliased_integer","type":"INT64","mode":"REQUIRED","description":"Aliased required integer"},
1190 {"name":"aliased_boolean","type":"BOOL","mode":"NULLABLE","description":"Aliased nullable boolean"},
1191 {"name":"aliased_float","type":"FLOAT64","mode":"REQUIRED","description":"Aliased required float"},
1192 {"name":"aliased_record","type":"STRUCT","mode":"NULLABLE","description":"Aliased nullable record"},
1193 {"name":"aliased_bignumeric","type":"BIGDECIMAL","mode":"NULLABLE","description":"Aliased nullable bignumeric"},
1194 {"name":"flat_interval","type":"INTERVAL","mode":"NULLABLE","description":"Flat nullable interval"}
1195 ]`),
1196 expectedSchema: Schema{
1197 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
1198 fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true, nil),
1199 fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false, nil),
1200 fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true, nil),
1201 fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false, nil),
1202 fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true, nil),
1203 fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil),
1204 fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil),
1205 fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil),
1206 fieldSchema("Flat required NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
1207 fieldSchema("Flat nullable BIGNUMERIC", "flat_bignumeric", "BIGNUMERIC", false, false, nil),
1208 fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil),
1209 fieldSchema("Aliased required integer", "aliased_integer", "INTEGER", false, true, nil),
1210 fieldSchema("Aliased nullable boolean", "aliased_boolean", "BOOLEAN", false, false, nil),
1211 fieldSchema("Aliased required float", "aliased_float", "FLOAT", false, true, nil),
1212 fieldSchema("Aliased nullable record", "aliased_record", "RECORD", false, false, nil),
1213 fieldSchema("Aliased nullable bignumeric", "aliased_bignumeric", "BIGNUMERIC", false, false, nil),
1214 fieldSchema("Flat nullable interval", "flat_interval", "INTERVAL", false, false, nil),
1215 },
1216 },
1217 {
1218 description: "Table with a nested RECORD",
1219 bqSchemaJSON: []byte(`
1220 [
1221 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"},
1222 {"name":"nested_record","type":"RECORD","mode":"NULLABLE","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]}
1223 ]`),
1224 expectedSchema: Schema{
1225 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
1226 &FieldSchema{
1227 Description: "Nested nullable RECORD",
1228 Name: "nested_record",
1229 Required: false,
1230 Type: "RECORD",
1231 Schema: Schema{
1232 {
1233 Description: "First nested record field",
1234 Name: "record_field_1",
1235 Required: false,
1236 Type: "STRING",
1237 },
1238 {
1239 Description: "Second nested record field",
1240 Name: "record_field_2",
1241 Required: true,
1242 Type: "INTEGER",
1243 },
1244 },
1245 },
1246 },
1247 },
1248 {
1249 description: "Table with a repeated RECORD",
1250 bqSchemaJSON: []byte(`
1251 [
1252 {"name":"flat_string","type":"STRING","mode":"NULLABLE","description":"Flat nullable string"},
1253 {"name":"nested_record","type":"RECORD","mode":"REPEATED","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]}
1254 ]`),
1255 expectedSchema: Schema{
1256 fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
1257 &FieldSchema{
1258 Description: "Nested nullable RECORD",
1259 Name: "nested_record",
1260 Repeated: true,
1261 Required: false,
1262 Type: "RECORD",
1263 Schema: Schema{
1264 {
1265 Description: "First nested record field",
1266 Name: "record_field_1",
1267 Required: false,
1268 Type: "STRING",
1269 },
1270 {
1271 Description: "Second nested record field",
1272 Name: "record_field_2",
1273 Required: true,
1274 Type: "INTEGER",
1275 },
1276 },
1277 },
1278 },
1279 },
1280 {
1281 description: "Table with advanced parameters",
1282 bqSchemaJSON: []byte(`
1283 [
1284 {"name":"strfield","type":"STRING","mode":"NULLABLE","description":"foo","maxLength":"100"},
1285 {"name":"numfield","type":"BIGNUMERIC","description":"bar","mode":"REPEATED","precision":"10","scale":"5","policyTags":{"names":["baz"]}}
1286 ]`),
1287 expectedSchema: Schema{
1288 &FieldSchema{
1289 Name: "strfield",
1290 Description: "foo",
1291 MaxLength: 100,
1292 Type: "STRING",
1293 },
1294 &FieldSchema{
1295 Name: "numfield",
1296 Description: "bar",
1297 Repeated: true,
1298 Type: "BIGNUMERIC",
1299 Precision: 10,
1300 Scale: 5,
1301 PolicyTags: &PolicyTagList{
1302 Names: []string{"baz"},
1303 },
1304 },
1305 },
1306 },
1307 }
1308 for _, tc := range testCasesExpectingSuccess {
1309 convertedSchema, err := SchemaFromJSON(tc.bqSchemaJSON)
1310 if err != nil {
1311 t.Errorf("encountered an error when converting JSON table schema (%s): %v", tc.description, err)
1312 continue
1313 }
1314 if diff := testutil.Diff(convertedSchema, tc.expectedSchema); diff != "" {
1315 t.Errorf("%s: %s", tc.description, diff)
1316 }
1317 }
1318
1319 testCasesExpectingFailure := []struct {
1320 bqSchemaJSON []byte
1321 description string
1322 }{
1323 {
1324 description: "Schema with invalid JSON",
1325 bqSchemaJSON: []byte(`This is not JSON`),
1326 },
1327 {
1328 description: "Schema with unknown field type",
1329 bqSchemaJSON: []byte(`[{"name":"strange_type","type":"STRANGE","description":"This type should not exist"}]`),
1330 },
1331 {
1332 description: "Schema with zero length",
1333 bqSchemaJSON: []byte(``),
1334 },
1335 }
1336 for _, tc := range testCasesExpectingFailure {
1337 _, err := SchemaFromJSON(tc.bqSchemaJSON)
1338 if err == nil {
1339 t.Errorf("converting this schema should have returned an error (%s): %v", tc.description, err)
1340 continue
1341 }
1342 }
1343 }
1344
1345 func TestSchemaToJSONFields(t *testing.T) {
1346
1347
1348
1349 normalizeJSON := cmp.FilterValues(func(x, y []byte) bool {
1350 return json.Valid(x) && json.Valid(y)
1351 }, cmp.Transformer("ParseJSON", func(in []byte) (out interface{}) {
1352 if err := json.Unmarshal(in, &out); err != nil {
1353 panic(err)
1354 }
1355 return out
1356 }))
1357
1358 testCases := []struct {
1359 description string
1360 inSchema Schema
1361 expectedJSON []byte
1362 }{
1363 {
1364 description: "basic schema",
1365 inSchema: Schema{
1366 fieldSchema("foo", "strfield", "STRING", false, false, nil),
1367 fieldSchema("bar", "intfield", "INTEGER", false, true, nil),
1368 fieldSchema("baz", "bool_arr", "INTEGER", true, false, []string{"tag1"}),
1369 },
1370 expectedJSON: []byte(`[
1371 {
1372 "description": "foo",
1373 "name": "strfield",
1374 "type": "STRING"
1375 },
1376 {
1377 "description": "bar",
1378 "mode": "REQUIRED",
1379 "name": "intfield",
1380 "type": "INTEGER"
1381 },
1382 {
1383 "description": "baz",
1384 "mode": "REPEATED",
1385 "name": "bool_arr",
1386 "policyTags": {
1387 "names": [
1388 "tag1"
1389 ]
1390 },
1391 "type": "INTEGER"
1392 }
1393 ]`),
1394 },
1395 }
1396 for _, tc := range testCases {
1397 got, err := tc.inSchema.ToJSONFields()
1398 if err != nil {
1399 t.Errorf("%s: %v", tc.description, err)
1400 }
1401
1402 if diff := cmp.Diff(got, tc.expectedJSON, normalizeJSON); diff != "" {
1403 t.Errorf("%s: %s", tc.description, diff)
1404 }
1405 }
1406 }
1407
View as plain text