1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package bigquery
16
17 import (
18 "testing"
19 "time"
20
21 "cloud.google.com/go/internal/testutil"
22 "github.com/google/go-cmp/cmp"
23 bq "google.golang.org/api/bigquery/v2"
24 )
25
26 func TestBQToTableMetadata(t *testing.T) {
27 bqClient := &Client{}
28 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
29 aTimeMillis := aTime.UnixNano() / 1e6
30 aDurationMillis := int64(1800000)
31 aDuration := time.Duration(aDurationMillis) * time.Millisecond
32 aStalenessValue, _ := ParseInterval("8:0:0")
33 for _, test := range []struct {
34 in *bq.Table
35 want *TableMetadata
36 }{
37 {&bq.Table{}, &TableMetadata{}},
38 {
39 &bq.Table{
40 CreationTime: aTimeMillis,
41 Description: "desc",
42 Etag: "etag",
43 ExpirationTime: aTimeMillis,
44 FriendlyName: "fname",
45 Id: "id",
46 LastModifiedTime: uint64(aTimeMillis),
47 Location: "loc",
48 NumBytes: 123,
49 NumLongTermBytes: 23,
50 NumRows: 7,
51 StreamingBuffer: &bq.Streamingbuffer{
52 EstimatedBytes: 11,
53 EstimatedRows: 3,
54 OldestEntryTime: uint64(aTimeMillis),
55 },
56 MaterializedView: &bq.MaterializedViewDefinition{
57 EnableRefresh: true,
58 Query: "mat view query",
59 LastRefreshTime: aTimeMillis,
60 RefreshIntervalMs: aDurationMillis,
61 AllowNonIncrementalDefinition: true,
62 MaxStaleness: "8:0:0",
63 },
64 TimePartitioning: &bq.TimePartitioning{
65 ExpirationMs: 7890,
66 Type: "DAY",
67 Field: "pfield",
68 },
69 Clustering: &bq.Clustering{
70 Fields: []string{"cfield1", "cfield2"},
71 },
72 RequirePartitionFilter: true,
73 EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
74 Type: "EXTERNAL",
75 View: &bq.ViewDefinition{Query: "view-query"},
76 Labels: map[string]string{"a": "b"},
77 ExternalDataConfiguration: &bq.ExternalDataConfiguration{
78 SourceFormat: "GOOGLE_SHEETS",
79 },
80 TableConstraints: &bq.TableConstraints{
81 PrimaryKey: &bq.TableConstraintsPrimaryKey{
82 Columns: []string{"id"},
83 },
84 ForeignKeys: []*bq.TableConstraintsForeignKeys{
85 {
86 Name: "fk",
87 ColumnReferences: []*bq.TableConstraintsForeignKeysColumnReferences{
88 {
89 ReferencedColumn: "id",
90 ReferencingColumn: "parent",
91 },
92 },
93 ReferencedTable: &bq.TableConstraintsForeignKeysReferencedTable{
94 DatasetId: "dataset_id",
95 ProjectId: "project_id",
96 TableId: "table_id",
97 },
98 },
99 },
100 },
101 ResourceTags: map[string]string{
102 "key1": "val1",
103 "key2": "val2",
104 },
105 },
106 &TableMetadata{
107 Description: "desc",
108 Name: "fname",
109 Location: "loc",
110 ViewQuery: "view-query",
111 FullID: "id",
112 Type: ExternalTable,
113 Labels: map[string]string{"a": "b"},
114 ExternalDataConfig: &ExternalDataConfig{SourceFormat: GoogleSheets},
115 ExpirationTime: aTime.Truncate(time.Millisecond),
116 CreationTime: aTime.Truncate(time.Millisecond),
117 LastModifiedTime: aTime.Truncate(time.Millisecond),
118 NumBytes: 123,
119 NumLongTermBytes: 23,
120 NumRows: 7,
121 MaterializedView: &MaterializedViewDefinition{
122 EnableRefresh: true,
123 Query: "mat view query",
124 LastRefreshTime: aTime,
125 RefreshInterval: aDuration,
126 AllowNonIncrementalDefinition: true,
127 MaxStaleness: aStalenessValue,
128 },
129 TimePartitioning: &TimePartitioning{
130 Type: DayPartitioningType,
131 Expiration: 7890 * time.Millisecond,
132 Field: "pfield",
133 },
134 Clustering: &Clustering{
135 Fields: []string{"cfield1", "cfield2"},
136 },
137 RequirePartitionFilter: true,
138 StreamingBuffer: &StreamingBuffer{
139 EstimatedBytes: 11,
140 EstimatedRows: 3,
141 OldestEntryTime: aTime,
142 },
143 EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
144 ETag: "etag",
145 TableConstraints: &TableConstraints{
146 PrimaryKey: &PrimaryKey{
147 Columns: []string{"id"},
148 },
149 ForeignKeys: []*ForeignKey{
150 {
151 Name: "fk",
152 ReferencedTable: &Table{
153 c: bqClient,
154 ProjectID: "project_id",
155 DatasetID: "dataset_id",
156 TableID: "table_id",
157 },
158 ColumnReferences: []*ColumnReference{
159 {
160 ReferencedColumn: "id",
161 ReferencingColumn: "parent",
162 },
163 },
164 },
165 },
166 },
167 ResourceTags: map[string]string{
168 "key1": "val1",
169 "key2": "val2",
170 },
171 },
172 },
173 } {
174 got, err := bqToTableMetadata(test.in, bqClient)
175 if err != nil {
176 t.Fatal(err)
177 }
178 if diff := testutil.Diff(got, test.want, cmp.AllowUnexported(Client{}, Table{})); diff != "" {
179 t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
180 }
181 }
182 }
183
184 func TestTableMetadataToBQ(t *testing.T) {
185 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
186 aTimeMillis := aTime.UnixNano() / 1e6
187 sc := Schema{fieldSchema("desc", "name", "STRING", false, true, nil)}
188
189 for _, test := range []struct {
190 in *TableMetadata
191 want *bq.Table
192 }{
193 {nil, &bq.Table{}},
194 {&TableMetadata{}, &bq.Table{}},
195 {
196 &TableMetadata{
197 Name: "n",
198 Description: "d",
199 Schema: sc,
200 ExpirationTime: aTime,
201 Labels: map[string]string{"a": "b"},
202 ExternalDataConfig: &ExternalDataConfig{SourceFormat: Bigtable},
203 EncryptionConfig: &EncryptionConfig{KMSKeyName: "keyName"},
204 ResourceTags: map[string]string{
205 "key1": "val1",
206 "key2": "val2",
207 },
208 },
209 &bq.Table{
210 FriendlyName: "n",
211 Description: "d",
212 Schema: &bq.TableSchema{
213 Fields: []*bq.TableFieldSchema{
214 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
215 },
216 },
217 ExpirationTime: aTimeMillis,
218 Labels: map[string]string{"a": "b"},
219 ExternalDataConfiguration: &bq.ExternalDataConfiguration{SourceFormat: "BIGTABLE"},
220 EncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
221 ResourceTags: map[string]string{
222 "key1": "val1",
223 "key2": "val2",
224 },
225 },
226 },
227 {
228 &TableMetadata{ViewQuery: "q"},
229 &bq.Table{
230 View: &bq.ViewDefinition{
231 Query: "q",
232 UseLegacySql: false,
233 ForceSendFields: []string{"UseLegacySql"},
234 },
235 },
236 },
237 {
238 &TableMetadata{
239 ViewQuery: "q",
240 UseLegacySQL: true,
241 TimePartitioning: &TimePartitioning{},
242 RequirePartitionFilter: true,
243 },
244 &bq.Table{
245 View: &bq.ViewDefinition{
246 Query: "q",
247 UseLegacySql: true,
248 },
249 TimePartitioning: &bq.TimePartitioning{
250 Type: "DAY",
251 ExpirationMs: 0,
252 },
253 RequirePartitionFilter: true,
254 },
255 },
256 {
257 &TableMetadata{
258 ViewQuery: "q",
259 UseStandardSQL: true,
260 TimePartitioning: &TimePartitioning{
261 Type: HourPartitioningType,
262 Expiration: time.Second,
263 Field: "ofDreams",
264 },
265 Clustering: &Clustering{
266 Fields: []string{"cfield1"},
267 },
268 },
269 &bq.Table{
270 View: &bq.ViewDefinition{
271 Query: "q",
272 UseLegacySql: false,
273 ForceSendFields: []string{"UseLegacySql"},
274 },
275 TimePartitioning: &bq.TimePartitioning{
276 Type: "HOUR",
277 ExpirationMs: 1000,
278 Field: "ofDreams",
279 },
280 Clustering: &bq.Clustering{
281 Fields: []string{"cfield1"},
282 },
283 },
284 },
285 {
286 &TableMetadata{
287 RangePartitioning: &RangePartitioning{
288 Field: "ofNumbers",
289 Range: &RangePartitioningRange{
290 Start: 1,
291 End: 100,
292 Interval: 5,
293 },
294 },
295 Clustering: &Clustering{
296 Fields: []string{"cfield1"},
297 },
298 },
299 &bq.Table{
300
301 RangePartitioning: &bq.RangePartitioning{
302 Field: "ofNumbers",
303 Range: &bq.RangePartitioningRange{
304 Start: 1,
305 End: 100,
306 Interval: 5,
307 ForceSendFields: []string{"Start", "End", "Interval"},
308 },
309 },
310 Clustering: &bq.Clustering{
311 Fields: []string{"cfield1"},
312 },
313 },
314 },
315 {
316 &TableMetadata{ExpirationTime: NeverExpire},
317 &bq.Table{ExpirationTime: 0},
318 },
319 } {
320 got, err := test.in.toBQ()
321 if err != nil {
322 t.Fatalf("%+v: %v", test.in, err)
323 }
324 if diff := testutil.Diff(got, test.want); diff != "" {
325 t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
326 }
327 }
328
329
330 for _, in := range []*TableMetadata{
331 {Schema: sc, ViewQuery: "q"},
332 {UseLegacySQL: true},
333 {UseStandardSQL: true},
334
335 {FullID: "x"},
336 {Type: "x"},
337 {CreationTime: aTime},
338 {LastModifiedTime: aTime},
339 {NumBytes: 1},
340 {NumLongTermBytes: 1},
341 {NumRows: 1},
342 {StreamingBuffer: &StreamingBuffer{}},
343 {ETag: "x"},
344
345
346 {ExpirationTime: time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC).Add(-1)},
347 {ExpirationTime: time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC).Add(1)},
348 } {
349 _, err := in.toBQ()
350 if err == nil {
351 t.Errorf("%+v: got nil, want error", in)
352 }
353 }
354 }
355
356 func TestTableMetadataToUpdateToBQ(t *testing.T) {
357 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
358 for _, test := range []struct {
359 tm TableMetadataToUpdate
360 want *bq.Table
361 }{
362 {
363 tm: TableMetadataToUpdate{},
364 want: &bq.Table{},
365 },
366 {
367 tm: TableMetadataToUpdate{
368 Description: "d",
369 Name: "n",
370 },
371 want: &bq.Table{
372 Description: "d",
373 FriendlyName: "n",
374 ForceSendFields: []string{"Description", "FriendlyName"},
375 },
376 },
377 {
378 tm: TableMetadataToUpdate{
379 Schema: Schema{fieldSchema("desc", "name", "STRING", false, true, nil)},
380 ExpirationTime: aTime,
381 },
382 want: &bq.Table{
383 Schema: &bq.TableSchema{
384 Fields: []*bq.TableFieldSchema{
385 bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
386 },
387 },
388 ExpirationTime: aTime.UnixNano() / 1e6,
389 ForceSendFields: []string{"Schema", "ExpirationTime"},
390 },
391 },
392 {
393 tm: TableMetadataToUpdate{ViewQuery: "q"},
394 want: &bq.Table{
395 View: &bq.ViewDefinition{Query: "q", ForceSendFields: []string{"Query"}},
396 },
397 },
398 {
399 tm: TableMetadataToUpdate{UseLegacySQL: false},
400 want: &bq.Table{
401 View: &bq.ViewDefinition{
402 UseLegacySql: false,
403 ForceSendFields: []string{"UseLegacySql"},
404 },
405 },
406 },
407 {
408 tm: TableMetadataToUpdate{ViewQuery: "q", UseLegacySQL: true},
409 want: &bq.Table{
410 View: &bq.ViewDefinition{
411 Query: "q",
412 UseLegacySql: true,
413 ForceSendFields: []string{"Query", "UseLegacySql"},
414 },
415 },
416 },
417 {
418 tm: func() (tm TableMetadataToUpdate) {
419 tm.SetLabel("L", "V")
420 tm.DeleteLabel("D")
421 return tm
422 }(),
423 want: &bq.Table{
424 Labels: map[string]string{"L": "V"},
425 NullFields: []string{"Labels.D"},
426 },
427 },
428 {
429 tm: TableMetadataToUpdate{ExpirationTime: NeverExpire},
430 want: &bq.Table{
431 NullFields: []string{"ExpirationTime"},
432 },
433 },
434 {
435 tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: 0}},
436 want: &bq.Table{
437 TimePartitioning: &bq.TimePartitioning{
438 Type: "DAY",
439 ForceSendFields: []string{"RequirePartitionFilter"},
440 NullFields: []string{"ExpirationMs"},
441 },
442 },
443 },
444 {
445 tm: TableMetadataToUpdate{TimePartitioning: &TimePartitioning{Expiration: time.Duration(time.Hour)}},
446 want: &bq.Table{
447 TimePartitioning: &bq.TimePartitioning{
448 ExpirationMs: 3600000,
449 Type: "DAY",
450 ForceSendFields: []string{"RequirePartitionFilter"},
451 },
452 },
453 },
454 {
455 tm: TableMetadataToUpdate{RequirePartitionFilter: false},
456 want: &bq.Table{
457 RequirePartitionFilter: false,
458 ForceSendFields: []string{"RequirePartitionFilter"},
459 },
460 },
461 {
462 tm: TableMetadataToUpdate{RequirePartitionFilter: true},
463 want: &bq.Table{
464 RequirePartitionFilter: true,
465 ForceSendFields: []string{"RequirePartitionFilter"},
466 },
467 },
468 {
469 tm: TableMetadataToUpdate{Clustering: &Clustering{Fields: []string{"foo", "bar"}}},
470 want: &bq.Table{
471 Clustering: &bq.Clustering{Fields: []string{"foo", "bar"}},
472 },
473 },
474 {
475 tm: TableMetadataToUpdate{
476 TableConstraints: &TableConstraints{
477 PrimaryKey: &PrimaryKey{
478 Columns: []string{"name"},
479 },
480 },
481 },
482 want: &bq.Table{
483 TableConstraints: &bq.TableConstraints{
484 PrimaryKey: &bq.TableConstraintsPrimaryKey{
485 Columns: []string{"name"},
486 ForceSendFields: []string{"Columns"},
487 },
488 ForceSendFields: []string{"PrimaryKey"},
489 },
490 },
491 },
492 {
493 tm: TableMetadataToUpdate{
494 TableConstraints: &TableConstraints{
495 ForeignKeys: []*ForeignKey{
496 {
497 Name: "fk",
498 ReferencedTable: &Table{
499 ProjectID: "projectID",
500 DatasetID: "datasetID",
501 TableID: "tableID",
502 },
503 ColumnReferences: []*ColumnReference{
504 {
505 ReferencedColumn: "id",
506 ReferencingColumn: "other_table_id",
507 },
508 },
509 },
510 },
511 },
512 },
513 want: &bq.Table{
514 TableConstraints: &bq.TableConstraints{
515 ForceSendFields: []string{"ForeignKeys"},
516 ForeignKeys: []*bq.TableConstraintsForeignKeys{
517 {
518 Name: "fk",
519 ReferencedTable: &bq.TableConstraintsForeignKeysReferencedTable{
520 ProjectId: "projectID",
521 DatasetId: "datasetID",
522 TableId: "tableID",
523 },
524 ColumnReferences: []*bq.TableConstraintsForeignKeysColumnReferences{
525 {
526 ReferencedColumn: "id",
527 ReferencingColumn: "other_table_id",
528 },
529 },
530 },
531 },
532 },
533 },
534 },
535 {
536 tm: TableMetadataToUpdate{
537 ResourceTags: map[string]string{
538 "key1": "val1",
539 "key2": "val2",
540 },
541 },
542 want: &bq.Table{
543 ResourceTags: map[string]string{
544 "key1": "val1",
545 "key2": "val2",
546 },
547 ForceSendFields: []string{"ResourceTags"},
548 },
549 },
550 } {
551 got, _ := test.tm.toBQ()
552 if !testutil.Equal(got, test.want) {
553 t.Errorf("%+v:\ngot %+v\nwant %+v", test.tm, got, test.want)
554 }
555 }
556 }
557
558 func TestTableMetadataToUpdateToBQErrors(t *testing.T) {
559
560 start := time.Date(1677, 9, 21, 0, 12, 43, 145224192, time.UTC)
561 end := time.Date(2262, 04, 11, 23, 47, 16, 854775807, time.UTC)
562
563 for _, test := range []struct {
564 desc string
565 aTime time.Time
566 wantErr bool
567 }{
568 {desc: "ignored zero value", aTime: time.Time{}, wantErr: false},
569 {desc: "earliest valid time", aTime: start, wantErr: false},
570 {desc: "latested valid time", aTime: end, wantErr: false},
571 {desc: "invalid times before 1678", aTime: start.Add(-1), wantErr: true},
572 {desc: "invalid times after 2262", aTime: end.Add(1), wantErr: true},
573 {desc: "valid times after 1678", aTime: start.Add(1), wantErr: false},
574 {desc: "valid times before 2262", aTime: end.Add(-1), wantErr: false},
575 } {
576 tm := &TableMetadataToUpdate{ExpirationTime: test.aTime}
577 _, err := tm.toBQ()
578 if test.wantErr && err == nil {
579 t.Errorf("[%s] got no error, want error", test.desc)
580 }
581 if !test.wantErr && err != nil {
582 t.Errorf("[%s] got error, want no error", test.desc)
583 }
584 }
585 }
586
587 func TestTableIdentifiers(t *testing.T) {
588 testTable := &Table{
589 ProjectID: "p",
590 DatasetID: "d",
591 TableID: "t",
592 c: nil,
593 }
594 for _, tc := range []struct {
595 description string
596 in *Table
597 format IdentifierFormat
598 want string
599 wantErr bool
600 }{
601 {
602 description: "empty format string",
603 in: testTable,
604 format: "",
605 wantErr: true,
606 },
607 {
608 description: "legacy",
609 in: testTable,
610 format: LegacySQLID,
611 want: "p:d.t",
612 },
613 {
614 description: "standard unquoted",
615 in: testTable,
616 format: StandardSQLID,
617 want: "p.d.t",
618 },
619 {
620 description: "standard w/dash",
621 in: &Table{ProjectID: "p-p", DatasetID: "d", TableID: "t"},
622 format: StandardSQLID,
623 want: "p-p.d.t",
624 },
625 {
626 description: "api resource",
627 in: testTable,
628 format: StorageAPIResourceID,
629 want: "projects/p/datasets/d/tables/t",
630 },
631 } {
632 got, err := tc.in.Identifier(tc.format)
633 if tc.wantErr && err == nil {
634 t.Errorf("case %q: wanted err, was success", tc.description)
635 }
636 if !tc.wantErr {
637 if err != nil {
638 t.Errorf("case %q: wanted success, got err: %v", tc.description, err)
639 } else {
640 if got != tc.want {
641 t.Errorf("case %q: got %s, want %s", tc.description, got, tc.want)
642 }
643 }
644 }
645 }
646 }
647
View as plain text