1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package bigquery
16
17 import (
18 "context"
19 "errors"
20 "fmt"
21 "strconv"
22 "testing"
23 "time"
24
25 "cloud.google.com/go/internal/testutil"
26 "github.com/google/go-cmp/cmp"
27 "github.com/google/go-cmp/cmp/cmpopts"
28 bq "google.golang.org/api/bigquery/v2"
29 itest "google.golang.org/api/iterator/testing"
30 )
31
32
33 type listTablesStub struct {
34 expectedProject, expectedDataset string
35 tables []*bq.TableListTables
36 }
37
38 func (s *listTablesStub) listTables(it *TableIterator, pageSize int, pageToken string) (*bq.TableList, error) {
39 if it.dataset.ProjectID != s.expectedProject {
40 return nil, fmt.Errorf("wrong project id: %q", it.dataset.ProjectID)
41 }
42 if it.dataset.DatasetID != s.expectedDataset {
43 return nil, fmt.Errorf("wrong dataset id: %q", it.dataset.DatasetID)
44 }
45 const maxPageSize = 2
46 if pageSize <= 0 || pageSize > maxPageSize {
47 pageSize = maxPageSize
48 }
49 start := 0
50 if pageToken != "" {
51 var err error
52 start, err = strconv.Atoi(pageToken)
53 if err != nil {
54 return nil, err
55 }
56 }
57 end := start + pageSize
58 if end > len(s.tables) {
59 end = len(s.tables)
60 }
61 nextPageToken := ""
62 if end < len(s.tables) {
63 nextPageToken = strconv.Itoa(end)
64 }
65 return &bq.TableList{
66 Tables: s.tables[start:end],
67 NextPageToken: nextPageToken,
68 }, nil
69 }
70
71 func TestTables(t *testing.T) {
72 c := &Client{projectID: "p1"}
73 inTables := []*bq.TableListTables{
74 {TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t1"}},
75 {TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t2"}},
76 {TableReference: &bq.TableReference{ProjectId: "p1", DatasetId: "d1", TableId: "t3"}},
77 }
78 outTables := []*Table{
79 {ProjectID: "p1", DatasetID: "d1", TableID: "t1", c: c},
80 {ProjectID: "p1", DatasetID: "d1", TableID: "t2", c: c},
81 {ProjectID: "p1", DatasetID: "d1", TableID: "t3", c: c},
82 }
83
84 lts := &listTablesStub{
85 expectedProject: "p1",
86 expectedDataset: "d1",
87 tables: inTables,
88 }
89 old := listTables
90 listTables = lts.listTables
91 defer func() { listTables = old }()
92
93 msg, ok := itest.TestIterator(outTables,
94 func() interface{} { return c.Dataset("d1").Tables(context.Background()) },
95 func(it interface{}) (interface{}, error) { return it.(*TableIterator).Next() })
96 if !ok {
97 t.Error(msg)
98 }
99 }
100
101
102 type listModelsStub struct {
103 expectedProject, expectedDataset string
104 models []*bq.Model
105 }
106
107 func (s *listModelsStub) listModels(it *ModelIterator, pageSize int, pageToken string) (*bq.ListModelsResponse, error) {
108 if it.dataset.ProjectID != s.expectedProject {
109 return nil, errors.New("wrong project id")
110 }
111 if it.dataset.DatasetID != s.expectedDataset {
112 return nil, errors.New("wrong dataset id")
113 }
114 const maxPageSize = 2
115 if pageSize <= 0 || pageSize > maxPageSize {
116 pageSize = maxPageSize
117 }
118 start := 0
119 if pageToken != "" {
120 var err error
121 start, err = strconv.Atoi(pageToken)
122 if err != nil {
123 return nil, err
124 }
125 }
126 end := start + pageSize
127 if end > len(s.models) {
128 end = len(s.models)
129 }
130 nextPageToken := ""
131 if end < len(s.models) {
132 nextPageToken = strconv.Itoa(end)
133 }
134 return &bq.ListModelsResponse{
135 Models: s.models[start:end],
136 NextPageToken: nextPageToken,
137 }, nil
138 }
139
140 func TestModels(t *testing.T) {
141 c := &Client{projectID: "p1"}
142 inModels := []*bq.Model{
143 {ModelReference: &bq.ModelReference{ProjectId: "p1", DatasetId: "d1", ModelId: "m1"}},
144 {ModelReference: &bq.ModelReference{ProjectId: "p1", DatasetId: "d1", ModelId: "m2"}},
145 {ModelReference: &bq.ModelReference{ProjectId: "p1", DatasetId: "d1", ModelId: "m3"}},
146 }
147 outModels := []*Model{
148 {ProjectID: "p1", DatasetID: "d1", ModelID: "m1", c: c},
149 {ProjectID: "p1", DatasetID: "d1", ModelID: "m2", c: c},
150 {ProjectID: "p1", DatasetID: "d1", ModelID: "m3", c: c},
151 }
152
153 lms := &listModelsStub{
154 expectedProject: "p1",
155 expectedDataset: "d1",
156 models: inModels,
157 }
158 old := listModels
159 listModels = lms.listModels
160 defer func() { listModels = old }()
161
162 msg, ok := itest.TestIterator(outModels,
163 func() interface{} { return c.Dataset("d1").Models(context.Background()) },
164 func(it interface{}) (interface{}, error) { return it.(*ModelIterator).Next() })
165 if !ok {
166 t.Error(msg)
167 }
168 }
169
170
171 type listRoutinesStub struct {
172 routines []*bq.Routine
173 }
174
175 func (s *listRoutinesStub) listRoutines(it *RoutineIterator, pageSize int, pageToken string) (*bq.ListRoutinesResponse, error) {
176 const maxPageSize = 2
177 if pageSize <= 0 || pageSize > maxPageSize {
178 pageSize = maxPageSize
179 }
180 start := 0
181 if pageToken != "" {
182 var err error
183 start, err = strconv.Atoi(pageToken)
184 if err != nil {
185 return nil, err
186 }
187 }
188 end := start + pageSize
189 if end > len(s.routines) {
190 end = len(s.routines)
191 }
192 nextPageToken := ""
193 if end < len(s.routines) {
194 nextPageToken = strconv.Itoa(end)
195 }
196 return &bq.ListRoutinesResponse{
197 Routines: s.routines[start:end],
198 NextPageToken: nextPageToken,
199 }, nil
200 }
201
202 func TestRoutines(t *testing.T) {
203 c := &Client{projectID: "p1"}
204 inRoutines := []*bq.Routine{
205 {RoutineReference: &bq.RoutineReference{ProjectId: "p1", DatasetId: "d1", RoutineId: "r1"}},
206 {RoutineReference: &bq.RoutineReference{ProjectId: "p1", DatasetId: "d1", RoutineId: "r2"}},
207 {RoutineReference: &bq.RoutineReference{ProjectId: "p1", DatasetId: "d1", RoutineId: "r3"}},
208 }
209 outRoutines := []*Routine{
210 {ProjectID: "p1", DatasetID: "d1", RoutineID: "r1", c: c},
211 {ProjectID: "p1", DatasetID: "d1", RoutineID: "r2", c: c},
212 {ProjectID: "p1", DatasetID: "d1", RoutineID: "r3", c: c},
213 }
214
215 lms := &listRoutinesStub{
216 routines: inRoutines,
217 }
218 old := listRoutines
219 listRoutines = lms.listRoutines
220 defer func() { listRoutines = old }()
221
222 msg, ok := itest.TestIterator(outRoutines,
223 func() interface{} { return c.Dataset("d1").Routines(context.Background()) },
224 func(it interface{}) (interface{}, error) { return it.(*RoutineIterator).Next() })
225 if !ok {
226 t.Error(msg)
227 }
228 }
229
230 type listDatasetsStub struct {
231 expectedProject string
232 datasets []*bq.DatasetListDatasets
233 hidden map[*bq.DatasetListDatasets]bool
234 }
235
236 func (s *listDatasetsStub) listDatasets(it *DatasetIterator, pageSize int, pageToken string) (*bq.DatasetList, error) {
237 const maxPageSize = 2
238 if pageSize <= 0 || pageSize > maxPageSize {
239 pageSize = maxPageSize
240 }
241 if it.Filter != "" {
242 return nil, errors.New("filter not supported")
243 }
244 if it.ProjectID != s.expectedProject {
245 return nil, errors.New("bad project ID")
246 }
247 start := 0
248 if pageToken != "" {
249 var err error
250 start, err = strconv.Atoi(pageToken)
251 if err != nil {
252 return nil, err
253 }
254 }
255 var (
256 i int
257 result []*bq.DatasetListDatasets
258 nextPageToken string
259 )
260 for i = start; len(result) < pageSize && i < len(s.datasets); i++ {
261 if s.hidden[s.datasets[i]] && !it.ListHidden {
262 continue
263 }
264 result = append(result, s.datasets[i])
265 }
266 if i < len(s.datasets) {
267 nextPageToken = strconv.Itoa(i)
268 }
269 return &bq.DatasetList{
270 Datasets: result,
271 NextPageToken: nextPageToken,
272 }, nil
273 }
274
275 func TestDatasets(t *testing.T) {
276 client := &Client{projectID: "p"}
277 inDatasets := []*bq.DatasetListDatasets{
278 {DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "a"}},
279 {DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "b"}},
280 {DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "hidden"}},
281 {DatasetReference: &bq.DatasetReference{ProjectId: "p", DatasetId: "c"}},
282 }
283 outDatasets := []*Dataset{
284 {"p", "a", client},
285 {"p", "b", client},
286 {"p", "hidden", client},
287 {"p", "c", client},
288 }
289 lds := &listDatasetsStub{
290 expectedProject: "p",
291 datasets: inDatasets,
292 hidden: map[*bq.DatasetListDatasets]bool{inDatasets[2]: true},
293 }
294 old := listDatasets
295 listDatasets = lds.listDatasets
296 defer func() { listDatasets = old }()
297
298 msg, ok := itest.TestIterator(outDatasets,
299 func() interface{} { it := client.Datasets(context.Background()); it.ListHidden = true; return it },
300 func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
301 if !ok {
302 t.Fatalf("ListHidden=true: %s", msg)
303 }
304
305 msg, ok = itest.TestIterator([]*Dataset{outDatasets[0], outDatasets[1], outDatasets[3]},
306 func() interface{} { it := client.Datasets(context.Background()); it.ListHidden = false; return it },
307 func(it interface{}) (interface{}, error) { return it.(*DatasetIterator).Next() })
308 if !ok {
309 t.Fatalf("ListHidden=false: %s", msg)
310 }
311 }
312
313 func TestDatasetToBQ(t *testing.T) {
314 testClient := &Client{projectID: "p"}
315 for _, test := range []struct {
316 in *DatasetMetadata
317 want *bq.Dataset
318 }{
319 {nil, &bq.Dataset{}},
320 {&DatasetMetadata{Name: "name"}, &bq.Dataset{FriendlyName: "name"}},
321 {&DatasetMetadata{
322 Name: "name",
323 Description: "desc",
324 DefaultTableExpiration: time.Hour,
325 MaxTimeTravel: time.Duration(181 * time.Minute),
326 DefaultPartitionExpiration: 24 * time.Hour,
327 DefaultEncryptionConfig: &EncryptionConfig{
328 KMSKeyName: "some_key",
329 },
330 ExternalDatasetReference: &ExternalDatasetReference{
331 Connection: "conn",
332 ExternalSource: "external_src",
333 },
334 Location: "EU",
335 Labels: map[string]string{"x": "y"},
336 Access: []*AccessEntry{
337 {Role: OwnerRole, Entity: "example.com", EntityType: DomainEntity},
338 {
339 EntityType: DatasetEntity,
340 Dataset: &DatasetAccessEntry{
341 Dataset: testClient.Dataset("otherdataset"),
342 TargetTypes: []string{"VIEWS"},
343 },
344 },
345 },
346 }, &bq.Dataset{
347 FriendlyName: "name",
348 Description: "desc",
349 DefaultTableExpirationMs: 60 * 60 * 1000,
350 MaxTimeTravelHours: 3,
351 DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
352 DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
353 KmsKeyName: "some_key",
354 },
355 ExternalDatasetReference: &bq.ExternalDatasetReference{
356 Connection: "conn",
357 ExternalSource: "external_src",
358 },
359 Location: "EU",
360 Labels: map[string]string{"x": "y"},
361 Access: []*bq.DatasetAccess{
362 {Role: "OWNER", Domain: "example.com"},
363 {
364 Dataset: &bq.DatasetAccessEntry{
365 Dataset: &bq.DatasetReference{
366 ProjectId: "p",
367 DatasetId: "otherdataset",
368 },
369 TargetTypes: []string{"VIEWS"},
370 },
371 },
372 },
373 }},
374 } {
375 got, err := test.in.toBQ()
376 if err != nil {
377 t.Fatal(err)
378 }
379 if diff := testutil.Diff(got, test.want, cmp.AllowUnexported(Dataset{})); diff != "" {
380 t.Errorf("got=-, want=+:\n%s", diff)
381 }
382 }
383
384
385 aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
386 for _, dm := range []*DatasetMetadata{
387 {CreationTime: aTime},
388 {LastModifiedTime: aTime},
389 {FullID: "x"},
390 {ETag: "e"},
391 } {
392 if _, err := dm.toBQ(); err == nil {
393 t.Errorf("%+v: got nil, want error", dm)
394 }
395 }
396 }
397
398 func TestBQToDatasetMetadata(t *testing.T) {
399 testClient := &Client{projectID: "p"}
400 cTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
401 cMillis := cTime.UnixNano() / 1e6
402 mTime := time.Date(2017, 10, 31, 0, 0, 0, 0, time.Local)
403 mMillis := mTime.UnixNano() / 1e6
404 q := &bq.Dataset{
405 CreationTime: cMillis,
406 LastModifiedTime: mMillis,
407 FriendlyName: "name",
408 Description: "desc",
409 DefaultTableExpirationMs: 60 * 60 * 1000,
410 MaxTimeTravelHours: 3,
411 DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
412 DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
413 KmsKeyName: "some_key",
414 },
415 ExternalDatasetReference: &bq.ExternalDatasetReference{
416 Connection: "conn",
417 ExternalSource: "external_src",
418 },
419 Location: "EU",
420 Labels: map[string]string{"x": "y"},
421 Access: []*bq.DatasetAccess{
422 {Role: "READER", UserByEmail: "joe@example.com"},
423 {Role: "WRITER", GroupByEmail: "users@example.com"},
424 {
425 Dataset: &bq.DatasetAccessEntry{
426 Dataset: &bq.DatasetReference{
427 ProjectId: "p",
428 DatasetId: "otherdataset",
429 },
430 TargetTypes: []string{"VIEWS"},
431 },
432 },
433 },
434 Tags: []*bq.DatasetTags{
435 {TagKey: "tag1", TagValue: "value1"},
436 {TagKey: "tag2", TagValue: "value2"},
437 },
438 Etag: "etag",
439 }
440 want := &DatasetMetadata{
441 CreationTime: cTime,
442 LastModifiedTime: mTime,
443 Name: "name",
444 Description: "desc",
445 DefaultTableExpiration: time.Hour,
446 MaxTimeTravel: time.Duration(3 * time.Hour),
447 DefaultPartitionExpiration: 24 * time.Hour,
448 DefaultEncryptionConfig: &EncryptionConfig{
449 KMSKeyName: "some_key",
450 },
451 ExternalDatasetReference: &ExternalDatasetReference{
452 Connection: "conn",
453 ExternalSource: "external_src",
454 },
455 StorageBillingModel: LogicalStorageBillingModel,
456 Location: "EU",
457 Labels: map[string]string{"x": "y"},
458 Access: []*AccessEntry{
459 {Role: ReaderRole, Entity: "joe@example.com", EntityType: UserEmailEntity},
460 {Role: WriterRole, Entity: "users@example.com", EntityType: GroupEmailEntity},
461 {
462 EntityType: DatasetEntity,
463 Dataset: &DatasetAccessEntry{
464 Dataset: testClient.Dataset("otherdataset"),
465 TargetTypes: []string{"VIEWS"},
466 },
467 },
468 },
469 Tags: []*DatasetTag{
470 {TagKey: "tag1", TagValue: "value1"},
471 {TagKey: "tag2", TagValue: "value2"},
472 },
473 ETag: "etag",
474 }
475 got, err := bqToDatasetMetadata(q, client)
476 if err != nil {
477 t.Fatal(err)
478 }
479 if diff := testutil.Diff(got, want, cmpopts.IgnoreUnexported(Dataset{})); diff != "" {
480 t.Errorf("-got, +want:\n%s", diff)
481 }
482 }
483
484 func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
485 dm := DatasetMetadataToUpdate{
486 Description: "desc",
487 Name: "name",
488 DefaultTableExpiration: time.Hour,
489 DefaultPartitionExpiration: 24 * time.Hour,
490 MaxTimeTravel: time.Duration(181 * time.Minute),
491 StorageBillingModel: PhysicalStorageBillingModel,
492 DefaultEncryptionConfig: &EncryptionConfig{
493 KMSKeyName: "some_key",
494 },
495 ExternalDatasetReference: &ExternalDatasetReference{
496 Connection: "conn",
497 ExternalSource: "external_src",
498 },
499 }
500 dm.SetLabel("label", "value")
501 dm.DeleteLabel("del")
502
503 got, err := dm.toBQ()
504 if err != nil {
505 t.Fatal(err)
506 }
507 want := &bq.Dataset{
508 Description: "desc",
509 FriendlyName: "name",
510 DefaultTableExpirationMs: 60 * 60 * 1000,
511 MaxTimeTravelHours: 3,
512 DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
513 StorageBillingModel: string(PhysicalStorageBillingModel),
514 DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
515 KmsKeyName: "some_key",
516 ForceSendFields: []string{"KmsKeyName"},
517 },
518 ExternalDatasetReference: &bq.ExternalDatasetReference{
519 Connection: "conn",
520 ExternalSource: "external_src",
521 },
522 Labels: map[string]string{"label": "value"},
523 ForceSendFields: []string{"Description", "FriendlyName", "ExternalDatasetReference", "StorageBillingModel"},
524 NullFields: []string{"Labels.del"},
525 }
526 if diff := testutil.Diff(got, want); diff != "" {
527 t.Errorf("-got, +want:\n%s", diff)
528 }
529 }
530
531 func TestConvertAccessEntry(t *testing.T) {
532 c := &Client{projectID: "pid"}
533 for _, e := range []*AccessEntry{
534 {Role: ReaderRole, Entity: "e", EntityType: DomainEntity},
535 {Role: WriterRole, Entity: "e", EntityType: GroupEmailEntity},
536 {Role: OwnerRole, Entity: "e", EntityType: UserEmailEntity},
537 {Role: ReaderRole, Entity: "e", EntityType: SpecialGroupEntity},
538 {Role: ReaderRole, Entity: "e", EntityType: IAMMemberEntity},
539 {Role: ReaderRole, EntityType: ViewEntity,
540 View: &Table{ProjectID: "p", DatasetID: "d", TableID: "t", c: c}},
541 {Role: ReaderRole, EntityType: RoutineEntity,
542 Routine: &Routine{ProjectID: "p", DatasetID: "d", RoutineID: "r", c: c}},
543 } {
544 q, err := e.toBQ()
545 if err != nil {
546 t.Fatal(err)
547 }
548 got, err := bqToAccessEntry(q, c)
549 if err != nil {
550 t.Fatal(err)
551 }
552 if diff := testutil.Diff(got, e, cmp.AllowUnexported(Table{}, Client{}, Routine{})); diff != "" {
553 t.Errorf("got=-, want=+:\n%s", diff)
554 }
555 }
556
557 e := &AccessEntry{Role: ReaderRole, Entity: "e"}
558 if _, err := e.toBQ(); err == nil {
559 t.Error("got nil, want error")
560 }
561 if _, err := bqToAccessEntry(&bq.DatasetAccess{Role: "WRITER"}, nil); err == nil {
562 t.Error("got nil, want error")
563 }
564 }
565
566 func TestDatasetIdentifiers(t *testing.T) {
567 testDataset := &Dataset{
568 ProjectID: "p",
569 DatasetID: "d",
570 c: nil,
571 }
572 for _, tc := range []struct {
573 description string
574 in *Dataset
575 format IdentifierFormat
576 want string
577 wantErr bool
578 }{
579 {
580 description: "empty format string",
581 in: testDataset,
582 format: "",
583 wantErr: true,
584 },
585 {
586 description: "legacy",
587 in: testDataset,
588 format: LegacySQLID,
589 want: "p:d",
590 },
591 {
592 description: "standard unquoted",
593 in: testDataset,
594 format: StandardSQLID,
595 want: "p.d",
596 },
597 {
598 description: "standard w/quoting",
599 in: &Dataset{ProjectID: "p-p", DatasetID: "d"},
600 format: StandardSQLID,
601 want: "`p-p`.d",
602 },
603 {
604 description: "api resource",
605 in: testDataset,
606 format: StorageAPIResourceID,
607 wantErr: true,
608 },
609 } {
610 got, err := tc.in.Identifier(tc.format)
611 if tc.wantErr && err == nil {
612 t.Errorf("case %q: wanted err, was success", tc.description)
613 }
614 if !tc.wantErr {
615 if err != nil {
616 t.Errorf("case %q: wanted success, got err: %v", tc.description, err)
617 } else {
618 if got != tc.want {
619 t.Errorf("case %q: got %s, want %s", tc.description, got, tc.want)
620 }
621 }
622 }
623 }
624 }
625
View as plain text