dataset_integration_test.go

Documentation: cloud.google.com/go/bigquery

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bigquery
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"net/http"
    21  	"testing"
    22  	"time"
    23  
    24  	"cloud.google.com/go/internal/testutil"
    25  	"github.com/google/go-cmp/cmp/cmpopts"
    26  )
    27  
    28  func TestIntegration_DatasetCreate(t *testing.T) {
    29  	if client == nil {
    30  		t.Skip("Integration tests skipped")
    31  	}
    32  	ctx := context.Background()
    33  	ds := client.Dataset(datasetIDs.New())
    34  	wmd := &DatasetMetadata{Name: "name", Location: "EU"}
    35  	err := ds.Create(ctx, wmd)
    36  	if err != nil {
    37  		t.Fatal(err)
    38  	}
    39  	gmd, err := ds.Metadata(ctx)
    40  	if err != nil {
    41  		t.Fatal(err)
    42  	}
    43  	if got, want := gmd.Name, wmd.Name; got != want {
    44  		t.Errorf("name: got %q, want %q", got, want)
    45  	}
    46  	if got, want := gmd.Location, wmd.Location; got != want {
    47  		t.Errorf("location: got %q, want %q", got, want)
    48  	}
    49  	if err := ds.Delete(ctx); err != nil {
    50  		t.Fatalf("deleting dataset %v: %v", ds, err)
    51  	}
    52  }
    53  
    54  func TestIntegration_DatasetMetadata(t *testing.T) {
    55  	if client == nil {
    56  		t.Skip("Integration tests skipped")
    57  	}
    58  	ctx := context.Background()
    59  	md, err := dataset.Metadata(ctx)
    60  	if err != nil {
    61  		t.Fatal(err)
    62  	}
    63  	if got, want := md.FullID, fmt.Sprintf("%s:%s", dataset.ProjectID, dataset.DatasetID); got != want {
    64  		t.Errorf("FullID: got %q, want %q", got, want)
    65  	}
    66  	jan2016 := time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC)
    67  	if md.CreationTime.Before(jan2016) {
    68  		t.Errorf("CreationTime: got %s, want > 2016-1-1", md.CreationTime)
    69  	}
    70  	if md.LastModifiedTime.Before(jan2016) {
    71  		t.Errorf("LastModifiedTime: got %s, want > 2016-1-1", md.LastModifiedTime)
    72  	}
    73  
    74  	// Verify that we get a NotFound for a nonexistent dataset.
    75  	_, err = client.Dataset("does_not_exist").Metadata(ctx)
    76  	if err == nil || !hasStatusCode(err, http.StatusNotFound) {
    77  		t.Errorf("got %v, want NotFound error", err)
    78  	}
    79  }
    80  
    81  func TestIntegration_DatasetDelete(t *testing.T) {
    82  	if client == nil {
    83  		t.Skip("Integration tests skipped")
    84  	}
    85  	ctx := context.Background()
    86  	ds := client.Dataset(datasetIDs.New())
    87  	if err := ds.Create(ctx, nil); err != nil {
    88  		t.Fatalf("creating dataset %s: %v", ds.DatasetID, err)
    89  	}
    90  	if err := ds.Delete(ctx); err != nil {
    91  		t.Fatalf("deleting dataset %s: %v", ds.DatasetID, err)
    92  	}
    93  }
    94  
    95  func TestIntegration_DatasetDeleteWithContents(t *testing.T) {
    96  	if client == nil {
    97  		t.Skip("Integration tests skipped")
    98  	}
    99  	ctx := context.Background()
   100  	ds := client.Dataset(datasetIDs.New())
   101  	if err := ds.Create(ctx, nil); err != nil {
   102  		t.Fatalf("creating dataset %s: %v", ds.DatasetID, err)
   103  	}
   104  	table := ds.Table(tableIDs.New())
   105  	if err := table.Create(ctx, nil); err != nil {
   106  		t.Fatalf("creating table %s in dataset %s: %v", table.TableID, table.DatasetID, err)
   107  	}
   108  	// We expect failure here
   109  	if err := ds.Delete(ctx); err == nil {
   110  		t.Fatalf("non-recursive delete of dataset %s succeeded unexpectedly.", ds.DatasetID)
   111  	}
   112  	if err := ds.DeleteWithContents(ctx); err != nil {
   113  		t.Fatalf("deleting recursively dataset %s: %v", ds.DatasetID, err)
   114  	}
   115  }
   116  
   117  func TestIntegration_DatasetUpdateETags(t *testing.T) {
   118  	if client == nil {
   119  		t.Skip("Integration tests skipped")
   120  	}
   121  
   122  	check := func(md *DatasetMetadata, wantDesc, wantName string) {
   123  		if md.Description != wantDesc {
   124  			t.Errorf("description: got %q, want %q", md.Description, wantDesc)
   125  		}
   126  		if md.Name != wantName {
   127  			t.Errorf("name: got %q, want %q", md.Name, wantName)
   128  		}
   129  	}
   130  
   131  	ctx := context.Background()
   132  	md, err := dataset.Metadata(ctx)
   133  	if err != nil {
   134  		t.Fatal(err)
   135  	}
   136  	if md.ETag == "" {
   137  		t.Fatal("empty ETag")
   138  	}
   139  	// Write without ETag succeeds.
   140  	desc := md.Description + "d2"
   141  	name := md.Name + "n2"
   142  	md2, err := dataset.Update(ctx, DatasetMetadataToUpdate{Description: desc, Name: name}, "")
   143  	if err != nil {
   144  		t.Fatal(err)
   145  	}
   146  	check(md2, desc, name)
   147  
   148  	// Write with original ETag fails because of intervening write.
   149  	_, err = dataset.Update(ctx, DatasetMetadataToUpdate{Description: "d", Name: "n"}, md.ETag)
   150  	if err == nil {
   151  		t.Fatal("got nil, want error")
   152  	}
   153  
   154  	// Write with most recent ETag succeeds.
   155  	md3, err := dataset.Update(ctx, DatasetMetadataToUpdate{Description: "", Name: ""}, md2.ETag)
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  	check(md3, "", "")
   160  }
   161  
   162  func TestIntegration_DatasetUpdateDefaultExpirationAndMaxTimeTravel(t *testing.T) {
   163  	if client == nil {
   164  		t.Skip("Integration tests skipped")
   165  	}
   166  	ctx := context.Background()
   167  	_, err := dataset.Metadata(ctx)
   168  	if err != nil {
   169  		t.Fatal(err)
   170  	}
   171  	wantExpiration := time.Hour
   172  	wantTimeTravel := 48 * time.Hour
   173  	// Set the default expiration time.
   174  	md, err := dataset.Update(ctx, DatasetMetadataToUpdate{
   175  		DefaultTableExpiration: wantExpiration,
   176  		MaxTimeTravel:          wantTimeTravel,
   177  	}, "")
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  	if got := md.DefaultTableExpiration; got != wantExpiration {
   182  		t.Fatalf("DefaultTableExpiration want %s got %s", wantExpiration, md.DefaultTableExpiration)
   183  	}
   184  	if got := md.MaxTimeTravel; got != wantTimeTravel {
   185  		t.Fatalf("MaxTimeTravelHours want %s got %s", wantTimeTravel, md.MaxTimeTravel)
   186  	}
   187  	// Omitting DefaultTableExpiration doesn't change it.
   188  	md, err = dataset.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")
   189  	if err != nil {
   190  		t.Fatal(err)
   191  	}
   192  	if md.DefaultTableExpiration != time.Hour {
   193  		t.Fatalf("got %s, want 1h", md.DefaultTableExpiration)
   194  	}
   195  	// Setting it to 0 deletes it (which looks like a 0 duration).
   196  	md, err = dataset.Update(ctx, DatasetMetadataToUpdate{DefaultTableExpiration: time.Duration(0)}, "")
   197  	if err != nil {
   198  		t.Fatal(err)
   199  	}
   200  	if md.DefaultTableExpiration != 0 {
   201  		t.Fatalf("got %s, want 0", md.DefaultTableExpiration)
   202  	}
   203  }
   204  
   205  func TestIntegration_DatasetUpdateDefaultPartitionExpiration(t *testing.T) {
   206  	if client == nil {
   207  		t.Skip("Integration tests skipped")
   208  	}
   209  	ctx := context.Background()
   210  	_, err := dataset.Metadata(ctx)
   211  	if err != nil {
   212  		t.Fatal(err)
   213  	}
   214  	// Set the default partition expiration time.
   215  	md, err := dataset.Update(ctx, DatasetMetadataToUpdate{DefaultPartitionExpiration: 24 * time.Hour}, "")
   216  	if err != nil {
   217  		t.Fatal(err)
   218  	}
   219  	if md.DefaultPartitionExpiration != 24*time.Hour {
   220  		t.Fatalf("got %v, want 24h", md.DefaultPartitionExpiration)
   221  	}
   222  	// Omitting DefaultPartitionExpiration doesn't change it.
   223  	md, err = dataset.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")
   224  	if err != nil {
   225  		t.Fatal(err)
   226  	}
   227  	if md.DefaultPartitionExpiration != 24*time.Hour {
   228  		t.Fatalf("got %s, want 24h", md.DefaultPartitionExpiration)
   229  	}
   230  	// Setting it to 0 deletes it (which looks like a 0 duration).
   231  	md, err = dataset.Update(ctx, DatasetMetadataToUpdate{DefaultPartitionExpiration: time.Duration(0)}, "")
   232  	if err != nil {
   233  		t.Fatal(err)
   234  	}
   235  	if md.DefaultPartitionExpiration != 0 {
   236  		t.Fatalf("got %s, want 0", md.DefaultPartitionExpiration)
   237  	}
   238  }
   239  
   240  func TestIntegration_DatasetUpdateDefaultCollation(t *testing.T) {
   241  	if client == nil {
   242  		t.Skip("Integration tests skipped")
   243  	}
   244  	caseInsensitiveCollation := "und:ci"
   245  	caseSensitiveCollation := ""
   246  
   247  	ctx := context.Background()
   248  	ds := client.Dataset(datasetIDs.New())
   249  	err := ds.Create(ctx, &DatasetMetadata{
   250  		DefaultCollation: caseSensitiveCollation,
   251  	})
   252  	if err != nil {
   253  		t.Fatal(err)
   254  	}
   255  	md, err := ds.Metadata(ctx)
   256  	if err != nil {
   257  		t.Fatal(err)
   258  	}
   259  	if md.DefaultCollation != caseSensitiveCollation {
   260  		t.Fatalf("got %q, want %q", md.DefaultCollation, caseSensitiveCollation)
   261  	}
   262  
   263  	// Update the default collation
   264  	md, err = ds.Update(ctx, DatasetMetadataToUpdate{
   265  		DefaultCollation: caseInsensitiveCollation,
   266  	}, "")
   267  	if err != nil {
   268  		t.Fatal(err)
   269  	}
   270  	if md.DefaultCollation != caseInsensitiveCollation {
   271  		t.Fatalf("got %q, want %q", md.DefaultCollation, caseInsensitiveCollation)
   272  	}
   273  
   274  	// Omitting DefaultCollation doesn't change it.
   275  	md, err = ds.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")
   276  	if err != nil {
   277  		t.Fatal(err)
   278  	}
   279  	if md.DefaultCollation != caseInsensitiveCollation {
   280  		t.Fatalf("got %q, want %q", md.DefaultCollation, caseInsensitiveCollation)
   281  	}
   282  
   283  	if err := ds.Delete(ctx); err != nil {
   284  		t.Fatalf("deleting dataset %v: %v", ds, err)
   285  	}
   286  }
   287  
   288  func TestIntegration_DatasetStorageBillingModel(t *testing.T) {
   289  	if client == nil {
   290  		t.Skip("Integration tests skipped")
   291  	}
   292  	t.Skip("BigQuery flat-rate commitments enabled for project, feature skipped")
   293  
   294  	ctx := context.Background()
   295  	md, err := dataset.Metadata(ctx)
   296  	if err != nil {
   297  		t.Fatal(err)
   298  	}
   299  	if md.StorageBillingModel != LogicalStorageBillingModel {
   300  		t.Fatalf("got %q, want %q", md.StorageBillingModel, LogicalStorageBillingModel)
   301  	}
   302  
   303  	ds := client.Dataset(datasetIDs.New())
   304  	err = ds.Create(ctx, &DatasetMetadata{
   305  		StorageBillingModel: PhysicalStorageBillingModel,
   306  	})
   307  	if err != nil {
   308  		t.Fatal(err)
   309  	}
   310  	md, err = ds.Metadata(ctx)
   311  	if err != nil {
   312  		t.Fatal(err)
   313  	}
   314  	if md.StorageBillingModel != PhysicalStorageBillingModel {
   315  		t.Fatalf("got %q, want %q", md.StorageBillingModel, PhysicalStorageBillingModel)
   316  	}
   317  	if err := ds.Delete(ctx); err != nil {
   318  		t.Fatalf("deleting dataset %v: %v", ds, err)
   319  	}
   320  }
   321  
   322  func TestIntegration_DatasetStorageUpdateBillingModel(t *testing.T) {
   323  	if client == nil {
   324  		t.Skip("Integration tests skipped")
   325  	}
   326  	t.Skip("BigQuery flat-rate commitments enabled for project, feature skipped")
   327  
   328  	ctx := context.Background()
   329  	ds := client.Dataset(datasetIDs.New())
   330  	err := ds.Create(ctx, &DatasetMetadata{
   331  		StorageBillingModel: LogicalStorageBillingModel,
   332  	})
   333  	if err != nil {
   334  		t.Fatal(err)
   335  	}
   336  
   337  	md, err := ds.Metadata(ctx)
   338  	if md.StorageBillingModel != LogicalStorageBillingModel {
   339  		t.Fatalf("got %q, want %q", md.StorageBillingModel, LogicalStorageBillingModel)
   340  	}
   341  
   342  	// Update the Storage billing model
   343  	md, err = ds.Update(ctx, DatasetMetadataToUpdate{
   344  		StorageBillingModel: PhysicalStorageBillingModel,
   345  	}, "")
   346  	if err != nil {
   347  		t.Fatal(err)
   348  	}
   349  	if md.StorageBillingModel != PhysicalStorageBillingModel {
   350  		t.Fatalf("got %q, want %q", md.StorageBillingModel, PhysicalStorageBillingModel)
   351  	}
   352  
   353  	// Omitting StorageBillingModel doesn't change it.
   354  	md, err = ds.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")
   355  	if err != nil {
   356  		t.Fatal(err)
   357  	}
   358  	if md.StorageBillingModel != PhysicalStorageBillingModel {
   359  		t.Fatalf("got %q, want %q", md.StorageBillingModel, PhysicalStorageBillingModel)
   360  	}
   361  
   362  	if err := ds.Delete(ctx); err != nil {
   363  		t.Fatalf("deleting dataset %v: %v", ds, err)
   364  	}
   365  }
   366  
   367  func TestIntegration_DatasetUpdateAccess(t *testing.T) {
   368  	if client == nil {
   369  		t.Skip("Integration tests skipped")
   370  	}
   371  	ctx := context.Background()
   372  	md, err := dataset.Metadata(ctx)
   373  	if err != nil {
   374  		t.Fatal(err)
   375  	}
   376  
   377  	// Create a sample UDF so we can verify adding authorized UDFs
   378  	routineID := routineIDs.New()
   379  	routine := dataset.Routine(routineID)
   380  	routineSQLID, _ := routine.Identifier(StandardSQLID)
   381  
   382  	sql := fmt.Sprintf(`
   383  			CREATE FUNCTION %s(x INT64) AS (x * 3);`,
   384  		routineSQLID)
   385  	if _, _, err := runQuerySQL(ctx, sql); err != nil {
   386  		t.Fatal(err)
   387  	}
   388  	defer routine.Delete(ctx)
   389  
   390  	origAccess := append([]*AccessEntry(nil), md.Access...)
   391  	newEntries := []*AccessEntry{
   392  		{
   393  			Role:       ReaderRole,
   394  			Entity:     "Joe@example.com",
   395  			EntityType: UserEmailEntity,
   396  		},
   397  		{
   398  			Role:       ReaderRole,
   399  			Entity:     "allUsers",
   400  			EntityType: IAMMemberEntity,
   401  		},
   402  		{
   403  			EntityType: RoutineEntity,
   404  			Routine:    routine,
   405  		},
   406  		{
   407  			EntityType: DatasetEntity,
   408  			Dataset: &DatasetAccessEntry{
   409  				Dataset:     otherDataset,
   410  				TargetTypes: []string{"VIEWS"},
   411  			},
   412  		},
   413  	}
   414  
   415  	newAccess := append(md.Access, newEntries...)
   416  	dm := DatasetMetadataToUpdate{Access: newAccess}
   417  	md, err = dataset.Update(ctx, dm, md.ETag)
   418  	if err != nil {
   419  		t.Fatal(err)
   420  	}
   421  	defer func() {
   422  		_, err := dataset.Update(ctx, DatasetMetadataToUpdate{Access: origAccess}, md.ETag)
   423  		if err != nil {
   424  			t.Log("could not restore dataset access list")
   425  		}
   426  	}()
   427  
   428  	if diff := testutil.Diff(md.Access, newAccess, cmpopts.SortSlices(lessAccessEntries), cmpopts.IgnoreUnexported(Routine{}, Dataset{})); diff != "" {
   429  		t.Errorf("got=-, want=+:\n%s", diff)
   430  	}
   431  }
   432  
   433  // Comparison function for AccessEntries to enable order insensitive equality checking.
   434  func lessAccessEntries(x, y *AccessEntry) bool {
   435  	if x.Entity < y.Entity {
   436  		return true
   437  	}
   438  	if x.Entity > y.Entity {
   439  		return false
   440  	}
   441  	if x.EntityType < y.EntityType {
   442  		return true
   443  	}
   444  	if x.EntityType > y.EntityType {
   445  		return false
   446  	}
   447  	if x.Role < y.Role {
   448  		return true
   449  	}
   450  	if x.Role > y.Role {
   451  		return false
   452  	}
   453  	if x.View == nil {
   454  		return y.View != nil
   455  	}
   456  	if x.Routine == nil {
   457  		return y.Routine == nil
   458  	}
   459  	if x.Dataset == nil {
   460  		return y.Dataset == nil
   461  	}
   462  	return false
   463  }
   464  
   465  func TestIntegration_DatasetUpdateLabels(t *testing.T) {
   466  	if client == nil {
   467  		t.Skip("Integration tests skipped")
   468  	}
   469  	ctx := context.Background()
   470  	_, err := dataset.Metadata(ctx)
   471  	if err != nil {
   472  		t.Fatal(err)
   473  	}
   474  	var dm DatasetMetadataToUpdate
   475  	dm.SetLabel("label", "value")
   476  	md, err := dataset.Update(ctx, dm, "")
   477  	if err != nil {
   478  		t.Fatal(err)
   479  	}
   480  	if got, want := md.Labels["label"], "value"; got != want {
   481  		t.Errorf("got %q, want %q", got, want)
   482  	}
   483  	dm = DatasetMetadataToUpdate{}
   484  	dm.DeleteLabel("label")
   485  	md, err = dataset.Update(ctx, dm, "")
   486  	if err != nil {
   487  		t.Fatal(err)
   488  	}
   489  	if _, ok := md.Labels["label"]; ok {
   490  		t.Error("label still present after deletion")
   491  	}
   492  }
   493
View as plain text