Source file src/cloud.google.com/go/storage/client_test.go

Documentation: cloud.google.com/go/storage

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package storage
    16  
    17  import (
    18  	"context"
    19  	"errors"
    20  	"fmt"
    21  	"log"
    22  	"net/url"
    23  	"os"
    24  	"strconv"
    25  	"strings"
    26  	"testing"
    27  	"time"
    28  
    29  	"cloud.google.com/go/iam/apiv1/iampb"
    30  	"github.com/google/go-cmp/cmp"
    31  	"github.com/googleapis/gax-go/v2"
    32  	"github.com/googleapis/gax-go/v2/apierror"
    33  	"github.com/googleapis/gax-go/v2/callctx"
    34  	"google.golang.org/api/iterator"
    35  	"google.golang.org/grpc/codes"
    36  )
    37  
    38  var emulatorClients map[string]storageClient
    39  var veneerClient *Client
    40  
    41  func TestCreateBucketEmulated(t *testing.T) {
    42  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
    43  		want := &BucketAttrs{
    44  			Name: bucket,
    45  			Logging: &BucketLogging{
    46  				LogBucket: bucket,
    47  			},
    48  		}
    49  		got, err := client.CreateBucket(context.Background(), project, want.Name, want, nil)
    50  		if err != nil {
    51  			t.Fatal(err)
    52  		}
    53  		want.Location = "US"
    54  		if diff := cmp.Diff(got.Name, want.Name); diff != "" {
    55  			t.Errorf("Name got(-),want(+):\n%s", diff)
    56  		}
    57  		if diff := cmp.Diff(got.Location, want.Location); diff != "" {
    58  			t.Errorf("Location got(-),want(+):\n%s", diff)
    59  		}
    60  		if diff := cmp.Diff(got.Logging.LogBucket, want.Logging.LogBucket); diff != "" {
    61  			t.Errorf("LogBucket got(-),want(+):\n%s", diff)
    62  		}
    63  	})
    64  }
    65  
    66  func TestDeleteBucketEmulated(t *testing.T) {
    67  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
    68  		b := &BucketAttrs{
    69  			Name: bucket,
    70  		}
    71  		// Create the bucket that will be deleted.
    72  		_, err := client.CreateBucket(context.Background(), project, b.Name, b, nil)
    73  		if err != nil {
    74  			t.Fatalf("client.CreateBucket: %v", err)
    75  		}
    76  		// Delete the bucket that was just created.
    77  		err = client.DeleteBucket(context.Background(), b.Name, nil)
    78  		if err != nil {
    79  			t.Fatalf("client.DeleteBucket: %v", err)
    80  		}
    81  	})
    82  }
    83  
    84  func TestGetBucketEmulated(t *testing.T) {
    85  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
    86  		want := &BucketAttrs{
    87  			Name: bucket,
    88  		}
    89  		// Create the bucket that will be retrieved.
    90  		_, err := client.CreateBucket(context.Background(), project, want.Name, want, nil)
    91  		if err != nil {
    92  			t.Fatalf("client.CreateBucket: %v", err)
    93  		}
    94  		got, err := client.GetBucket(context.Background(), want.Name, &BucketConditions{MetagenerationMatch: 1})
    95  		if err != nil {
    96  			t.Fatal(err)
    97  		}
    98  		if diff := cmp.Diff(got.Name, want.Name); diff != "" {
    99  			t.Errorf("got(-),want(+):\n%s", diff)
   100  		}
   101  	})
   102  }
   103  
   104  func TestUpdateBucketEmulated(t *testing.T) {
   105  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   106  		bkt := &BucketAttrs{
   107  			Name: bucket,
   108  		}
   109  		// Create the bucket that will be updated.
   110  		_, err := client.CreateBucket(context.Background(), project, bkt.Name, bkt, nil)
   111  		if err != nil {
   112  			t.Fatalf("client.CreateBucket: %v", err)
   113  		}
   114  
   115  		ua := &BucketAttrsToUpdate{
   116  			VersioningEnabled:     false,
   117  			RequesterPays:         false,
   118  			DefaultEventBasedHold: false,
   119  			Encryption:            &BucketEncryption{DefaultKMSKeyName: "key2"},
   120  			Lifecycle: &Lifecycle{
   121  				Rules: []LifecycleRule{
   122  					{
   123  						Action:    LifecycleAction{Type: "Delete"},
   124  						Condition: LifecycleCondition{AgeInDays: 30},
   125  					},
   126  				},
   127  			},
   128  			Logging:      &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
   129  			Website:      &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
   130  			StorageClass: "NEARLINE",
   131  			RPO:          RPOAsyncTurbo,
   132  		}
   133  		want := &BucketAttrs{
   134  			Name:                  bucket,
   135  			VersioningEnabled:     false,
   136  			RequesterPays:         false,
   137  			DefaultEventBasedHold: false,
   138  			Encryption:            &BucketEncryption{DefaultKMSKeyName: "key2"},
   139  			Lifecycle: Lifecycle{
   140  				Rules: []LifecycleRule{
   141  					{
   142  						Action:    LifecycleAction{Type: "Delete"},
   143  						Condition: LifecycleCondition{AgeInDays: 30},
   144  					},
   145  				},
   146  			},
   147  			Logging:      &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
   148  			Website:      &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
   149  			StorageClass: "NEARLINE",
   150  			RPO:          RPOAsyncTurbo,
   151  		}
   152  
   153  		got, err := client.UpdateBucket(context.Background(), bucket, ua, &BucketConditions{MetagenerationMatch: 1})
   154  		if err != nil {
   155  			t.Fatal(err)
   156  		}
   157  		if diff := cmp.Diff(got.Name, want.Name); diff != "" {
   158  			t.Errorf("Name: got(-),want(+):\n%s", diff)
   159  		}
   160  		if diff := cmp.Diff(got.VersioningEnabled, want.VersioningEnabled); diff != "" {
   161  			t.Errorf("VersioningEnabled: got(-),want(+):\n%s", diff)
   162  		}
   163  		if diff := cmp.Diff(got.RequesterPays, want.RequesterPays); diff != "" {
   164  			t.Errorf("RequesterPays: got(-),want(+):\n%s", diff)
   165  		}
   166  		if diff := cmp.Diff(got.DefaultEventBasedHold, want.DefaultEventBasedHold); diff != "" {
   167  			t.Errorf("DefaultEventBasedHold: got(-),want(+):\n%s", diff)
   168  		}
   169  		if diff := cmp.Diff(got.Encryption, want.Encryption); diff != "" {
   170  			t.Errorf("Encryption: got(-),want(+):\n%s", diff)
   171  		}
   172  		if diff := cmp.Diff(got.Lifecycle, want.Lifecycle); diff != "" {
   173  			t.Errorf("Lifecycle: got(-),want(+):\n%s", diff)
   174  		}
   175  		if diff := cmp.Diff(got.Logging, want.Logging); diff != "" {
   176  			t.Errorf("Logging: got(-),want(+):\n%s", diff)
   177  		}
   178  		if diff := cmp.Diff(got.Website, want.Website); diff != "" {
   179  			t.Errorf("Website: got(-),want(+):\n%s", diff)
   180  		}
   181  		if diff := cmp.Diff(got.RPO, want.RPO); diff != "" {
   182  			t.Errorf("RPO: got(-),want(+):\n%s", diff)
   183  		}
   184  		if diff := cmp.Diff(got.StorageClass, want.StorageClass); diff != "" {
   185  			t.Errorf("StorageClass: got(-),want(+):\n%s", diff)
   186  		}
   187  	})
   188  }
   189  
   190  func TestGetServiceAccountEmulated(t *testing.T) {
   191  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   192  		_, err := client.GetServiceAccount(context.Background(), project)
   193  		if err != nil {
   194  			t.Fatalf("client.GetServiceAccount: %v", err)
   195  		}
   196  	})
   197  }
   198  
   199  func TestGetSetTestIamPolicyEmulated(t *testing.T) {
   200  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   201  		battrs, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   202  			Name: bucket,
   203  		}, nil)
   204  		if err != nil {
   205  			t.Fatalf("client.CreateBucket: %v", err)
   206  		}
   207  		got, err := client.GetIamPolicy(context.Background(), battrs.Name, 0)
   208  		if err != nil {
   209  			t.Fatalf("client.GetIamPolicy: %v", err)
   210  		}
   211  		err = client.SetIamPolicy(context.Background(), battrs.Name, &iampb.Policy{
   212  			Etag:     got.GetEtag(),
   213  			Bindings: []*iampb.Binding{{Role: "roles/viewer", Members: []string{"allUsers"}}},
   214  		})
   215  		if err != nil {
   216  			t.Fatalf("client.SetIamPolicy: %v", err)
   217  		}
   218  		want := []string{"storage.foo", "storage.bar"}
   219  		perms, err := client.TestIamPermissions(context.Background(), battrs.Name, want)
   220  		if err != nil {
   221  			t.Fatalf("client.TestIamPermissions: %v", err)
   222  		}
   223  		if diff := cmp.Diff(perms, want); diff != "" {
   224  			t.Errorf("got(-),want(+):\n%s", diff)
   225  		}
   226  	})
   227  }
   228  
   229  func TestDeleteObjectEmulated(t *testing.T) {
   230  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   231  		// Populate test object that will be deleted.
   232  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   233  			Name: bucket,
   234  		}, nil)
   235  		if err != nil {
   236  			t.Fatalf("client.CreateBucket: %v", err)
   237  		}
   238  		want := ObjectAttrs{
   239  			Bucket: bucket,
   240  			Name:   fmt.Sprintf("testObject-%d", time.Now().Nanosecond()),
   241  		}
   242  		w := veneerClient.Bucket(bucket).Object(want.Name).NewWriter(context.Background())
   243  		if _, err := w.Write(randomBytesToWrite); err != nil {
   244  			t.Fatalf("failed to populate test object: %v", err)
   245  		}
   246  		if err := w.Close(); err != nil {
   247  			t.Fatalf("closing object: %v", err)
   248  		}
   249  		err = client.DeleteObject(context.Background(), bucket, want.Name, defaultGen, nil)
   250  		if err != nil {
   251  			t.Fatalf("client.DeleteBucket: %v", err)
   252  		}
   253  	})
   254  }
   255  
   256  func TestGetObjectEmulated(t *testing.T) {
   257  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   258  		// Populate test object.
   259  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   260  			Name: bucket,
   261  		}, nil)
   262  		if err != nil {
   263  			t.Fatalf("client.CreateBucket: %v", err)
   264  		}
   265  		want := ObjectAttrs{
   266  			Bucket: bucket,
   267  			Name:   fmt.Sprintf("testObject-%d", time.Now().Nanosecond()),
   268  		}
   269  		w := veneerClient.Bucket(bucket).Object(want.Name).NewWriter(context.Background())
   270  		if _, err := w.Write(randomBytesToWrite); err != nil {
   271  			t.Fatalf("failed to populate test object: %v", err)
   272  		}
   273  		if err := w.Close(); err != nil {
   274  			t.Fatalf("closing object: %v", err)
   275  		}
   276  		got, err := client.GetObject(context.Background(), &getObjectParams{bucket: bucket, object: want.Name, gen: defaultGen})
   277  		if err != nil {
   278  			t.Fatal(err)
   279  		}
   280  		if diff := cmp.Diff(got.Name, want.Name); diff != "" {
   281  			t.Errorf("got(-),want(+):\n%s", diff)
   282  		}
   283  	})
   284  }
   285  
   286  func TestRewriteObjectEmulated(t *testing.T) {
   287  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   288  		// Populate test object.
   289  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   290  			Name: bucket,
   291  		}, nil)
   292  		if err != nil {
   293  			t.Fatalf("client.CreateBucket: %v", err)
   294  		}
   295  		src := ObjectAttrs{
   296  			Bucket: bucket,
   297  			Name:   fmt.Sprintf("testObject-%d", time.Now().Nanosecond()),
   298  		}
   299  		w := veneerClient.Bucket(bucket).Object(src.Name).NewWriter(context.Background())
   300  		if _, err := w.Write(randomBytesToWrite); err != nil {
   301  			t.Fatalf("failed to populate test object: %v", err)
   302  		}
   303  		if err := w.Close(); err != nil {
   304  			t.Fatalf("closing object: %v", err)
   305  		}
   306  		req := &rewriteObjectRequest{
   307  			dstObject: destinationObject{
   308  				bucket: bucket,
   309  				name:   fmt.Sprintf("copy-of-%s", src.Name),
   310  				attrs:  &ObjectAttrs{},
   311  			},
   312  			srcObject: sourceObject{
   313  				bucket: bucket,
   314  				name:   src.Name,
   315  				gen:    defaultGen,
   316  			},
   317  		}
   318  		got, err := client.RewriteObject(context.Background(), req)
   319  		if err != nil {
   320  			t.Fatal(err)
   321  		}
   322  		if !got.done {
   323  			t.Fatal("didn't finish writing!")
   324  		}
   325  		if want := int64(len(randomBytesToWrite)); got.written != want {
   326  			t.Errorf("Bytes written: got %d, want %d", got.written, want)
   327  		}
   328  	})
   329  }
   330  
   331  func TestUpdateObjectEmulated(t *testing.T) {
   332  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   333  		// Populate test object.
   334  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   335  			Name: bucket,
   336  		}, nil)
   337  		if err != nil {
   338  			t.Fatalf("client.CreateBucket: %v", err)
   339  		}
   340  		ct := time.Date(2022, 5, 25, 12, 12, 12, 0, time.UTC)
   341  		o := ObjectAttrs{
   342  			Bucket:     bucket,
   343  			Name:       fmt.Sprintf("testObject-%d", time.Now().Nanosecond()),
   344  			CustomTime: ct,
   345  		}
   346  		w := veneerClient.Bucket(bucket).Object(o.Name).NewWriter(context.Background())
   347  		if _, err := w.Write(randomBytesToWrite); err != nil {
   348  			t.Fatalf("failed to populate test object: %v", err)
   349  		}
   350  		if err := w.Close(); err != nil {
   351  			t.Fatalf("closing object: %v", err)
   352  		}
   353  		want := &ObjectAttrsToUpdate{
   354  			EventBasedHold:     false,
   355  			TemporaryHold:      false,
   356  			ContentType:        "text/html",
   357  			ContentLanguage:    "en",
   358  			ContentEncoding:    "gzip",
   359  			ContentDisposition: "",
   360  			CacheControl:       "",
   361  			CustomTime:         ct.Add(10 * time.Hour),
   362  		}
   363  
   364  		params := &updateObjectParams{bucket: bucket, object: o.Name, uattrs: want, gen: defaultGen, conds: &Conditions{MetagenerationMatch: 1}}
   365  		got, err := client.UpdateObject(context.Background(), params)
   366  		if err != nil {
   367  			t.Fatalf("client.UpdateObject: %v", err)
   368  		}
   369  		if diff := cmp.Diff(got.Name, o.Name); diff != "" {
   370  			t.Errorf("Name: got(-),want(+):\n%s", diff)
   371  		}
   372  		if diff := cmp.Diff(got.EventBasedHold, want.EventBasedHold); diff != "" {
   373  			t.Errorf("EventBasedHold: got(-),want(+):\n%s", diff)
   374  		}
   375  		if diff := cmp.Diff(got.TemporaryHold, want.TemporaryHold); diff != "" {
   376  			t.Errorf("TemporaryHold: got(-),want(+):\n%s", diff)
   377  		}
   378  		if diff := cmp.Diff(got.ContentType, want.ContentType); diff != "" {
   379  			t.Errorf("ContentType: got(-),want(+):\n%s", diff)
   380  		}
   381  		if diff := cmp.Diff(got.ContentLanguage, want.ContentLanguage); diff != "" {
   382  			t.Errorf("ContentLanguage: got(-),want(+):\n%s", diff)
   383  		}
   384  		if diff := cmp.Diff(got.ContentEncoding, want.ContentEncoding); diff != "" {
   385  			t.Errorf("ContentEncoding: got(-),want(+):\n%s", diff)
   386  		}
   387  		if diff := cmp.Diff(got.ContentDisposition, want.ContentDisposition); diff != "" {
   388  			t.Errorf("ContentDisposition: got(-),want(+):\n%s", diff)
   389  		}
   390  		if diff := cmp.Diff(got.CacheControl, want.CacheControl); diff != "" {
   391  			t.Errorf("CacheControl: got(-),want(+):\n%s", diff)
   392  		}
   393  		if diff := cmp.Diff(got.CustomTime, want.CustomTime); diff != "" {
   394  			t.Errorf("CustomTime: got(-),want(+):\n%s", diff)
   395  		}
   396  	})
   397  }
   398  
   399  func TestListObjectsEmulated(t *testing.T) {
   400  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   401  		// Populate test data.
   402  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   403  			Name: bucket,
   404  		}, nil)
   405  		if err != nil {
   406  			t.Fatalf("client.CreateBucket: %v", err)
   407  		}
   408  		prefix := time.Now().Nanosecond()
   409  		want := []*ObjectAttrs{
   410  			{
   411  				Bucket: bucket,
   412  				Name:   fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   413  			},
   414  			{
   415  				Bucket: bucket,
   416  				Name:   fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   417  			},
   418  			{
   419  				Bucket: bucket,
   420  				Name:   fmt.Sprintf("object-%d", time.Now().Nanosecond()),
   421  			},
   422  		}
   423  		for _, obj := range want {
   424  			w := veneerClient.Bucket(bucket).Object(obj.Name).NewWriter(context.Background())
   425  			if _, err := w.Write(randomBytesToWrite); err != nil {
   426  				t.Fatalf("failed to populate test data: %v", err)
   427  			}
   428  			if err := w.Close(); err != nil {
   429  				t.Fatalf("closing object: %v", err)
   430  			}
   431  		}
   432  
   433  		// Simple list, no query.
   434  		it := client.ListObjects(context.Background(), bucket, nil)
   435  		var o *ObjectAttrs
   436  		var got int
   437  		for i := 0; err == nil && i <= len(want); i++ {
   438  			o, err = it.Next()
   439  			if err != nil {
   440  				break
   441  			}
   442  			got++
   443  			if diff := cmp.Diff(o.Name, want[i].Name); diff != "" {
   444  				t.Errorf("got(-),want(+):\n%s", diff)
   445  			}
   446  		}
   447  		if err != iterator.Done {
   448  			t.Fatalf("expected %q but got %q", iterator.Done, err)
   449  		}
   450  		expected := len(want)
   451  		if got != expected {
   452  			t.Errorf("expected to get %d objects, but got %d", expected, got)
   453  		}
   454  	})
   455  }
   456  
   457  func TestListObjectsWithPrefixEmulated(t *testing.T) {
   458  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   459  		// Populate test data.
   460  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   461  			Name: bucket,
   462  		}, nil)
   463  		if err != nil {
   464  			t.Fatalf("client.CreateBucket: %v", err)
   465  		}
   466  		prefix := time.Now().Nanosecond()
   467  		want := []*ObjectAttrs{
   468  			{
   469  				Bucket: bucket,
   470  				Name:   fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   471  			},
   472  			{
   473  				Bucket: bucket,
   474  				Name:   fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   475  			},
   476  			{
   477  				Bucket: bucket,
   478  				Name:   fmt.Sprintf("object-%d", time.Now().Nanosecond()),
   479  			},
   480  		}
   481  		for _, obj := range want {
   482  			w := veneerClient.Bucket(bucket).Object(obj.Name).NewWriter(context.Background())
   483  			if _, err := w.Write(randomBytesToWrite); err != nil {
   484  				t.Fatalf("failed to populate test data: %v", err)
   485  			}
   486  			if err := w.Close(); err != nil {
   487  				t.Fatalf("closing object: %v", err)
   488  			}
   489  		}
   490  
   491  		// Query with Prefix.
   492  		it := client.ListObjects(context.Background(), bucket, &Query{Prefix: strconv.Itoa(prefix)})
   493  		var o *ObjectAttrs
   494  		var got int
   495  		want = want[:2]
   496  		for i := 0; i <= len(want); i++ {
   497  			o, err = it.Next()
   498  			if err != nil {
   499  				break
   500  			}
   501  			got++
   502  			if diff := cmp.Diff(o.Name, want[i].Name); diff != "" {
   503  				t.Errorf("got(-),want(+):\n%s", diff)
   504  			}
   505  		}
   506  		if err != iterator.Done {
   507  			t.Fatalf("expected %q but got %q", iterator.Done, err)
   508  		}
   509  		expected := len(want)
   510  		if got != expected {
   511  			t.Errorf("expected to get %d objects, but got %d", expected, got)
   512  		}
   513  	})
   514  }
   515  
   516  func TestListBucketsEmulated(t *testing.T) {
   517  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   518  		prefix := time.Now().Nanosecond()
   519  		want := []*BucketAttrs{
   520  			{Name: fmt.Sprintf("%d-%s-%d", prefix, bucket, time.Now().Nanosecond())},
   521  			{Name: fmt.Sprintf("%d-%s-%d", prefix, bucket, time.Now().Nanosecond())},
   522  			{Name: fmt.Sprintf("%s-%d", bucket, time.Now().Nanosecond())},
   523  		}
   524  		// Create the buckets that will be listed.
   525  		for _, b := range want {
   526  			_, err := client.CreateBucket(context.Background(), project, b.Name, b, nil)
   527  			if err != nil {
   528  				t.Fatalf("client.CreateBucket: %v", err)
   529  			}
   530  		}
   531  
   532  		it := client.ListBuckets(context.Background(), project)
   533  		it.Prefix = strconv.Itoa(prefix)
   534  		// Drop the non-prefixed bucket from the expected results.
   535  		want = want[:2]
   536  		var err error
   537  		var b *BucketAttrs
   538  		var got int
   539  		for i := 0; err == nil && i <= len(want); i++ {
   540  			b, err = it.Next()
   541  			if err != nil {
   542  				break
   543  			}
   544  			got++
   545  			if diff := cmp.Diff(b.Name, want[i].Name); diff != "" {
   546  				t.Errorf("got(-),want(+):\n%s", diff)
   547  			}
   548  		}
   549  		if err != iterator.Done {
   550  			t.Fatalf("expected %q but got %q", iterator.Done, err)
   551  		}
   552  		expected := len(want)
   553  		if got != expected {
   554  			t.Errorf("expected to get %d buckets, but got %d", expected, got)
   555  		}
   556  	})
   557  }
   558  
   559  func TestListBucketACLsEmulated(t *testing.T) {
   560  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   561  		ctx := context.Background()
   562  		attrs := &BucketAttrs{
   563  			Name:          bucket,
   564  			PredefinedACL: "publicRead",
   565  		}
   566  		// Create the bucket that will be retrieved.
   567  		if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil {
   568  			t.Fatalf("client.CreateBucket: %v", err)
   569  		}
   570  
   571  		acls, err := client.ListBucketACLs(ctx, bucket)
   572  		if err != nil {
   573  			t.Fatalf("client.ListBucketACLs: %v", err)
   574  		}
   575  		if want, got := len(acls), 2; want != got {
   576  			t.Errorf("ListBucketACLs: got %v, want %v items", acls, want)
   577  		}
   578  	})
   579  }
   580  
   581  func TestUpdateBucketACLEmulated(t *testing.T) {
   582  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   583  		ctx := context.Background()
   584  		attrs := &BucketAttrs{
   585  			Name:          bucket,
   586  			PredefinedACL: "authenticatedRead",
   587  		}
   588  		// Create the bucket that will be retrieved.
   589  		if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil {
   590  			t.Fatalf("client.CreateBucket: %v", err)
   591  		}
   592  		var listAcls []ACLRule
   593  		var err error
   594  		// Assert bucket has two BucketACL entities, including project owner and predefinedACL.
   595  		if listAcls, err = client.ListBucketACLs(ctx, bucket); err != nil {
   596  			t.Fatalf("client.ListBucketACLs: %v", err)
   597  		}
   598  		if got, want := len(listAcls), 2; got != want {
   599  			t.Errorf("ListBucketACLs: got %v, want %v items", listAcls, want)
   600  		}
   601  		entity := AllUsers
   602  		role := RoleReader
   603  		err = client.UpdateBucketACL(ctx, bucket, entity, role)
   604  		if err != nil {
   605  			t.Fatalf("client.UpdateBucketACL: %v", err)
   606  		}
   607  		// Assert bucket now has three BucketACL entities, including existing ACLs.
   608  		if listAcls, err = client.ListBucketACLs(ctx, bucket); err != nil {
   609  			t.Fatalf("client.ListBucketACLs: %v", err)
   610  		}
   611  		if got, want := len(listAcls), 3; got != want {
   612  			t.Errorf("ListBucketACLs: got %v, want %v items", listAcls, want)
   613  		}
   614  	})
   615  }
   616  
   617  func TestDeleteBucketACLEmulated(t *testing.T) {
   618  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   619  		ctx := context.Background()
   620  		attrs := &BucketAttrs{
   621  			Name:          bucket,
   622  			PredefinedACL: "publicRead",
   623  		}
   624  		// Create the bucket that will be retrieved.
   625  		if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil {
   626  			t.Fatalf("client.CreateBucket: %v", err)
   627  		}
   628  		// Assert bucket has two BucketACL entities, including project owner and predefinedACL.
   629  		acls, err := client.ListBucketACLs(ctx, bucket)
   630  		if err != nil {
   631  			t.Fatalf("client.ListBucketACLs: %v", err)
   632  		}
   633  		if got, want := len(acls), 2; got != want {
   634  			t.Errorf("ListBucketACLs: got %v, want %v items", acls, want)
   635  		}
   636  		// Delete one BucketACL with AllUsers entity.
   637  		if err := client.DeleteBucketACL(ctx, bucket, AllUsers); err != nil {
   638  			t.Fatalf("client.DeleteBucketACL: %v", err)
   639  		}
   640  		// Assert bucket has one BucketACL.
   641  		acls, err = client.ListBucketACLs(ctx, bucket)
   642  		if err != nil {
   643  			t.Fatalf("client.ListBucketACLs: %v", err)
   644  		}
   645  		if got, want := len(acls), 1; got != want {
   646  			t.Errorf("ListBucketACLs: got %v, want %v items", acls, want)
   647  		}
   648  	})
   649  }
   650  
   651  func TestDefaultObjectACLCRUDEmulated(t *testing.T) {
   652  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   653  		ctx := context.Background()
   654  		attrs := &BucketAttrs{
   655  			Name:                       bucket,
   656  			PredefinedDefaultObjectACL: "publicRead",
   657  		}
   658  		// Create the bucket that will be retrieved.
   659  		if _, err := client.CreateBucket(ctx, project, attrs.Name, attrs, nil); err != nil {
   660  			t.Fatalf("client.CreateBucket: %v", err)
   661  		}
   662  		// Assert bucket has 2 DefaultObjectACL entities, including project owner and PredefinedDefaultObjectACL.
   663  		acls, err := client.ListDefaultObjectACLs(ctx, bucket)
   664  		if err != nil {
   665  			t.Fatalf("client.ListDefaultObjectACLs: %v", err)
   666  		}
   667  		if got, want := len(acls), 2; got != want {
   668  			t.Errorf("ListDefaultObjectACLs: got %v, want %v items", acls, want)
   669  		}
   670  		entity := AllAuthenticatedUsers
   671  		role := RoleOwner
   672  		err = client.UpdateDefaultObjectACL(ctx, bucket, entity, role)
   673  		if err != nil {
   674  			t.Fatalf("UpdateDefaultObjectCL: %v", err)
   675  		}
   676  		// Assert there are now 3 DefaultObjectACL entities, including existing DefaultObjectACLs.
   677  		acls, err = client.ListDefaultObjectACLs(ctx, bucket)
   678  		if err != nil {
   679  			t.Fatalf("client.ListDefaultObjectACLs: %v", err)
   680  		}
   681  		if got, want := len(acls), 3; got != want {
   682  			t.Errorf("ListDefaultObjectACLs: %v got %v, want %v items", len(acls), acls, want)
   683  		}
   684  		// Delete 1 DefaultObjectACL with AllUsers entity.
   685  		if err := client.DeleteDefaultObjectACL(ctx, bucket, AllUsers); err != nil {
   686  			t.Fatalf("client.DeleteDefaultObjectACL: %v", err)
   687  		}
   688  		// Assert bucket has 2 DefaultObjectACL entities.
   689  		acls, err = client.ListDefaultObjectACLs(ctx, bucket)
   690  		if err != nil {
   691  			t.Fatalf("client.ListDefaultObjectACLs: %v", err)
   692  		}
   693  		if got, want := len(acls), 2; got != want {
   694  			t.Errorf("ListDefaultObjectACLs: %v got %v, want %v items", len(acls), acls, want)
   695  		}
   696  	})
   697  }
   698  
   699  func TestObjectACLCRUDEmulated(t *testing.T) {
   700  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   701  		// Populate test object.
   702  		ctx := context.Background()
   703  		_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{
   704  			Name: bucket,
   705  		}, nil)
   706  		if err != nil {
   707  			t.Fatalf("CreateBucket: %v", err)
   708  		}
   709  		o := ObjectAttrs{
   710  			Bucket: bucket,
   711  			Name:   fmt.Sprintf("testObject-%d", time.Now().Nanosecond()),
   712  		}
   713  		w := veneerClient.Bucket(bucket).Object(o.Name).NewWriter(context.Background())
   714  		if _, err := w.Write(randomBytesToWrite); err != nil {
   715  			t.Fatalf("failed to populate test object: %v", err)
   716  		}
   717  		if err := w.Close(); err != nil {
   718  			t.Fatalf("closing object: %v", err)
   719  		}
   720  		var listAcls []ACLRule
   721  		// Assert there are 4 ObjectACL entities, including object owner and project owners/editors/viewers.
   722  		if listAcls, err = client.ListObjectACLs(ctx, bucket, o.Name); err != nil {
   723  			t.Fatalf("ListObjectACLs: %v", err)
   724  		}
   725  		if got, want := len(listAcls), 4; got != want {
   726  			t.Errorf("ListObjectACLs: got %v, want %v items", listAcls, want)
   727  		}
   728  		entity := AllUsers
   729  		role := RoleReader
   730  		err = client.UpdateObjectACL(ctx, bucket, o.Name, entity, role)
   731  		if err != nil {
   732  			t.Fatalf("UpdateObjectCL: %v", err)
   733  		}
   734  		// Assert there are now 5 ObjectACL entities, including existing ACLs.
   735  		if listAcls, err = client.ListObjectACLs(ctx, bucket, o.Name); err != nil {
   736  			t.Fatalf("ListObjectACLs: %v", err)
   737  		}
   738  		if got, want := len(listAcls), 5; got != want {
   739  			t.Errorf("ListObjectACLs: got %v, want %v items", listAcls, want)
   740  		}
   741  		if err = client.DeleteObjectACL(ctx, bucket, o.Name, AllUsers); err != nil {
   742  			t.Fatalf("client.DeleteObjectACL: %v", err)
   743  		}
   744  		// Assert there are now 4 ObjectACL entities after deletion.
   745  		if listAcls, err = client.ListObjectACLs(ctx, bucket, o.Name); err != nil {
   746  			t.Fatalf("ListObjectACLs: %v", err)
   747  		}
   748  		if got, want := len(listAcls), 4; got != want {
   749  			t.Errorf("ListObjectACLs: got %v, want %v items", listAcls, want)
   750  		}
   751  	})
   752  }
   753  
   754  func TestOpenReaderEmulated(t *testing.T) {
   755  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   756  		// Populate test data.
   757  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   758  			Name: bucket,
   759  		}, nil)
   760  		if err != nil {
   761  			t.Fatalf("client.CreateBucket: %v", err)
   762  		}
   763  		prefix := time.Now().Nanosecond()
   764  		want := &ObjectAttrs{
   765  			Bucket: bucket,
   766  			Name:   fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   767  		}
   768  		w := veneerClient.Bucket(bucket).Object(want.Name).NewWriter(context.Background())
   769  		if _, err := w.Write(randomBytesToWrite); err != nil {
   770  			t.Fatalf("failed to populate test data: %v", err)
   771  		}
   772  		if err := w.Close(); err != nil {
   773  			t.Fatalf("closing object: %v", err)
   774  		}
   775  
   776  		params := &newRangeReaderParams{
   777  			bucket: bucket,
   778  			object: want.Name,
   779  			gen:    defaultGen,
   780  			offset: 0,
   781  			length: -1,
   782  		}
   783  		r, err := client.NewRangeReader(context.Background(), params)
   784  		if err != nil {
   785  			t.Fatalf("opening reading: %v", err)
   786  		}
   787  		wantLen := len(randomBytesToWrite)
   788  		got := make([]byte, wantLen)
   789  		n, err := r.Read(got)
   790  		if n != wantLen {
   791  			t.Fatalf("expected to read %d bytes, but got %d", wantLen, n)
   792  		}
   793  		if diff := cmp.Diff(got, randomBytesToWrite); diff != "" {
   794  			t.Fatalf("Read: got(-),want(+):\n%s", diff)
   795  		}
   796  	})
   797  }
   798  
   799  func TestOpenWriterEmulated(t *testing.T) {
   800  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   801  		// Populate test data.
   802  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
   803  			Name: bucket,
   804  		}, nil)
   805  		if err != nil {
   806  			t.Fatalf("client.CreateBucket: %v", err)
   807  		}
   808  		prefix := time.Now().Nanosecond()
   809  		want := &ObjectAttrs{
   810  			Bucket:     bucket,
   811  			Name:       fmt.Sprintf("%d-object-%d", prefix, time.Now().Nanosecond()),
   812  			Generation: defaultGen,
   813  		}
   814  
   815  		var gotAttrs *ObjectAttrs
   816  		params := &openWriterParams{
   817  			attrs:    want,
   818  			bucket:   bucket,
   819  			ctx:      context.Background(),
   820  			donec:    make(chan struct{}),
   821  			setError: func(_ error) {}, // no-op
   822  			progress: func(_ int64) {}, // no-op
   823  			setObj:   func(o *ObjectAttrs) { gotAttrs = o },
   824  		}
   825  		pw, err := client.OpenWriter(params)
   826  		if err != nil {
   827  			t.Fatalf("failed to open writer: %v", err)
   828  		}
   829  		if _, err := pw.Write(randomBytesToWrite); err != nil {
   830  			t.Fatalf("failed to populate test data: %v", err)
   831  		}
   832  		if err := pw.Close(); err != nil {
   833  			t.Fatalf("closing object: %v", err)
   834  		}
   835  		select {
   836  		case <-params.donec:
   837  		}
   838  		if gotAttrs == nil {
   839  			t.Fatalf("Writer finished, but resulting object wasn't set")
   840  		}
   841  		if diff := cmp.Diff(gotAttrs.Name, want.Name); diff != "" {
   842  			t.Fatalf("Resulting object name: got(-),want(+):\n%s", diff)
   843  		}
   844  
   845  		r, err := veneerClient.Bucket(bucket).Object(want.Name).NewReader(context.Background())
   846  		if err != nil {
   847  			t.Fatalf("opening reading: %v", err)
   848  		}
   849  		wantLen := len(randomBytesToWrite)
   850  		got := make([]byte, wantLen)
   851  		n, err := r.Read(got)
   852  		if n != wantLen {
   853  			t.Fatalf("expected to read %d bytes, but got %d", wantLen, n)
   854  		}
   855  		if diff := cmp.Diff(got, randomBytesToWrite); diff != "" {
   856  			t.Fatalf("checking written content: got(-),want(+):\n%s", diff)
   857  		}
   858  	})
   859  }
   860  
   861  func TestListNotificationsEmulated(t *testing.T) {
   862  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   863  		// Populate test object.
   864  		ctx := context.Background()
   865  		_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{
   866  			Name: bucket,
   867  		}, nil)
   868  		if err != nil {
   869  			t.Fatalf("client.CreateBucket: %v", err)
   870  		}
   871  		_, err = client.CreateNotification(ctx, bucket, &Notification{
   872  			TopicProjectID: project,
   873  			TopicID:        "go-storage-notification-test",
   874  			PayloadFormat:  "JSON_API_V1",
   875  		})
   876  		if err != nil {
   877  			t.Fatalf("client.CreateNotification: %v", err)
   878  		}
   879  		n, err := client.ListNotifications(ctx, bucket)
   880  		if err != nil {
   881  			t.Fatalf("client.ListNotifications: %v", err)
   882  		}
   883  		if want, got := 1, len(n); want != got {
   884  			t.Errorf("ListNotifications: got %v, want %v items", n, want)
   885  		}
   886  	})
   887  }
   888  
   889  func TestCreateNotificationEmulated(t *testing.T) {
   890  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   891  		// Populate test object.
   892  		ctx := context.Background()
   893  		_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{
   894  			Name: bucket,
   895  		}, nil)
   896  		if err != nil {
   897  			t.Fatalf("client.CreateBucket: %v", err)
   898  		}
   899  
   900  		want := &Notification{
   901  			TopicProjectID: project,
   902  			TopicID:        "go-storage-notification-test",
   903  			PayloadFormat:  "JSON_API_V1",
   904  		}
   905  		got, err := client.CreateNotification(ctx, bucket, want)
   906  		if err != nil {
   907  			t.Fatalf("client.CreateNotification: %v", err)
   908  		}
   909  		if diff := cmp.Diff(got.TopicID, want.TopicID); diff != "" {
   910  			t.Errorf("CreateNotification topic: got(-),want(+):\n%s", diff)
   911  		}
   912  	})
   913  }
   914  
   915  func TestDeleteNotificationEmulated(t *testing.T) {
   916  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   917  		// Populate test object.
   918  		ctx := context.Background()
   919  		_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{
   920  			Name: bucket,
   921  		}, nil)
   922  		if err != nil {
   923  			t.Fatalf("client.CreateBucket: %v", err)
   924  		}
   925  		var n *Notification
   926  		n, err = client.CreateNotification(ctx, bucket, &Notification{
   927  			TopicProjectID: project,
   928  			TopicID:        "go-storage-notification-test",
   929  			PayloadFormat:  "JSON_API_V1",
   930  		})
   931  		if err != nil {
   932  			t.Fatalf("client.CreateNotification: %v", err)
   933  		}
   934  		err = client.DeleteNotification(ctx, bucket, n.ID)
   935  		if err != nil {
   936  			t.Fatalf("client.DeleteNotification: %v", err)
   937  		}
   938  	})
   939  }
   940  
   941  func initEmulatorClients() func() error {
   942  	noopCloser := func() error { return nil }
   943  	if !isEmulatorEnvironmentSet() {
   944  		return noopCloser
   945  	}
   946  	ctx := context.Background()
   947  
   948  	grpcClient, err := newGRPCStorageClient(ctx)
   949  	if err != nil {
   950  		log.Fatalf("Error setting up gRPC client for emulator tests: %v", err)
   951  		return noopCloser
   952  	}
   953  	httpClient, err := newHTTPStorageClient(ctx)
   954  	if err != nil {
   955  		log.Fatalf("Error setting up HTTP client for emulator tests: %v", err)
   956  		return noopCloser
   957  	}
   958  
   959  	emulatorClients = map[string]storageClient{
   960  		"http": httpClient,
   961  		"grpc": grpcClient,
   962  	}
   963  
   964  	veneerClient, err = NewClient(ctx)
   965  	if err != nil {
   966  		log.Fatalf("Error setting up Veneer client for emulator tests: %v", err)
   967  		return noopCloser
   968  	}
   969  
   970  	return func() error {
   971  		gerr := grpcClient.Close()
   972  		herr := httpClient.Close()
   973  		verr := veneerClient.Close()
   974  
   975  		if gerr != nil {
   976  			return gerr
   977  		} else if herr != nil {
   978  			return herr
   979  		}
   980  		return verr
   981  	}
   982  }
   983  
   984  func TestLockBucketRetentionPolicyEmulated(t *testing.T) {
   985  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
   986  		b := &BucketAttrs{
   987  			Name: bucket,
   988  			RetentionPolicy: &RetentionPolicy{
   989  				RetentionPeriod: time.Minute,
   990  			},
   991  		}
   992  		// Create the bucket that will be locked.
   993  		_, err := client.CreateBucket(context.Background(), project, b.Name, b, nil)
   994  		if err != nil {
   995  			t.Fatalf("client.CreateBucket: %v", err)
   996  		}
   997  		// Lock the bucket's retention policy.
   998  		err = client.LockBucketRetentionPolicy(context.Background(), b.Name, &BucketConditions{MetagenerationMatch: 1})
   999  		if err != nil {
  1000  			t.Fatalf("client.LockBucketRetentionPolicy: %v", err)
  1001  		}
  1002  		got, err := client.GetBucket(context.Background(), bucket, nil)
  1003  		if err != nil {
  1004  			t.Fatalf("client.GetBucket: %v", err)
  1005  		}
  1006  		if !got.RetentionPolicy.IsLocked {
  1007  			t.Error("Expected bucket retention policy to be locked, but was not.")
  1008  		}
  1009  	})
  1010  }
  1011  
  1012  func TestComposeEmulated(t *testing.T) {
  1013  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1014  		ctx := context.Background()
  1015  
  1016  		// Populate test data.
  1017  		_, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{
  1018  			Name: bucket,
  1019  		}, nil)
  1020  		if err != nil {
  1021  			t.Fatalf("client.CreateBucket: %v", err)
  1022  		}
  1023  		prefix := time.Now().Nanosecond()
  1024  		srcNames := []string{
  1025  			fmt.Sprintf("%d-object1", prefix),
  1026  			fmt.Sprintf("%d-object2", prefix),
  1027  		}
  1028  
  1029  		for _, n := range srcNames {
  1030  			w := veneerClient.Bucket(bucket).Object(n).NewWriter(ctx)
  1031  			if _, err := w.Write(randomBytesToWrite); err != nil {
  1032  				t.Fatalf("failed to populate test data: %v", err)
  1033  			}
  1034  			if err := w.Close(); err != nil {
  1035  				t.Fatalf("closing object: %v", err)
  1036  			}
  1037  		}
  1038  
  1039  		dstName := fmt.Sprintf("%d-object3", prefix)
  1040  		req := composeObjectRequest{
  1041  			dstBucket: bucket,
  1042  			dstObject: destinationObject{
  1043  				name:  dstName,
  1044  				attrs: &ObjectAttrs{StorageClass: "COLDLINE"},
  1045  			},
  1046  			srcs: []sourceObject{
  1047  				{name: srcNames[0]},
  1048  				{name: srcNames[1]},
  1049  			},
  1050  		}
  1051  		attrs, err := client.ComposeObject(ctx, &req)
  1052  		if err != nil {
  1053  			t.Fatalf("client.ComposeObject(): %v", err)
  1054  		}
  1055  		if got := attrs.Name; got != dstName {
  1056  			t.Errorf("attrs.Name: got %v, want %v", got, dstName)
  1057  		}
  1058  		// Check that the destination object size is equal to the sum of its
  1059  		// sources.
  1060  		if got, want := attrs.Size, 2*len(randomBytesToWrite); got != int64(want) {
  1061  			t.Errorf("attrs.Size: got %v, want %v", got, want)
  1062  		}
  1063  		// Check that destination attrs set via object attrs are preserved.
  1064  		if got, want := attrs.StorageClass, "COLDLINE"; got != want {
  1065  			t.Errorf("attrs.StorageClass: got %v, want %v", got, want)
  1066  		}
  1067  	})
  1068  }
  1069  
  1070  func TestHMACKeyCRUDEmulated(t *testing.T) {
  1071  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1072  		ctx := context.Background()
  1073  		serviceAccountEmail := "test@test-project.iam.gserviceaccount.com"
  1074  		want, err := client.CreateHMACKey(ctx, project, serviceAccountEmail)
  1075  		if err != nil {
  1076  			t.Fatalf("CreateHMACKey: %v", err)
  1077  		}
  1078  		if want == nil {
  1079  			t.Fatal("CreateHMACKey: Unexpectedly got back a nil HMAC key")
  1080  		}
  1081  		if want.State != Active {
  1082  			t.Fatalf("CreateHMACKey: Unexpected state %q, expected %q", want.State, Active)
  1083  		}
  1084  		got, err := client.GetHMACKey(ctx, project, want.AccessID)
  1085  		if err != nil {
  1086  			t.Fatalf("GetHMACKey: %v", err)
  1087  		}
  1088  		if diff := cmp.Diff(got.ID, want.ID); diff != "" {
  1089  			t.Errorf("GetHMACKey ID:got(-),want(+):\n%s", diff)
  1090  		}
  1091  		if diff := cmp.Diff(got.UpdatedTime, want.UpdatedTime); diff != "" {
  1092  			t.Errorf("GetHMACKey UpdatedTime: got(-),want(+):\n%s", diff)
  1093  		}
  1094  		attr := &HMACKeyAttrsToUpdate{
  1095  			State: Inactive,
  1096  		}
  1097  		got, err = client.UpdateHMACKey(ctx, project, serviceAccountEmail, want.AccessID, attr)
  1098  		if err != nil {
  1099  			t.Fatalf("UpdateHMACKey: %v", err)
  1100  		}
  1101  		if got.State != attr.State {
  1102  			t.Errorf("UpdateHMACKey State: got %v, want %v", got.State, attr.State)
  1103  		}
  1104  		showDeletedKeys := false
  1105  		it := client.ListHMACKeys(ctx, project, serviceAccountEmail, showDeletedKeys)
  1106  		var count int
  1107  		var e error
  1108  		for ; ; count++ {
  1109  			_, e = it.Next()
  1110  			if e != nil {
  1111  				break
  1112  			}
  1113  		}
  1114  		if e != iterator.Done {
  1115  			t.Fatalf("ListHMACKeys: expected %q but got %q", iterator.Done, err)
  1116  		}
  1117  		if expected := 1; count != expected {
  1118  			t.Errorf("ListHMACKeys: expected to get %d hmacKeys, but got %d", expected, count)
  1119  		}
  1120  		err = client.DeleteHMACKey(ctx, project, want.AccessID)
  1121  		if err != nil {
  1122  			t.Fatalf("DeleteHMACKey: %v", err)
  1123  		}
  1124  		got, err = client.GetHMACKey(ctx, project, want.AccessID)
  1125  		if err == nil {
  1126  			t.Fatalf("GetHMACKey unexcepted error: wanted 404")
  1127  		}
  1128  	})
  1129  }
  1130  
  1131  func TestBucketConditionsEmulated(t *testing.T) {
  1132  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1133  		ctx := context.Background()
  1134  		cases := []struct {
  1135  			name string
  1136  			call func(bucket string, metaGen int64) error
  1137  		}{
  1138  			{
  1139  				name: "get",
  1140  				call: func(bucket string, metaGen int64) error {
  1141  					_, err := client.GetBucket(ctx, bucket, &BucketConditions{MetagenerationMatch: metaGen})
  1142  					return err
  1143  				},
  1144  			},
  1145  			{
  1146  				name: "update",
  1147  				call: func(bucket string, metaGen int64) error {
  1148  					_, err := client.UpdateBucket(ctx, bucket, &BucketAttrsToUpdate{StorageClass: "ARCHIVE"}, &BucketConditions{MetagenerationMatch: metaGen})
  1149  					return err
  1150  				},
  1151  			},
  1152  			{
  1153  				name: "delete",
  1154  				call: func(bucket string, metaGen int64) error {
  1155  					return client.DeleteBucket(ctx, bucket, &BucketConditions{MetagenerationMatch: metaGen})
  1156  				},
  1157  			},
  1158  			{
  1159  				name: "lockRetentionPolicy",
  1160  				call: func(bucket string, metaGen int64) error {
  1161  					return client.LockBucketRetentionPolicy(ctx, bucket, &BucketConditions{MetagenerationMatch: metaGen})
  1162  				},
  1163  			},
  1164  		}
  1165  		for _, c := range cases {
  1166  			t.Run(c.name, func(r *testing.T) {
  1167  				bucket, metaGen, err := createBucket(ctx, project)
  1168  				if err != nil {
  1169  					r.Fatalf("creating bucket: %v", err)
  1170  				}
  1171  				if err := c.call(bucket, metaGen); err != nil {
  1172  					r.Errorf("error: %v", err)
  1173  				}
  1174  			})
  1175  		}
  1176  	})
  1177  }
  1178  
  1179  func TestObjectConditionsEmulated(t *testing.T) {
  1180  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1181  		ctx := context.Background()
  1182  
  1183  		// Create test bucket
  1184  		if _, err := client.CreateBucket(context.Background(), project, bucket, &BucketAttrs{Name: bucket}, nil); err != nil {
  1185  			t.Fatalf("client.CreateBucket: %v", err)
  1186  		}
  1187  
  1188  		cases := []struct {
  1189  			name string
  1190  			call func() error
  1191  		}{
  1192  			{
  1193  				name: "update generation",
  1194  				call: func() error {
  1195  					objName, gen, _, err := createObject(ctx, bucket)
  1196  					if err != nil {
  1197  						return fmt.Errorf("creating object: %w", err)
  1198  					}
  1199  					uattrs := &ObjectAttrsToUpdate{CustomTime: time.Now()}
  1200  					_, err = client.UpdateObject(ctx, &updateObjectParams{bucket: bucket, object: objName, uattrs: uattrs, gen: gen})
  1201  					return err
  1202  				},
  1203  			},
  1204  			{
  1205  				name: "update ifMetagenerationMatch",
  1206  				call: func() error {
  1207  					objName, gen, metaGen, err := createObject(ctx, bucket)
  1208  					if err != nil {
  1209  						return fmt.Errorf("creating object: %w", err)
  1210  					}
  1211  					uattrs := &ObjectAttrsToUpdate{CustomTime: time.Now()}
  1212  					conds := &Conditions{
  1213  						GenerationMatch:     gen,
  1214  						MetagenerationMatch: metaGen,
  1215  					}
  1216  					_, err = client.UpdateObject(ctx, &updateObjectParams{bucket: bucket, object: objName, uattrs: uattrs, gen: gen, conds: conds})
  1217  					return err
  1218  				},
  1219  			},
  1220  			{
  1221  				name: "write ifGenerationMatch",
  1222  				call: func() error {
  1223  					var err error
  1224  					_, err = client.OpenWriter(&openWriterParams{
  1225  						ctx:                ctx,
  1226  						chunkSize:          256 * 1024,
  1227  						chunkRetryDeadline: 0,
  1228  						bucket:             bucket,
  1229  						attrs:              &ObjectAttrs{},
  1230  						conds:              &Conditions{DoesNotExist: true},
  1231  						encryptionKey:      nil,
  1232  						sendCRC32C:         false,
  1233  						donec:              nil,
  1234  						setError: func(e error) {
  1235  							if e != nil {
  1236  								err = e
  1237  							}
  1238  						},
  1239  						progress: nil,
  1240  						setObj:   nil,
  1241  					})
  1242  					return err
  1243  				},
  1244  			},
  1245  			{
  1246  				name: "rewrite ifMetagenerationMatch",
  1247  				call: func() error {
  1248  					objName, gen, metaGen, err := createObject(ctx, bucket)
  1249  					if err != nil {
  1250  						return fmt.Errorf("creating object: %w", err)
  1251  					}
  1252  					_, err = client.RewriteObject(ctx, &rewriteObjectRequest{
  1253  						srcObject: sourceObject{
  1254  							name:   objName,
  1255  							bucket: bucket,
  1256  							gen:    gen,
  1257  							conds: &Conditions{
  1258  								GenerationMatch:     gen,
  1259  								MetagenerationMatch: metaGen,
  1260  							},
  1261  						},
  1262  						dstObject: destinationObject{
  1263  							name:   fmt.Sprintf("%d-object", time.Now().Nanosecond()),
  1264  							bucket: bucket,
  1265  							conds: &Conditions{
  1266  								DoesNotExist: true,
  1267  							},
  1268  							attrs: &ObjectAttrs{},
  1269  						},
  1270  					})
  1271  					return err
  1272  				},
  1273  			},
  1274  			{
  1275  				name: "compose ifGenerationMatch",
  1276  				call: func() error {
  1277  					obj1, obj1Gen, _, err := createObject(ctx, bucket)
  1278  					if err != nil {
  1279  						return fmt.Errorf("creating object: %w", err)
  1280  					}
  1281  					obj2, obj2Gen, _, err := createObject(ctx, bucket)
  1282  					if err != nil {
  1283  						return fmt.Errorf("creating object: %w", err)
  1284  					}
  1285  					_, err = client.ComposeObject(ctx, &composeObjectRequest{
  1286  						dstBucket: bucket,
  1287  						dstObject: destinationObject{
  1288  							name:   fmt.Sprintf("%d-object", time.Now().Nanosecond()),
  1289  							bucket: bucket,
  1290  							conds:  &Conditions{DoesNotExist: true},
  1291  							attrs:  &ObjectAttrs{},
  1292  						},
  1293  						srcs: []sourceObject{
  1294  							{
  1295  								name:   obj1,
  1296  								bucket: bucket,
  1297  								gen:    obj1Gen,
  1298  								conds: &Conditions{
  1299  									GenerationMatch: obj1Gen,
  1300  								},
  1301  							},
  1302  							{
  1303  								name:   obj2,
  1304  								bucket: bucket,
  1305  								conds: &Conditions{
  1306  									GenerationMatch: obj2Gen,
  1307  								},
  1308  							},
  1309  						},
  1310  					})
  1311  					return err
  1312  				},
  1313  			},
  1314  			{
  1315  				name: "delete ifGenerationMatch",
  1316  				call: func() error {
  1317  					objName, gen, _, err := createObject(ctx, bucket)
  1318  					if err != nil {
  1319  						return fmt.Errorf("creating object: %w", err)
  1320  					}
  1321  					err = client.DeleteObject(ctx, bucket, objName, gen, &Conditions{GenerationMatch: gen})
  1322  					return err
  1323  				},
  1324  			},
  1325  			{
  1326  				name: "get ifMetagenerationMatch",
  1327  				call: func() error {
  1328  					objName, gen, metaGen, err := createObject(ctx, bucket)
  1329  					if err != nil {
  1330  						return fmt.Errorf("creating object: %w", err)
  1331  					}
  1332  					_, err = client.GetObject(ctx, &getObjectParams{bucket: bucket, object: objName, gen: gen, conds: &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen}})
  1333  					return err
  1334  				},
  1335  			},
  1336  		}
  1337  		for _, c := range cases {
  1338  			t.Run(c.name, func(r *testing.T) {
  1339  				if err := c.call(); err != nil {
  1340  					r.Errorf("error: %v", err)
  1341  				}
  1342  			})
  1343  		}
  1344  	})
  1345  }
  1346  
  1347  // Test that RetryNever prevents any retries from happening in both transports.
  1348  func TestRetryNeverEmulated(t *testing.T) {
  1349  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1350  		ctx := context.Background()
  1351  		instructions := map[string][]string{"storage.buckets.get": {"return-503"}}
  1352  		testID := createRetryTest(t, project, bucket, client, instructions)
  1353  		ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
  1354  		_, err := client.GetBucket(ctx, bucket, nil, withRetryConfig(&retryConfig{policy: RetryNever}))
  1355  
  1356  		var ae *apierror.APIError
  1357  		if errors.As(err, &ae) {
  1358  			// We expect a 503/UNAVAILABLE error. For anything else including a nil
  1359  			// error, the test should fail.
  1360  			if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
  1361  				t.Errorf("GetBucket: got unexpected error %v; want 503", err)
  1362  			}
  1363  		}
  1364  	})
  1365  }
  1366  
  1367  // Test that errors are wrapped correctly if retry happens until a timeout.
  1368  func TestRetryTimeoutEmulated(t *testing.T) {
  1369  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1370  		ctx := context.Background()
  1371  		instructions := map[string][]string{"storage.buckets.get": {"return-503", "return-503", "return-503", "return-503", "return-503"}}
  1372  		testID := createRetryTest(t, project, bucket, client, instructions)
  1373  		ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
  1374  		ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
  1375  		defer cancel()
  1376  		_, err := client.GetBucket(ctx, bucket, nil, idempotent(true))
  1377  
  1378  		var ae *apierror.APIError
  1379  		if errors.As(err, &ae) {
  1380  			// We expect a 503/UNAVAILABLE error. For anything else including a nil
  1381  			// error, the test should fail.
  1382  			if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
  1383  				t.Errorf("GetBucket: got unexpected error: %v; want 503", err)
  1384  			}
  1385  		}
  1386  		// Error should be wrapped so it's also equivalent to a context timeout.
  1387  		if !errors.Is(err, context.DeadlineExceeded) {
  1388  			t.Errorf("GetBucket: got unexpected error %v, want to match DeadlineExceeded.", err)
  1389  		}
  1390  	})
  1391  }
  1392  
  1393  // Test that errors are wrapped correctly if retry happens until max attempts.
  1394  func TestRetryMaxAttemptsEmulated(t *testing.T) {
  1395  	transportClientTest(t, func(t *testing.T, project, bucket string, client storageClient) {
  1396  		ctx := context.Background()
  1397  		instructions := map[string][]string{"storage.buckets.get": {"return-503", "return-503", "return-503", "return-503", "return-503"}}
  1398  		testID := createRetryTest(t, project, bucket, client, instructions)
  1399  		ctx = callctx.SetHeaders(ctx, "x-retry-test-id", testID)
  1400  		config := &retryConfig{maxAttempts: expectedAttempts(3), backoff: &gax.Backoff{Initial: 10 * time.Millisecond}}
  1401  		_, err := client.GetBucket(ctx, bucket, nil, idempotent(true), withRetryConfig(config))
  1402  
  1403  		var ae *apierror.APIError
  1404  		if errors.As(err, &ae) {
  1405  			// We expect a 503/UNAVAILABLE error. For anything else including a nil
  1406  			// error, the test should fail.
  1407  			if ae.GRPCStatus().Code() != codes.Unavailable && ae.HTTPCode() != 503 {
  1408  				t.Errorf("GetBucket: got unexpected error %v; want 503", err)
  1409  			}
  1410  		}
  1411  		// Error should be wrapped so it indicates that MaxAttempts has been reached.
  1412  		if got, want := err.Error(), "retry failed after 3 attempts"; !strings.Contains(got, want) {
  1413  			t.Errorf("got error: %q, want to contain: %q", got, want)
  1414  		}
  1415  	})
  1416  }
  1417  
  1418  // createRetryTest creates a bucket in the emulator and sets up a test using the
  1419  // Retry Test API for the given instructions. This is intended for emulator tests
  1420  // of retry behavior that are not covered by conformance tests.
  1421  func createRetryTest(t *testing.T, project, bucket string, client storageClient, instructions map[string][]string) string {
  1422  	t.Helper()
  1423  	ctx := context.Background()
  1424  
  1425  	_, err := client.CreateBucket(ctx, project, bucket, &BucketAttrs{}, nil)
  1426  	if err != nil {
  1427  		t.Fatalf("creating bucket: %v", err)
  1428  	}
  1429  
  1430  	// Need the HTTP hostname to set up a retry test, as well as knowledge of
  1431  	// underlying transport to specify instructions.
  1432  	host := os.Getenv("STORAGE_EMULATOR_HOST")
  1433  	endpoint, err := url.Parse(host)
  1434  	if err != nil {
  1435  		t.Fatalf("parsing endpoint: %v", err)
  1436  	}
  1437  	var transport string
  1438  	if _, ok := client.(*httpStorageClient); ok {
  1439  		transport = "http"
  1440  	} else {
  1441  		transport = "grpc"
  1442  	}
  1443  
  1444  	et := emulatorTest{T: t, name: t.Name(), resources: resources{}, host: endpoint}
  1445  	et.create(instructions, transport)
  1446  	t.Cleanup(func() {
  1447  		et.delete()
  1448  	})
  1449  	return et.id
  1450  }
  1451  
  1452  // createObject creates an object in the emulator and returns its name, generation, and
  1453  // metageneration.
  1454  func createObject(ctx context.Context, bucket string) (string, int64, int64, error) {
  1455  	prefix := time.Now().Nanosecond()
  1456  	objName := fmt.Sprintf("%d-object", prefix)
  1457  
  1458  	w := veneerClient.Bucket(bucket).Object(objName).NewWriter(ctx)
  1459  	if _, err := w.Write(randomBytesToWrite); err != nil {
  1460  		return "", 0, 0, fmt.Errorf("failed to populate test data: %w", err)
  1461  	}
  1462  	if err := w.Close(); err != nil {
  1463  		return "", 0, 0, fmt.Errorf("closing object: %w", err)
  1464  	}
  1465  	attrs, err := veneerClient.Bucket(bucket).Object(objName).Attrs(ctx)
  1466  	if err != nil {
  1467  		return "", 0, 0, fmt.Errorf("get object: %w", err)
  1468  	}
  1469  	return objName, attrs.Generation, attrs.Metageneration, nil
  1470  }
  1471  
  1472  // createBucket creates a new bucket in the emulator and returns its name and
  1473  // metageneration.
  1474  func createBucket(ctx context.Context, projectID string) (string, int64, error) {
  1475  	prefix := time.Now().Nanosecond()
  1476  	bucket := fmt.Sprintf("%d-bucket", prefix)
  1477  
  1478  	if err := veneerClient.Bucket(bucket).Create(ctx, projectID, nil); err != nil {
  1479  		return "", 0, fmt.Errorf("Bucket.Create: %w", err)
  1480  	}
  1481  	attrs, err := veneerClient.Bucket(bucket).Attrs(ctx)
  1482  	if err != nil {
  1483  		return "", 0, fmt.Errorf("Bucket.Attrs: %w", err)
  1484  	}
  1485  	return bucket, attrs.MetaGeneration, nil
  1486  }
  1487  
  1488  // transportClienttest executes the given function with a sub-test, a project name
  1489  // based on the transport, a unique bucket name also based on the transport, and
  1490  // the transport-specific client to run the test with. It also checks the environment
  1491  // to ensure it is suitable for emulator-based tests, or skips.
  1492  func transportClientTest(t *testing.T, test func(*testing.T, string, string, storageClient)) {
  1493  	checkEmulatorEnvironment(t)
  1494  
  1495  	for transport, client := range emulatorClients {
  1496  		t.Run(transport, func(t *testing.T) {
  1497  			project := fmt.Sprintf("%s-project", transport)
  1498  			bucket := fmt.Sprintf("%s-bucket-%d", transport, time.Now().Nanosecond())
  1499  			test(t, project, bucket, client)
  1500  		})
  1501  	}
  1502  }
  1503  
  1504  // checkEmulatorEnvironment skips the test if the emulator environment variables
  1505  // are not set.
  1506  func checkEmulatorEnvironment(t *testing.T) {
  1507  	if !isEmulatorEnvironmentSet() {
  1508  		t.Skip("Emulator tests skipped without emulator environment variables set")
  1509  	}
  1510  }
  1511  
  1512  // isEmulatorEnvironmentSet checks if the emulator environment variables are set.
  1513  func isEmulatorEnvironmentSet() bool {
  1514  	return os.Getenv("STORAGE_EMULATOR_HOST_GRPC") != "" && os.Getenv("STORAGE_EMULATOR_HOST") != ""
  1515  }
  1516  

View as plain text