1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package storage
16
17 import (
18 "bytes"
19 "compress/gzip"
20 "context"
21 "crypto"
22 "crypto/md5"
23 cryptorand "crypto/rand"
24 "crypto/rsa"
25 "crypto/sha256"
26 "encoding/base64"
27 "encoding/json"
28 "errors"
29 "flag"
30 "fmt"
31 "hash/crc32"
32 "io"
33 "log"
34 "math"
35 "math/rand"
36 "mime/multipart"
37 "net/http"
38 "net/http/httputil"
39 "os"
40 "sort"
41 "strconv"
42 "strings"
43 "testing"
44 "time"
45
46 "cloud.google.com/go/httpreplay"
47 "cloud.google.com/go/iam"
48 "cloud.google.com/go/iam/apiv1/iampb"
49 "cloud.google.com/go/internal/testutil"
50 "cloud.google.com/go/internal/uid"
51 "github.com/google/go-cmp/cmp"
52 "github.com/google/go-cmp/cmp/cmpopts"
53 "github.com/googleapis/gax-go/v2/apierror"
54 "golang.org/x/oauth2/google"
55 "google.golang.org/api/googleapi"
56 "google.golang.org/api/iterator"
57 itesting "google.golang.org/api/iterator/testing"
58 "google.golang.org/api/option"
59 raw "google.golang.org/api/storage/v1"
60 "google.golang.org/api/transport"
61 "google.golang.org/grpc"
62 "google.golang.org/grpc/codes"
63 "google.golang.org/grpc/status"
64 )
65
66 type skipTransportTestKey string
67
68 const (
69 testPrefix = "go-integration-test"
70 replayFilename = "storage.replay"
71
72 envFirestoreProjID = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID"
73 envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY"
74 grpcTestPrefix = "golang-grpc-test"
75 )
76
77 var (
78 record = flag.Bool("record", false, "record RPCs")
79
80 uidSpace *uid.Space
81 uidSpaceObjects *uid.Space
82 bucketName string
83 grpcBucketName string
84
85
86
87 rng *rand.Rand
88 newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error)
89 replaying bool
90 testTime time.Time
91 )
92
93 func TestMain(m *testing.M) {
94 grpc.EnableTracing = true
95 cleanup := initIntegrationTest()
96 cleanupEmulatorClients := initEmulatorClients()
97 exit := m.Run()
98 if err := cleanup(); err != nil {
99
100 log.Printf("Post-test cleanup failed: %v", err)
101 }
102 if err := cleanupEmulatorClients(); err != nil {
103
104 log.Printf("Post-test cleanup failed for emulator clients: %v", err)
105 }
106
107 os.Exit(exit)
108 }
109
110
111
112
113 func initIntegrationTest() func() error {
114 flag.Parse()
115 switch {
116 case testing.Short() && *record:
117 log.Fatal("cannot combine -short and -record")
118 return nil
119
120 case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "":
121
122
123 replaying = true
124 httpreplay.DebugHeaders()
125 replayer, err := httpreplay.NewReplayer(replayFilename)
126 if err != nil {
127 log.Fatal(err)
128 }
129 var t time.Time
130 if err := json.Unmarshal(replayer.Initial(), &t); err != nil {
131 log.Fatal(err)
132 }
133 initUIDsAndRand(t)
134 newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) {
135 hc, err := replayer.Client(ctx)
136 if err != nil {
137 return nil, err
138 }
139 return NewClient(ctx, option.WithHTTPClient(hc))
140 }
141 log.Printf("replaying from %s", replayFilename)
142 return func() error { return replayer.Close() }
143
144 case testing.Short():
145
146 if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" {
147 log.Print("replay not supported for Go versions before 1.8")
148 }
149 newTestClient = nil
150 return func() error { return nil }
151
152 default:
153 now := time.Now().UTC()
154 initUIDsAndRand(now)
155 var cleanup func() error
156 if *record && httpreplay.Supported() {
157
158 nowBytes, err := json.Marshal(now)
159 if err != nil {
160 log.Fatal(err)
161 }
162 recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes)
163 if err != nil {
164 log.Fatalf("could not record: %v", err)
165 }
166 newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
167 hc, err := recorder.Client(ctx, opts...)
168 if err != nil {
169 return nil, err
170 }
171 return NewClient(ctx, option.WithHTTPClient(hc))
172 }
173 cleanup = func() error {
174 err1 := cleanupBuckets()
175 err2 := recorder.Close()
176 if err1 != nil {
177 return err1
178 }
179 return err2
180 }
181 log.Printf("recording to %s", replayFilename)
182 } else {
183 if *record {
184 log.Print("record not supported for Go versions before 1.8")
185 }
186 newTestClient = NewClient
187 cleanup = cleanupBuckets
188 }
189 ctx := context.Background()
190 client, err := newTestClient(ctx)
191 if err != nil {
192 log.Fatalf("NewClient: %v", err)
193 }
194 if client == nil {
195 return func() error { return nil }
196 }
197 defer client.Close()
198 if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil {
199 log.Fatalf("creating bucket %q: %v", bucketName, err)
200 }
201 if err := client.Bucket(grpcBucketName).Create(ctx, testutil.ProjID(), nil); err != nil {
202 log.Fatalf("creating bucket %q: %v", grpcBucketName, err)
203 }
204 return cleanup
205 }
206 }
207
208 func initUIDsAndRand(t time.Time) {
209 uidSpace = uid.NewSpace("", &uid.Options{Time: t, Short: true})
210 bucketName = testPrefix + uidSpace.New()
211 uidSpaceObjects = uid.NewSpace("obj", &uid.Options{Time: t})
212 grpcBucketName = grpcTestPrefix + uidSpace.New()
213
214
215
216 rng = testutil.NewRand(t)
217 testTime = t
218 }
219
220
221
222
223 func testConfig(ctx context.Context, t *testing.T, opts ...option.ClientOption) *Client {
224 if testing.Short() && !replaying {
225 t.Skip("Integration tests skipped in short mode")
226 }
227 client, err := newTestClient(ctx, opts...)
228 if err != nil {
229 t.Fatalf("NewClient: %v", err)
230 }
231 if client == nil {
232 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
233 }
234 return client
235 }
236
237
238
239 func testConfigGRPC(ctx context.Context, t *testing.T, opts ...option.ClientOption) (gc *Client) {
240 if testing.Short() {
241 t.Skip("Integration tests skipped in short mode")
242 }
243
244 gc, err := NewGRPCClient(ctx, opts...)
245 if err != nil {
246 t.Fatalf("NewGRPCClient: %v", err)
247 }
248
249 return
250 }
251
252
253 func initTransportClients(ctx context.Context, t *testing.T, opts ...option.ClientOption) map[string]*Client {
254 withJSON := append(opts, WithJSONReads())
255 return map[string]*Client{
256 "http": testConfig(ctx, t, opts...),
257 "grpc": testConfigGRPC(ctx, t, opts...),
258
259 "jsonReads": testConfig(ctx, t, withJSON...),
260 }
261 }
262
263
264
265
266
267
268 func multiTransportTest(ctx context.Context, t *testing.T,
269 test func(*testing.T, context.Context, string, string, *Client),
270 opts ...option.ClientOption) {
271 for transport, client := range initTransportClients(ctx, t, opts...) {
272 t.Run(transport, func(t *testing.T) {
273 t.Cleanup(func() {
274 client.Close()
275 })
276
277 if reason := ctx.Value(skipTransportTestKey(transport)); reason != nil {
278 t.Skip("transport", fmt.Sprintf("%q", transport), "explicitly skipped:", reason)
279 }
280
281 bucket := bucketName
282 prefix := testPrefix
283 if transport == "grpc" {
284 bucket = grpcBucketName
285 prefix = grpcTestPrefix
286 }
287
288 test(t, ctx, bucket, prefix, client)
289 })
290 }
291 }
292
293
294 type readCase struct {
295 desc string
296 readFunc (func(io.Reader) ([]byte, error))
297 }
298
299 var readCases = []readCase{
300 {
301 desc: "Read",
302 readFunc: io.ReadAll,
303 },
304 {
305 desc: "WriteTo",
306 readFunc: func(r io.Reader) ([]byte, error) {
307 b := new(bytes.Buffer)
308 _, err := io.Copy(b, r)
309 return b.Bytes(), err
310 },
311 },
312 }
313
314 func TestIntegration_BucketCreateDelete(t *testing.T) {
315 ctx := skipJSONReads(context.Background(), "no reads in test")
316 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
317 projectID := testutil.ProjID()
318
319 labels := map[string]string{
320 "l1": "v1",
321 "empty": "",
322 }
323
324 lifecycle := Lifecycle{
325 Rules: []LifecycleRule{{
326 Action: LifecycleAction{
327 Type: SetStorageClassAction,
328 StorageClass: "NEARLINE",
329 },
330 Condition: LifecycleCondition{
331 AgeInDays: 10,
332 Liveness: Archived,
333 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
334 MatchesStorageClasses: []string{"STANDARD"},
335 NumNewerVersions: 3,
336 },
337 }, {
338 Action: LifecycleAction{
339 Type: SetStorageClassAction,
340 StorageClass: "ARCHIVE",
341 },
342 Condition: LifecycleCondition{
343 CustomTimeBefore: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
344 DaysSinceCustomTime: 20,
345 Liveness: Live,
346 MatchesStorageClasses: []string{"STANDARD"},
347 },
348 }, {
349 Action: LifecycleAction{
350 Type: DeleteAction,
351 },
352 Condition: LifecycleCondition{
353 DaysSinceNoncurrentTime: 30,
354 Liveness: Live,
355 NoncurrentTimeBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
356 MatchesStorageClasses: []string{"NEARLINE"},
357 NumNewerVersions: 10,
358 },
359 }, {
360 Action: LifecycleAction{
361 Type: DeleteAction,
362 },
363 Condition: LifecycleCondition{
364 AgeInDays: 10,
365 MatchesPrefix: []string{"testPrefix"},
366 MatchesSuffix: []string{"testSuffix"},
367 NumNewerVersions: 3,
368 },
369 }, {
370 Action: LifecycleAction{
371 Type: DeleteAction,
372 },
373 Condition: LifecycleCondition{
374 AllObjects: true,
375 },
376 }},
377 }
378
379
380 type testedAttrs struct {
381 StorageClass string
382 VersioningEnabled bool
383 LocationType string
384 Labels map[string]string
385 Location string
386 Lifecycle Lifecycle
387 CustomPlacementConfig *CustomPlacementConfig
388 }
389
390 for _, test := range []struct {
391 name string
392 attrs *BucketAttrs
393 wantAttrs testedAttrs
394 }{
395 {
396 name: "no attrs",
397 attrs: nil,
398 wantAttrs: testedAttrs{
399 StorageClass: "STANDARD",
400 VersioningEnabled: false,
401 LocationType: "multi-region",
402 Location: "US",
403 },
404 },
405 {
406 name: "with attrs",
407 attrs: &BucketAttrs{
408 StorageClass: "NEARLINE",
409 VersioningEnabled: true,
410 Labels: labels,
411 Lifecycle: lifecycle,
412 Location: "SOUTHAMERICA-EAST1",
413 },
414 wantAttrs: testedAttrs{
415 StorageClass: "NEARLINE",
416 VersioningEnabled: true,
417 Labels: labels,
418 Location: "SOUTHAMERICA-EAST1",
419 LocationType: "region",
420 Lifecycle: lifecycle,
421 },
422 },
423 {
424 name: "dual-region",
425 attrs: &BucketAttrs{
426 Location: "US",
427 CustomPlacementConfig: &CustomPlacementConfig{
428 DataLocations: []string{"US-EAST1", "US-WEST1"},
429 },
430 },
431 wantAttrs: testedAttrs{
432 Location: "US",
433 LocationType: "dual-region",
434 StorageClass: "STANDARD",
435 CustomPlacementConfig: &CustomPlacementConfig{
436 DataLocations: []string{"US-EAST1", "US-WEST1"},
437 },
438 },
439 },
440 } {
441 t.Run(test.name, func(t *testing.T) {
442 newBucketName := prefix + uidSpace.New()
443 b := client.Bucket(newBucketName)
444
445 if err := b.Create(ctx, projectID, test.attrs); err != nil {
446 t.Fatalf("bucket create: %v", err)
447 }
448
449 gotAttrs, err := b.Attrs(ctx)
450 if err != nil {
451 t.Fatalf("bucket attrs: %v", err)
452 }
453
454
455 if gotAttrs.MetaGeneration != 1 {
456 t.Errorf("metageneration: got %d, should be 1", gotAttrs.MetaGeneration)
457 }
458 if gotAttrs.ProjectNumber == 0 {
459 t.Errorf("got a zero ProjectNumber")
460 }
461
462
463 if gotAttrs.VersioningEnabled != test.wantAttrs.VersioningEnabled {
464 t.Errorf("versioning enabled: got %t, want %t", gotAttrs.VersioningEnabled, test.wantAttrs.VersioningEnabled)
465 }
466 if got, want := gotAttrs.Labels, test.wantAttrs.Labels; !testutil.Equal(got, want) {
467 t.Errorf("labels: got %v, want %v", got, want)
468 }
469 if diff := cmp.Diff(gotAttrs.Lifecycle, test.wantAttrs.Lifecycle); diff != "" {
470 t.Errorf("lifecycle: diff got vs. want: %v", diff)
471 }
472 if gotAttrs.LocationType != test.wantAttrs.LocationType {
473 t.Errorf("location type: got %s, want %s", gotAttrs.LocationType, test.wantAttrs.LocationType)
474 }
475 if gotAttrs.StorageClass != test.wantAttrs.StorageClass {
476 t.Errorf("storage class: got %s, want %s", gotAttrs.StorageClass, test.wantAttrs.StorageClass)
477 }
478 if gotAttrs.Location != test.wantAttrs.Location {
479 t.Errorf("location: got %s, want %s", gotAttrs.Location, test.wantAttrs.Location)
480 }
481 if got, want := gotAttrs.CustomPlacementConfig, test.wantAttrs.CustomPlacementConfig; !testutil.Equal(got, want) {
482 t.Errorf("customPlacementConfig: \ngot\t%v\nwant\t%v", got, want)
483 }
484
485
486 if err := b.Delete(ctx); err != nil {
487 t.Fatalf("bucket delete: %v", err)
488 }
489 _, err = b.Attrs(ctx)
490 if err != ErrBucketNotExist {
491 t.Fatalf("expected ErrBucketNotExist, got %v", err)
492 }
493 })
494 }
495 })
496 }
497
498 func TestIntegration_BucketLifecycle(t *testing.T) {
499 ctx := skipJSONReads(context.Background(), "no reads in test")
500 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
501 h := testHelper{t}
502
503 wantLifecycle := Lifecycle{
504 Rules: []LifecycleRule{
505 {
506 Action: LifecycleAction{Type: AbortIncompleteMPUAction},
507 Condition: LifecycleCondition{AgeInDays: 30},
508 },
509 {
510 Action: LifecycleAction{Type: DeleteAction},
511 Condition: LifecycleCondition{AllObjects: true},
512 },
513 },
514 }
515
516 bucket := client.Bucket(prefix + uidSpace.New())
517
518
519 h.mustCreate(bucket, testutil.ProjID(), &BucketAttrs{
520 Lifecycle: wantLifecycle,
521 })
522 defer h.mustDeleteBucket(bucket)
523
524 attrs := h.mustBucketAttrs(bucket)
525 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
526 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
527 }
528
529
530 ua := BucketAttrsToUpdate{Lifecycle: &Lifecycle{}}
531 attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration)
532 if !testutil.Equal(attrs.Lifecycle, Lifecycle{}) {
533 t.Fatalf("got %v, want %v", attrs.Lifecycle, Lifecycle{})
534 }
535
536
537 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle}
538 attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration)
539 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
540 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
541 }
542 })
543 }
544
545 func TestIntegration_BucketUpdate(t *testing.T) {
546 ctx := skipJSONReads(context.Background(), "no reads in test")
547 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
548 h := testHelper{t}
549
550 b := client.Bucket(prefix + uidSpace.New())
551 h.mustCreate(b, testutil.ProjID(), nil)
552 defer h.mustDeleteBucket(b)
553
554 attrs := h.mustBucketAttrs(b)
555 if attrs.VersioningEnabled {
556 t.Fatal("bucket should not have versioning by default")
557 }
558 if len(attrs.Labels) > 0 {
559 t.Fatal("bucket should not have labels initially")
560 }
561
562
563 ua := BucketAttrsToUpdate{VersioningEnabled: true}
564 ua.SetLabel("l1", "v1")
565 ua.SetLabel("empty", "")
566 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
567 if !attrs.VersioningEnabled {
568 t.Fatal("should have versioning now")
569 }
570 wantLabels := map[string]string{
571 "l1": "v1",
572 "empty": "",
573 }
574 if !testutil.Equal(attrs.Labels, wantLabels) {
575 t.Fatalf("add labels: got %v, want %v", attrs.Labels, wantLabels)
576 }
577
578
579 ua = BucketAttrsToUpdate{VersioningEnabled: false}
580 ua.SetLabel("l1", "v2")
581 ua.SetLabel("new", "new")
582 ua.DeleteLabel("empty")
583 ua.DeleteLabel("absent")
584 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
585 if attrs.VersioningEnabled {
586 t.Fatal("should have versioning off")
587 }
588 wantLabels = map[string]string{
589 "l1": "v2",
590 "new": "new",
591 }
592 if !testutil.Equal(attrs.Labels, wantLabels) {
593 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels)
594 }
595
596
597 wantLifecycle := Lifecycle{
598 Rules: []LifecycleRule{
599 {
600 Action: LifecycleAction{Type: "Delete"},
601 Condition: LifecycleCondition{
602 AgeInDays: 30,
603 MatchesPrefix: []string{"testPrefix"},
604 MatchesSuffix: []string{"testSuffix"},
605 },
606 },
607 },
608 }
609 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle}
610 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
611 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
612 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
613 }
614
615
616 wantStorageClass := "STANDARD"
617 if !testutil.Equal(attrs.StorageClass, wantStorageClass) {
618 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass)
619 }
620 wantStorageClass = "NEARLINE"
621 ua = BucketAttrsToUpdate{StorageClass: wantStorageClass}
622 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
623 if !testutil.Equal(attrs.StorageClass, wantStorageClass) {
624 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass)
625 }
626
627
628 gotAttrs, err := b.Update(ctx, BucketAttrsToUpdate{})
629 if err != nil {
630 t.Fatalf("empty update: %v", err)
631 }
632 if !testutil.Equal(attrs, gotAttrs) {
633 t.Fatalf("empty update: got %v, want %v", gotAttrs, attrs)
634 }
635 })
636 }
637
638 func TestIntegration_BucketPolicyOnly(t *testing.T) {
639 ctx := skipJSONReads(context.Background(), "no reads in test")
640 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
641 h := testHelper{t}
642
643 bkt := client.Bucket(prefix + uidSpace.New())
644 h.mustCreate(bkt, testutil.ProjID(), nil)
645 defer h.mustDeleteBucket(bkt)
646
647
648 o := bkt.Object("bucketPolicyOnly")
649 defer func() {
650 if err := o.Delete(ctx); err != nil {
651 log.Printf("failed to delete test object: %v", err)
652 }
653 }()
654 wc := o.NewWriter(ctx)
655 wc.ContentType = "text/plain"
656 h.mustWrite(wc, []byte("test"))
657 a := o.ACL()
658 aclEntity := ACLEntity("user-test@example.com")
659 err := a.Set(ctx, aclEntity, RoleReader)
660 if err != nil {
661 t.Fatalf("set ACL failed: %v", err)
662 }
663
664
665 ua := BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: true}}
666 attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration)
667 if got, want := attrs.BucketPolicyOnly.Enabled, true; got != want {
668 t.Fatalf("got %v, want %v", got, want)
669 }
670 if got := attrs.BucketPolicyOnly.LockedTime; got.IsZero() {
671 t.Fatal("got a zero time value, want a populated value")
672 }
673
674
675
676
677
678
679
680 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
681 b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs))
682 _, err = b.ACL().List(ctxWithTimeout)
683 cancelCtx()
684 if err == nil {
685 t.Errorf("ACL.List: expected bucket ACL list to fail")
686 }
687
688
689 ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10)
690 _, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout)
691 cancelCtx()
692 if err == nil {
693 t.Errorf("ACL.List: expected object ACL list to fail")
694 }
695
696
697 ua = BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: false}}
698 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
699 if got, want := attrs.BucketPolicyOnly.Enabled, false; got != want {
700 t.Fatalf("attrs.BucketPolicyOnly.Enabled: got %v, want %v", got, want)
701 }
702
703
704
705
706
707
708 var acl []ACLRule
709 err = retry(ctx, func() error {
710 var err error
711 acl, err = o.ACL().List(ctx)
712 if err != nil {
713 return fmt.Errorf("ACL.List: object ACL list failed: %v", err)
714 }
715 return nil
716 }, func() error {
717 if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) {
718 return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader})
719 }
720 return nil
721 })
722 if err != nil {
723 t.Fatal(err)
724 }
725 })
726 }
727
728 func TestIntegration_UniformBucketLevelAccess(t *testing.T) {
729 ctx := skipJSONReads(context.Background(), "no reads in test")
730 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
731 h := testHelper{t}
732 bkt := client.Bucket(prefix + uidSpace.New())
733 h.mustCreate(bkt, testutil.ProjID(), nil)
734 defer h.mustDeleteBucket(bkt)
735
736
737 o := bkt.Object("uniformBucketLevelAccess")
738 defer func() {
739 if err := o.Delete(ctx); err != nil {
740 log.Printf("failed to delete test object: %v", err)
741 }
742 }()
743 wc := o.NewWriter(ctx)
744 wc.ContentType = "text/plain"
745 h.mustWrite(wc, []byte("test"))
746 a := o.ACL()
747 aclEntity := ACLEntity("user-test@example.com")
748 err := a.Set(ctx, aclEntity, RoleReader)
749 if err != nil {
750 t.Fatalf("set ACL failed: %v", err)
751 }
752
753
754 ua := BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}}
755 attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration)
756 if got, want := attrs.UniformBucketLevelAccess.Enabled, true; got != want {
757 t.Fatalf("got %v, want %v", got, want)
758 }
759 if got := attrs.UniformBucketLevelAccess.LockedTime; got.IsZero() {
760 t.Fatal("got a zero time value, want a populated value")
761 }
762
763
764
765 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
766 b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs))
767 _, err = b.ACL().List(ctxWithTimeout)
768 cancelCtx()
769 if err == nil {
770 t.Errorf("ACL.List: expected bucket ACL list to fail")
771 }
772
773
774 ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10)
775 _, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout)
776 cancelCtx()
777 if err == nil {
778 t.Errorf("ACL.List: expected object ACL list to fail")
779 }
780
781
782 ua = BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: false}}
783 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
784 if got, want := attrs.UniformBucketLevelAccess.Enabled, false; got != want {
785 t.Fatalf("got %v, want %v", got, want)
786 }
787
788
789
790
791 var acl []ACLRule
792 err = retry(ctx, func() error {
793 var err error
794 acl, err = o.ACL().List(ctx)
795 if err != nil {
796 return fmt.Errorf("ACL.List: object ACL list failed: %v", err)
797 }
798 return nil
799 }, func() error {
800 if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) {
801 return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader})
802 }
803 return nil
804 })
805 if err != nil {
806 t.Fatal(err)
807 }
808 })
809 }
810
811 func TestIntegration_PublicAccessPrevention(t *testing.T) {
812 ctx := skipJSONReads(context.Background(), "no reads in test")
813 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
814 h := testHelper{t}
815
816
817 bkt := client.Bucket(prefix + uidSpace.New())
818 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{PublicAccessPrevention: PublicAccessPreventionEnforced})
819 defer h.mustDeleteBucket(bkt)
820
821
822 policy, err := bkt.IAM().V3().Policy(ctx)
823 if err != nil {
824 t.Fatalf("fetching bucket IAM policy: %v", err)
825 }
826 policy.Bindings = append(policy.Bindings, &iampb.Binding{
827 Role: "roles/storage.objectViewer",
828 Members: []string{iam.AllUsers},
829 })
830 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err == nil {
831 t.Error("SetPolicy: expected adding AllUsers policy to bucket should fail")
832 }
833
834
835 o := bkt.Object("publicAccessPrevention")
836 defer func() {
837 if err := o.Delete(ctx); err != nil {
838 log.Printf("failed to delete test object: %v", err)
839 }
840 }()
841 wc := o.NewWriter(ctx)
842 wc.ContentType = "text/plain"
843 h.mustWrite(wc, []byte("test"))
844 a := o.ACL()
845 if err := a.Set(ctx, AllUsers, RoleReader); err == nil {
846 t.Error("ACL.Set: expected adding AllUsers ACL to object should fail")
847 }
848
849
850 attrs, err := bkt.Update(ctx, BucketAttrsToUpdate{PublicAccessPrevention: PublicAccessPreventionInherited})
851 if err != nil {
852 t.Fatalf("updating PublicAccessPrevention failed: %v", err)
853 }
854 if attrs.PublicAccessPrevention != PublicAccessPreventionInherited {
855 t.Errorf("updating PublicAccessPrevention: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited)
856 }
857 if attrs.UniformBucketLevelAccess.Enabled || attrs.BucketPolicyOnly.Enabled {
858 t.Error("updating PublicAccessPrevention changed UBLA setting")
859 }
860
861
862
863 retrier := func(err error) bool {
864
865
866
867
868 return ShouldRetry(err) || status.Code(err) == codes.FailedPrecondition || extractErrCode(err) == http.StatusPreconditionFailed
869 }
870
871 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
872 a = o.Retryer(WithErrorFunc(retrier), WithPolicy(RetryAlways)).ACL()
873 err = a.Set(ctxWithTimeout, AllUsers, RoleReader)
874 cancelCtx()
875 if err != nil {
876 t.Errorf("ACL.Set: making object public failed: %v", err)
877 }
878
879 policy, err = bkt.IAM().V3().Policy(ctx)
880 if err != nil {
881 t.Fatalf("fetching bucket IAM policy: %v", err)
882 }
883 policy.Bindings = append(policy.Bindings, &iampb.Binding{
884 Role: "roles/storage.objectViewer",
885 Members: []string{iam.AllUsers},
886 })
887 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err != nil {
888 t.Errorf("SetPolicy: making bucket public failed: %v", err)
889 }
890
891
892 attrs, err = bkt.Update(ctx, BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}})
893 if err != nil {
894 t.Fatalf("updating UBLA failed: %v", err)
895 }
896 if !attrs.UniformBucketLevelAccess.Enabled {
897 t.Error("updating UBLA: got UBLA not enabled, want enabled")
898 }
899 if attrs.PublicAccessPrevention != PublicAccessPreventionInherited {
900 t.Errorf("updating UBLA: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited)
901 }
902 })
903 }
904
905 func TestIntegration_Autoclass(t *testing.T) {
906 ctx := skipJSONReads(context.Background(), "no reads in test")
907 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
908 h := testHelper{t}
909
910
911 bkt := client.Bucket(prefix + uidSpace.New())
912 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{Autoclass: &Autoclass{Enabled: true}})
913 defer h.mustDeleteBucket(bkt)
914
915
916
917 attrs, err := bkt.Attrs(ctx)
918 if err != nil {
919 t.Fatalf("get bucket attrs failed: %v", err)
920 }
921 var toggleTime time.Time
922 var tscUpdateTime time.Time
923 if attrs != nil && attrs.Autoclass != nil {
924 if got, want := attrs.Autoclass.Enabled, true; got != want {
925 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
926 }
927 if toggleTime = attrs.Autoclass.ToggleTime; toggleTime.IsZero() {
928 t.Error("got a zero time value, want a populated value")
929 }
930 if got, want := attrs.Autoclass.TerminalStorageClass, "NEARLINE"; got != want {
931 t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
932 }
933 if tscUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime; tscUpdateTime.IsZero() {
934 t.Error("got a zero time value, want a populated value")
935 }
936 }
937
938
939 ua := BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}}
940 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
941 if got, want := attrs.Autoclass.Enabled, true; got != want {
942 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
943 }
944 if got, want := attrs.Autoclass.TerminalStorageClass, "ARCHIVE"; got != want {
945 t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
946 }
947 latestTSCUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime
948 if latestTSCUpdateTime.IsZero() {
949 t.Error("got a zero time value, want a populated value")
950 }
951 if !latestTSCUpdateTime.After(tscUpdateTime) {
952 t.Error("latestTSCUpdateTime should be newer than bucket creation tscUpdateTime")
953 }
954
955
956 ua = BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: false}}
957 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
958 if got, want := attrs.Autoclass.Enabled, false; got != want {
959 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
960 }
961 latestToggleTime := attrs.Autoclass.ToggleTime
962 if latestToggleTime.IsZero() {
963 t.Error("got a zero time value, want a populated value")
964 }
965 if !latestToggleTime.After(toggleTime) {
966 t.Error("latestToggleTime should be newer than bucket creation toggleTime")
967 }
968 })
969 }
970
971 func TestIntegration_ConditionalDelete(t *testing.T) {
972 ctx := skipJSONReads(context.Background(), "no reads in test")
973 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
974 h := testHelper{t}
975
976 o := client.Bucket(bucket).Object("conddel")
977
978 wc := o.NewWriter(ctx)
979 wc.ContentType = "text/plain"
980 h.mustWrite(wc, []byte("foo"))
981
982 gen := wc.Attrs().Generation
983 metaGen := wc.Attrs().Metageneration
984
985 if err := o.Generation(gen - 1).Delete(ctx); err == nil {
986 t.Fatalf("Unexpected successful delete with Generation")
987 }
988 if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil {
989 t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch")
990 }
991 if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil {
992 t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch")
993 }
994 if err := o.Generation(gen).Delete(ctx); err != nil {
995 t.Fatalf("final delete failed: %v", err)
996 }
997 })
998 }
999
1000 func TestIntegration_ObjectsRangeReader(t *testing.T) {
1001 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1002 bkt := client.Bucket(bucket)
1003
1004 objName := uidSpaceObjects.New()
1005 obj := bkt.Object(objName)
1006 contents := []byte("Hello, world this is a range request")
1007
1008 w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx)
1009 if _, err := w.Write(contents); err != nil {
1010 t.Errorf("Failed to write contents: %v", err)
1011 }
1012 if err := w.Close(); err != nil {
1013 t.Errorf("Failed to close writer: %v", err)
1014 }
1015
1016 last5s := []struct {
1017 name string
1018 start int64
1019 length int64
1020 }{
1021 {name: "negative offset", start: -5, length: -1},
1022 {name: "offset with specified length", start: int64(len(contents)) - 5, length: 5},
1023 {name: "offset and read till end", start: int64(len(contents)) - 5, length: -1},
1024 }
1025
1026 for _, last5 := range last5s {
1027 t.Run(last5.name, func(t *testing.T) {
1028
1029 for _, c := range readCases {
1030 t.Run(c.desc, func(t *testing.T) {
1031 wantBuf := contents[len(contents)-5:]
1032 r, err := obj.NewRangeReader(ctx, last5.start, last5.length)
1033 if err != nil {
1034 t.Fatalf("Failed to make range read: %v", err)
1035 }
1036 defer r.Close()
1037
1038 if got, want := r.Attrs.StartOffset, int64(len(contents))-5; got != want {
1039 t.Errorf("StartOffset mismatch, got %d want %d", got, want)
1040 }
1041
1042 gotBuf, err := c.readFunc(r)
1043 if err != nil {
1044 t.Fatalf("reading object: %v", err)
1045 }
1046 if got, want := len(gotBuf), 5; got != want {
1047 t.Errorf("Body length mismatch, got %d want %d", got, want)
1048 } else if diff := cmp.Diff(string(gotBuf), string(wantBuf)); diff != "" {
1049 t.Errorf("Content read does not match - got(-),want(+):\n%s", diff)
1050 }
1051 })
1052 }
1053
1054 })
1055 }
1056 })
1057 }
1058
1059 func TestIntegration_ObjectReadChunksGRPC(t *testing.T) {
1060 multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1061 h := testHelper{t}
1062
1063 content := make([]byte, 5<<20)
1064 rand.New(rand.NewSource(0)).Read(content)
1065
1066
1067 obj := client.Bucket(bucket).Object(uidSpaceObjects.New())
1068 if err := writeObject(ctx, obj, "text/plain", content); err != nil {
1069 t.Fatal(err)
1070 }
1071 defer h.mustDeleteObject(obj)
1072
1073 r, err := obj.NewReader(ctx)
1074 if err != nil {
1075 t.Fatal(err)
1076 }
1077 defer r.Close()
1078
1079 if size := r.Size(); size != int64(len(content)) {
1080 t.Errorf("got size = %v, want %v", size, len(content))
1081 }
1082 if rem := r.Remain(); rem != int64(len(content)) {
1083 t.Errorf("got %v bytes remaining, want %v", rem, len(content))
1084 }
1085
1086 bufSize := len(content)
1087 buf := make([]byte, bufSize)
1088
1089
1090 chunk := 4<<10 + 1234
1091 offset := 0
1092 for {
1093 end := math.Min(float64(offset+chunk), float64(bufSize))
1094 n, err := r.Read(buf[offset:int(end)])
1095 if err == io.EOF {
1096 break
1097 }
1098 if err != nil {
1099 t.Fatal(err)
1100 }
1101 offset += n
1102 }
1103
1104 if rem := r.Remain(); rem != 0 {
1105 t.Errorf("got %v bytes remaining, want 0", rem)
1106 }
1107 if !bytes.Equal(buf, content) {
1108 t.Errorf("content mismatch")
1109 }
1110 })
1111 }
1112
1113 func TestIntegration_MultiMessageWriteGRPC(t *testing.T) {
1114 multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1115 h := testHelper{t}
1116
1117 name := uidSpaceObjects.New()
1118 obj := client.Bucket(bucket).Object(name).Retryer(WithPolicy(RetryAlways))
1119 defer h.mustDeleteObject(obj)
1120
1121
1122 content := bytes.Repeat([]byte("a"), 5<<20)
1123
1124 crc32c := crc32.Checksum(content, crc32cTable)
1125 w := obj.NewWriter(ctx)
1126 w.ProgressFunc = func(p int64) {
1127 t.Logf("%s: committed %d\n", t.Name(), p)
1128 }
1129 w.SendCRC32C = true
1130 w.CRC32C = crc32c
1131 got, err := w.Write(content)
1132 if err != nil {
1133 t.Fatalf("Writer.Write: %v", err)
1134 }
1135
1136 if err := w.Close(); err != nil {
1137 t.Fatalf("Writer.Close: %v", err)
1138 }
1139
1140 want := len(content)
1141 if got != want {
1142 t.Errorf("While writing got: %d want %d", got, want)
1143 }
1144
1145
1146 reader, err := client.Bucket(bucket).Object(name).NewReader(ctx)
1147 if err != nil {
1148 t.Fatal(err)
1149 }
1150 defer reader.Close()
1151
1152 buf := make([]byte, want+4<<10)
1153 b := bytes.NewBuffer(buf)
1154 gotr, err := io.Copy(b, reader)
1155 if err != nil {
1156 t.Fatal(err)
1157 }
1158 if gotr != int64(want) {
1159 t.Errorf("While reading got: %d want %d", gotr, want)
1160 }
1161 })
1162 }
1163
1164 func TestIntegration_MultiChunkWrite(t *testing.T) {
1165 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1166 h := testHelper{t}
1167 obj := client.Bucket(bucket).Object(uidSpaceObjects.New()).Retryer(WithPolicy(RetryAlways))
1168 defer h.mustDeleteObject(obj)
1169
1170
1171 content := bytes.Repeat([]byte("a"), 5<<20)
1172 crc32c := crc32.Checksum(content, crc32cTable)
1173
1174 w := obj.NewWriter(ctx)
1175 w.SendCRC32C = true
1176 w.CRC32C = crc32c
1177
1178 w.ChunkSize = 1 << 20
1179 w.ProgressFunc = func(p int64) {
1180 t.Logf("%s: committed %d\n", t.Name(), p)
1181 }
1182 got, err := w.Write(content)
1183 if err != nil {
1184 t.Fatalf("Writer.Write: %v", err)
1185 }
1186
1187 if err := w.Close(); err != nil {
1188 t.Fatalf("Writer.Close: %v", err)
1189 }
1190
1191 want := len(content)
1192 if got != want {
1193 t.Errorf("While writing got: %d want %d", got, want)
1194 }
1195
1196 r, err := obj.NewReader(ctx)
1197 if err != nil {
1198 t.Fatal(err)
1199 }
1200 defer r.Close()
1201
1202 buf := make([]byte, want+4<<10)
1203 b := bytes.NewBuffer(buf)
1204 gotr, err := io.Copy(b, r)
1205 if err != nil {
1206 t.Fatal(err)
1207 }
1208 if gotr != int64(want) {
1209 t.Errorf("While reading got: %d want %d", gotr, want)
1210 }
1211 })
1212 }
1213
1214 func TestIntegration_ConditionalDownload(t *testing.T) {
1215 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1216 h := testHelper{t}
1217
1218 o := client.Bucket(bucket).Object("condread")
1219 defer o.Delete(ctx)
1220
1221 wc := o.NewWriter(ctx)
1222 wc.ContentType = "text/plain"
1223 h.mustWrite(wc, []byte("foo"))
1224
1225 gen := wc.Attrs().Generation
1226 metaGen := wc.Attrs().Metageneration
1227
1228 if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil {
1229 t.Fatalf("Unexpected successful download with nonexistent Generation")
1230 }
1231 if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil {
1232 t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch")
1233 }
1234 if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil {
1235 t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch")
1236 }
1237 if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil {
1238 t.Fatalf("Download failed: %v", err)
1239 }
1240 })
1241 }
1242
1243 func TestIntegration_ObjectIteration(t *testing.T) {
1244 ctx := skipJSONReads(context.Background(), "no reads in test")
1245 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
1246
1247
1248 testTime = time.Now().UTC()
1249 newBucketName := prefix + uidSpace.New()
1250 h := testHelper{t}
1251 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
1252
1253 h.mustCreate(bkt, testutil.ProjID(), nil)
1254 defer func() {
1255 if err := killBucket(ctx, client, newBucketName); err != nil {
1256 log.Printf("deleting %q: %v", newBucketName, err)
1257 }
1258 }()
1259 const defaultType = "text/plain"
1260
1261
1262 objects := []string{
1263 "obj1",
1264 "obj2",
1265 "obj/with/slashes",
1266 "obj/",
1267 }
1268 contents := make(map[string][]byte)
1269
1270
1271 for _, obj := range objects {
1272 c := randomContents()
1273 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
1274 t.Errorf("Write for %v failed with %v", obj, err)
1275 }
1276 contents[obj] = c
1277 }
1278
1279 testObjectIterator(t, bkt, objects)
1280 testObjectsIterateSelectedAttrs(t, bkt, objects)
1281 testObjectsIterateAllSelectedAttrs(t, bkt, objects)
1282 testObjectIteratorWithOffset(t, bkt, objects)
1283 testObjectsIterateWithProjection(t, bkt)
1284 t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) {
1285 query := &Query{Prefix: "", Delimiter: "/"}
1286 if err := query.SetAttrSelection([]string{"Name"}); err != nil {
1287 t.Fatalf("selecting query attrs: %v", err)
1288 }
1289
1290 var gotNames []string
1291 var gotPrefixes []string
1292 it := bkt.Objects(context.Background(), query)
1293 for {
1294 attrs, err := it.Next()
1295 if err == iterator.Done {
1296 break
1297 }
1298 if err != nil {
1299 t.Fatalf("iterator.Next: %v", err)
1300 }
1301 if attrs.Name != "" {
1302 gotNames = append(gotNames, attrs.Name)
1303 } else if attrs.Prefix != "" {
1304 gotPrefixes = append(gotPrefixes, attrs.Prefix)
1305 }
1306
1307 if attrs.Bucket != "" {
1308 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
1309 }
1310 }
1311
1312 sortedNames := []string{"obj1", "obj2"}
1313 if !cmp.Equal(sortedNames, gotNames) {
1314 t.Errorf("names = %v, want %v", gotNames, sortedNames)
1315 }
1316 sortedPrefixes := []string{"obj/"}
1317 if !cmp.Equal(sortedPrefixes, gotPrefixes) {
1318 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes)
1319 }
1320 })
1321 t.Run("testObjectsIterateSelectedAttrsDelimiterIncludeTrailingDelimiter", func(t *testing.T) {
1322 query := &Query{Prefix: "", Delimiter: "/", IncludeTrailingDelimiter: true}
1323 if err := query.SetAttrSelection([]string{"Name"}); err != nil {
1324 t.Fatalf("selecting query attrs: %v", err)
1325 }
1326
1327 var gotNames []string
1328 var gotPrefixes []string
1329 it := bkt.Objects(context.Background(), query)
1330 for {
1331 attrs, err := it.Next()
1332 if err == iterator.Done {
1333 break
1334 }
1335 if err != nil {
1336 t.Fatalf("iterator.Next: %v", err)
1337 }
1338 if attrs.Name != "" {
1339 gotNames = append(gotNames, attrs.Name)
1340 } else if attrs.Prefix != "" {
1341 gotPrefixes = append(gotPrefixes, attrs.Prefix)
1342 }
1343
1344 if attrs.Bucket != "" {
1345 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
1346 }
1347 }
1348
1349 sortedNames := []string{"obj/", "obj1", "obj2"}
1350 if !cmp.Equal(sortedNames, gotNames) {
1351 t.Errorf("names = %v, want %v", gotNames, sortedNames)
1352 }
1353 sortedPrefixes := []string{"obj/"}
1354 if !cmp.Equal(sortedPrefixes, gotPrefixes) {
1355 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes)
1356 }
1357 })
1358 })
1359 }
1360
1361 func TestIntegration_ObjectIterationMatchGlob(t *testing.T) {
1362 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
1363
1364
1365 testTime = time.Now().UTC()
1366 newBucketName := prefix + uidSpace.New()
1367 h := testHelper{t}
1368 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
1369
1370 h.mustCreate(bkt, testutil.ProjID(), nil)
1371 defer func() {
1372 if err := killBucket(ctx, client, newBucketName); err != nil {
1373 log.Printf("deleting %q: %v", newBucketName, err)
1374 }
1375 }()
1376 const defaultType = "text/plain"
1377
1378
1379 objects := []string{
1380 "obj1",
1381 "obj2",
1382 "obj/with/slashes",
1383 "obj/",
1384 "other/obj1",
1385 }
1386 contents := make(map[string][]byte)
1387
1388
1389 for _, obj := range objects {
1390 c := randomContents()
1391 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
1392 t.Errorf("Write for %v failed with %v", obj, err)
1393 }
1394 contents[obj] = c
1395 }
1396 query := &Query{MatchGlob: "**obj1"}
1397
1398 var gotNames []string
1399 it := bkt.Objects(context.Background(), query)
1400 for {
1401 attrs, err := it.Next()
1402 if err == iterator.Done {
1403 break
1404 }
1405 if err != nil {
1406 t.Fatalf("iterator.Next: %v", err)
1407 }
1408 if attrs.Name != "" {
1409 gotNames = append(gotNames, attrs.Name)
1410 }
1411 }
1412
1413 sortedNames := []string{"obj1", "other/obj1"}
1414 if !cmp.Equal(sortedNames, gotNames) {
1415 t.Errorf("names = %v, want %v", gotNames, sortedNames)
1416 }
1417 })
1418 }
1419
1420 func TestIntegration_ObjectIterationManagedFolder(t *testing.T) {
1421 ctx := skipGRPC("not yet implemented in gRPC")
1422 multiTransportTest(skipJSONReads(ctx, "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
1423 newBucketName := prefix + uidSpace.New()
1424 h := testHelper{t}
1425 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
1426
1427
1428 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
1429 UniformBucketLevelAccess: UniformBucketLevelAccess{
1430 Enabled: true,
1431 },
1432 })
1433
1434 t.Cleanup(func() {
1435 if err := killBucket(ctx, client, newBucketName); err != nil {
1436 log.Printf("deleting %q: %v", newBucketName, err)
1437 }
1438 })
1439 const defaultType = "text/plain"
1440
1441
1442 objects := []string{
1443 "obj1",
1444 "obj2",
1445 "obj/with/slashes",
1446 "obj/",
1447 "other/obj1",
1448 }
1449 contents := make(map[string][]byte)
1450
1451
1452 for _, obj := range objects {
1453 c := randomContents()
1454 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
1455 t.Errorf("Write for %v failed with %v", obj, err)
1456 }
1457 contents[obj] = c
1458 }
1459
1460
1461
1462
1463 call := client.raw.ManagedFolders.Insert(newBucketName, &raw.ManagedFolder{Name: "mf"})
1464 mf, err := call.Context(ctx).Do()
1465 if err != nil {
1466 t.Fatalf("creating managed folder: %v", err)
1467 }
1468
1469 t.Cleanup(func() {
1470
1471 call := client.raw.ManagedFolders.Delete(newBucketName, mf.Name)
1472 call.Context(ctx).Do()
1473 })
1474
1475
1476 cases := []struct {
1477 name string
1478 query *Query
1479 want []string
1480 }{
1481 {
1482 name: "include folders",
1483 query: &Query{Delimiter: "/", IncludeFoldersAsPrefixes: true},
1484 want: []string{"mf/", "obj/", "other/"},
1485 },
1486 {
1487 name: "no folders",
1488 query: &Query{Delimiter: "/"},
1489 want: []string{"obj/", "other/"},
1490 },
1491 }
1492
1493 for _, c := range cases {
1494 t.Run(c.name, func(t *testing.T) {
1495 var gotNames []string
1496 var gotPrefixes []string
1497 it := bkt.Objects(context.Background(), c.query)
1498 for {
1499 attrs, err := it.Next()
1500 if err == iterator.Done {
1501 break
1502 }
1503 if err != nil {
1504 t.Fatalf("iterator.Next: %v", err)
1505 }
1506 if attrs.Name != "" {
1507 gotNames = append(gotNames, attrs.Name)
1508 }
1509 if attrs.Prefix != "" {
1510 gotPrefixes = append(gotPrefixes, attrs.Prefix)
1511 }
1512 }
1513
1514 sortedNames := []string{"obj1", "obj2"}
1515 if !cmp.Equal(sortedNames, gotNames) {
1516 t.Errorf("names = %v, want %v", gotNames, sortedNames)
1517 }
1518
1519 if !cmp.Equal(c.want, gotPrefixes) {
1520 t.Errorf("prefixes = %v, want %v", gotPrefixes, c.want)
1521 }
1522 })
1523 }
1524 })
1525 }
1526
1527 func TestIntegration_ObjectUpdate(t *testing.T) {
1528 ctx := skipJSONReads(context.Background(), "no reads in test")
1529 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1530 b := client.Bucket(bucket)
1531
1532 o := b.Object("update-obj" + uidSpaceObjects.New())
1533 w := o.NewWriter(ctx)
1534 _, err := io.Copy(w, bytes.NewReader(randomContents()))
1535 if err != nil {
1536 t.Fatalf("io.Copy: %v", err)
1537 }
1538 if err := w.Close(); err != nil {
1539 t.Fatalf("w.Close: %v", err)
1540 }
1541 defer func() {
1542 if err := o.Delete(ctx); err != nil {
1543 t.Errorf("o.Delete : %v", err)
1544 }
1545 }()
1546
1547 attrs, err := o.Attrs(ctx)
1548 if err != nil {
1549 t.Fatalf("o.Attrs: %v", err)
1550 }
1551
1552
1553 metadata := map[string]string{"key": "value"}
1554
1555 updated, err := o.If(Conditions{MetagenerationMatch: attrs.Metageneration}).Update(ctx, ObjectAttrsToUpdate{
1556 ContentType: "text/html",
1557 ContentLanguage: "en",
1558 Metadata: metadata,
1559 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
1560 })
1561 if err != nil {
1562 t.Fatalf("o.Update: %v", err)
1563 }
1564
1565 if got, want := updated.ContentType, "text/html"; got != want {
1566 t.Errorf("updated.ContentType == %q; want %q", got, want)
1567 }
1568 if got, want := updated.ContentLanguage, "en"; got != want {
1569 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want)
1570 }
1571 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) {
1572 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want)
1573 }
1574 if got, want := updated.Created, attrs.Created; got != want {
1575 t.Errorf("updated.Created == %q; want %q", got, want)
1576 }
1577 if !updated.Created.Before(updated.Updated) {
1578 t.Errorf("updated.Updated should be newer than update.Created")
1579 }
1580
1581
1582 anotherKey := map[string]string{"key2": "value2"}
1583 metadata["key2"] = "value2"
1584
1585 updated, err = o.Update(ctx, ObjectAttrsToUpdate{
1586 Metadata: anotherKey,
1587 })
1588 if err != nil {
1589 t.Fatalf("o.Update: %v", err)
1590 }
1591
1592 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) {
1593 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want)
1594 }
1595
1596
1597 updated, err = o.If(Conditions{MetagenerationMatch: updated.Metageneration}).Update(ctx, ObjectAttrsToUpdate{
1598 ContentType: "",
1599 ContentLanguage: "",
1600 Metadata: map[string]string{},
1601 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
1602 })
1603 if err != nil {
1604 t.Fatalf("o.Update: %v", err)
1605 }
1606
1607 if got, want := updated.ContentType, ""; got != want {
1608 t.Errorf("updated.ContentType == %q; want %q", got, want)
1609 }
1610 if got, want := updated.ContentLanguage, ""; got != want {
1611 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want)
1612 }
1613 if updated.Metadata != nil {
1614 t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata)
1615 }
1616 if got, want := updated.Created, attrs.Created; got != want {
1617 t.Errorf("updated.Created == %q; want %q", got, want)
1618 }
1619 if !updated.Created.Before(updated.Updated) {
1620 t.Errorf("updated.Updated should be newer than update.Created")
1621 }
1622
1623
1624
1625 wantAttrs := updated
1626 gotAttrs, err := o.Update(ctx, ObjectAttrsToUpdate{})
1627 if err != nil {
1628 t.Fatalf("empty update: %v", err)
1629 }
1630 if diff := testutil.Diff(gotAttrs, wantAttrs, cmpopts.IgnoreFields(ObjectAttrs{}, "Etag", "Metageneration", "Updated")); diff != "" {
1631 t.Errorf("empty update: got=-, want=+:\n%s", diff)
1632 }
1633 })
1634 }
1635
1636 func TestIntegration_ObjectChecksums(t *testing.T) {
1637 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1638 b := client.Bucket(bucket)
1639 checksumCases := []struct {
1640 name string
1641 contents [][]byte
1642 size int64
1643 md5 string
1644 crc32c uint32
1645 }{
1646 {
1647 name: "checksum-object",
1648 contents: [][]byte{[]byte("hello"), []byte("world")},
1649 size: 10,
1650 md5: "fc5e038d38a57032085441e7fe7010b0",
1651 crc32c: 1456190592,
1652 },
1653 {
1654 name: "zero-object",
1655 contents: [][]byte{},
1656 size: 0,
1657 md5: "d41d8cd98f00b204e9800998ecf8427e",
1658 crc32c: 0,
1659 },
1660 }
1661 for _, c := range checksumCases {
1662 wc := b.Object(c.name + uidSpaceObjects.New()).NewWriter(ctx)
1663 for _, data := range c.contents {
1664 if _, err := wc.Write(data); err != nil {
1665 t.Fatalf("Write(%q) failed with %q", data, err)
1666 }
1667 }
1668 if err := wc.Close(); err != nil {
1669 t.Fatalf("%q: close failed with %q", c.name, err)
1670 }
1671 obj := wc.Attrs()
1672 if got, want := obj.Size, c.size; got != want {
1673 t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want)
1674 }
1675 if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want {
1676 t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want)
1677 }
1678 if got, want := obj.CRC32C, c.crc32c; got != want {
1679 t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want)
1680 }
1681 }
1682 })
1683 }
1684
1685 func TestIntegration_ObjectCompose(t *testing.T) {
1686 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1687 b := client.Bucket(bucket)
1688
1689 objects := []*ObjectHandle{
1690 b.Object("obj1" + uidSpaceObjects.New()),
1691 b.Object("obj2" + uidSpaceObjects.New()),
1692 b.Object("obj/with/slashes" + uidSpaceObjects.New()),
1693 b.Object("obj/" + uidSpaceObjects.New()),
1694 }
1695 var compSrcs []*ObjectHandle
1696 wantContents := make([]byte, 0)
1697
1698
1699 for _, obj := range objects {
1700 c := randomContents()
1701 if err := writeObject(ctx, obj, "text/plain", c); err != nil {
1702 t.Errorf("Write for %v failed with %v", obj, err)
1703 }
1704 compSrcs = append(compSrcs, obj)
1705 wantContents = append(wantContents, c...)
1706 defer obj.Delete(ctx)
1707 }
1708
1709 checkCompose := func(obj *ObjectHandle, contentTypeSet *string) {
1710 r, err := obj.NewReader(ctx)
1711 if err != nil {
1712 t.Fatalf("new reader: %v", err)
1713 }
1714
1715 slurp, err := io.ReadAll(r)
1716 if err != nil {
1717 t.Fatalf("io.ReadAll: %v", err)
1718 }
1719 defer r.Close()
1720 if !bytes.Equal(slurp, wantContents) {
1721 t.Errorf("Composed object contents\ngot: %q\nwant: %q", slurp, wantContents)
1722 }
1723 got := r.ContentType()
1724
1725
1726 if !(contentTypeSet == nil && (got == "" || got == "application/octet-stream")) && got != *contentTypeSet {
1727 t.Errorf("Composed object content-type = %q, want %q", got, *contentTypeSet)
1728 }
1729 }
1730
1731
1732 compDst := b.Object("composed1")
1733 c := compDst.ComposerFrom(compSrcs...)
1734 attrs, err := c.Run(ctx)
1735 if err != nil {
1736 t.Fatalf("ComposeFrom error: %v", err)
1737 }
1738 if attrs.ComponentCount != int64(len(objects)) {
1739 t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects)))
1740 }
1741 checkCompose(compDst, nil)
1742
1743
1744 contentType := "text/json"
1745 compDst = b.Object("composed2")
1746 c = compDst.ComposerFrom(compSrcs...)
1747 c.ContentType = contentType
1748 attrs, err = c.Run(ctx)
1749 if err != nil {
1750 t.Fatalf("ComposeFrom error: %v", err)
1751 }
1752 if attrs.ComponentCount != int64(len(objects)) {
1753 t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects)))
1754 }
1755 checkCompose(compDst, &contentType)
1756 })
1757 }
1758
1759 func TestIntegration_Copy(t *testing.T) {
1760 ctx := skipJSONReads(context.Background(), "no reads in test")
1761 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) {
1762 h := testHelper{t}
1763
1764 bucketFrom := client.Bucket(bucket)
1765 bucketInSameRegion := client.Bucket(prefix + uidSpace.New())
1766 bucketInDifferentRegion := client.Bucket(prefix + uidSpace.New())
1767
1768
1769 if err := bucketInSameRegion.Create(ctx, testutil.ProjID(), nil); err != nil {
1770 t.Fatalf("bucket.Create: %v", err)
1771 }
1772 t.Cleanup(func() {
1773 h.mustDeleteBucket(bucketInSameRegion)
1774 })
1775
1776
1777 if err := bucketInDifferentRegion.Create(ctx, testutil.ProjID(), &BucketAttrs{Location: "NORTHAMERICA-NORTHEAST2"}); err != nil {
1778 t.Fatalf("bucket.Create: %v", err)
1779 }
1780 t.Cleanup(func() {
1781 h.mustDeleteBucket(bucketInDifferentRegion)
1782 })
1783
1784
1785 minObjectSize := 2500000
1786 obj := bucketFrom.Object("copy-object-original" + uidSpaceObjects.New())
1787
1788
1789 w := obj.NewWriter(ctx)
1790 c := randomContents()
1791 for written := 0; written < minObjectSize; {
1792 n, err := w.Write(c)
1793 if err != nil {
1794 t.Fatalf("w.Write: %v", err)
1795 }
1796 written += n
1797 }
1798 if err := w.Close(); err != nil {
1799 t.Fatalf("w.Close: %v", err)
1800 }
1801 t.Cleanup(func() {
1802 h.mustDeleteObject(obj)
1803 })
1804
1805 attrs, err := obj.Attrs(ctx)
1806 if err != nil {
1807 t.Fatalf("obj.Attrs: %v", err)
1808 }
1809
1810 crc32c := attrs.CRC32C
1811
1812 type copierAttrs struct {
1813 contentEncoding string
1814 maxBytesPerCall int64
1815 }
1816
1817 for _, test := range []struct {
1818 desc string
1819 toObj string
1820 toBucket *BucketHandle
1821 copierAttrs *copierAttrs
1822 numExpectedRewriteCalls int
1823 }{
1824 {
1825 desc: "copy within bucket",
1826 toObj: "copy-within-bucket",
1827 toBucket: bucketFrom,
1828 numExpectedRewriteCalls: 1,
1829 },
1830 {
1831 desc: "copy to new bucket",
1832 toObj: "copy-new-bucket",
1833 toBucket: bucketInSameRegion,
1834 numExpectedRewriteCalls: 1,
1835 },
1836 {
1837 desc: "copy with attributes",
1838 toObj: "copy-with-attributes",
1839 toBucket: bucketInSameRegion,
1840 copierAttrs: &copierAttrs{contentEncoding: "identity"},
1841 numExpectedRewriteCalls: 1,
1842 },
1843 {
1844
1845
1846 desc: "copy to new region",
1847 toObj: "copy-new-region",
1848 toBucket: bucketInDifferentRegion,
1849 copierAttrs: &copierAttrs{maxBytesPerCall: 1048576},
1850 numExpectedRewriteCalls: 3,
1851 },
1852 } {
1853 t.Run(test.desc, func(t *testing.T) {
1854 copyObj := test.toBucket.Object(test.toObj)
1855 copier := copyObj.CopierFrom(obj)
1856
1857 if attrs := test.copierAttrs; attrs != nil {
1858 if attrs.contentEncoding != "" {
1859 copier.ContentEncoding = attrs.contentEncoding
1860 }
1861 if attrs.maxBytesPerCall != 0 {
1862 copier.maxBytesRewrittenPerCall = attrs.maxBytesPerCall
1863 }
1864 }
1865
1866 rewriteCallsCount := 0
1867 copier.ProgressFunc = func(_, _ uint64) {
1868 rewriteCallsCount++
1869 }
1870
1871 attrs, err = copier.Run(ctx)
1872 if err != nil {
1873 t.Fatalf("Copier.Run failed with %v", err)
1874 }
1875 t.Cleanup(func() {
1876 h.mustDeleteObject(copyObj)
1877 })
1878
1879
1880 if attrs.Bucket != test.toBucket.name || attrs.Name != test.toObj {
1881 t.Errorf("unexpected copy behaviour: got: %s in bucket %s, want: %s in bucket %s", attrs.Name, attrs.Bucket, attrs.Name, test.toBucket.name)
1882 }
1883
1884
1885 if test.copierAttrs != nil {
1886 if attrs.ContentEncoding != test.copierAttrs.contentEncoding {
1887 t.Errorf("unexpected ContentEncoding; got: %s, want: %s", attrs.ContentEncoding, test.copierAttrs.contentEncoding)
1888 }
1889 }
1890
1891
1892 if attrs.CRC32C != crc32c {
1893 t.Errorf("mismatching checksum: got %v, want %v", attrs.CRC32C, crc32c)
1894 }
1895
1896
1897 if rewriteCallsCount != test.numExpectedRewriteCalls {
1898 t.Errorf("unexpected number of rewrite calls: got %v, want %v", rewriteCallsCount, test.numExpectedRewriteCalls)
1899 }
1900 })
1901 }
1902 })
1903 }
1904
1905 func TestIntegration_Encoding(t *testing.T) {
1906 multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
1907 bkt := client.Bucket(bucket)
1908
1909
1910 const zeroCount = 20 << 1
1911 obj := bkt.Object("gzip-test")
1912 w := obj.NewWriter(ctx)
1913 w.ContentEncoding = "gzip"
1914 gw := gzip.NewWriter(w)
1915 if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil {
1916 t.Fatalf("io.Copy, upload: %v", err)
1917 }
1918 if err := gw.Close(); err != nil {
1919 t.Errorf("gzip.Close(): %v", err)
1920 }
1921 if err := w.Close(); err != nil {
1922 t.Errorf("w.Close(): %v", err)
1923 }
1924 r, err := obj.NewReader(ctx)
1925 if err != nil {
1926 t.Fatalf("NewReader(gzip-test): %v", err)
1927 }
1928 n, err := io.Copy(io.Discard, r)
1929 if err != nil {
1930 t.Errorf("io.Copy, download: %v", err)
1931 }
1932 if n != zeroCount {
1933 t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount)
1934 }
1935
1936
1937 _, err = bkt.Object("obj-not-exists").NewReader(ctx)
1938 if err != ErrObjectNotExist {
1939 t.Errorf("Object should not exist, err found to be %v", err)
1940 }
1941 })
1942 }
1943
1944 func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) {
1945 ctx := context.Background()
1946 h := testHelper{t}
1947
1948 names := make([]string, len(objects))
1949 copy(names, objects)
1950 sort.Strings(names)
1951 var attrs []*ObjectAttrs
1952 for _, name := range names {
1953 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name)))
1954 }
1955 msg, ok := itesting.TestIterator(attrs,
1956 func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) },
1957 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
1958 if !ok {
1959 t.Errorf("ObjectIterator.Next: %s", msg)
1960 }
1961
1962 }
1963
1964 func testObjectIteratorWithOffset(t *testing.T, bkt *BucketHandle, objects []string) {
1965 ctx := context.Background()
1966 h := testHelper{t}
1967
1968 names := make([]string, len(objects))
1969 copy(names, objects)
1970 sort.Strings(names)
1971 var attrs []*ObjectAttrs
1972 for _, name := range names {
1973 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name)))
1974 }
1975 m := make(map[string][]*ObjectAttrs)
1976 for i, name := range names {
1977
1978
1979
1980
1981 m[name] = attrs[i:]
1982 msg, ok := itesting.TestIterator(m[name],
1983 func() interface{} { return bkt.Objects(ctx, &Query{StartOffset: name}) },
1984 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
1985 if !ok {
1986 t.Errorf("ObjectIterator.Next: %s", msg)
1987 }
1988
1989
1990
1991
1992 m[name] = attrs[:i]
1993 msg, ok = itesting.TestIterator(m[name],
1994 func() interface{} { return bkt.Objects(ctx, &Query{EndOffset: name}) },
1995 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
1996 if !ok {
1997 t.Errorf("ObjectIterator.Next: %s", msg)
1998 }
1999 }
2000 }
2001
2002 func testObjectsIterateSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) {
2003
2004
2005 query := &Query{Prefix: ""}
2006 query.SetAttrSelection([]string{"Name"})
2007
2008 var gotNames []string
2009 it := bkt.Objects(context.Background(), query)
2010 for {
2011 attrs, err := it.Next()
2012 if err == iterator.Done {
2013 break
2014 }
2015 if err != nil {
2016 t.Fatalf("iterator.Next: %v", err)
2017 }
2018 gotNames = append(gotNames, attrs.Name)
2019
2020 if len(attrs.Bucket) > 0 {
2021 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
2022 }
2023 }
2024
2025 sortedNames := make([]string, len(objects))
2026 copy(sortedNames, objects)
2027 sort.Strings(sortedNames)
2028 sort.Strings(gotNames)
2029
2030 if !cmp.Equal(sortedNames, gotNames) {
2031 t.Errorf("names = %v, want %v", gotNames, sortedNames)
2032 }
2033 }
2034
2035 func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) {
2036
2037
2038 query := &Query{
2039 Prefix: "",
2040 StartOffset: "obj/",
2041 EndOffset: "obj2",
2042 }
2043 var selectedAttrs []string
2044 for k := range attrToFieldMap {
2045 selectedAttrs = append(selectedAttrs, k)
2046 }
2047 query.SetAttrSelection(selectedAttrs)
2048
2049 count := 0
2050 it := bkt.Objects(context.Background(), query)
2051 for {
2052 _, err := it.Next()
2053 if err == iterator.Done {
2054 break
2055 }
2056 if err != nil {
2057 t.Fatalf("iterator.Next: %v", err)
2058 }
2059 count++
2060 }
2061
2062 if count != len(objects)-1 {
2063 t.Errorf("count = %v, want %v", count, len(objects)-1)
2064 }
2065 }
2066
2067 func testObjectsIterateWithProjection(t *testing.T, bkt *BucketHandle) {
2068 projections := map[Projection]bool{
2069 ProjectionDefault: true,
2070 ProjectionFull: true,
2071 ProjectionNoACL: false,
2072 }
2073
2074 for projection, expectACL := range projections {
2075 query := &Query{Projection: projection}
2076 it := bkt.Objects(context.Background(), query)
2077 attrs, err := it.Next()
2078 if err == iterator.Done {
2079 t.Fatalf("iterator: no objects")
2080 }
2081 if err != nil {
2082 t.Fatalf("iterator.Next: %v", err)
2083 }
2084
2085 if expectACL {
2086 if attrs.Owner == "" {
2087 t.Errorf("projection %q: Owner is empty, want nonempty Owner", projection)
2088 }
2089 if len(attrs.ACL) == 0 {
2090 t.Errorf("projection %q: ACL is empty, want at least one ACL rule", projection)
2091 }
2092 } else {
2093 if attrs.Owner != "" {
2094 t.Errorf("projection %q: got Owner = %q, want empty Owner", projection, attrs.Owner)
2095 }
2096 if len(attrs.ACL) != 0 {
2097 t.Errorf("projection %q: got %d ACL rules, want empty ACL", projection, len(attrs.ACL))
2098 }
2099 }
2100 }
2101 }
2102
2103 func TestIntegration_SignedURL(t *testing.T) {
2104 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
2105
2106
2107 jwtConf, err := testutil.JWTConfig()
2108 if err != nil {
2109 t.Fatal(err)
2110 }
2111 if jwtConf == nil {
2112 t.Skip("JSON key file is not present")
2113 }
2114
2115 bkt := client.Bucket(bucket)
2116 obj := "signedURL"
2117 contents := []byte("This is a test of SignedURL.\n")
2118 md5 := "Jyxvgwm9n2MsrGTMPbMeYA=="
2119 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil {
2120 t.Fatalf("writing: %v", err)
2121 }
2122 for _, test := range []struct {
2123 desc string
2124 opts SignedURLOptions
2125 headers map[string][]string
2126 fail bool
2127 }{
2128 {
2129 desc: "basic v2",
2130 },
2131 {
2132 desc: "basic v4",
2133 opts: SignedURLOptions{Scheme: SigningSchemeV4},
2134 },
2135 {
2136 desc: "MD5 sent and matches",
2137 opts: SignedURLOptions{MD5: md5},
2138 headers: map[string][]string{"Content-MD5": {md5}},
2139 },
2140 {
2141 desc: "MD5 not sent",
2142 opts: SignedURLOptions{MD5: md5},
2143 fail: true,
2144 },
2145 {
2146 desc: "Content-Type sent and matches",
2147 opts: SignedURLOptions{ContentType: "text/plain"},
2148 headers: map[string][]string{"Content-Type": {"text/plain"}},
2149 },
2150 {
2151 desc: "Content-Type sent but does not match",
2152 opts: SignedURLOptions{ContentType: "text/plain"},
2153 headers: map[string][]string{"Content-Type": {"application/json"}},
2154 fail: true,
2155 },
2156 {
2157 desc: "Canonical headers sent and match",
2158 opts: SignedURLOptions{Headers: []string{
2159 " X-Goog-Foo: Bar baz ",
2160 "X-Goog-Novalue",
2161 "X-Google-Foo",
2162 "x-goog-meta-start-time: 2023-02-10T02:00:00Z",
2163 }},
2164 headers: map[string][]string{"X-Goog-foo": {"Bar baz "}, "x-goog-meta-start-time": {"2023-02-10T02:00:00Z"}},
2165 },
2166 {
2167 desc: "Canonical headers sent and match using V4",
2168 opts: SignedURLOptions{Headers: []string{
2169 "x-goog-meta-start-time: 2023-02-10T02:",
2170 " X-Goog-Foo: Bar baz ",
2171 "X-Goog-Novalue",
2172 "X-Google-Foo",
2173 },
2174 Scheme: SigningSchemeV4,
2175 },
2176 headers: map[string][]string{"x-goog-meta-start-time": {"2023-02-10T02:"}, "X-Goog-foo": {"Bar baz "}},
2177 },
2178 {
2179 desc: "Canonical headers sent but don't match",
2180 opts: SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}},
2181 headers: map[string][]string{"X-Goog-Foo": {"bar baz"}},
2182 fail: true,
2183 },
2184 {
2185 desc: "Virtual hosted style with custom hostname",
2186 opts: SignedURLOptions{
2187 Style: VirtualHostedStyle(),
2188 Hostname: "storage.googleapis.com:443",
2189 },
2190 fail: false,
2191 },
2192 {
2193 desc: "Hostname v4",
2194 opts: SignedURLOptions{
2195 Hostname: "storage.googleapis.com:443",
2196 Scheme: SigningSchemeV4,
2197 },
2198 fail: false,
2199 },
2200 } {
2201 opts := test.opts
2202 opts.GoogleAccessID = jwtConf.Email
2203 opts.PrivateKey = jwtConf.PrivateKey
2204 opts.Method = "GET"
2205 opts.Expires = time.Now().Add(time.Hour)
2206
2207 u, err := bkt.SignedURL(obj, &opts)
2208 if err != nil {
2209 t.Errorf("%s: SignedURL: %v", test.desc, err)
2210 continue
2211 }
2212
2213 err = verifySignedURL(u, test.headers, contents)
2214 if err != nil && !test.fail {
2215 t.Errorf("%s: wanted success but got error:\n%v", test.desc, err)
2216 } else if err == nil && test.fail {
2217 t.Errorf("%s: wanted failure but test succeeded", test.desc)
2218 }
2219 }
2220 })
2221 }
2222
2223 func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
2224 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
2225
2226
2227
2228 jwtConf, err := testutil.JWTConfig()
2229 if err != nil {
2230 t.Fatal(err)
2231 }
2232 if jwtConf == nil {
2233 t.Skip("JSON key file is not present")
2234 }
2235
2236 bkt := client.Bucket(bucket)
2237
2238
2239 encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI="
2240 encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k="
2241 headers := map[string][]string{
2242 "x-goog-encryption-algorithm": {"AES256"},
2243 "x-goog-encryption-key": {encryptionKey},
2244 "x-goog-encryption-key-sha256": {encryptionKeySha256},
2245 }
2246 contents := []byte(`{"message":"encryption with csek works"}`)
2247 tests := []struct {
2248 desc string
2249 opts *SignedURLOptions
2250 }{
2251 {
2252 desc: "v4 URL with customer supplied encryption keys for PUT",
2253 opts: &SignedURLOptions{
2254 Method: "PUT",
2255 Headers: []string{
2256 "x-goog-encryption-algorithm:AES256",
2257 "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=",
2258 "x-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=",
2259 },
2260 Scheme: SigningSchemeV4,
2261 },
2262 },
2263 {
2264 desc: "v4 URL with customer supplied encryption keys for GET",
2265 opts: &SignedURLOptions{
2266 Method: "GET",
2267 Headers: []string{
2268 "x-goog-encryption-algorithm:AES256",
2269 fmt.Sprintf("x-goog-encryption-key:%s", encryptionKey),
2270 fmt.Sprintf("x-goog-encryption-key-sha256:%s", encryptionKeySha256),
2271 },
2272 Scheme: SigningSchemeV4,
2273 },
2274 },
2275 }
2276 defer func() {
2277
2278 err := bkt.Object("csek.json").Delete(ctx)
2279 if err != nil {
2280 log.Printf("failed to deleted encrypted file: %v", err)
2281 }
2282 }()
2283
2284 for _, test := range tests {
2285 opts := test.opts
2286 opts.GoogleAccessID = jwtConf.Email
2287 opts.PrivateKey = jwtConf.PrivateKey
2288 opts.Expires = time.Now().Add(time.Hour)
2289
2290 u, err := bkt.SignedURL("csek.json", test.opts)
2291 if err != nil {
2292 t.Fatalf("%s: %v", test.desc, err)
2293 }
2294
2295 if test.opts.Method == "PUT" {
2296 if _, err := putURL(u, headers, bytes.NewReader(contents)); err != nil {
2297 t.Fatalf("%s: %v", test.desc, err)
2298 }
2299 }
2300
2301 if test.opts.Method == "GET" {
2302 if err := verifySignedURL(u, headers, contents); err != nil {
2303 t.Fatalf("%s: %v", test.desc, err)
2304 }
2305 }
2306 }
2307 })
2308 }
2309
2310 func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) {
2311 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
2312
2313
2314
2315 jwtConf, err := testutil.JWTConfig()
2316 if err != nil {
2317 t.Fatal(err)
2318 }
2319 if jwtConf == nil {
2320 t.Skip("JSON key file is not present")
2321 }
2322
2323 opts := &SignedURLOptions{
2324 Scheme: SigningSchemeV4,
2325 Method: "GET",
2326 GoogleAccessID: jwtConf.Email,
2327 PrivateKey: jwtConf.PrivateKey,
2328 Expires: time.Now().Add(time.Hour),
2329 }
2330
2331 bkt := client.Bucket(bucket)
2332 u, err := bkt.SignedURL("", opts)
2333 if err != nil {
2334 t.Fatal(err)
2335 }
2336
2337
2338 _, err = getURL(u, nil)
2339 if err != nil {
2340 t.Fatal(err)
2341 }
2342 })
2343
2344 }
2345
2346 func TestIntegration_BucketACL(t *testing.T) {
2347 ctx := skipJSONReads(context.Background(), "no reads in test")
2348 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
2349 h := testHelper{t}
2350
2351 bucket := prefix + uidSpace.New()
2352 bkt := client.Bucket(bucket)
2353 h.mustCreate(bkt, testutil.ProjID(), nil)
2354 defer h.mustDeleteBucket(bkt)
2355
2356 entity := ACLEntity("domain-google.com")
2357 rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"}
2358
2359 if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil {
2360 t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err)
2361 }
2362
2363 acl, err := bkt.DefaultObjectACL().List(ctx)
2364 if err != nil {
2365 t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucket, err)
2366 }
2367 if !containsACLRule(acl, testACLRule(rule)) {
2368 t.Fatalf("default ACL rule missing; want: %#v, got rules: %+v", rule, acl)
2369 }
2370
2371 o := bkt.Object("acl1")
2372 defer h.mustDeleteObject(o)
2373
2374
2375 err = retry(ctx, func() error {
2376 if err := writeObject(ctx, o, "", randomContents()); err != nil {
2377 return fmt.Errorf("Write for %v failed with %v", o.ObjectName(), err)
2378 }
2379 acl, err = o.ACL().List(ctx)
2380 return err
2381 }, func() error {
2382 if !containsACLRule(acl, testACLRule(rule)) {
2383 return fmt.Errorf("object ACL rule missing %+v from ACL \n%+v", rule, acl)
2384 }
2385 return nil
2386 })
2387 if err != nil {
2388 t.Error(err)
2389 }
2390
2391 if err := o.ACL().Delete(ctx, entity); err != nil {
2392 t.Errorf("object ACL: could not delete entity %s", entity)
2393 }
2394
2395
2396
2397 if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil {
2398 t.Errorf("default ACL: could not delete entity %s", entity)
2399 }
2400
2401 entity2 := AllAuthenticatedUsers
2402 rule2 := ACLRule{Entity: entity2, Role: RoleReader}
2403 if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil {
2404 t.Errorf("Error while putting bucket ACL rule: %v", err)
2405 }
2406
2407 var bACL []ACLRule
2408
2409
2410 err = retry(ctx, func() error {
2411 bACL, err = bkt.ACL().List(ctx)
2412 return err
2413 }, func() error {
2414 if !containsACLRule(bACL, testACLRule(rule2)) {
2415 return fmt.Errorf("bucket ACL missing %+v", rule2)
2416 }
2417 return nil
2418 })
2419 if err != nil {
2420 t.Error(err)
2421 }
2422
2423 if err := bkt.ACL().Delete(ctx, entity2); err != nil {
2424 t.Errorf("Error while deleting bucket ACL rule: %v", err)
2425 }
2426 })
2427 }
2428
2429 func TestIntegration_ValidObjectNames(t *testing.T) {
2430 ctx := skipJSONReads(context.Background(), "no reads in test")
2431 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2432 bkt := client.Bucket(bucket)
2433
2434 validNames := []string{
2435 "gopher",
2436 "Гоферови",
2437 "a",
2438 strings.Repeat("a", 1024),
2439 }
2440 for _, name := range validNames {
2441 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil {
2442 t.Errorf("Object %q write failed: %v. Want success", name, err)
2443 continue
2444 }
2445 defer bkt.Object(name).Delete(ctx)
2446 }
2447
2448 invalidNames := []string{
2449 "",
2450 strings.Repeat("a", 1025),
2451 "new\nlines",
2452 "bad\xffunicode",
2453 }
2454 for _, name := range invalidNames {
2455
2456 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil {
2457 continue
2458 }
2459 defer bkt.Object(name).Delete(ctx)
2460 t.Errorf("%q should have failed. Didn't", name)
2461 }
2462 })
2463 }
2464
2465 func TestIntegration_WriterContentType(t *testing.T) {
2466 ctx := skipJSONReads(context.Background(), "no reads in test")
2467 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2468 obj := client.Bucket(bucket).Object("content")
2469 testCases := []struct {
2470 content string
2471 setType, wantType string
2472 forceEmptyContentType bool
2473 }{
2474 {
2475
2476 content: "It was the best of times, it was the worst of times.",
2477 wantType: "text/plain; charset=utf-8",
2478 },
2479 {
2480
2481 content: "<html><head><title>My first page</title></head></html>",
2482 wantType: "text/html; charset=utf-8",
2483 },
2484 {
2485 content: "<html><head><title>My first page</title></head></html>",
2486 setType: "text/html",
2487 wantType: "text/html",
2488 },
2489 {
2490 content: "<html><head><title>My first page</title></head></html>",
2491 setType: "image/jpeg",
2492 wantType: "image/jpeg",
2493 },
2494 {
2495
2496 content: "<html><head><title>My first page</title></head></html>",
2497 setType: "",
2498 wantType: "",
2499 forceEmptyContentType: true,
2500 },
2501 }
2502 for i, tt := range testCases {
2503 writer := newWriter(ctx, obj, tt.setType, tt.forceEmptyContentType)
2504 if err := writeContents(writer, []byte(tt.content)); err != nil {
2505 t.Errorf("writing #%d: %v", i, err)
2506 }
2507 attrs, err := obj.Attrs(ctx)
2508 if err != nil {
2509 t.Errorf("obj.Attrs: %v", err)
2510 continue
2511 }
2512 if got := attrs.ContentType; got != tt.wantType {
2513 t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType)
2514 }
2515 }
2516 })
2517 }
2518
2519 func TestIntegration_WriterChunksize(t *testing.T) {
2520 ctx := skipJSONReads(context.Background(), "no reads in test")
2521 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2522 obj := client.Bucket(bucket).Object("writer-chunksize-test" + uidSpaceObjects.New())
2523 objSize := 1<<10<<10 + 1
2524 contents := bytes.Repeat([]byte("a"), objSize)
2525
2526 for _, test := range []struct {
2527 desc string
2528 chunksize int
2529 wantBytesPerCall int64
2530 wantCallbacks int
2531 }{
2532 {
2533 desc: "default chunksize",
2534 chunksize: 16 << 10 << 10,
2535 wantBytesPerCall: 16 << 10 << 10,
2536 wantCallbacks: 0,
2537 },
2538 {
2539 desc: "small chunksize rounds up to 256kib",
2540 chunksize: 1,
2541 wantBytesPerCall: 256 << 10,
2542 wantCallbacks: 5,
2543 },
2544 {
2545 desc: "chunksize of 256kib",
2546 chunksize: 256 << 10,
2547 wantBytesPerCall: 256 << 10,
2548 wantCallbacks: 5,
2549 },
2550 {
2551 desc: "chunksize of just over 256kib rounds up",
2552 chunksize: 256<<10 + 1,
2553 wantBytesPerCall: 256 * 2 << 10,
2554 wantCallbacks: 3,
2555 },
2556 {
2557 desc: "multiple of 256kib",
2558 chunksize: 256 * 3 << 10,
2559 wantBytesPerCall: 256 * 3 << 10,
2560 wantCallbacks: 2,
2561 },
2562 {
2563 desc: "chunksize 0 uploads everything",
2564 chunksize: 0,
2565 wantBytesPerCall: int64(objSize),
2566 wantCallbacks: 0,
2567 },
2568 } {
2569 t.Run(test.desc, func(t *testing.T) {
2570 t.Cleanup(func() { obj.Delete(ctx) })
2571
2572 w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
2573 w.ChunkSize = test.chunksize
2574
2575 bytesWrittenSoFar := int64(0)
2576 callbacks := 0
2577
2578 w.ProgressFunc = func(i int64) {
2579 bytesWrittenByCall := i - bytesWrittenSoFar
2580
2581
2582 if i != int64(objSize) && bytesWrittenByCall != test.wantBytesPerCall {
2583 t.Errorf("unexpected number of bytes written by call; wanted: %d, written: %d", test.wantBytesPerCall, bytesWrittenByCall)
2584 }
2585
2586 bytesWrittenSoFar = i
2587 callbacks++
2588 }
2589
2590 if _, err := w.Write(contents); err != nil {
2591 _ = w.Close()
2592 t.Fatalf("writer.Write: %v", err)
2593 }
2594 if err := w.Close(); err != nil {
2595 t.Fatalf("writer.Close: %v", err)
2596 }
2597
2598 if callbacks != test.wantCallbacks {
2599 t.Errorf("ProgressFunc was called %d times, expected %d", callbacks, test.wantCallbacks)
2600 }
2601
2602
2603 attrs, err := obj.Attrs(ctx)
2604 if err != nil {
2605 t.Fatalf("obj.Attrs: %v", err)
2606 }
2607 if attrs.Size != int64(objSize) {
2608 t.Errorf("incorrect number of bytes written; got %v, want %v", attrs.Size, objSize)
2609 }
2610 })
2611 }
2612 })
2613 }
2614
2615 func TestIntegration_ZeroSizedObject(t *testing.T) {
2616 t.Parallel()
2617 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2618 obj := client.Bucket(bucket).Object("zero")
2619
2620
2621 w := obj.NewWriter(ctx)
2622 if err := w.Close(); err != nil {
2623 t.Fatalf("Writer.Close: %v", err)
2624 }
2625 defer obj.Delete(ctx)
2626
2627
2628 for _, c := range readCases {
2629 t.Run(c.desc, func(t *testing.T) {
2630 r, err := obj.NewReader(ctx)
2631 if err != nil {
2632 t.Fatalf("NewReader: %v", err)
2633 }
2634 body, err := c.readFunc(r)
2635 if err != nil {
2636 t.Fatalf("reading object: %v", err)
2637 }
2638 if len(body) != 0 {
2639 t.Errorf("Body is %v, want empty []byte{}", body)
2640 }
2641 })
2642 }
2643 })
2644 }
2645
2646 func TestIntegration_Encryption(t *testing.T) {
2647
2648
2649
2650 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2651 h := testHelper{t}
2652
2653 obj := client.Bucket(bucket).Object("customer-encryption")
2654 key := []byte("my-secret-AES-256-encryption-key")
2655 keyHash := sha256.Sum256(key)
2656 keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:])
2657 key2 := []byte("My-Secret-AES-256-Encryption-Key")
2658 contents := "top secret."
2659
2660 checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) {
2661
2662 attrs, err := f(obj)
2663 if err != nil {
2664 t.Fatalf("%s: %v", msg, err)
2665 }
2666
2667 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want {
2668 t.Errorf("%s: key hash: got %q, want %q", msg, got, want)
2669 }
2670
2671 if attrs.CRC32C != 0 {
2672 t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C)
2673 }
2674 if len(attrs.MD5) > 0 {
2675 t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5)
2676 }
2677
2678
2679 attrs, err = f(obj.Key(key))
2680 if err != nil {
2681 t.Fatalf("%s: %v", msg, err)
2682 }
2683
2684 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want {
2685 t.Errorf("%s: key hash: got %q, want %q", msg, got, want)
2686 }
2687 if attrs.CRC32C == 0 {
2688 t.Errorf("%s: CRC: got 0, want non-zero", msg)
2689 }
2690 if len(attrs.MD5) == 0 {
2691 t.Errorf("%s: MD5: got len == 0, want len > 0", msg)
2692 }
2693 }
2694
2695 checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) {
2696
2697 if _, err := readObject(ctx, o); err == nil {
2698 t.Errorf("%s: reading without key: want error, got nil", msg)
2699 }
2700
2701 got := h.mustRead(o.Key(k))
2702 gotContents := string(got)
2703
2704 if gotContents != wantContents {
2705 t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents)
2706 }
2707 }
2708
2709 checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) {
2710 got := h.mustRead(obj)
2711 gotContents := string(got)
2712 if gotContents != wantContents {
2713 t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents)
2714 }
2715 }
2716
2717
2718
2719 h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents))
2720
2721 checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) {
2722 return o.Attrs(ctx)
2723 })
2724
2725 checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) {
2726 return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
2727 })
2728
2729 checkRead("first object", obj, key, contents)
2730
2731
2732
2733 obj2 := client.Bucket(bucket).Object("customer-encryption-2")
2734 obj4 := client.Bucket(bucket).Object("customer-encryption-4")
2735
2736
2737 if _, err := obj4.CopierFrom(obj).Run(ctx); err == nil {
2738 t.Fatal("want error, got nil")
2739 }
2740
2741 if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil {
2742 t.Fatal(err)
2743 }
2744
2745 checkReadUnencrypted("copy dest", obj2, contents)
2746
2747
2748
2749 if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil {
2750 t.Fatal("want error, got nil")
2751 }
2752
2753
2754 if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil {
2755 t.Fatal(err)
2756 }
2757
2758
2759 checkRead("copy destination", obj2, key2, contents)
2760
2761
2762
2763
2764 if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil {
2765 t.Fatal(err)
2766 }
2767 obj3 := client.Bucket(bucket).Object("customer-encryption-3")
2768
2769 if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil {
2770 t.Fatal("want error, got nil")
2771 }
2772
2773 if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil {
2774 t.Fatal("want error, got nil")
2775 }
2776
2777
2778 if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil {
2779 t.Fatalf("got %v, want nil", err)
2780 }
2781
2782 checkRead("compose destination", obj3, key, contents+contents)
2783
2784
2785
2786 _, err := obj4.CopierFrom(obj2.Key(key)).Run(ctx)
2787 if err != nil {
2788 t.Fatal(err)
2789 }
2790 if _, err := obj3.Key(key).ComposerFrom(obj4).Run(ctx); err == nil {
2791 t.Fatal("got nil, want error")
2792 }
2793 })
2794 }
2795
2796 func TestIntegration_NonexistentObjectRead(t *testing.T) {
2797 t.Parallel()
2798 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2799 _, err := client.Bucket(bucket).Object("object-does-not-exist").NewReader(ctx)
2800 if !errors.Is(err, ErrObjectNotExist) {
2801 t.Errorf("Objects: got %v, want ErrObjectNotExist", err)
2802 }
2803 })
2804 }
2805
2806 func TestIntegration_NonexistentBucket(t *testing.T) {
2807 t.Parallel()
2808 ctx := skipJSONReads(context.Background(), "no reads in test")
2809 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
2810 bkt := client.Bucket(prefix + uidSpace.New())
2811 if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist {
2812 t.Errorf("Attrs: got %v, want ErrBucketNotExist", err)
2813 }
2814 it := bkt.Objects(ctx, nil)
2815 if _, err := it.Next(); err != ErrBucketNotExist {
2816 t.Errorf("Objects: got %v, want ErrBucketNotExist", err)
2817 }
2818 })
2819 }
2820
2821 func TestIntegration_PerObjectStorageClass(t *testing.T) {
2822 const (
2823 defaultStorageClass = "STANDARD"
2824 newStorageClass = "NEARLINE"
2825 )
2826 ctx := skipJSONReads(context.Background(), "no reads in test")
2827
2828 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2829 h := testHelper{t}
2830
2831 bkt := client.Bucket(bucket)
2832
2833
2834 battrs := h.mustBucketAttrs(bkt)
2835 if battrs.StorageClass != defaultStorageClass {
2836 t.Fatalf("bucket storage class: got %q, want %q",
2837 battrs.StorageClass, defaultStorageClass)
2838 }
2839
2840 obj := bkt.Object("posc")
2841 h.mustWrite(obj.NewWriter(ctx), []byte("foo"))
2842 oattrs, err := obj.Attrs(ctx)
2843 if err != nil {
2844 t.Fatal(err)
2845 }
2846 if oattrs.StorageClass != defaultStorageClass {
2847 t.Fatalf("object storage class: got %q, want %q",
2848 oattrs.StorageClass, defaultStorageClass)
2849 }
2850
2851 copier := obj.CopierFrom(obj)
2852 copier.StorageClass = newStorageClass
2853 oattrs2, err := copier.Run(ctx)
2854 if err != nil {
2855 log.Fatal(err)
2856 }
2857 if oattrs2.StorageClass != newStorageClass {
2858 t.Fatalf("new object storage class: got %q, want %q",
2859 oattrs2.StorageClass, newStorageClass)
2860 }
2861
2862
2863 obj2 := bkt.Object("posc2")
2864 w := obj2.NewWriter(ctx)
2865 w.StorageClass = newStorageClass
2866 h.mustWrite(w, []byte("xxx"))
2867 if w.Attrs().StorageClass != newStorageClass {
2868 t.Fatalf("new object storage class: got %q, want %q",
2869 w.Attrs().StorageClass, newStorageClass)
2870 }
2871 })
2872 }
2873
2874 func TestIntegration_NoUnicodeNormalization(t *testing.T) {
2875 t.Parallel()
2876 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2877 bkt := client.Bucket(bucket)
2878 h := testHelper{t}
2879
2880 for _, tst := range []struct {
2881 nameQuoted, content string
2882 }{
2883 {`"Caf\u00e9"`, "Normalization Form C"},
2884 {`"Cafe\u0301"`, "Normalization Form D"},
2885 } {
2886 name, err := strconv.Unquote(tst.nameQuoted)
2887 w := bkt.Object(name).NewWriter(ctx)
2888 h.mustWrite(w, []byte(tst.content))
2889 if err != nil {
2890 t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err)
2891 }
2892 if got := string(h.mustRead(bkt.Object(name))); got != tst.content {
2893 t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content)
2894 }
2895 }
2896 })
2897 }
2898
2899 func TestIntegration_HashesOnUpload(t *testing.T) {
2900
2901 ctx := skipJSONReads(context.Background(), "no reads in test")
2902 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
2903 obj := client.Bucket(bucket).Object("hashesOnUpload-1")
2904 data := []byte("I can't wait to be verified")
2905
2906 write := func(w *Writer) error {
2907 if _, err := w.Write(data); err != nil {
2908 _ = w.Close()
2909 return err
2910 }
2911 return w.Close()
2912 }
2913
2914 crc32c := crc32.Checksum(data, crc32cTable)
2915
2916 w := obj.NewWriter(ctx)
2917 w.CRC32C = crc32c
2918 w.SendCRC32C = true
2919 if err := write(w); err != nil {
2920 t.Error(err)
2921 }
2922
2923
2924 w = obj.NewWriter(ctx)
2925 w.CRC32C = crc32c + 1
2926 w.SendCRC32C = true
2927 if err := write(w); err == nil {
2928 t.Error("write with bad CRC32c: want error, got nil")
2929 }
2930
2931
2932 w = obj.NewWriter(ctx)
2933 w.CRC32C = crc32c + 1
2934 if err := write(w); err != nil {
2935 t.Error(err)
2936 }
2937
2938
2939 md5 := md5.Sum(data)
2940
2941 w = obj.NewWriter(ctx)
2942 w.MD5 = md5[:]
2943 if err := write(w); err != nil {
2944 t.Error(err)
2945 }
2946
2947
2948 w = obj.NewWriter(ctx)
2949 w.MD5 = append([]byte(nil), md5[:]...)
2950 w.MD5[0]++
2951 if err := write(w); err == nil {
2952 t.Error("write with bad MD5: want error, got nil")
2953 }
2954 })
2955 }
2956
2957 func TestIntegration_BucketIAM(t *testing.T) {
2958 ctx := skipJSONReads(context.Background(), "no reads in test")
2959 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
2960 h := testHelper{t}
2961 bkt := client.Bucket(prefix + uidSpace.New())
2962 h.mustCreate(bkt, testutil.ProjID(), nil)
2963 defer h.mustDeleteBucket(bkt)
2964
2965
2966
2967
2968 member := "projectViewer:" + testutil.ProjID()
2969 role := iam.RoleName("roles/storage.objectViewer")
2970
2971 policy, err := bkt.IAM().Policy(ctx)
2972 if err != nil {
2973 t.Fatalf("Getting policy: %v", err)
2974 }
2975
2976 if policy.HasRole(member, role) {
2977 t.Errorf("member %q has role %q", member, role)
2978 }
2979
2980 policy.Add(member, role)
2981 if err := bkt.IAM().SetPolicy(ctx, policy); err != nil {
2982 t.Fatalf("SetPolicy: %v", err)
2983 }
2984
2985 policy, err = bkt.IAM().Policy(ctx)
2986 if err != nil {
2987 t.Fatalf("Getting policy: %v", err)
2988 }
2989 if !policy.HasRole(member, role) {
2990 t.Errorf("member %q does not have role %q", member, role)
2991 }
2992
2993
2994
2995 perms := []string{"storage.buckets.get", "storage.buckets.delete"}
2996 got, err := bkt.IAM().TestPermissions(ctx, perms)
2997 if err != nil {
2998 t.Fatalf("TestPermissions: %v", err)
2999 }
3000 sort.Strings(perms)
3001 sort.Strings(got)
3002 if !testutil.Equal(got, perms) {
3003 t.Errorf("got %v, want %v", got, perms)
3004 }
3005 })
3006 }
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025 func TestIntegration_RequesterPaysOwner(t *testing.T) {
3026 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
3027 jwt, err := testutil.JWTConfig()
3028 if err != nil {
3029 t.Fatalf("testutil.JWTConfig: %v", err)
3030 }
3031
3032 mainUserEmail := jwt.Email
3033
3034
3035 mainProjectID := testutil.ProjID()
3036
3037 client.SetRetry(WithPolicy(RetryAlways))
3038
3039
3040
3041 secondaryProject := os.Getenv(envFirestoreProjID)
3042 if secondaryProject == "" {
3043 t.Fatalf("need a second project (env var %s)", envFirestoreProjID)
3044 }
3045
3046 for _, test := range []struct {
3047 desc string
3048 userProject *string
3049 expectSuccess bool
3050 }{
3051 {
3052 desc: "user is Owner on the project that owns the bucket",
3053 userProject: nil,
3054 expectSuccess: true,
3055 },
3056 {
3057 desc: "userProject is unnecessary but allowed",
3058 userProject: &mainProjectID,
3059 expectSuccess: true,
3060 },
3061 {
3062 desc: "cannot use someone else's project for billing",
3063 userProject: &secondaryProject,
3064 expectSuccess: false,
3065 },
3066 } {
3067 t.Run(test.desc, func(t *testing.T) {
3068 h := testHelper{t}
3069 ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
3070 defer cancel()
3071
3072 printTestCase := func() string {
3073 userProject := "none"
3074 if test.userProject != nil {
3075 userProject = *test.userProject
3076 }
3077 return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", mainUserEmail, mainProjectID, userProject)
3078 }
3079
3080 checkforErrors := func(desc string, err error) {
3081 if err != nil && test.expectSuccess {
3082 t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase())
3083 } else if err == nil && !test.expectSuccess {
3084 t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase())
3085 }
3086 }
3087
3088 bucketName := prefix + uidSpace.New()
3089 requesterPaysBucket := client.Bucket(bucketName)
3090
3091
3092 h.mustCreate(requesterPaysBucket, mainProjectID, &BucketAttrs{RequesterPays: true})
3093 t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) })
3094
3095
3096
3097
3098
3099
3100
3101 objectName := "acl-go-test" + uidSpaceObjects.New()
3102 h.mustWrite(requesterPaysBucket.Object(objectName).NewWriter(ctx), []byte("hello"))
3103
3104
3105 bucket := client.Bucket(bucketName)
3106 if test.userProject != nil {
3107 bucket = bucket.UserProject(*test.userProject)
3108 }
3109
3110
3111 attrs, err := bucket.Attrs(ctx)
3112 checkforErrors("get bucket attrs", err)
3113 if attrs != nil {
3114 if got, want := attrs.RequesterPays, true; got != want {
3115 t.Fatalf("attr.RequesterPays = %t, want %t", got, want)
3116 }
3117 }
3118
3119
3120 entity := ACLEntity("domain-google.com")
3121
3122 checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader))
3123 _, err = bucket.ACL().List(ctx)
3124 checkforErrors("bucket acl list", err)
3125 checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity))
3126
3127
3128
3129
3130 o := bucket.Object(objectName)
3131 ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*10)
3132 defer cancel()
3133
3134
3135 if test.expectSuccess {
3136 o = o.Retryer(WithErrorFunc(retryOnTransient400and403))
3137 }
3138 checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello")))
3139 _, err = readObject(ctx, bucket.Object(objectName))
3140 checkforErrors("read object", err)
3141 _, err = bucket.Object(objectName).Attrs(ctx)
3142 checkforErrors("get object attrs", err)
3143 _, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
3144 checkforErrors("update object", err)
3145
3146
3147 checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader))
3148 _, err = bucket.Object(objectName).ACL().List(ctx)
3149 checkforErrors("object acl list", err)
3150 checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity))
3151
3152
3153
3154 checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader))
3155 _, err = bucket.DefaultObjectACL().List(ctx)
3156 checkforErrors("default object acl list", err)
3157 checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity))
3158
3159
3160 _, err = bucket.Object("copy").CopierFrom(bucket.Object(objectName)).Run(ctx)
3161 checkforErrors("copy", err)
3162
3163 if err == nil {
3164 t.Cleanup(func() {
3165 h.mustDeleteObject(bucket.Object("copy"))
3166 })
3167 }
3168
3169
3170 _, err = bucket.Object("compose").ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx)
3171 checkforErrors("compose", err)
3172
3173 if err == nil {
3174 t.Cleanup(func() {
3175 h.mustDeleteObject(bucket.Object("compose"))
3176 })
3177 }
3178
3179
3180 if err = bucket.Object(objectName).Delete(ctx); err != nil {
3181
3182 h.mustDeleteObject(requesterPaysBucket.Object(objectName))
3183 }
3184 checkforErrors("delete object", err)
3185 })
3186 }
3187 })
3188 }
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208 func TestIntegration_RequesterPaysNonOwner(t *testing.T) {
3209 if testing.Short() && !replaying {
3210 t.Skip("Integration tests skipped in short mode")
3211 }
3212 ctx := context.Background()
3213
3214
3215 mainProject := testutil.ProjID()
3216
3217
3218
3219
3220 secondaryProject := os.Getenv(envFirestoreProjID)
3221 if secondaryProject == "" {
3222 t.Fatalf("need a second project (env var %s)", envFirestoreProjID)
3223 }
3224
3225
3226
3227
3228
3229
3230 secondaryUserEmail, err := keyFileEmail(os.Getenv(envFirestorePrivateKey))
3231 if err != nil {
3232 t.Fatalf("keyFileEmail error getting second account (env var %s): %v", envFirestorePrivateKey, err)
3233 }
3234
3235
3236 ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl)
3237 if ts == nil {
3238 t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey)
3239 }
3240
3241 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
3242 client.SetRetry(WithPolicy(RetryAlways))
3243
3244 for _, test := range []struct {
3245 desc string
3246 userProject *string
3247 expectSuccess bool
3248 wantErrorCode int
3249 wantErrorCodeGRPC codes.Code
3250 }{
3251 {
3252 desc: "no UserProject",
3253 userProject: nil,
3254 expectSuccess: false,
3255 },
3256 {
3257 desc: "user is an Editor on UserProject",
3258 userProject: &secondaryProject,
3259 expectSuccess: true,
3260 },
3261 {
3262 desc: "user is not an Editor on UserProject",
3263 userProject: &mainProject,
3264 expectSuccess: false,
3265 wantErrorCode: 403,
3266 wantErrorCodeGRPC: codes.PermissionDenied,
3267 },
3268 } {
3269 t.Run(test.desc, func(t *testing.T) {
3270 ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
3271 t.Cleanup(cancel)
3272
3273 printTestCase := func() string {
3274 userProject := "none"
3275 if test.userProject != nil {
3276 userProject = *test.userProject
3277 }
3278 return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", secondaryUserEmail, mainProject, userProject)
3279 }
3280
3281 checkforErrors := func(desc string, err error) {
3282 errCode := extractErrCode(err)
3283 if err != nil && test.expectSuccess {
3284 t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase())
3285 } else if err == nil && !test.expectSuccess {
3286 t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase())
3287 } else if !test.expectSuccess && test.wantErrorCode != 0 {
3288 if (status.Code(err) != codes.OK && status.Code(err) != codes.Unknown && status.Code(err) != test.wantErrorCodeGRPC) || (errCode > 0 && errCode != test.wantErrorCode) {
3289 fmt.Println(status.Code(err), " ", status.Code(err) != test.wantErrorCodeGRPC)
3290 t.Errorf("%s: mismatched errors; want error code: %d or grpc error: %s, got error: %v \n\t\t%s\n",
3291 desc, test.wantErrorCode, test.wantErrorCodeGRPC, err, printTestCase())
3292 }
3293 }
3294 }
3295
3296 bucketName := prefix + uidSpace.New()
3297 objectName := "acl-go-test" + uidSpaceObjects.New()
3298
3299 setUpRequesterPaysBucket(ctx, t, bucketName, objectName, secondaryUserEmail)
3300
3301
3302 bucket := client.Bucket(bucketName)
3303 if test.userProject != nil {
3304 bucket = bucket.UserProject(*test.userProject)
3305 }
3306
3307
3308 attrs, err := bucket.Attrs(ctx)
3309 checkforErrors("get bucket attrs", err)
3310 if attrs != nil {
3311 if got, want := attrs.RequesterPays, true; got != want {
3312 t.Fatalf("attr.RequesterPays = %t, want %t", got, want)
3313 }
3314 }
3315
3316
3317 entity := ACLEntity("domain-google.com")
3318
3319 checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader))
3320 _, err = bucket.ACL().List(ctx)
3321 checkforErrors("bucket acl list", err)
3322 checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity))
3323
3324
3325
3326
3327 o := bucket.Object(objectName)
3328 ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*15)
3329 defer cancel()
3330
3331
3332 if test.expectSuccess {
3333 o = o.Retryer(WithErrorFunc(retryOnTransient400and403))
3334 }
3335 checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello")))
3336 _, err = readObject(ctx, bucket.Object(objectName))
3337 checkforErrors("read object", err)
3338 _, err = bucket.Object(objectName).Attrs(ctx)
3339 checkforErrors("get object attrs", err)
3340 _, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
3341 checkforErrors("update object", err)
3342
3343
3344 checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader))
3345 _, err = bucket.Object(objectName).ACL().List(ctx)
3346 checkforErrors("object acl list", err)
3347 checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity))
3348
3349
3350
3351 checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader))
3352 _, err = bucket.DefaultObjectACL().List(ctx)
3353 checkforErrors("default object acl list", err)
3354 checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity))
3355
3356
3357 copyObj := bucket.Object("copy")
3358 _, err = copyObj.CopierFrom(bucket.Object(objectName)).Run(ctx)
3359 checkforErrors("copy", err)
3360
3361 if err == nil {
3362 t.Cleanup(func() {
3363 if err := deleteObjectIfExists(copyObj, WithErrorFunc(retryOnTransient400and403)); err != nil {
3364 t.Error(err)
3365 }
3366 })
3367 }
3368
3369
3370 composeObj := bucket.Object("compose")
3371 _, err = composeObj.ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx)
3372 checkforErrors("compose", err)
3373
3374 if err == nil {
3375 t.Cleanup(func() {
3376 if err := deleteObjectIfExists(composeObj, WithErrorFunc(retryOnTransient400and403)); err != nil {
3377 t.Error(err)
3378 }
3379 })
3380 }
3381
3382
3383 checkforErrors("delete object", bucket.Object(objectName).Delete(ctx))
3384 })
3385 }
3386 }, option.WithTokenSource(ts))
3387 }
3388
3389 func TestIntegration_Notifications(t *testing.T) {
3390 multiTransportTest(skipGRPC("notifications not implemented"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3391 bkt := client.Bucket(bucket)
3392
3393 checkNotifications := func(msg string, want map[string]*Notification) {
3394 got, err := bkt.Notifications(ctx)
3395 if err != nil {
3396 t.Fatal(err)
3397 }
3398 if diff := testutil.Diff(got, want); diff != "" {
3399 t.Errorf("%s: got=-, want=+:\n%s", msg, diff)
3400 }
3401 }
3402 checkNotifications("initial", map[string]*Notification{})
3403
3404 nArg := &Notification{
3405 TopicProjectID: testutil.ProjID(),
3406 TopicID: "go-storage-notification-test",
3407 PayloadFormat: NoPayload,
3408 }
3409 n, err := bkt.AddNotification(ctx, nArg)
3410 if err != nil {
3411 t.Fatal(err)
3412 }
3413 if n.ID == "" {
3414 t.Fatal("expected created Notification to have non-empty ID")
3415 }
3416 nArg.ID = n.ID
3417 if !testutil.Equal(n, nArg) {
3418 t.Errorf("got %+v, want %+v", n, nArg)
3419 }
3420 checkNotifications("after add", map[string]*Notification{n.ID: n})
3421
3422 if err := bkt.DeleteNotification(ctx, n.ID); err != nil {
3423 t.Fatal(err)
3424 }
3425 checkNotifications("after delete", map[string]*Notification{})
3426 })
3427 }
3428
3429 func TestIntegration_PublicBucket(t *testing.T) {
3430
3431
3432
3433 multiTransportTest(skipGRPC("no public buckets for gRPC"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3434 const landsatBucket = "gcp-public-data-landsat"
3435 const landsatPrefix = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/"
3436 const landsatObject = landsatPrefix + "LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt"
3437
3438 h := testHelper{t}
3439 bkt := client.Bucket(landsatBucket)
3440 obj := bkt.Object(landsatObject)
3441
3442
3443 bytes := h.mustRead(obj)
3444 if got, want := len(bytes), 117255; got != want {
3445 t.Errorf("len(bytes) = %d, want %d", got, want)
3446 }
3447
3448
3449 iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix})
3450 gotCount := 0
3451 for {
3452 _, err := iter.Next()
3453 if err == iterator.Done {
3454 break
3455 }
3456 if err != nil {
3457 t.Fatal(err)
3458 }
3459 gotCount++
3460 }
3461 if wantCount := 14; gotCount != wantCount {
3462 t.Errorf("object count: got %d, want %d", gotCount, wantCount)
3463 }
3464
3465 errCode := func(err error) int {
3466 var err2 *googleapi.Error
3467 if ok := errors.As(err, &err2); !ok {
3468 return -1
3469 }
3470 return err2.Code
3471 }
3472
3473
3474 c := testConfig(ctx, t)
3475 defer c.Close()
3476 nonPublicObj := client.Bucket(bucket).Object("noauth")
3477
3478
3479 _, err := readObject(ctx, nonPublicObj)
3480 if got := errCode(err); got != 403 && got != 401 {
3481 t.Errorf("got code %d; want %v\nerror: %v", got, "401 or 403", err)
3482 }
3483 err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b"))
3484 if got, want := errCode(err), 401; got != want {
3485 t.Errorf("got code %d; want %d\nerror: %v", got, want, err)
3486 }
3487 }, option.WithoutAuthentication())
3488 }
3489
3490 func TestIntegration_PublicObject(t *testing.T) {
3491 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3492 publicObj := client.Bucket(bucket).Object("public-obj" + uidSpaceObjects.New())
3493 contents := randomContents()
3494
3495 w := publicObj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
3496 if _, err := w.Write(contents); err != nil {
3497 t.Fatalf("writer.Write: %v", err)
3498 }
3499 if err := w.Close(); err != nil {
3500 t.Errorf("writer.Close: %v", err)
3501 }
3502
3503
3504 if err := publicObj.ACL().Set(ctx, AllUsers, RoleReader); err != nil {
3505 t.Fatalf("PutACLEntry failed with %v", err)
3506 }
3507
3508
3509 publicClient, err := newTestClient(ctx, option.WithoutAuthentication())
3510 if err != nil {
3511 t.Fatalf("newTestClient: %v", err)
3512 }
3513
3514
3515 publicObjUnauthenticated := publicClient.Bucket(bucket).Object(publicObj.ObjectName())
3516 data, err := readObject(context.Background(), publicObjUnauthenticated)
3517 if err != nil {
3518 t.Fatalf("readObject: %v", err)
3519 }
3520
3521 if !bytes.Equal(data, contents) {
3522 t.Errorf("Public object's content: got %q, want %q", data, contents)
3523 }
3524
3525
3526 wc := publicObjUnauthenticated.NewWriter(ctx)
3527 if _, err := wc.Write([]byte("hello")); err != nil {
3528 t.Errorf("Write unexpectedly failed with %v", err)
3529 }
3530 if err = wc.Close(); err == nil {
3531 t.Error("Close expected an error, found none")
3532 }
3533 })
3534 }
3535
3536 func TestIntegration_ReadCRC(t *testing.T) {
3537
3538
3539 ctx := skipJSONReads(skipGRPC("transcoding not supported"), "https://github.com/googleapis/google-cloud-go/issues/7786")
3540 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3541 const (
3542
3543
3544 uncompressedBucket = "gcp-public-data-landsat"
3545 uncompressedObject = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt"
3546
3547 gzippedObject = "gzipped-text.txt"
3548 )
3549
3550 h := testHelper{t}
3551
3552
3553 var buf bytes.Buffer
3554 zw := gzip.NewWriter(&buf)
3555 zw.Name = gzippedObject
3556 if _, err := zw.Write([]byte("gzipped object data")); err != nil {
3557 t.Fatalf("creating gzip: %v", err)
3558 }
3559 if err := zw.Close(); err != nil {
3560 t.Fatalf("closing gzip writer: %v", err)
3561 }
3562 w := client.Bucket(bucket).Object(gzippedObject).NewWriter(ctx)
3563 w.ContentEncoding = "gzip"
3564 w.ContentType = "text/plain"
3565 h.mustWrite(w, buf.Bytes())
3566
3567 for _, test := range []struct {
3568 desc string
3569 obj *ObjectHandle
3570 offset, length int64
3571 readCompressed bool
3572
3573 wantErr bool
3574 wantCheck bool
3575 }{
3576 {
3577 desc: "uncompressed, entire file",
3578 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject),
3579 offset: 0,
3580 length: -1,
3581 readCompressed: false,
3582 wantCheck: true,
3583 },
3584 {
3585 desc: "uncompressed, entire file, don't decompress",
3586 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject),
3587 offset: 0,
3588 length: -1,
3589 readCompressed: true,
3590 wantCheck: true,
3591 },
3592 {
3593 desc: "uncompressed, suffix",
3594 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject),
3595 offset: 1,
3596 length: -1,
3597 readCompressed: false,
3598 wantCheck: false,
3599 },
3600 {
3601 desc: "uncompressed, prefix",
3602 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject),
3603 offset: 0,
3604 length: 18,
3605 readCompressed: false,
3606 wantCheck: false,
3607 },
3608 {
3609
3610
3611
3612 desc: "compressed, entire file, unzipped",
3613 obj: client.Bucket(bucket).Object(gzippedObject),
3614 offset: 0,
3615 length: -1,
3616 readCompressed: false,
3617 wantCheck: false,
3618 },
3619 {
3620
3621
3622 desc: "compressed, entire file, read compressed",
3623 obj: client.Bucket(bucket).Object(gzippedObject),
3624 offset: 0,
3625 length: -1,
3626 readCompressed: true,
3627 wantCheck: true,
3628 },
3629 {
3630 desc: "compressed, partial, server unzips",
3631 obj: client.Bucket(bucket).Object(gzippedObject),
3632 offset: 1,
3633 length: 8,
3634 readCompressed: false,
3635 wantErr: true,
3636 wantCheck: false,
3637 },
3638 {
3639 desc: "compressed, partial, read compressed",
3640 obj: client.Bucket(bucket).Object(gzippedObject),
3641 offset: 1,
3642 length: 8,
3643 readCompressed: true,
3644 wantCheck: false,
3645 },
3646 } {
3647 t.Run(test.desc, func(t *testing.T) {
3648
3649 for _, c := range readCases {
3650 t.Run(c.desc, func(t *testing.T) {
3651 obj := test.obj.ReadCompressed(test.readCompressed)
3652 r, err := obj.NewRangeReader(ctx, test.offset, test.length)
3653 if err != nil {
3654 if test.wantErr {
3655 return
3656 }
3657 t.Fatalf("%s: %v", test.desc, err)
3658 }
3659 if got, want := r.checkCRC, test.wantCheck; got != want {
3660 t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want)
3661 }
3662 _, err = c.readFunc(r)
3663 _ = r.Close()
3664 if err != nil {
3665 t.Fatalf("%s: %v", test.desc, err)
3666 }
3667 })
3668
3669 }
3670 })
3671 }
3672 })
3673 }
3674
3675 func TestIntegration_CancelWrite(t *testing.T) {
3676
3677 ctx := skipJSONReads(context.Background(), "no reads in test")
3678 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
3679 bkt := client.Bucket(bucket)
3680
3681 cctx, cancel := context.WithCancel(ctx)
3682 defer cancel()
3683 obj := bkt.Object("cancel-write")
3684 w := obj.NewWriter(cctx)
3685 w.ChunkSize = googleapi.MinUploadChunkSize
3686 buf := make([]byte, w.ChunkSize)
3687
3688
3689
3690 _, err := w.Write(buf)
3691 if err != nil {
3692 t.Fatal(err)
3693 }
3694
3695 cancel()
3696
3697 _, err = w.Write(buf)
3698 if !errors.Is(err, context.Canceled) {
3699 t.Fatalf("got %v, wanted context.Canceled", err)
3700 }
3701
3702 err = w.Close()
3703 if !errors.Is(err, context.Canceled) {
3704 t.Fatalf("got %v, wanted context.Canceled", err)
3705 }
3706 })
3707 }
3708
3709 func TestIntegration_UpdateCORS(t *testing.T) {
3710 ctx := skipJSONReads(context.Background(), "no reads in test")
3711 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
3712 initialSettings := []CORS{
3713 {
3714 MaxAge: time.Hour,
3715 Methods: []string{"POST"},
3716 Origins: []string{"some-origin.com"},
3717 ResponseHeaders: []string{"foo-bar"},
3718 },
3719 }
3720
3721 for _, test := range []struct {
3722 desc string
3723 input []CORS
3724 want []CORS
3725 }{
3726 {
3727 desc: "set new values",
3728 input: []CORS{
3729 {
3730 MaxAge: time.Hour,
3731 Methods: []string{"GET"},
3732 Origins: []string{"*"},
3733 ResponseHeaders: []string{"some-header"},
3734 },
3735 },
3736 want: []CORS{
3737 {
3738 MaxAge: time.Hour,
3739 Methods: []string{"GET"},
3740 Origins: []string{"*"},
3741 ResponseHeaders: []string{"some-header"},
3742 },
3743 },
3744 },
3745 {
3746 desc: "set to empty to remove existing policies",
3747 input: []CORS{},
3748 want: nil,
3749 },
3750 {
3751 desc: "do not set to keep existing policies",
3752 input: nil,
3753 want: []CORS{
3754 {
3755 MaxAge: time.Hour,
3756 Methods: []string{"POST"},
3757 Origins: []string{"some-origin.com"},
3758 ResponseHeaders: []string{"foo-bar"},
3759 },
3760 },
3761 },
3762 } {
3763 t.Run(test.desc, func(t *testing.T) {
3764 h := testHelper{t}
3765
3766 bkt := client.Bucket(prefix + uidSpace.New())
3767 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings})
3768 defer h.mustDeleteBucket(bkt)
3769 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input}, h.mustBucketAttrs(bkt).MetaGeneration)
3770 attrs := h.mustBucketAttrs(bkt)
3771 if diff := testutil.Diff(attrs.CORS, test.want); diff != "" {
3772 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff)
3773 }
3774 })
3775 }
3776 })
3777 }
3778
3779 func TestIntegration_UpdateDefaultEventBasedHold(t *testing.T) {
3780 ctx := skipJSONReads(context.Background(), "no reads in test")
3781 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
3782 h := testHelper{t}
3783
3784 bkt := client.Bucket(prefix + uidSpace.New())
3785 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{})
3786 defer h.mustDeleteBucket(bkt)
3787 attrs := h.mustBucketAttrs(bkt)
3788 if attrs.DefaultEventBasedHold != false {
3789 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, false)
3790 }
3791
3792 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{DefaultEventBasedHold: true}, attrs.MetaGeneration)
3793 attrs = h.mustBucketAttrs(bkt)
3794 if attrs.DefaultEventBasedHold != true {
3795 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true)
3796 }
3797
3798
3799 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RequesterPays: true}, attrs.MetaGeneration)
3800 attrs = h.mustBucketAttrs(bkt)
3801 if attrs.DefaultEventBasedHold != true {
3802 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true)
3803 }
3804 })
3805 }
3806
3807 func TestIntegration_UpdateEventBasedHold(t *testing.T) {
3808 ctx := skipJSONReads(context.Background(), "no reads in test")
3809 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3810 h := testHelper{t}
3811
3812 obj := client.Bucket(bucket).Object("some-obj")
3813 h.mustWrite(obj.NewWriter(ctx), randomContents())
3814
3815 defer func() {
3816 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: false}, h.mustObjectAttrs(obj).Metageneration)
3817 h.mustDeleteObject(obj)
3818 }()
3819
3820 attrs := h.mustObjectAttrs(obj)
3821 if attrs.EventBasedHold != false {
3822 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, false)
3823 }
3824
3825 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: true}, attrs.Metageneration)
3826 attrs = h.mustObjectAttrs(obj)
3827 if attrs.EventBasedHold != true {
3828 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true)
3829 }
3830
3831
3832 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration)
3833 attrs = h.mustObjectAttrs(obj)
3834 if attrs.EventBasedHold != true {
3835 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true)
3836 }
3837 })
3838 }
3839
3840 func TestIntegration_UpdateTemporaryHold(t *testing.T) {
3841 ctx := skipJSONReads(context.Background(), "no reads in test")
3842 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3843 h := testHelper{t}
3844
3845 obj := client.Bucket(bucket).Object("updatetemporaryhold-obj")
3846 h.mustWrite(obj.NewWriter(ctx), randomContents())
3847
3848 defer func() {
3849 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: false}, h.mustObjectAttrs(obj).Metageneration)
3850 h.mustDeleteObject(obj)
3851 }()
3852
3853 attrs := h.mustObjectAttrs(obj)
3854 if attrs.TemporaryHold != false {
3855 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, false)
3856 }
3857
3858 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: true}, attrs.Metageneration)
3859 attrs = h.mustObjectAttrs(obj)
3860 if attrs.TemporaryHold != true {
3861 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true)
3862 }
3863
3864
3865 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration)
3866 attrs = h.mustObjectAttrs(obj)
3867 if attrs.TemporaryHold != true {
3868 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true)
3869 }
3870 })
3871 }
3872
3873 func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) {
3874 ctx := skipJSONReads(context.Background(), "no reads in test")
3875 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
3876 h := testHelper{t}
3877
3878 bkt := client.Bucket(prefix + uidSpace.New())
3879 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}})
3880 obj := bkt.Object("some-obj")
3881 h.mustWrite(obj.NewWriter(ctx), randomContents())
3882
3883 defer func() {
3884 t.Helper()
3885 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}, h.mustBucketAttrs(bkt).MetaGeneration)
3886
3887
3888
3889
3890 if err := obj.Delete(context.Background()); err != nil {
3891 t.Logf("object delete: %v", err)
3892 }
3893 if err := bkt.Delete(context.Background()); err != nil {
3894 t.Logf("bucket delete: %v", err)
3895 }
3896 }()
3897
3898 attrs := h.mustObjectAttrs(obj)
3899 if attrs.RetentionExpirationTime == (time.Time{}) {
3900 t.Fatalf("got=%v, wanted a non-zero value", attrs.RetentionExpirationTime)
3901 }
3902 })
3903 }
3904
3905 func TestIntegration_CustomTime(t *testing.T) {
3906 ctx := skipJSONReads(context.Background(), "no reads in test")
3907 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
3908 h := testHelper{t}
3909
3910
3911 bkt := client.Bucket(bucket)
3912 obj := bkt.Object("custom-time-obj")
3913 w := obj.NewWriter(ctx)
3914 ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC)
3915 w.ObjectAttrs.CustomTime = ct
3916 h.mustWrite(w, randomContents())
3917
3918
3919 checkCustomTime := func(want time.Time) error {
3920 attrs, err := obj.Attrs(ctx)
3921 if err != nil {
3922 return fmt.Errorf("failed to get object attrs: %v", err)
3923 }
3924 if got := attrs.CustomTime; got != want {
3925 return fmt.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct)
3926 }
3927 return nil
3928 }
3929
3930 if err := checkCustomTime(ct); err != nil {
3931 t.Fatalf("checking CustomTime: %v", err)
3932 }
3933
3934
3935 laterTime := ct.Add(10 * time.Hour)
3936 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: laterTime}); err != nil {
3937 t.Fatalf("updating CustomTime: %v", err)
3938 }
3939
3940
3941 earlierTime := ct.Add(5 * time.Hour)
3942 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: earlierTime}); err == nil {
3943 t.Fatalf("backdating CustomTime: expected error, got none")
3944 }
3945
3946
3947
3948 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{TemporaryHold: false}); err != nil {
3949 t.Fatalf("empty update: %v", err)
3950 }
3951 if err := checkCustomTime(laterTime); err != nil {
3952 t.Fatalf("after sending zero value: %v", err)
3953 }
3954 })
3955 }
3956
3957 func TestIntegration_UpdateRetentionPolicy(t *testing.T) {
3958 ctx := skipJSONReads(context.Background(), "no reads in test")
3959 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
3960 initial := &RetentionPolicy{RetentionPeriod: time.Minute}
3961
3962 for _, test := range []struct {
3963 desc string
3964 input *RetentionPolicy
3965 want *RetentionPolicy
3966 }{
3967 {
3968 desc: "update",
3969 input: &RetentionPolicy{RetentionPeriod: time.Hour},
3970 want: &RetentionPolicy{RetentionPeriod: time.Hour},
3971 },
3972 {
3973 desc: "update even with timestamp (EffectiveTime should be ignored)",
3974 input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()},
3975 want: &RetentionPolicy{RetentionPeriod: time.Hour},
3976 },
3977 {
3978 desc: "remove",
3979 input: &RetentionPolicy{},
3980 want: nil,
3981 },
3982 {
3983 desc: "remove even with timestamp (EffectiveTime should be ignored)",
3984 input: &RetentionPolicy{EffectiveTime: time.Now().Add(time.Hour)},
3985 want: nil,
3986 },
3987 {
3988 desc: "ignore",
3989 input: nil,
3990 want: initial,
3991 },
3992 } {
3993 t.Run(test.desc, func(t *testing.T) {
3994 h := testHelper{t}
3995 bkt := client.Bucket(prefix + uidSpace.New())
3996 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial})
3997 defer h.mustDeleteBucket(bkt)
3998 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input}, h.mustBucketAttrs(bkt).MetaGeneration)
3999
4000 attrs := h.mustBucketAttrs(bkt)
4001 if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 {
4002
4003 t.Fatal("EffectiveTime should be set, but it was not")
4004 }
4005 if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" {
4006 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff)
4007 }
4008 })
4009 }
4010 })
4011 }
4012
4013 func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) {
4014 ctx := skipJSONReads(context.Background(), "no reads in test")
4015 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4016 h := testHelper{t}
4017
4018 bkt := client.Bucket(prefix + uidSpace.New())
4019 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}})
4020 defer h.mustDeleteBucket(bkt)
4021
4022 o := bkt.Object("some-object")
4023 if err := writeObject(ctx, o, "text/plain", []byte("hello world")); err != nil {
4024 t.Fatal(err)
4025 }
4026
4027 if err := o.Delete(ctx); err == nil {
4028 t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil")
4029 }
4030
4031
4032 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{}}, h.mustBucketAttrs(bkt).MetaGeneration)
4033
4034
4035
4036 retry := func(err error) bool { return err != nil }
4037 ctx, cancel := context.WithTimeout(ctx, time.Second*10)
4038 defer cancel()
4039
4040 o = o.Retryer(WithErrorFunc(retry), WithPolicy(RetryAlways))
4041 if err := o.Delete(ctx); err != nil {
4042 t.Fatalf("object delete: %v", err)
4043 }
4044 })
4045 }
4046
4047 func TestIntegration_LockBucket(t *testing.T) {
4048 ctx := skipJSONReads(context.Background(), "no reads in test")
4049 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4050 h := testHelper{t}
4051
4052 bkt := client.Bucket(prefix + uidSpace.New())
4053 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}})
4054 attrs := h.mustBucketAttrs(bkt)
4055 if attrs.RetentionPolicy.IsLocked {
4056 t.Fatal("Expected bucket to begin unlocked, but it was not")
4057 }
4058 err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx)
4059 if err != nil {
4060 t.Fatal("could not lock", err)
4061 }
4062
4063 attrs = h.mustBucketAttrs(bkt)
4064 if !attrs.RetentionPolicy.IsLocked {
4065 t.Fatal("Expected bucket to be locked, but it was not")
4066 }
4067
4068 _, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}})
4069 if err == nil {
4070 t.Fatal("Expected error updating locked bucket, got nil")
4071 }
4072 })
4073 }
4074
4075 func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) {
4076 ctx := skipJSONReads(context.Background(), "no reads in test")
4077 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4078 h := testHelper{t}
4079
4080 bkt := client.Bucket(prefix + uidSpace.New())
4081 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
4082 RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25},
4083 })
4084 err := bkt.LockRetentionPolicy(ctx)
4085 if err == nil {
4086 t.Fatal("expected error locking bucket without metageneration condition, got nil")
4087 }
4088 })
4089 }
4090
4091 func TestIntegration_BucketObjectRetention(t *testing.T) {
4092 ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test")
4093 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4094 setTrue, setFalse := true, false
4095
4096 for _, test := range []struct {
4097 desc string
4098 enable *bool
4099 wantRetentionMode string
4100 }{
4101 {
4102 desc: "ObjectRetentionMode is not enabled by default",
4103 wantRetentionMode: "",
4104 },
4105 {
4106 desc: "Enable retention",
4107 enable: &setTrue,
4108 wantRetentionMode: "Enabled",
4109 },
4110 {
4111 desc: "Set object retention to false",
4112 enable: &setFalse,
4113 wantRetentionMode: "",
4114 },
4115 } {
4116 t.Run(test.desc, func(t *testing.T) {
4117 b := client.Bucket(prefix + uidSpace.New())
4118 if test.enable != nil {
4119 b = b.SetObjectRetention(*test.enable)
4120 }
4121
4122 err := b.Create(ctx, testutil.ProjID(), nil)
4123 if err != nil {
4124 t.Fatalf("error creating bucket: %v", err)
4125 }
4126 t.Cleanup(func() { b.Delete(ctx) })
4127
4128 attrs, err := b.Attrs(ctx)
4129 if err != nil {
4130 t.Fatalf("b.Attrs: %v", err)
4131 }
4132 if got, want := attrs.ObjectRetentionMode, test.wantRetentionMode; got != want {
4133 t.Errorf("expected ObjectRetentionMode to be %q, got %q", want, got)
4134 }
4135 })
4136 }
4137 })
4138 }
4139
4140 func TestIntegration_ObjectRetention(t *testing.T) {
4141 ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test")
4142 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4143 h := testHelper{t}
4144
4145 b := client.Bucket(prefix + uidSpace.New()).SetObjectRetention(true)
4146
4147 if err := b.Create(ctx, testutil.ProjID(), nil); err != nil {
4148 t.Fatalf("error creating bucket: %v", err)
4149 }
4150 t.Cleanup(func() { h.mustDeleteBucket(b) })
4151
4152 retentionUnlocked := &ObjectRetention{
4153 Mode: "Unlocked",
4154 RetainUntil: time.Now().Add(time.Minute * 20).Truncate(time.Second),
4155 }
4156 retentionUnlockedExtended := &ObjectRetention{
4157 Mode: "Unlocked",
4158 RetainUntil: time.Now().Add(time.Hour).Truncate(time.Second),
4159 }
4160
4161
4162 o := b.Object("retention-on-create" + uidSpaceObjects.New())
4163 w := o.NewWriter(ctx)
4164 w.Retention = retentionUnlocked
4165 h.mustWrite(w, []byte("contents"))
4166 t.Cleanup(func() {
4167 if _, err := o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}); err != nil {
4168 t.Fatalf("failed to remove retention from object: %v", err)
4169 }
4170 h.mustDeleteObject(o)
4171 })
4172
4173 if got, want := w.Attrs().Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
4174 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
4175 }
4176
4177
4178 if err := o.Delete(ctx); err == nil || extractErrCode(err) != http.StatusForbidden {
4179 t.Fatalf("delete should have failed with: %v, instead got:%v", http.StatusForbidden, err)
4180 }
4181
4182
4183 attrs, err := o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlockedExtended})
4184 if err != nil {
4185 t.Fatalf("failed to add retention to object: %v", err)
4186 }
4187
4188 if got, want := attrs.Retention, retentionUnlockedExtended; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
4189 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
4190 }
4191
4192
4193
4194 _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked})
4195 if err == nil || extractErrCode(err) != http.StatusForbidden {
4196 t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4197 }
4198
4199
4200
4201 _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}})
4202 if err == nil || extractErrCode(err) != http.StatusForbidden {
4203 t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4204 }
4205
4206
4207 attrs, err = o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{
4208 Retention: retentionUnlocked,
4209 })
4210 if err != nil {
4211 t.Fatalf("failed to add retention to object: %v", err)
4212 }
4213
4214 if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
4215 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
4216 }
4217
4218
4219 objectWithRetentionOnUpdate := b.Object("retention-on-update" + uidSpaceObjects.New())
4220 w = objectWithRetentionOnUpdate.NewWriter(ctx)
4221 h.mustWrite(w, []byte("contents"))
4222
4223
4224 if got := w.Attrs().Retention; got != nil {
4225 t.Errorf("expected no ObjectRetention, got: %+v", got)
4226 }
4227
4228
4229 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}})
4230 if err == nil || extractErrCode(err) != http.StatusBadRequest {
4231 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4232 }
4233
4234 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}})
4235 if err == nil || extractErrCode(err) != http.StatusBadRequest {
4236 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4237 }
4238
4239
4240 attrs, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked})
4241 if err != nil {
4242 t.Errorf("o.Update: %v", err)
4243 }
4244
4245 if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
4246 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
4247 }
4248
4249
4250 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(-time.Second)}})
4251 if err == nil || extractErrCode(err) != http.StatusBadRequest {
4252 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4253 }
4254
4255
4256 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}})
4257 if err == nil || extractErrCode(err) != http.StatusBadRequest {
4258 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4259 }
4260
4261 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}})
4262 if err == nil || extractErrCode(err) != http.StatusBadRequest {
4263 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
4264 }
4265
4266
4267 attrs, err = objectWithRetentionOnUpdate.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{
4268 Retention: &ObjectRetention{},
4269 })
4270 if err != nil {
4271 t.Fatalf("failed to remove retention from object: %v", err)
4272 }
4273
4274 if got := attrs.Retention; got != nil {
4275 t.Errorf("mismatching retention config, got: %+v, wanted nil", got)
4276 }
4277
4278
4279 if err := objectWithRetentionOnUpdate.Delete(ctx); err != nil {
4280 t.Errorf("object.Delete:%v", err)
4281 }
4282 })
4283 }
4284
4285 func TestIntegration_SoftDelete(t *testing.T) {
4286 multiTransportTest(skipJSONReads(context.Background(), "does not test reads"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4287 h := testHelper{t}
4288 testStart := time.Now()
4289
4290 policy := &SoftDeletePolicy{
4291 RetentionDuration: time.Hour * 24 * 8,
4292 }
4293
4294 b := client.Bucket(prefix + uidSpace.New())
4295
4296
4297 if err := b.Create(ctx, testutil.ProjID(), &BucketAttrs{SoftDeletePolicy: policy}); err != nil {
4298 t.Fatalf("error creating bucket with soft delete policy set: %v", err)
4299 }
4300 t.Cleanup(func() { h.mustDeleteBucket(b) })
4301
4302
4303 attrs, err := b.Attrs(ctx)
4304 if err != nil {
4305 t.Fatalf("b.Attrs(%q): %v", b.name, err)
4306 }
4307
4308 got := attrs.SoftDeletePolicy
4309 if got == nil {
4310 t.Fatal("got nil soft delete policy")
4311 }
4312 if got.RetentionDuration != policy.RetentionDuration {
4313 t.Fatalf("mismatching retention duration; got soft delete policy: %+v, expected: %+v", got, policy)
4314 }
4315 if got.EffectiveTime.Before(testStart) {
4316 t.Fatalf("effective time of soft delete policy should not be in the past, got: %v, test start: %v", got.EffectiveTime, testStart.UTC())
4317 }
4318
4319
4320 policy.RetentionDuration = time.Hour * 24 * 9
4321
4322 attrs, err = b.Update(ctx, BucketAttrsToUpdate{SoftDeletePolicy: policy})
4323 if err != nil {
4324 t.Fatalf("b.Update: %v", err)
4325 }
4326
4327 if got, expect := attrs.SoftDeletePolicy.RetentionDuration, policy.RetentionDuration; got != expect {
4328 t.Fatalf("mismatching retention duration; got: %+v, expected: %+v", got, expect)
4329 }
4330
4331
4332 deletedObject := b.Object("soft-delete" + uidSpaceObjects.New())
4333 liveObject := b.Object("not-soft-delete" + uidSpaceObjects.New())
4334
4335 h.mustWrite(deletedObject.NewWriter(ctx), []byte("soft-deleted"))
4336 h.mustWrite(liveObject.NewWriter(ctx), []byte("soft-delete"))
4337 t.Cleanup(func() {
4338 h.mustDeleteObject(liveObject)
4339 h.mustDeleteObject(deletedObject)
4340 })
4341
4342 h.mustDeleteObject(deletedObject)
4343
4344 var gen int64
4345
4346 it := b.Objects(ctx, &Query{SoftDeleted: true})
4347 var gotNames []string
4348 for {
4349 attrs, err := it.Next()
4350 if err == iterator.Done {
4351 break
4352 }
4353 if err != nil {
4354 t.Fatalf("iterator.Next: %v", err)
4355 }
4356 gotNames = append(gotNames, attrs.Name)
4357
4358
4359 gen = attrs.Generation
4360 }
4361 if len(gotNames) != 1 || gotNames[0] != deletedObject.ObjectName() {
4362 t.Fatalf("list soft deleted objects; got: %v, expected only one object named: %s", gotNames, deletedObject.ObjectName())
4363 }
4364
4365
4366 gotNames = []string{}
4367 it = b.Objects(ctx, nil)
4368 for {
4369 attrs, err := it.Next()
4370 if err == iterator.Done {
4371 break
4372 }
4373 if err != nil {
4374 t.Fatalf("iterator.Next: %v", err)
4375 }
4376 gotNames = append(gotNames, attrs.Name)
4377 }
4378 if len(gotNames) != 1 || gotNames[0] != liveObject.ObjectName() {
4379 t.Fatalf("list objects that are not soft deleted; got: %v, expected only one object named: %s", gotNames, liveObject.ObjectName())
4380 }
4381
4382
4383 oAttrs, err := deletedObject.Generation(gen).SoftDeleted().Attrs(ctx)
4384 if err != nil {
4385 t.Fatalf("deletedObject.SoftDeleted().Attrs: %v", err)
4386 }
4387 if oAttrs.SoftDeleteTime.Before(testStart) {
4388 t.Fatalf("SoftDeleteTime of soft deleted object should not be in the past, got: %v, test start: %v", oAttrs.SoftDeleteTime, testStart.UTC())
4389 }
4390 if got, expected := oAttrs.HardDeleteTime, oAttrs.SoftDeleteTime.Add(policy.RetentionDuration); !expected.Equal(got) {
4391 t.Fatalf("HardDeleteTime of soft deleted object should be equal to SoftDeleteTime+RetentionDuration, got: %v, expected: %v", got, expected)
4392 }
4393
4394
4395 _, err = deletedObject.Generation(gen).Restore(ctx, &RestoreOptions{CopySourceACL: true})
4396 if err != nil {
4397 t.Fatalf("Object(deletedObject).Restore: %v", err)
4398 }
4399
4400
4401 attrs, err = b.Update(ctx, BucketAttrsToUpdate{SoftDeletePolicy: &SoftDeletePolicy{}})
4402 if err != nil {
4403 t.Fatalf("b.Update: %v", err)
4404 }
4405
4406 if got, expect := attrs.SoftDeletePolicy.RetentionDuration, time.Duration(0); got != expect {
4407 t.Fatalf("mismatching retention duration; got: %+v, expected: %+v", got, expect)
4408 }
4409 })
4410 }
4411
4412 func TestIntegration_KMS(t *testing.T) {
4413 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, prefix string, client *Client) {
4414 h := testHelper{t}
4415
4416 keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING")
4417 if keyRingName == "" {
4418 t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details")
4419 }
4420 keyName1 := keyRingName + "/cryptoKeys/key1"
4421 keyName2 := keyRingName + "/cryptoKeys/key2"
4422 contents := []byte("my secret")
4423
4424 write := func(obj *ObjectHandle, setKey bool) {
4425 w := obj.NewWriter(ctx)
4426 if setKey {
4427 w.KMSKeyName = keyName1
4428 }
4429 h.mustWrite(w, contents)
4430 }
4431
4432 checkRead := func(obj *ObjectHandle) {
4433 got := h.mustRead(obj)
4434 if !bytes.Equal(got, contents) {
4435 t.Errorf("got %v, want %v", got, contents)
4436 }
4437 attrs := h.mustObjectAttrs(obj)
4438 if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 {
4439 t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1)
4440 }
4441 }
4442
4443
4444 bkt := client.Bucket(bucket)
4445 obj := bkt.Object("kms")
4446 write(obj, true)
4447 checkRead(obj)
4448 h.mustDeleteObject(obj)
4449
4450
4451 src := bkt.Object("csek").Key(testEncryptionKey)
4452 if err := writeObject(ctx, src, "text/plain", contents); err != nil {
4453 t.Fatal(err)
4454 }
4455 dest := bkt.Object("cmek")
4456 c := dest.CopierFrom(src)
4457 c.DestinationKMSKeyName = keyName1
4458 if _, err := c.Run(ctx); err != nil {
4459 t.Fatal(err)
4460 }
4461 checkRead(dest)
4462 src.Delete(ctx)
4463 dest.Delete(ctx)
4464
4465
4466 bkt = client.Bucket(prefix + uidSpace.New())
4467 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
4468 Location: "US",
4469 Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1},
4470 })
4471 defer h.mustDeleteBucket(bkt)
4472
4473 attrs := h.mustBucketAttrs(bkt)
4474 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want {
4475 t.Fatalf("got %q, want %q", got, want)
4476 }
4477 obj = bkt.Object("kms")
4478 write(obj, false)
4479 checkRead(obj)
4480 h.mustDeleteObject(obj)
4481
4482
4483
4484 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}, attrs.MetaGeneration)
4485 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want {
4486 t.Fatalf("got %q, want %q", got, want)
4487 }
4488 attrs = h.mustBucketAttrs(bkt)
4489 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want {
4490 t.Fatalf("got %q, want %q", got, want)
4491 }
4492
4493
4494 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}, attrs.MetaGeneration)
4495 if attrs.Encryption != nil {
4496 t.Fatalf("got %#v, want nil", attrs.Encryption)
4497 }
4498 })
4499 }
4500
4501 func TestIntegration_PredefinedACLs(t *testing.T) {
4502 projectOwners := prefixRoleACL{prefix: "project-owners", role: RoleOwner}
4503 userOwner := prefixRoleACL{prefix: "user", role: RoleOwner}
4504 authenticatedRead := entityRoleACL{entity: AllAuthenticatedUsers, role: RoleReader}
4505
4506 ctx := skipJSONReads(context.Background(), "no reads in test")
4507 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
4508 h := testHelper{t}
4509
4510 bkt := client.Bucket(prefix + uidSpace.New())
4511 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
4512 PredefinedACL: "authenticatedRead",
4513 PredefinedDefaultObjectACL: "publicRead",
4514 })
4515 defer h.mustDeleteBucket(bkt)
4516 attrs := h.mustBucketAttrs(bkt)
4517
4518 if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) {
4519 t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
4520 }
4521 if acl, want := attrs.ACL, authenticatedRead; !containsACLRule(acl, want) {
4522 t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
4523 }
4524 if acl := attrs.DefaultObjectACL; !containsACLRule(acl, entityRoleACL{AllUsers, RoleReader}) {
4525 t.Fatalf("DefaultObjectACL: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, acl)
4526 }
4527
4528
4529 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{
4530 PredefinedACL: "private",
4531 PredefinedDefaultObjectACL: "authenticatedRead",
4532 }, attrs.MetaGeneration)
4533 if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) {
4534 t.Fatalf("Bucket.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
4535 }
4536 if acl, want := attrs.DefaultObjectACL, authenticatedRead; !containsACLRule(acl, want) {
4537 t.Fatalf("DefaultObjectACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
4538 }
4539
4540
4541 obj := bkt.Object("private")
4542 w := obj.NewWriter(ctx)
4543 w.PredefinedACL = "authenticatedRead"
4544 h.mustWrite(w, []byte("hello"))
4545 defer h.mustDeleteObject(obj)
4546 var acl []ACLRule
4547 err := retry(ctx, func() error {
4548 attrs, err := obj.Attrs(ctx)
4549 if err != nil {
4550 return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err)
4551 }
4552 acl = attrs.ACL
4553 return nil
4554 }, func() error {
4555 if want := userOwner; !containsACLRule(acl, want) {
4556 return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
4557 }
4558 return nil
4559 })
4560 if err != nil {
4561 t.Fatal(err)
4562 }
4563 err = retry(ctx, func() error {
4564 attrs, err := obj.Attrs(ctx)
4565 if err != nil {
4566 return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err)
4567 }
4568 acl = attrs.ACL
4569 return nil
4570 }, func() error {
4571 if want := authenticatedRead; !containsACLRule(acl, want) {
4572 return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
4573 }
4574 return nil
4575 })
4576 if err != nil {
4577 t.Fatal(err)
4578 }
4579
4580
4581 oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}, h.mustObjectAttrs(obj).Metageneration)
4582 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
4583 t.Fatalf("Object.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
4584 }
4585 if got := len(oattrs.ACL); got != 1 {
4586 t.Errorf("got %d ACL rules, want 1", got)
4587 }
4588
4589
4590 dst := bkt.Object("dst")
4591 copier := dst.CopierFrom(obj)
4592 copier.PredefinedACL = "publicRead"
4593 oattrs, err = copier.Run(ctx)
4594 if err != nil {
4595 t.Fatal(err)
4596 }
4597 defer h.mustDeleteObject(dst)
4598
4599 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
4600 t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", want, acl)
4601 }
4602 if !containsACLRule(oattrs.ACL, entityRoleACL{AllUsers, RoleReader}) {
4603 t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, oattrs.ACL)
4604 }
4605
4606
4607 comp := bkt.Object("comp")
4608
4609 composer := comp.ComposerFrom(obj, dst)
4610 composer.PredefinedACL = "authenticatedRead"
4611 oattrs, err = composer.Run(ctx)
4612 if err != nil {
4613 t.Fatal(err)
4614 }
4615 defer h.mustDeleteObject(comp)
4616
4617 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
4618 t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl)
4619 }
4620 if acl, want := oattrs.ACL, authenticatedRead; !containsACLRule(acl, want) {
4621 t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl)
4622 }
4623 })
4624 }
4625
4626 func TestIntegration_ServiceAccount(t *testing.T) {
4627 ctx := skipJSONReads(context.Background(), "no reads in test")
4628 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) {
4629 s, err := client.ServiceAccount(ctx, testutil.ProjID())
4630 if err != nil {
4631 t.Fatal(err)
4632 }
4633 want := "@gs-project-accounts.iam.gserviceaccount.com"
4634 if !strings.Contains(s, want) {
4635 t.Fatalf("got %v, want to contain %v", s, want)
4636 }
4637 })
4638 }
4639
4640 func TestIntegration_Reader(t *testing.T) {
4641 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
4642 b := client.Bucket(bucket)
4643 const defaultType = "text/plain"
4644
4645
4646 objects := []string{
4647 "obj1",
4648 "obj2",
4649 "obj/with/slashes",
4650 "obj/",
4651 "./obj",
4652 "!#$&'()*+,/:;=,?@,[] and spaces",
4653 }
4654 contents := make(map[string][]byte)
4655
4656
4657 for _, obj := range objects {
4658 c := randomContents()
4659 if err := writeObject(ctx, b.Object(obj), defaultType, c); err != nil {
4660 t.Errorf("Write for %v failed with %v", obj, err)
4661 }
4662 contents[obj] = c
4663 }
4664
4665
4666
4667 for _, obj := range objects {
4668 t.Run(obj, func(t *testing.T) {
4669
4670 for _, c := range readCases {
4671 t.Run(c.desc, func(t *testing.T) {
4672 rc, err := b.Object(obj).NewReader(ctx)
4673 if err != nil {
4674 t.Fatalf("Can't create a reader for %v, errored with %v", obj, err)
4675 }
4676 if !rc.checkCRC {
4677 t.Errorf("%v: not checking CRC", obj)
4678 }
4679
4680 slurp, err := c.readFunc(rc)
4681 if err != nil {
4682 t.Errorf("Can't read object %v, errored with %v", obj, err)
4683 }
4684 if got, want := slurp, contents[obj]; !bytes.Equal(got, want) {
4685 t.Errorf("Contents (%q) = %q; want %q", obj, got, want)
4686 }
4687 if got, want := rc.Size(), len(contents[obj]); got != int64(want) {
4688 t.Errorf("Size (%q) = %d; want %d", obj, got, want)
4689 }
4690 if got, want := rc.ContentType(), "text/plain"; got != want {
4691 t.Errorf("ContentType (%q) = %q; want %q", obj, got, want)
4692 }
4693 rc.Close()
4694
4695
4696 buf := make([]byte, 1)
4697 rc, err = b.Object(obj).NewReader(ctx)
4698 if err != nil {
4699 t.Fatalf("%v: %v", obj, err)
4700 }
4701 _, err = rc.Read(buf)
4702 if err != nil {
4703 t.Fatalf("%v: %v", obj, err)
4704 }
4705 if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) {
4706 t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want)
4707 }
4708 if err := rc.Close(); err != nil {
4709 t.Errorf("%v Close: %v", obj, err)
4710 }
4711 })
4712 }
4713
4714 })
4715 }
4716
4717 obj := objects[0]
4718 objlen := int64(len(contents[obj]))
4719
4720
4721 for _, r := range []struct {
4722 desc string
4723 offset, length, want int64
4724 }{
4725 {"entire object", 0, objlen, objlen},
4726 {"first half of object", 0, objlen / 2, objlen / 2},
4727 {"second half of object", objlen / 2, objlen, objlen / 2},
4728 {"no bytes - start at beginning", 0, 0, 0},
4729 {"no bytes - start halfway through", objlen / 2, 0, 0},
4730 {"start halfway through - use negative to get rest of obj", objlen / 2, -1, objlen / 2},
4731 {"2 times object length", 0, objlen * 2, objlen},
4732 {"-2 offset", -2, -1, 2},
4733 {"-object length offset", -objlen, -1, objlen},
4734 {"-half of object length offset", -(objlen / 2), -1, objlen / 2},
4735 } {
4736 t.Run(r.desc, func(t *testing.T) {
4737
4738 for _, c := range readCases {
4739 t.Run(c.desc, func(t *testing.T) {
4740 rc, err := b.Object(obj).NewRangeReader(ctx, r.offset, r.length)
4741 if err != nil {
4742 t.Fatalf("%+v: Can't create a range reader for %v, errored with %v", r.desc, obj, err)
4743 }
4744 if rc.Size() != objlen {
4745 t.Errorf("%+v: Reader has a content-size of %d, want %d", r.desc, rc.Size(), objlen)
4746 }
4747 if rc.Remain() != r.want {
4748 t.Errorf("%+v: Reader's available bytes reported as %d, want %d", r.desc, rc.Remain(), r.want)
4749 }
4750 slurp, err := c.readFunc(rc)
4751 if err != nil {
4752 t.Fatalf("%+v: can't read object %v, errored with %v", r, obj, err)
4753 }
4754 if len(slurp) != int(r.want) {
4755 t.Fatalf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", r.desc, r.offset, r.length, len(slurp), r.want)
4756 }
4757
4758 switch {
4759 case r.offset < 0:
4760 start := objlen + r.offset
4761 if got, want := slurp, contents[obj][start:]; !bytes.Equal(got, want) {
4762 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want)
4763 }
4764
4765 default:
4766 if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) {
4767 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want)
4768 }
4769 }
4770 rc.Close()
4771 })
4772 }
4773 })
4774 }
4775
4776 objName := objects[0]
4777
4778
4779
4780 realLen := len(contents[objName])
4781 _, err := b.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10)
4782
4783 var e *googleapi.Error
4784 if !errors.As(err, &e) {
4785
4786 if !(status.Code(err) == codes.OutOfRange) {
4787 t.Errorf("NewRangeReader did not return a googleapi.Error nor GRPC OutOfRange error; got: %v", err)
4788 }
4789 } else {
4790 if e.Code != 416 {
4791 t.Errorf("Code = %d; want %d", e.Code, 416)
4792 }
4793 if len(e.Header) == 0 {
4794 t.Error("Missing googleapi.Error.Header")
4795 }
4796 if len(e.Body) == 0 {
4797 t.Error("Missing googleapi.Error.Body")
4798 }
4799 }
4800 })
4801 }
4802
4803 func TestIntegration_ReaderAttrs(t *testing.T) {
4804 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
4805 bkt := client.Bucket(bucket)
4806
4807 const defaultType = "text/plain"
4808 o := bkt.Object("reader-attrs-obj")
4809 c := randomContents()
4810 if err := writeObject(ctx, o, defaultType, c); err != nil {
4811 t.Errorf("Write for %v failed with %v", o.ObjectName(), err)
4812 }
4813 defer func() {
4814 if err := o.Delete(ctx); err != nil {
4815 log.Printf("failed to delete test object: %v", err)
4816 }
4817 }()
4818
4819 rc, err := o.NewReader(ctx)
4820 if err != nil {
4821 t.Fatal(err)
4822 }
4823
4824 attrs, err := o.Attrs(ctx)
4825 if err != nil {
4826 t.Fatal(err)
4827 }
4828
4829 got := rc.Attrs
4830 want := ReaderObjectAttrs{
4831 Size: attrs.Size,
4832 ContentType: attrs.ContentType,
4833 ContentEncoding: attrs.ContentEncoding,
4834 CacheControl: got.CacheControl,
4835 LastModified: got.LastModified,
4836 Generation: attrs.Generation,
4837 Metageneration: attrs.Metageneration,
4838 }
4839 if got != want {
4840 t.Fatalf("got\t%v,\nwanted\t%v", got, want)
4841 }
4842 })
4843 }
4844
4845 func TestIntegration_ReaderLastModified(t *testing.T) {
4846 ctx := skipJSONReads(context.Background(), "LastModified not populated by json response")
4847 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
4848 testStart := time.Now()
4849 b := client.Bucket(bucket)
4850 o := b.Object("reader-lm-obj" + uidSpaceObjects.New())
4851
4852 if err := writeObject(ctx, o, "text/plain", randomContents()); err != nil {
4853 t.Errorf("Write for %v failed with %v", o.ObjectName(), err)
4854 }
4855 defer func() {
4856 if err := o.Delete(ctx); err != nil {
4857 log.Printf("failed to delete test object: %v", err)
4858 }
4859 }()
4860
4861 r, err := o.NewReader(ctx)
4862 if err != nil {
4863 t.Fatal(err)
4864 }
4865
4866 lm := r.Attrs.LastModified
4867 if lm.IsZero() {
4868 t.Fatal("LastModified is 0, should be >0")
4869 }
4870
4871
4872
4873
4874 expectedVariance := time.Minute
4875
4876 if lm.After(testStart.Add(expectedVariance)) {
4877 t.Errorf("LastModified (%q): got %s, which is not within %v from test start (%v)", o.ObjectName(), lm, expectedVariance, testStart)
4878 }
4879 })
4880 }
4881
4882 func TestIntegration_ReaderCacheControl(t *testing.T) {
4883 ctx := skipJSONReads(context.Background(), "Cache control header is populated differently by the json api")
4884 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
4885 b := client.Bucket(bucket)
4886 o := b.Object("reader-cc" + uidSpaceObjects.New())
4887
4888 cacheControl := "public, max-age=60"
4889
4890
4891 w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
4892 w.CacheControl = cacheControl
4893 if _, err := w.Write(randomContents()); err != nil {
4894 t.Fatalf("Write for %v failed with %v", o.ObjectName(), err)
4895 }
4896 if err := w.Close(); err != nil {
4897 t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err)
4898 }
4899 defer func() {
4900 if err := o.Delete(ctx); err != nil {
4901 log.Printf("failed to delete test object: %v", err)
4902 }
4903 }()
4904
4905
4906 r, err := o.NewReader(ctx)
4907 if err != nil {
4908 t.Fatal(err)
4909 }
4910
4911 if got, want := r.Attrs.CacheControl, cacheControl; got != want {
4912 t.Fatalf("cache control; got: %s, want: %s", got, want)
4913 }
4914 })
4915 }
4916
4917 func TestIntegration_ReaderErrObjectNotExist(t *testing.T) {
4918 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
4919 o := client.Bucket(bucket).Object("non-existing")
4920
4921 _, err := o.NewReader(ctx)
4922 if !errors.Is(err, ErrObjectNotExist) {
4923 t.Fatalf("expected ErrObjectNotExist, got %v", err)
4924 }
4925 })
4926 }
4927
4928
4929
4930 func TestIntegration_JSONReaderConditions(t *testing.T) {
4931 ctx := skipXMLReads(skipGRPC("json-only test"), "json-only test")
4932 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
4933 b := client.Bucket(bucket)
4934 o := b.Object("reader-conditions" + uidSpaceObjects.New())
4935
4936
4937 w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
4938 if _, err := w.Write(randomContents()); err != nil {
4939 t.Fatalf("Write for %v failed with %v", o.ObjectName(), err)
4940 }
4941 if err := w.Close(); err != nil {
4942 t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err)
4943 }
4944
4945 t.Cleanup(func() {
4946 if err := o.Delete(ctx); err != nil {
4947 log.Printf("failed to delete test object: %v", err)
4948 }
4949 })
4950
4951
4952 attrs, err := o.Attrs(ctx)
4953 if err != nil {
4954 t.Fatalf("o.Attrs(%s): %v", o.ObjectName(), err)
4955 }
4956 currGen := attrs.Generation
4957 currMetagen := attrs.Metageneration
4958
4959
4960 for _, test := range []struct {
4961 desc string
4962 conds Conditions
4963 wantErrCode int
4964 }{
4965 {
4966 desc: "GenerationMatch incorrect gen",
4967 conds: Conditions{GenerationMatch: currGen + 2},
4968 wantErrCode: 412,
4969 },
4970 {
4971 desc: "GenerationNotMatch current gen",
4972 conds: Conditions{GenerationNotMatch: currGen},
4973 wantErrCode: 304,
4974 },
4975 {
4976 desc: "DoesNotExist set to true",
4977 conds: Conditions{DoesNotExist: true},
4978 wantErrCode: 412,
4979 },
4980 {
4981 desc: "MetagenerationMatch incorrect gen",
4982 conds: Conditions{MetagenerationMatch: currMetagen + 1},
4983 wantErrCode: 412,
4984 },
4985 {
4986 desc: "MetagenerationNotMatch current gen",
4987 conds: Conditions{MetagenerationNotMatch: currMetagen},
4988 wantErrCode: 304,
4989 },
4990 } {
4991 t.Run(test.desc, func(t *testing.T) {
4992 o := o.If(test.conds)
4993 _, err := o.NewReader(ctx)
4994
4995 got := extractErrCode(err)
4996 if test.wantErrCode != got {
4997 t.Errorf("want err code: %v, got err: %v", test.wantErrCode, err)
4998 }
4999 })
5000 }
5001 })
5002 }
5003
5004
5005 func TestIntegration_ReaderCancel(t *testing.T) {
5006 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5007 ctx, close := context.WithDeadline(ctx, time.Now().Add(time.Second*30))
5008 defer close()
5009
5010 bkt := client.Bucket(bucket)
5011 obj := bkt.Object("reader-cancel-obj")
5012
5013 minObjectSize := 5000000
5014
5015 w := obj.NewWriter(ctx)
5016 c := randomContents()
5017 for written := 0; written < minObjectSize; {
5018 n, err := w.Write(c)
5019 if err != nil {
5020 t.Fatalf("w.Write: %v", err)
5021 }
5022 written += n
5023 }
5024
5025 if err := w.Close(); err != nil {
5026 t.Fatalf("writer close: %v", err)
5027 }
5028 defer func() {
5029 if err := obj.Delete(ctx); err != nil {
5030 log.Printf("failed to delete test object: %v", err)
5031 }
5032 }()
5033
5034
5035
5036 readerCtx, cancel := context.WithCancel(ctx)
5037 r, err := obj.NewReader(readerCtx)
5038 if err != nil {
5039 t.Fatalf("obj.NewReader: %v", err)
5040 }
5041 defer func() {
5042 if err := r.Close(); err != nil {
5043 log.Printf("r.Close(): %v", err)
5044 }
5045 }()
5046
5047 cancel()
5048
5049 _, err = io.Copy(io.Discard, r)
5050 if err == nil || !errors.Is(err, context.Canceled) && !(status.Code(err) == codes.Canceled) {
5051 t.Fatalf("r.Read: got error %v, want context.Canceled", err)
5052 }
5053 })
5054 }
5055
5056
5057
5058
5059
5060
5061
5062
5063 func TestIntegration_NewReaderWithContentEncodingGzip(t *testing.T) {
5064 multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
5065 h := testHelper{t}
5066
5067 projectID := testutil.ProjID()
5068 bkt := client.Bucket(prefix + uidSpace.New())
5069 h.mustCreate(bkt, projectID, nil)
5070 defer h.mustDeleteBucket(bkt)
5071 obj := bkt.Object("decompressive-transcoding")
5072 original := bytes.Repeat([]byte("a"), 4<<10)
5073
5074
5075 w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx)
5076
5077 gzw := gzip.NewWriter(w)
5078 if _, err := gzw.Write(original); err != nil {
5079 t.Fatalf("Failed to compress content: %v", err)
5080 }
5081 if err := gzw.Close(); err != nil {
5082 t.Errorf("Failed to compress content: %v", err)
5083 }
5084 if err := w.Close(); err != nil {
5085 t.Errorf("Failed to finish uploading the file: %v", err)
5086 }
5087
5088 defer h.mustDeleteObject(obj)
5089
5090
5091
5092 updatedAttrs, err := obj.Update(ctx, ObjectAttrsToUpdate{
5093 ContentEncoding: "gzip",
5094 ContentType: "text/plain",
5095 })
5096 if err != nil {
5097 t.Fatalf("Attribute update failure: %v", err)
5098 }
5099 if g, w := updatedAttrs.ContentEncoding, "gzip"; g != w {
5100 t.Fatalf("ContentEncoding mismtach:\nGot: %q\nWant: %q", g, w)
5101 }
5102 if g, w := updatedAttrs.ContentType, "text/plain"; g != w {
5103 t.Fatalf("ContentType mismtach:\nGot: %q\nWant: %q", g, w)
5104 }
5105
5106
5107 for _, c := range readCases {
5108 t.Run(c.desc, func(t *testing.T) {
5109 rWhole, err := obj.NewReader(ctx)
5110 if err != nil {
5111 t.Fatalf("Failed to create wholesome reader: %v", err)
5112 }
5113 blobWhole, err := c.readFunc(rWhole)
5114 rWhole.Close()
5115 if err != nil {
5116 t.Fatalf("Failed to read the whole body: %v", err)
5117 }
5118 if g, w := blobWhole, original; !bytes.Equal(g, w) {
5119 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w)
5120 }
5121
5122
5123
5124 r2kBTo3kB, err := obj.NewRangeReader(ctx, 2<<10, 3<<10)
5125 if err != nil {
5126 t.Fatalf("Failed to create range reader: %v", err)
5127 }
5128 blob2kBTo3kB, err := c.readFunc(r2kBTo3kB)
5129 r2kBTo3kB.Close()
5130 if err != nil {
5131 t.Fatalf("Failed to read with the 2kB to 3kB range request: %v", err)
5132 }
5133
5134 if g, w := blob2kBTo3kB, original; !bytes.Equal(g, w) {
5135 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w)
5136 }
5137 })
5138 }
5139 })
5140 }
5141
5142 func TestIntegration_HMACKey(t *testing.T) {
5143 ctx := skipJSONReads(context.Background(), "no reads in test")
5144 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) {
5145 client.SetRetry(WithPolicy(RetryAlways))
5146
5147 projectID := testutil.ProjID()
5148
5149
5150
5151
5152
5153
5154 credentials := testutil.CredentialsEnv(ctx, "GCLOUD_TESTS_GOLANG_KEY")
5155 if credentials == nil {
5156 t.Fatal("credentials could not be determined, is GCLOUD_TESTS_GOLANG_KEY set correctly?")
5157 }
5158 if credentials.JSON == nil {
5159 t.Fatal("could not read the JSON key file, is GCLOUD_TESTS_GOLANG_KEY set correctly?")
5160 }
5161 conf, err := google.JWTConfigFromJSON(credentials.JSON)
5162 if err != nil {
5163 t.Fatal(err)
5164 }
5165 serviceAccountEmail := conf.Email
5166
5167 hmacKey, err := client.CreateHMACKey(ctx, projectID, serviceAccountEmail)
5168 if err != nil {
5169 t.Fatalf("Failed to create HMACKey: %v", err)
5170 }
5171 if hmacKey == nil {
5172 t.Fatal("Unexpectedly got back a nil HMAC key")
5173 }
5174
5175 if hmacKey.State != Active {
5176 t.Fatalf("Unexpected state %q, expected %q", hmacKey.State, Active)
5177 }
5178
5179 hkh := client.HMACKeyHandle(projectID, hmacKey.AccessID)
5180
5181 if err := hkh.Delete(ctx); err == nil {
5182 t.Fatal("Unexpectedly deleted key whose state is ACTIVE: No error from Delete.")
5183 }
5184
5185 invalidStates := []HMACState{"", Deleted, "active", "inactive", "foo_bar"}
5186 for _, invalidState := range invalidStates {
5187 t.Run("invalid-"+string(invalidState), func(t *testing.T) {
5188 _, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{
5189 State: invalidState,
5190 })
5191 if err == nil {
5192 t.Fatal("Unexpectedly succeeded")
5193 }
5194 invalidStateMsg := fmt.Sprintf(`storage: invalid state %q for update, must be either "ACTIVE" or "INACTIVE"`, invalidState)
5195 if err.Error() != invalidStateMsg {
5196 t.Fatalf("Mismatched error: got: %q\nwant: %q", err, invalidStateMsg)
5197 }
5198 })
5199 }
5200
5201
5202 hu, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{
5203 State: Inactive,
5204 })
5205 if err != nil {
5206 t.Fatalf("Unexpected Update failure: %v", err)
5207 }
5208 if got, want := hu.State, Inactive; got != want {
5209 t.Fatalf("Unexpected updated state %q, expected %q", got, want)
5210 }
5211
5212
5213 hu, err = hkh.Update(ctx, HMACKeyAttrsToUpdate{
5214 State: Active,
5215 })
5216 if err != nil {
5217 t.Fatalf("Unexpected Update failure: %v", err)
5218 }
5219 if got, want := hu.State, Active; got != want {
5220 t.Fatalf("Unexpected updated state %q, expected %q", got, want)
5221 }
5222
5223
5224 iter := client.ListHMACKeys(ctx, projectID)
5225 count := 0
5226 for ; ; count++ {
5227 _, err := iter.Next()
5228 if err == iterator.Done {
5229 break
5230 }
5231 if err != nil {
5232 t.Fatalf("Failed to ListHMACKeys: %v", err)
5233 }
5234 }
5235 if count == 0 {
5236 t.Fatal("Failed to list any HMACKeys")
5237 }
5238
5239
5240
5241 _, _ = hkh.Update(ctx, HMACKeyAttrsToUpdate{
5242 State: Inactive,
5243 })
5244 if err := hkh.Delete(ctx); err != nil {
5245 t.Fatalf("Unexpected deletion failure: %v", err)
5246 }
5247
5248 _, err = hkh.Get(ctx)
5249 if err != nil && !strings.Contains(err.Error(), "404") {
5250
5251
5252 t.Fatalf("Unexpected error: %v", err)
5253 }
5254 })
5255 }
5256
5257 func TestIntegration_PostPolicyV4(t *testing.T) {
5258 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
5259 jwtConf, err := testutil.JWTConfig()
5260 if err != nil {
5261 t.Fatal(err)
5262 }
5263 if jwtConf == nil {
5264 t.Skip("JSON key file is not present")
5265 }
5266
5267 projectID := testutil.ProjID()
5268 newBucketName := prefix + uidSpace.New()
5269 b := client.Bucket(newBucketName)
5270 h := testHelper{t}
5271 h.mustCreate(b, projectID, nil)
5272 defer h.mustDeleteBucket(b)
5273
5274 statusCodeToRespond := 200
5275 opts := &PostPolicyV4Options{
5276 GoogleAccessID: jwtConf.Email,
5277 PrivateKey: jwtConf.PrivateKey,
5278
5279 Expires: time.Now().Add(30 * time.Minute),
5280
5281 Fields: &PolicyV4Fields{
5282 StatusCodeOnSuccess: statusCodeToRespond,
5283 ContentType: "text/plain",
5284 ACL: "public-read",
5285 },
5286
5287
5288 Conditions: []PostPolicyV4Condition{
5289
5290 ConditionContentLengthRange(0, 10<<20),
5291 ConditionStartsWith("$acl", "public"),
5292 },
5293 }
5294
5295 objectName := uidSpaceObjects.New()
5296 object := b.Object(objectName)
5297 defer h.mustDeleteObject(object)
5298
5299 pv4, err := b.GenerateSignedPostPolicyV4(objectName, opts)
5300 if err != nil {
5301 t.Fatal(err)
5302 }
5303
5304 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
5305 t.Fatal(err)
5306 }
5307 })
5308 }
5309
5310
5311 func TestIntegration_Scopes(t *testing.T) {
5312 ctx := skipJSONReads(context.Background(), "no reads in test")
5313
5314 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5315 bkt := client.Bucket(bucket)
5316 obj := bkt.Object("test-scopes")
5317 contents := []byte("This object should not be written.\n")
5318
5319
5320 if _, err := bkt.Attrs(ctx); err != nil {
5321 t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err)
5322 }
5323
5324
5325 if err := writeObject(ctx, obj, "text/plain", contents); err == nil {
5326 if err := obj.Delete(ctx); err != nil {
5327 t.Logf("obj.Delete: %v", err)
5328 }
5329 t.Error("client with ScopeReadOnly was able to write an object unexpectedly.")
5330 }
5331
5332
5333 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
5334 t.Error("client with ScopeReadWrite was able to change unexpectedly.")
5335 }
5336 }, option.WithScopes(ScopeReadOnly))
5337
5338 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5339 bkt := client.Bucket(bucket)
5340 obj := bkt.Object("test-scopes")
5341 contents := []byte("This object should be written.\n")
5342
5343
5344 if _, err := bkt.Attrs(ctx); err != nil {
5345 t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err)
5346 }
5347
5348
5349 if err := writeObject(ctx, obj, "text/plain", contents); err != nil {
5350 t.Errorf("client with ScopeReadWrite was not able to write: %v", err)
5351 }
5352 defer func() {
5353 if err := obj.Delete(ctx); err != nil {
5354 t.Logf("obj.Delete: %v", err)
5355 }
5356 }()
5357
5358
5359 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
5360 t.Error("client with ScopeReadWrite was able to change permissions unexpectedly")
5361 }
5362 }, option.WithScopes(ScopeReadWrite))
5363
5364 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5365 bkt := client.Bucket(bucket)
5366 obj := bkt.Object("test-scopes")
5367 contents := []byte("This object should be written.\n")
5368
5369
5370 if _, err := bkt.Attrs(ctx); err == nil {
5371 t.Errorf("client with no scopes was able to read attrs unexpectedly")
5372 }
5373
5374 if err := writeObject(ctx, obj, "text/plain", contents); err == nil {
5375 if err := obj.Delete(ctx); err != nil {
5376 t.Logf("obj.Delete: %v", err)
5377 }
5378 t.Error("client with no scopes was able to write an object unexpectedly.")
5379 }
5380
5381 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
5382 t.Error("client with no scopes was able to change permissions unexpectedly")
5383 }
5384 }, option.WithScopes(""))
5385 }
5386
5387 func TestIntegration_SignedURL_WithCreds(t *testing.T) {
5388
5389 if testing.Short() {
5390 t.Skip("Integration tests skipped in short mode")
5391 }
5392
5393 ctx := context.Background()
5394
5395 creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
5396 if err != nil {
5397 t.Fatalf("unable to find test credentials: %v", err)
5398 }
5399
5400 multiTransportTest(skipGRPC("creds capture logic must be implemented for gRPC constructor"), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5401
5402 obj := "testBucketSignedURL"
5403 contents := []byte("test")
5404 if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil {
5405 t.Fatalf("writing: %v", err)
5406 }
5407 opts := SignedURLOptions{
5408 Method: "GET",
5409 Expires: time.Now().Add(30 * time.Second),
5410 }
5411 bkt := client.Bucket(bucket)
5412 url, err := bkt.SignedURL(obj, &opts)
5413 if err != nil {
5414 t.Fatalf("unable to create signed URL: %v", err)
5415 }
5416
5417 if err := verifySignedURL(url, nil, contents); err != nil {
5418 t.Fatalf("problem with the signed URL: %v", err)
5419 }
5420 }, option.WithCredentials(creds))
5421 }
5422
5423 func TestIntegration_SignedURL_DefaultSignBytes(t *testing.T) {
5424
5425 if testing.Short() {
5426 t.Skip("Integration tests skipped in short mode")
5427 }
5428
5429 ctx := context.Background()
5430
5431
5432 scopes := []string{ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"}
5433 ts := testutil.TokenSource(ctx, scopes...)
5434 if ts == nil {
5435 t.Fatalf("Cannot get token source to create client")
5436 }
5437
5438 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
5439 jwt, err := testutil.JWTConfig()
5440 if err != nil {
5441 t.Fatalf("unable to find test credentials: %v", err)
5442 }
5443
5444 obj := "testBucketSignedURL"
5445 contents := []byte("test")
5446 if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil {
5447 t.Fatalf("writing: %v", err)
5448 }
5449
5450 opts := SignedURLOptions{
5451 Method: "GET",
5452 Expires: time.Now().Add(30 * time.Second),
5453 GoogleAccessID: jwt.Email,
5454 }
5455 bkt := client.Bucket(bucket)
5456 url, err := bkt.SignedURL(obj, &opts)
5457 if err != nil {
5458 t.Fatalf("unable to create signed URL: %v", err)
5459 }
5460
5461 if err := verifySignedURL(url, nil, contents); err != nil {
5462 t.Fatalf("problem with the signed URL: %v", err)
5463 }
5464 }, option.WithTokenSource(ts))
5465
5466 }
5467
5468 func TestIntegration_PostPolicyV4_WithCreds(t *testing.T) {
5469
5470 if testing.Short() {
5471 t.Skip("Integration tests skipped in short mode")
5472 }
5473
5474
5475
5476
5477 creds, err := findTestCredentials(context.Background(), "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
5478 if err != nil {
5479 t.Fatalf("unable to find test credentials: %v", err)
5480 }
5481
5482 ctx := skipJSONReads(skipGRPC("creds capture logic must be implemented for gRPC constructor"), "test is not testing the read behaviour")
5483 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithCredentials *Client) {
5484 h := testHelper{t}
5485
5486 statusCodeToRespond := 200
5487
5488 for _, test := range []struct {
5489 desc string
5490 opts PostPolicyV4Options
5491 client *Client
5492 }{
5493 {
5494 desc: "signing with the private key",
5495 opts: PostPolicyV4Options{
5496 Expires: time.Now().Add(30 * time.Minute),
5497
5498 Fields: &PolicyV4Fields{
5499 StatusCodeOnSuccess: statusCodeToRespond,
5500 ContentType: "text/plain",
5501 ACL: "public-read",
5502 },
5503 },
5504 client: clientWithCredentials,
5505 },
5506 } {
5507 t.Run(test.desc, func(t *testing.T) {
5508 objectName := uidSpace.New()
5509 object := test.client.Bucket(bucket).Object(objectName)
5510 defer h.mustDeleteObject(object)
5511
5512 pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(objectName, &test.opts)
5513 if err != nil {
5514 t.Fatal(err)
5515 }
5516
5517 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
5518 t.Fatal(err)
5519 }
5520 })
5521 }
5522 }, option.WithCredentials(creds))
5523
5524 }
5525
5526 func TestIntegration_PostPolicyV4_BucketDefault(t *testing.T) {
5527 ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour")
5528 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithoutPrivateKey *Client) {
5529 h := testHelper{t}
5530
5531 jwt, err := testutil.JWTConfig()
5532 if err != nil {
5533 t.Fatalf("unable to find test credentials: %v", err)
5534 }
5535
5536 statusCodeToRespond := 200
5537
5538 for _, test := range []struct {
5539 desc string
5540 opts PostPolicyV4Options
5541 client *Client
5542 }{
5543 {
5544 desc: "signing with the default sign bytes func",
5545 opts: PostPolicyV4Options{
5546 Expires: time.Now().Add(30 * time.Minute),
5547 GoogleAccessID: jwt.Email,
5548 Fields: &PolicyV4Fields{
5549 StatusCodeOnSuccess: statusCodeToRespond,
5550 ContentType: "text/plain",
5551 ACL: "public-read",
5552 },
5553 },
5554 client: clientWithoutPrivateKey,
5555 },
5556 } {
5557 t.Run(test.desc, func(t *testing.T) {
5558 objectName := uidSpaceObjects.New()
5559 object := test.client.Bucket(bucket).Object(objectName)
5560 defer h.mustDeleteObject(object)
5561
5562 pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(object.ObjectName(), &test.opts)
5563 if err != nil {
5564 t.Fatal(err)
5565 }
5566
5567 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
5568 t.Fatal(err)
5569 }
5570 })
5571 }
5572 })
5573
5574 }
5575
5576
5577
5578 func TestIntegration_PostPolicyV4_SignedURL_WithSignBytes(t *testing.T) {
5579 ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour")
5580 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
5581
5582 h := testHelper{t}
5583 projectID := testutil.ProjID()
5584 bucketName := prefix + uidSpace.New()
5585 objectName := uidSpaceObjects.New()
5586 fileBody := bytes.Repeat([]byte("b"), 25)
5587 bucket := client.Bucket(bucketName)
5588
5589 h.mustCreate(bucket, projectID, nil)
5590 defer h.mustDeleteBucket(bucket)
5591
5592 object := bucket.Object(objectName)
5593 defer h.mustDeleteObject(object)
5594
5595 jwtConf, err := testutil.JWTConfig()
5596 if err != nil {
5597 t.Fatal(err)
5598 }
5599 if jwtConf == nil {
5600 t.Skip("JSON key file is not present")
5601 }
5602
5603 signingFunc := func(b []byte) ([]byte, error) {
5604 parsedRSAPrivKey, err := parseKey(jwtConf.PrivateKey)
5605 if err != nil {
5606 return nil, err
5607 }
5608 sum := sha256.Sum256(b)
5609 return rsa.SignPKCS1v15(cryptorand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:])
5610 }
5611
5612
5613 successStatusCode := 200
5614 ppv4Opts := &PostPolicyV4Options{
5615 GoogleAccessID: jwtConf.Email,
5616 SignRawBytes: signingFunc,
5617 Expires: time.Now().Add(30 * time.Minute),
5618 Fields: &PolicyV4Fields{
5619 StatusCodeOnSuccess: successStatusCode,
5620 ContentType: "text/plain",
5621 ACL: "public-read",
5622 },
5623 }
5624
5625 pv4, err := GenerateSignedPostPolicyV4(bucketName, objectName, ppv4Opts)
5626 if err != nil {
5627 t.Fatal(err)
5628 }
5629
5630 if err := verifyPostPolicy(pv4, object, fileBody, successStatusCode); err != nil {
5631 t.Fatal(err)
5632 }
5633
5634
5635 signURLOpts := &SignedURLOptions{
5636 GoogleAccessID: jwtConf.Email,
5637 SignBytes: signingFunc,
5638 Method: "GET",
5639 Expires: time.Now().Add(30 * time.Second),
5640 }
5641
5642 url, err := bucket.SignedURL(objectName, signURLOpts)
5643 if err != nil {
5644 t.Fatalf("unable to create signed URL: %v", err)
5645 }
5646
5647 if err := verifySignedURL(url, nil, fileBody); err != nil {
5648 t.Fatal(err)
5649 }
5650 })
5651 }
5652
5653 func TestIntegration_OCTracing(t *testing.T) {
5654 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
5655 te := testutil.NewTestExporter()
5656 defer te.Unregister()
5657
5658 bkt := client.Bucket(bucket)
5659 bkt.Attrs(ctx)
5660
5661 if len(te.Spans) == 0 {
5662 t.Fatalf("Expected some spans to be created, but got %d", 0)
5663 }
5664 })
5665 }
5666
5667
5668
5669 func verifySignedURL(url string, headers map[string][]string, expectedFileBody []byte) error {
5670 got, err := getURL(url, headers)
5671 if err != nil {
5672 return fmt.Errorf("getURL %q: %v", url, err)
5673 }
5674 if !bytes.Equal(got, expectedFileBody) {
5675 return fmt.Errorf("got %q, want %q", got, expectedFileBody)
5676 }
5677 return nil
5678 }
5679
5680
5681
5682 func verifyPostPolicy(pv4 *PostPolicyV4, obj *ObjectHandle, bytesToWrite []byte, statusCodeOnSuccess int) error {
5683 ctx := context.Background()
5684 var res *http.Response
5685
5686
5687
5688 return retry(ctx,
5689 func() error {
5690 formBuf := new(bytes.Buffer)
5691 mw := multipart.NewWriter(formBuf)
5692 for fieldName, value := range pv4.Fields {
5693 if err := mw.WriteField(fieldName, value); err != nil {
5694 return fmt.Errorf("Failed to write form field: %q: %v", fieldName, err)
5695 }
5696 }
5697
5698
5699 mf, err := mw.CreateFormFile("file", "myfile.txt")
5700 if err != nil {
5701 return err
5702 }
5703 if _, err := mf.Write(bytesToWrite); err != nil {
5704 return err
5705 }
5706 if err := mw.Close(); err != nil {
5707 return err
5708 }
5709
5710
5711 req, err := http.NewRequest("POST", pv4.URL, formBuf)
5712 if err != nil {
5713 return fmt.Errorf("Failed to compose HTTP request: %v", err)
5714 }
5715
5716
5717 req.Header.Set("Content-Type", mw.FormDataContentType())
5718
5719
5720 res, err = http.DefaultClient.Do(req)
5721 if err != nil {
5722 return err
5723 }
5724 return nil
5725 },
5726 func() error {
5727
5728 if g, w := res.StatusCode, statusCodeOnSuccess; g != w {
5729 blob, _ := httputil.DumpResponse(res, true)
5730 return fmt.Errorf("Status code in response mismatch: got %d want %d\nBody: %s", g, w, blob)
5731 }
5732 io.Copy(io.Discard, res.Body)
5733
5734
5735
5736 attrs, err := obj.Attrs(ctx)
5737 if err != nil {
5738 return fmt.Errorf("Failed to retrieve attributes: %v", err)
5739 }
5740 if g, w := attrs.Size, int64(len(bytesToWrite)); g != w {
5741 return fmt.Errorf("ContentLength mismatch: got %d want %d", g, w)
5742 }
5743 if g, w := attrs.MD5, md5.Sum(bytesToWrite); !bytes.Equal(g, w[:]) {
5744 return fmt.Errorf("MD5Checksum mismatch\nGot: %x\nWant: %x", g, w)
5745 }
5746
5747
5748 rd, err := obj.NewReader(ctx)
5749 if err != nil {
5750 return fmt.Errorf("Failed to create a reader: %v", err)
5751 }
5752 gotBody, err := io.ReadAll(rd)
5753 if err != nil {
5754 return fmt.Errorf("Failed to read the body: %v", err)
5755 }
5756 if diff := testutil.Diff(string(gotBody), string(bytesToWrite)); diff != "" {
5757 return fmt.Errorf("Body mismatch: got - want +\n%s", diff)
5758 }
5759 return nil
5760 })
5761 }
5762
5763 func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) {
5764 key := os.Getenv(envVar)
5765 var opts []option.ClientOption
5766 if len(scopes) > 0 {
5767 opts = append(opts, option.WithScopes(scopes...))
5768 }
5769 if key != "" {
5770 opts = append(opts, option.WithCredentialsFile(key))
5771 }
5772 return transport.Creds(ctx, opts...)
5773 }
5774
5775 type testHelper struct {
5776 t *testing.T
5777 }
5778
5779 func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) {
5780 h.t.Helper()
5781 if err := b.Create(context.Background(), projID, attrs); err != nil {
5782 h.t.Fatalf("bucket create: %v", err)
5783 }
5784 }
5785
5786 func (h testHelper) mustDeleteBucket(b *BucketHandle) {
5787 h.t.Helper()
5788 if err := b.Delete(context.Background()); err != nil {
5789 h.t.Fatalf("bucket delete: %v", err)
5790 }
5791 }
5792
5793 func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs {
5794 h.t.Helper()
5795 attrs, err := b.Attrs(context.Background())
5796 if err != nil {
5797 h.t.Fatalf("bucket attrs: %v", err)
5798 }
5799 return attrs
5800 }
5801
5802
5803 func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate, metageneration int64) *BucketAttrs {
5804 h.t.Helper()
5805 attrs, err := b.If(BucketConditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua)
5806 if err != nil {
5807 h.t.Fatalf("update: %v", err)
5808 }
5809 return attrs
5810 }
5811
5812 func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs {
5813 h.t.Helper()
5814 attrs, err := o.Attrs(context.Background())
5815 if err != nil {
5816 h.t.Fatalf("object attrs: %v", err)
5817 }
5818 return attrs
5819 }
5820
5821 func (h testHelper) mustDeleteObject(o *ObjectHandle) {
5822 h.t.Helper()
5823 if err := o.Retryer(WithPolicy(RetryAlways)).Delete(context.Background()); err != nil {
5824 var apiErr *apierror.APIError
5825 if ok := errors.As(err, &apiErr); ok {
5826
5827 if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound {
5828 return
5829 }
5830 }
5831 h.t.Fatalf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err)
5832 }
5833 }
5834
5835
5836 func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate, metageneration int64) *ObjectAttrs {
5837 h.t.Helper()
5838 attrs, err := o.If(Conditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua)
5839 if err != nil {
5840 h.t.Fatalf("update: %v", err)
5841 }
5842 return attrs
5843 }
5844
5845 func (h testHelper) mustWrite(w *Writer, data []byte) {
5846 h.t.Helper()
5847 if _, err := w.Write(data); err != nil {
5848 w.Close()
5849 h.t.Fatalf("write: %v", err)
5850 }
5851 if err := w.Close(); err != nil {
5852 h.t.Fatalf("close write: %v", err)
5853 }
5854 }
5855
5856 func (h testHelper) mustRead(obj *ObjectHandle) []byte {
5857 h.t.Helper()
5858 data, err := readObject(context.Background(), obj)
5859 if err != nil {
5860 h.t.Fatalf("read: %v", err)
5861 }
5862 return data
5863 }
5864
5865
5866
5867
5868 func deleteObjectIfExists(o *ObjectHandle, retryOpts ...RetryOption) error {
5869 ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
5870 defer cancel()
5871 retryOpts = append([]RetryOption{WithPolicy(RetryAlways)}, retryOpts...)
5872
5873 if err := o.Retryer(retryOpts...).Delete(ctx); err != nil {
5874 var apiErr *apierror.APIError
5875 if ok := errors.As(err, &apiErr); ok {
5876
5877 if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound {
5878 return nil
5879 }
5880 }
5881 return fmt.Errorf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err)
5882 }
5883 return nil
5884 }
5885
5886 func writeContents(w *Writer, contents []byte) error {
5887 if contents != nil {
5888 if _, err := w.Write(contents); err != nil {
5889 _ = w.Close()
5890 return err
5891 }
5892 }
5893 return w.Close()
5894 }
5895
5896 func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error {
5897 w := newWriter(ctx, obj, contentType, false)
5898
5899 return writeContents(w, contents)
5900 }
5901
5902 func newWriter(ctx context.Context, obj *ObjectHandle, contentType string, forceEmptyContentType bool) *Writer {
5903 w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
5904 w.ContentType = contentType
5905 w.ForceEmptyContentType = forceEmptyContentType
5906
5907 return w
5908 }
5909
5910 func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) {
5911 r, err := obj.NewReader(ctx)
5912 if err != nil {
5913 return nil, err
5914 }
5915 defer r.Close()
5916 return io.ReadAll(r)
5917 }
5918
5919
5920
5921 func cleanupBuckets() error {
5922 if testing.Short() {
5923 return nil
5924 }
5925 ctx := context.Background()
5926 client, err := newTestClient(ctx)
5927 if err != nil {
5928 log.Fatalf("NewClient: %v", err)
5929 }
5930 if client == nil {
5931 return nil
5932 }
5933 defer client.Close()
5934 if err := killBucket(ctx, client, bucketName); err != nil {
5935 return err
5936 }
5937 if err := killBucket(ctx, client, grpcBucketName); err != nil {
5938 return err
5939 }
5940
5941
5942
5943
5944 if err := deleteExpiredBuckets(ctx, client, testPrefix); err != nil {
5945 return err
5946 }
5947 return deleteExpiredBuckets(ctx, client, grpcTestPrefix)
5948 }
5949
5950 func deleteExpiredBuckets(ctx context.Context, client *Client, prefix string) error {
5951 const expireAge = 24 * time.Hour
5952 projectID := testutil.ProjID()
5953 it := client.Buckets(ctx, projectID)
5954 it.Prefix = prefix
5955 for {
5956 bktAttrs, err := it.Next()
5957 if err == iterator.Done {
5958 break
5959 }
5960 if err != nil {
5961 return err
5962 }
5963 if time.Since(bktAttrs.Created) > expireAge {
5964 log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge)
5965 if err := killBucket(ctx, client, bktAttrs.Name); err != nil {
5966 return err
5967 }
5968 }
5969 }
5970 return nil
5971 }
5972
5973
5974 func killBucket(ctx context.Context, client *Client, bucketName string) error {
5975 bkt := client.Bucket(bucketName)
5976
5977 it := bkt.Objects(ctx, nil)
5978 for {
5979 objAttrs, err := it.Next()
5980 if err == iterator.Done {
5981 break
5982 }
5983 if err != nil {
5984 return err
5985 }
5986
5987 if objAttrs.EventBasedHold || objAttrs.TemporaryHold {
5988 obj := bkt.Object(objAttrs.Name)
5989 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{EventBasedHold: false, TemporaryHold: false}); err != nil {
5990 return fmt.Errorf("removing hold from %q: %v", bucketName+"/"+objAttrs.Name, err)
5991 }
5992 }
5993 if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil {
5994 return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err)
5995 }
5996 }
5997
5998
5999
6000 if err := bkt.Delete(ctx); err != nil {
6001 log.Printf("deleting %q: %v", bucketName, err)
6002 }
6003 return nil
6004 }
6005
6006 func randomContents() []byte {
6007 h := md5.New()
6008 io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000)))
6009 return h.Sum(nil)
6010 }
6011
6012 type zeros struct{}
6013
6014 func (zeros) Read(p []byte) (int, error) { return len(p), nil }
6015
6016
6017 func getURL(url string, headers map[string][]string) ([]byte, error) {
6018 req, err := http.NewRequest("GET", url, nil)
6019 if err != nil {
6020 return nil, err
6021 }
6022 req.Header = headers
6023 res, err := http.DefaultClient.Do(req)
6024 if err != nil {
6025 return nil, err
6026 }
6027 defer res.Body.Close()
6028 bytes, err := io.ReadAll(res.Body)
6029 if err != nil {
6030 return nil, err
6031 }
6032 if res.StatusCode != 200 {
6033 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes))
6034 }
6035 return bytes, nil
6036 }
6037
6038
6039 func putURL(url string, headers map[string][]string, payload io.Reader) ([]byte, error) {
6040 req, err := http.NewRequest("PUT", url, payload)
6041 if err != nil {
6042 return nil, err
6043 }
6044 req.Header = headers
6045 res, err := http.DefaultClient.Do(req)
6046 if err != nil {
6047 return nil, err
6048 }
6049 defer res.Body.Close()
6050 bytes, err := io.ReadAll(res.Body)
6051 if err != nil {
6052 return nil, err
6053 }
6054 if res.StatusCode != 200 {
6055 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes))
6056 }
6057 return bytes, nil
6058 }
6059
6060 func keyFileEmail(filename string) (string, error) {
6061 bytes, err := os.ReadFile(filename)
6062 if err != nil {
6063 return "", err
6064 }
6065 var v struct {
6066 ClientEmail string `json:"client_email"`
6067 }
6068 if err := json.Unmarshal(bytes, &v); err != nil {
6069 return "", err
6070 }
6071 return v.ClientEmail, nil
6072 }
6073
6074 type comparableACL interface {
6075 equals(ACLRule) bool
6076 }
6077
6078 type testACLRule ACLRule
6079
6080 func (acl testACLRule) equals(a ACLRule) bool {
6081 return cmp.Equal(a, ACLRule(acl))
6082 }
6083
6084 type entityRoleACL struct {
6085 entity ACLEntity
6086 role ACLRole
6087 }
6088
6089 func (er entityRoleACL) equals(a ACLRule) bool {
6090 return a.Entity == er.entity && a.Role == er.role
6091 }
6092
6093 type prefixRoleACL struct {
6094 prefix string
6095 role ACLRole
6096 }
6097
6098 func (pr prefixRoleACL) equals(a ACLRule) bool {
6099 return strings.HasPrefix(string(a.Entity), pr.prefix) && a.Role == pr.role
6100 }
6101
6102 func containsACLRule(acl []ACLRule, want comparableACL) bool {
6103 for _, acl := range acl {
6104 if want.equals(acl) {
6105 return true
6106 }
6107 }
6108 return false
6109 }
6110
6111
6112
6113
6114
6115
6116 func retry(ctx context.Context, call func() error, check func() error) error {
6117 timeout := time.After(60 * time.Second)
6118 var err error
6119 for {
6120 select {
6121 case <-timeout:
6122 return err
6123 default:
6124 }
6125 err = call()
6126 if err == nil {
6127 if check == nil || check() == nil {
6128 return nil
6129 }
6130 err = check()
6131 }
6132 time.Sleep(200 * time.Millisecond)
6133 }
6134 }
6135
6136 func retryOnNilAndTransientErrs(err error) bool {
6137 return err == nil || ShouldRetry(err)
6138 }
6139 func retryOnTransient400and403(err error) bool {
6140 var e *googleapi.Error
6141 var ae *apierror.APIError
6142 return ShouldRetry(err) ||
6143 errors.As(err, &e) && (e.Code == 400 || e.Code == 403) ||
6144 errors.As(err, &ae) && (ae.GRPCStatus().Code() == codes.InvalidArgument || ae.GRPCStatus().Code() == codes.PermissionDenied)
6145 }
6146
6147 func skipGRPC(reason string) context.Context {
6148 return context.WithValue(context.Background(), skipTransportTestKey("grpc"), reason)
6149 }
6150
6151 func skipHTTP(reason string) context.Context {
6152 ctx := context.WithValue(context.Background(), skipTransportTestKey("http"), reason)
6153 return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason)
6154 }
6155
6156 func skipJSONReads(ctx context.Context, reason string) context.Context {
6157 return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason)
6158 }
6159
6160 func skipXMLReads(ctx context.Context, reason string) context.Context {
6161 return context.WithValue(ctx, skipTransportTestKey("http"), reason)
6162 }
6163
6164
6165 func extractErrCode(err error) int {
6166 if err == nil {
6167 return 0
6168 }
6169 var e *googleapi.Error
6170 if errors.As(err, &e) {
6171 return e.Code
6172 }
6173
6174 return -1
6175 }
6176
6177 func setUpRequesterPaysBucket(ctx context.Context, t *testing.T, bucket, object string, addOwnerEmail string) {
6178 t.Helper()
6179 client := testConfig(ctx, t)
6180 h := testHelper{t}
6181
6182 requesterPaysBucket := client.Bucket(bucket)
6183
6184
6185 h.mustCreate(requesterPaysBucket, testutil.ProjID(), &BucketAttrs{RequesterPays: true})
6186 t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) })
6187
6188
6189 if err := requesterPaysBucket.ACL().Set(ctx, ACLEntity("user-"+addOwnerEmail), RoleOwner); err != nil {
6190 t.Fatalf("set ACL: %v", err)
6191 }
6192
6193 h.mustWrite(requesterPaysBucket.Object(object).NewWriter(ctx), []byte("hello"))
6194 t.Cleanup(func() {
6195 err := requesterPaysBucket.Object(object).Delete(ctx)
6196 if err != nil {
6197
6198 t.Logf("could not delete object: %v", err)
6199 }
6200 })
6201 }
6202
View as plain text