Source file src/google.golang.org/api/integration-tests/storage/integration_test.go

Documentation: google.golang.org/api/integration-tests/storage

     1  // Copyright 2017 Google LLC.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build integration
     6  // +build integration
     7  
     8  package storage
     9  
    10  import (
    11  	"bytes"
    12  	"context"
    13  	"encoding/base64"
    14  	"errors"
    15  	"fmt"
    16  	"io/ioutil"
    17  	"log"
    18  	"net/http"
    19  	"os"
    20  	"strings"
    21  	"testing"
    22  
    23  	"golang.org/x/oauth2"
    24  	"golang.org/x/oauth2/google"
    25  	"google.golang.org/api/googleapi"
    26  	storage "google.golang.org/api/storage/v1"
    27  )
    28  
    29  type object struct {
    30  	name, contents string
    31  }
    32  
    33  var (
    34  	projectID string
    35  	bucket    string
    36  	objects   = []object{
    37  		{"obj1", testContents},
    38  		{"obj2", testContents},
    39  		{"obj/with/slashes", testContents},
    40  		{"resumable", testContents},
    41  		{"large", strings.Repeat("a", 514)}, // larger than the first section of content that is sniffed by ContentSniffer.
    42  	}
    43  	aclObjects = []string{"acl1", "acl2"}
    44  	copyObj    = "copy-object"
    45  )
    46  
    47  const (
    48  	envProject    = "GCLOUD_TESTS_GOLANG_PROJECT_ID"
    49  	envPrivateKey = "GCLOUD_TESTS_GOLANG_KEY"
    50  	// NOTE that running this test on a bucket deletes ALL contents of the bucket!
    51  	envBucket    = "GCLOUD_TESTS_GOLANG_DESTRUCTIVE_TEST_BUCKET_NAME"
    52  	testContents = "some text that will be saved to a bucket object"
    53  )
    54  
    55  func verifyAcls(obj *storage.Object, wantDomainRole, wantAllUsersRole string) (err error) {
    56  	var gotDomainRole, gotAllUsersRole string
    57  	for _, acl := range obj.Acl {
    58  		if acl.Entity == "domain-google.com" {
    59  			gotDomainRole = acl.Role
    60  		}
    61  		if acl.Entity == "allUsers" {
    62  			gotAllUsersRole = acl.Role
    63  		}
    64  	}
    65  	if gotDomainRole != wantDomainRole {
    66  		err = fmt.Errorf("domain-google.com role = %q; want %q", gotDomainRole, wantDomainRole)
    67  	}
    68  	if gotAllUsersRole != wantAllUsersRole {
    69  		err = fmt.Errorf("allUsers role = %q; want %q; %v", gotAllUsersRole, wantAllUsersRole, err)
    70  	}
    71  	return err
    72  }
    73  
    74  // TODO(gmlewis): Move this to a common location.
    75  func tokenSource(ctx context.Context, scopes ...string) (oauth2.TokenSource, error) {
    76  	keyFile := os.Getenv(envPrivateKey)
    77  	if keyFile == "" {
    78  		return nil, errors.New(envPrivateKey + " not set")
    79  	}
    80  	jsonKey, err := ioutil.ReadFile(keyFile)
    81  	if err != nil {
    82  		return nil, fmt.Errorf("unable to read %q: %v", keyFile, err)
    83  	}
    84  	conf, err := google.JWTConfigFromJSON(jsonKey, scopes...)
    85  	if err != nil {
    86  		return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
    87  	}
    88  	return conf.TokenSource(ctx), nil
    89  }
    90  
    91  const defaultType = "text/plain; charset=utf-8"
    92  
    93  // writeObject writes some data and default metadata to the specified object.
    94  // Resumable upload is used if resumable is true.
    95  // The written data is returned.
    96  func writeObject(s *storage.Service, bucket, obj string, resumable bool, contents string) error {
    97  	o := &storage.Object{
    98  		Bucket:          bucket,
    99  		Name:            obj,
   100  		ContentType:     defaultType,
   101  		ContentEncoding: "utf-8",
   102  		ContentLanguage: "en",
   103  		Metadata:        map[string]string{"foo": "bar"},
   104  	}
   105  	f := strings.NewReader(contents)
   106  	insert := s.Objects.Insert(bucket, o)
   107  	if resumable {
   108  		insert.ResumableMedia(context.Background(), f, int64(len(contents)), defaultType)
   109  	} else {
   110  		insert.Media(f)
   111  	}
   112  	_, err := insert.Do()
   113  	return err
   114  }
   115  
   116  func checkMetadata(t *testing.T, s *storage.Service, bucket, obj string) {
   117  	o, err := s.Objects.Get(bucket, obj).Do()
   118  	if err != nil {
   119  		t.Error(err)
   120  	}
   121  	if got, want := o.Name, obj; got != want {
   122  		t.Errorf("name of %q = %q; want %q", obj, got, want)
   123  	}
   124  	if got, want := o.ContentType, defaultType; got != want {
   125  		t.Errorf("contentType of %q = %q; want %q", obj, got, want)
   126  	}
   127  	if got, want := o.Metadata["foo"], "bar"; got != want {
   128  		t.Errorf("metadata entry foo of %q = %q; want %q", obj, got, want)
   129  	}
   130  }
   131  
   132  func createService() *storage.Service {
   133  	if projectID = os.Getenv(envProject); projectID == "" {
   134  		log.Print("no project ID specified")
   135  		return nil
   136  	}
   137  	if bucket = os.Getenv(envBucket); bucket == "" {
   138  		log.Print("no bucket specified")
   139  		return nil
   140  	}
   141  
   142  	ctx := context.Background()
   143  	ts, err := tokenSource(ctx, storage.DevstorageFullControlScope)
   144  	if err != nil {
   145  		log.Printf("tokenSource: %v", err)
   146  		return nil
   147  	}
   148  	client := oauth2.NewClient(ctx, ts)
   149  	s, err := storage.New(client)
   150  	if err != nil {
   151  		log.Printf("unable to create service: %v", err)
   152  		return nil
   153  	}
   154  	return s
   155  }
   156  
   157  func TestMain(m *testing.M) {
   158  	if err := cleanup(); err != nil {
   159  		log.Fatalf("Pre-test cleanup failed: %v", err)
   160  	}
   161  	exit := m.Run()
   162  	if err := cleanup(); err != nil {
   163  		log.Fatalf("Post-test cleanup failed: %v", err)
   164  	}
   165  	os.Exit(exit)
   166  }
   167  
   168  func TestContentType(t *testing.T) {
   169  	s := createService()
   170  	if s == nil {
   171  		t.Fatal("Could not create service")
   172  	}
   173  
   174  	type testCase struct {
   175  		objectContentType    string
   176  		useOptionContentType bool
   177  		optionContentType    string
   178  
   179  		wantContentType string
   180  	}
   181  
   182  	// The Media method will use resumable upload if the supplied data is
   183  	// larger than googleapi.DefaultUploadChunkSize We run the following
   184  	// tests with two different file contents: one that will trigger
   185  	// resumable upload, and one that won't.
   186  	forceResumableData := bytes.Repeat([]byte("a"), googleapi.DefaultUploadChunkSize+1)
   187  	smallData := bytes.Repeat([]byte("a"), 2)
   188  
   189  	// In the following test, the content type, if any, in the Object struct is always "text/plain".
   190  	// The content type configured via googleapi.ContentType, if any, is always "text/html".
   191  	for _, tc := range []testCase{
   192  		// With content type specified in the object struct
   193  		// Temporarily disable this test during rollout of strict Content-Type.
   194  		// TODO(djd): Re-enable once strict check is 100%.
   195  		// {
   196  		// 	objectContentType:    "text/plain",
   197  		// 	useOptionContentType: true,
   198  		// 	optionContentType:    "text/html",
   199  		// 	wantContentType:      "text/html",
   200  		// },
   201  		{
   202  			objectContentType:    "text/plain",
   203  			useOptionContentType: true,
   204  			optionContentType:    "",
   205  			wantContentType:      "text/plain",
   206  		},
   207  		{
   208  			objectContentType:    "text/plain",
   209  			useOptionContentType: false,
   210  			wantContentType:      "text/plain",
   211  		},
   212  
   213  		// Without content type specified in the object struct
   214  		{
   215  			useOptionContentType: true,
   216  			optionContentType:    "text/html",
   217  			wantContentType:      "text/html",
   218  		},
   219  		{
   220  			useOptionContentType: true,
   221  			optionContentType:    "",
   222  			wantContentType:      "", // Result is an object without a content type.
   223  		},
   224  		{
   225  			useOptionContentType: false,
   226  			wantContentType:      "text/plain; charset=utf-8", // sniffed.
   227  		},
   228  	} {
   229  		// The behavior should be the same, regardless of whether resumable upload is used or not.
   230  		for _, data := range [][]byte{smallData, forceResumableData} {
   231  			o := &storage.Object{
   232  				Bucket:      bucket,
   233  				Name:        "test-content-type",
   234  				ContentType: tc.objectContentType,
   235  			}
   236  			call := s.Objects.Insert(bucket, o)
   237  			var opts []googleapi.MediaOption
   238  			if tc.useOptionContentType {
   239  				opts = append(opts, googleapi.ContentType(tc.optionContentType))
   240  			}
   241  			call.Media(bytes.NewReader(data), opts...)
   242  
   243  			_, err := call.Do()
   244  			if err != nil {
   245  				t.Fatalf("unable to insert object %q: %v", o.Name, err)
   246  			}
   247  
   248  			readObj, err := s.Objects.Get(bucket, o.Name).Do()
   249  			if err != nil {
   250  				t.Error(err)
   251  			}
   252  			if got, want := readObj.ContentType, tc.wantContentType; got != want {
   253  				t.Errorf("contentType of %q; got %q; want %q", o.Name, got, want)
   254  			}
   255  		}
   256  	}
   257  }
   258  
   259  func TestFunctions(t *testing.T) {
   260  	s := createService()
   261  	if s == nil {
   262  		t.Fatal("Could not create service")
   263  	}
   264  
   265  	t.Logf("Listing buckets for project %q", projectID)
   266  	var numBuckets int
   267  	pageToken := ""
   268  	for {
   269  		call := s.Buckets.List(projectID)
   270  		if pageToken != "" {
   271  			call.PageToken(pageToken)
   272  		}
   273  		resp, err := call.Do()
   274  		if err != nil {
   275  			t.Fatalf("unable to list buckets for project %q: %v", projectID, err)
   276  		}
   277  		numBuckets += len(resp.Items)
   278  		if pageToken = resp.NextPageToken; pageToken == "" {
   279  			break
   280  		}
   281  	}
   282  	if numBuckets == 0 {
   283  		t.Fatalf("no buckets found for project %q", projectID)
   284  	}
   285  
   286  	for _, obj := range objects {
   287  		t.Logf("Writing %q", obj.name)
   288  		// TODO(mcgreevy): stop relying on "resumable" name to determine whether to
   289  		// do a resumable upload.
   290  		err := writeObject(s, bucket, obj.name, obj.name == "resumable", obj.contents)
   291  		if err != nil {
   292  			t.Fatalf("unable to insert object %q: %v", obj.name, err)
   293  		}
   294  	}
   295  
   296  	for _, obj := range objects {
   297  		t.Logf("Reading %q", obj.name)
   298  		resp, err := s.Objects.Get(bucket, obj.name).Download()
   299  		if err != nil {
   300  			t.Fatalf("unable to get object %q: %v", obj.name, err)
   301  		}
   302  		slurp, err := ioutil.ReadAll(resp.Body)
   303  		if err != nil {
   304  			t.Fatalf("unable to read response body %q: %v", obj.name, err)
   305  		}
   306  		resp.Body.Close()
   307  		if got, want := string(slurp), obj.contents; got != want {
   308  			t.Errorf("contents of %q = %q; want %q", obj.name, got, want)
   309  		}
   310  	}
   311  
   312  	name := "obj-not-exists"
   313  	if _, err := s.Objects.Get(bucket, name).Download(); !isError(err, http.StatusNotFound) {
   314  		t.Errorf("object %q should not exist, err = %v", name, err)
   315  	} else {
   316  		t.Log("Successfully tested StatusNotFound.")
   317  	}
   318  
   319  	for _, obj := range objects {
   320  		t.Logf("Checking %q metadata", obj.name)
   321  		checkMetadata(t, s, bucket, obj.name)
   322  	}
   323  
   324  	name = objects[0].name
   325  
   326  	t.Logf("Rewriting %q to %q", name, copyObj)
   327  	copy, err := s.Objects.Rewrite(bucket, name, bucket, copyObj, nil).Do()
   328  	if err != nil {
   329  		t.Fatalf("unable to rewrite object %q to %q: %v", name, copyObj, err)
   330  	}
   331  	if copy.Resource.Name != copyObj {
   332  		t.Errorf("copy object's name = %q; want %q", copy.Resource.Name, copyObj)
   333  	}
   334  	if copy.Resource.Bucket != bucket {
   335  		t.Errorf("copy object's bucket = %q; want %q", copy.Resource.Bucket, bucket)
   336  	}
   337  
   338  	// Note that arrays such as ACLs below are completely overwritten using Patch
   339  	// semantics, so these must be updated in a read-modify-write sequence of operations.
   340  	// See https://cloud.google.com/storage/docs/json_api/v1/how-tos/performance#patch-semantics
   341  	// for more details.
   342  	t.Logf("Updating attributes of %q", name)
   343  	obj, err := s.Objects.Get(bucket, name).Projection("full").Fields("acl").Do()
   344  	if err != nil {
   345  		t.Fatalf("Objects.Get(%q, %q): %v", bucket, name, err)
   346  	}
   347  	if err := verifyAcls(obj, "", ""); err != nil {
   348  		t.Errorf("before update ACLs: %v", err)
   349  	}
   350  	obj.ContentType = "text/html"
   351  	for _, entity := range []string{"domain-google.com", "allUsers"} {
   352  		obj.Acl = append(obj.Acl, &storage.ObjectAccessControl{Entity: entity, Role: "READER"})
   353  	}
   354  	updated, err := s.Objects.Patch(bucket, name, obj).Projection("full").Fields("contentType", "acl").Do()
   355  	if err != nil {
   356  		t.Fatalf("Objects.Patch(%q, %q, %#v) failed with %v", bucket, name, obj, err)
   357  	}
   358  	if want := "text/html"; updated.ContentType != want {
   359  		t.Errorf("updated.ContentType == %q; want %q", updated.ContentType, want)
   360  	}
   361  	if err := verifyAcls(updated, "READER", "READER"); err != nil {
   362  		t.Errorf("after update ACLs: %v", err)
   363  	}
   364  
   365  	t.Log("Testing checksums")
   366  	checksumCases := []struct {
   367  		name     string
   368  		contents string
   369  		size     uint64
   370  		md5      string
   371  		crc32c   uint32
   372  	}{
   373  		{
   374  			name:     "checksum-object",
   375  			contents: "helloworld",
   376  			size:     10,
   377  			md5:      "fc5e038d38a57032085441e7fe7010b0",
   378  			crc32c:   1456190592,
   379  		},
   380  		{
   381  			name:     "zero-object",
   382  			contents: "",
   383  			size:     0,
   384  			md5:      "d41d8cd98f00b204e9800998ecf8427e",
   385  			crc32c:   0,
   386  		},
   387  	}
   388  	for _, c := range checksumCases {
   389  		f := strings.NewReader(c.contents)
   390  		o := &storage.Object{
   391  			Bucket:          bucket,
   392  			Name:            c.name,
   393  			ContentType:     defaultType,
   394  			ContentEncoding: "utf-8",
   395  			ContentLanguage: "en",
   396  		}
   397  		obj, err := s.Objects.Insert(bucket, o).Media(f).Do()
   398  		if err != nil {
   399  			t.Fatalf("unable to insert object %v: %v", obj, err)
   400  		}
   401  		if got, want := obj.Size, c.size; got != want {
   402  			t.Errorf("object %q size = %v; want %v", c.name, got, want)
   403  		}
   404  		md5, err := base64.StdEncoding.DecodeString(obj.Md5Hash)
   405  		if err != nil {
   406  			t.Fatalf("object %q base64 decode of MD5 %q: %v", c.name, obj.Md5Hash, err)
   407  		}
   408  		if got, want := fmt.Sprintf("%x", md5), c.md5; got != want {
   409  			t.Errorf("object %q MD5 = %q; want %q", c.name, got, want)
   410  		}
   411  		var crc32c uint32
   412  		d, err := base64.StdEncoding.DecodeString(obj.Crc32c)
   413  		if err != nil {
   414  			t.Errorf("object %q base64 decode of CRC32 %q: %v", c.name, obj.Crc32c, err)
   415  		}
   416  		if err == nil && len(d) == 4 {
   417  			crc32c = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3])
   418  		}
   419  		if got, want := crc32c, c.crc32c; got != want {
   420  			t.Errorf("object %q CRC32C = %v; want %v", c.name, got, want)
   421  		}
   422  	}
   423  }
   424  
   425  // cleanup destroys ALL objects in the bucket!
   426  func cleanup() error {
   427  	s := createService()
   428  	if s == nil {
   429  		return errors.New("Could not create service")
   430  	}
   431  
   432  	var pageToken string
   433  	var failed bool
   434  	for {
   435  		call := s.Objects.List(bucket)
   436  		if pageToken != "" {
   437  			call.PageToken(pageToken)
   438  		}
   439  		resp, err := call.Do()
   440  		if err != nil {
   441  			return fmt.Errorf("cleanup list failed: %v", err)
   442  		}
   443  		for _, obj := range resp.Items {
   444  			log.Printf("Cleanup deletion of %q", obj.Name)
   445  			if err := s.Objects.Delete(bucket, obj.Name).Do(); err != nil {
   446  				// Print the error out, but keep going.
   447  				log.Printf("Cleanup deletion of %q failed: %v", obj.Name, err)
   448  				failed = true
   449  			}
   450  			if _, err := s.Objects.Get(bucket, obj.Name).Download(); !isError(err, http.StatusNotFound) {
   451  				log.Printf("object %q should not exist, err = %v", obj.Name, err)
   452  				failed = true
   453  			} else {
   454  				log.Printf("Successfully deleted %q.", obj.Name)
   455  			}
   456  		}
   457  		if pageToken = resp.NextPageToken; pageToken == "" {
   458  			break
   459  		}
   460  	}
   461  	if failed {
   462  		return errors.New("Failed to delete at least one object")
   463  	}
   464  	return nil
   465  }
   466  
   467  func isError(err error, code int) bool {
   468  	if err == nil {
   469  		return false
   470  	}
   471  	ae, ok := err.(*googleapi.Error)
   472  	return ok && ae.Code == code
   473  }
   474  

View as plain text