client.go

Documentation: cloud.google.com/go/storage

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package storage
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  	"time"
    21  
    22  	"cloud.google.com/go/iam/apiv1/iampb"
    23  	gax "github.com/googleapis/gax-go/v2"
    24  	"google.golang.org/api/option"
    25  )
    26  
    27  // TODO(noahdietz): Move existing factory methods to this file.
    28  
    29  // storageClient is an internal-only interface designed to separate the
    30  // transport-specific logic of making Storage API calls from the logic of the
    31  // client library.
    32  //
    33  // Implementation requirements beyond implementing the interface include:
    34  // * factory method(s) must accept a `userProject string` param
    35  // * `settings` must be retained per instance
    36  // * `storageOption`s must be resolved in the order they are received
    37  // * all API errors must be wrapped in the gax-go APIError type
    38  // * any unimplemented interface methods must return a StorageUnimplementedErr
    39  //
    40  // TODO(noahdietz): This interface is currently not used in the production code
    41  // paths
    42  type storageClient interface {
    43  
    44  	// Top-level methods.
    45  
    46  	GetServiceAccount(ctx context.Context, project string, opts ...storageOption) (string, error)
    47  	CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error)
    48  	ListBuckets(ctx context.Context, project string, opts ...storageOption) *BucketIterator
    49  	Close() error
    50  
    51  	// Bucket methods.
    52  
    53  	DeleteBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) error
    54  	GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error)
    55  	UpdateBucket(ctx context.Context, bucket string, uattrs *BucketAttrsToUpdate, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error)
    56  	LockBucketRetentionPolicy(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) error
    57  	ListObjects(ctx context.Context, bucket string, q *Query, opts ...storageOption) *ObjectIterator
    58  
    59  	// Object metadata methods.
    60  
    61  	DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
    62  	GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error)
    63  	UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error)
    64  	RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error)
    65  
    66  	// Default Object ACL methods.
    67  
    68  	DeleteDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error
    69  	ListDefaultObjectACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error)
    70  	UpdateDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, role ACLRole, opts ...storageOption) error
    71  
    72  	// Bucket ACL methods.
    73  
    74  	DeleteBucketACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error
    75  	ListBucketACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error)
    76  	UpdateBucketACL(ctx context.Context, bucket string, entity ACLEntity, role ACLRole, opts ...storageOption) error
    77  
    78  	// Object ACL methods.
    79  
    80  	DeleteObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, opts ...storageOption) error
    81  	ListObjectACLs(ctx context.Context, bucket, object string, opts ...storageOption) ([]ACLRule, error)
    82  	UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) error
    83  
    84  	// Media operations.
    85  
    86  	ComposeObject(ctx context.Context, req *composeObjectRequest, opts ...storageOption) (*ObjectAttrs, error)
    87  	RewriteObject(ctx context.Context, req *rewriteObjectRequest, opts ...storageOption) (*rewriteObjectResponse, error)
    88  
    89  	NewRangeReader(ctx context.Context, params *newRangeReaderParams, opts ...storageOption) (*Reader, error)
    90  	OpenWriter(params *openWriterParams, opts ...storageOption) (*io.PipeWriter, error)
    91  
    92  	// IAM methods.
    93  
    94  	GetIamPolicy(ctx context.Context, resource string, version int32, opts ...storageOption) (*iampb.Policy, error)
    95  	SetIamPolicy(ctx context.Context, resource string, policy *iampb.Policy, opts ...storageOption) error
    96  	TestIamPermissions(ctx context.Context, resource string, permissions []string, opts ...storageOption) ([]string, error)
    97  
    98  	// HMAC Key methods.
    99  
   100  	GetHMACKey(ctx context.Context, project, accessID string, opts ...storageOption) (*HMACKey, error)
   101  	ListHMACKeys(ctx context.Context, project, serviceAccountEmail string, showDeletedKeys bool, opts ...storageOption) *HMACKeysIterator
   102  	UpdateHMACKey(ctx context.Context, project, serviceAccountEmail, accessID string, attrs *HMACKeyAttrsToUpdate, opts ...storageOption) (*HMACKey, error)
   103  	CreateHMACKey(ctx context.Context, project, serviceAccountEmail string, opts ...storageOption) (*HMACKey, error)
   104  	DeleteHMACKey(ctx context.Context, project, accessID string, opts ...storageOption) error
   105  
   106  	// Notification methods.
   107  	ListNotifications(ctx context.Context, bucket string, opts ...storageOption) (map[string]*Notification, error)
   108  	CreateNotification(ctx context.Context, bucket string, n *Notification, opts ...storageOption) (*Notification, error)
   109  	DeleteNotification(ctx context.Context, bucket string, id string, opts ...storageOption) error
   110  }
   111  
   112  // settings contains transport-agnostic configuration for API calls made via
   113  // the storageClient inteface. All implementations must utilize settings
   114  // and respect those that are applicable.
   115  type settings struct {
   116  	// retry is the complete retry configuration to use when evaluating if an
   117  	// API call should be retried.
   118  	retry *retryConfig
   119  
   120  	// gax is a set of gax.CallOption to be conveyed to gax.Invoke.
   121  	// Note: Not all storageClient interfaces will must use gax.Invoke.
   122  	gax []gax.CallOption
   123  
   124  	// idempotent indicates if the call is idempotent or not when considering
   125  	// if the call should be retired or not.
   126  	idempotent bool
   127  
   128  	// clientOption is a set of option.ClientOption to be used during client
   129  	// transport initialization. See https://pkg.go.dev/google.golang.org/api/option
   130  	// for a list of supported options.
   131  	clientOption []option.ClientOption
   132  
   133  	// userProject is the user project that should be billed for the request.
   134  	userProject string
   135  }
   136  
   137  func initSettings(opts ...storageOption) *settings {
   138  	s := &settings{}
   139  	resolveOptions(s, opts...)
   140  	return s
   141  }
   142  
   143  func resolveOptions(s *settings, opts ...storageOption) {
   144  	for _, o := range opts {
   145  		o.Apply(s)
   146  	}
   147  }
   148  
   149  // callSettings is a helper for resolving storage options against the settings
   150  // in the context of an individual call. This is to ensure that client-level
   151  // default settings are not mutated by two different calls getting options.
   152  //
   153  // Example: s := callSettings(c.settings, opts...)
   154  func callSettings(defaults *settings, opts ...storageOption) *settings {
   155  	if defaults == nil {
   156  		return nil
   157  	}
   158  	// This does not make a deep copy of the pointer/slice fields, but all
   159  	// options replace the settings fields rather than modify their values in
   160  	// place.
   161  	cs := *defaults
   162  	resolveOptions(&cs, opts...)
   163  	return &cs
   164  }
   165  
   166  // makeStorageOpts is a helper for generating a set of storageOption based on
   167  // idempotency, retryConfig, and userProject. All top-level client operations
   168  // will generally have to pass these options through the interface.
   169  func makeStorageOpts(isIdempotent bool, retry *retryConfig, userProject string) []storageOption {
   170  	opts := []storageOption{idempotent(isIdempotent)}
   171  	if retry != nil {
   172  		opts = append(opts, withRetryConfig(retry))
   173  	}
   174  	if userProject != "" {
   175  		opts = append(opts, withUserProject(userProject))
   176  	}
   177  	return opts
   178  }
   179  
   180  // storageOption is the transport-agnostic call option for the storageClient
   181  // interface.
   182  type storageOption interface {
   183  	Apply(s *settings)
   184  }
   185  
   186  func withRetryConfig(rc *retryConfig) storageOption {
   187  	return &retryOption{rc}
   188  }
   189  
   190  type retryOption struct {
   191  	rc *retryConfig
   192  }
   193  
   194  func (o *retryOption) Apply(s *settings) { s.retry = o.rc }
   195  
   196  func idempotent(i bool) storageOption {
   197  	return &idempotentOption{i}
   198  }
   199  
   200  type idempotentOption struct {
   201  	idempotency bool
   202  }
   203  
   204  func (o *idempotentOption) Apply(s *settings) { s.idempotent = o.idempotency }
   205  
   206  func withClientOptions(opts ...option.ClientOption) storageOption {
   207  	return &clientOption{opts: opts}
   208  }
   209  
   210  type clientOption struct {
   211  	opts []option.ClientOption
   212  }
   213  
   214  func (o *clientOption) Apply(s *settings) { s.clientOption = o.opts }
   215  
   216  func withUserProject(project string) storageOption {
   217  	return &userProjectOption{project}
   218  }
   219  
   220  type userProjectOption struct {
   221  	project string
   222  }
   223  
   224  func (o *userProjectOption) Apply(s *settings) { s.userProject = o.project }
   225  
   226  type openWriterParams struct {
   227  	// Writer configuration
   228  
   229  	// ctx is the context used by the writer routine to make all network calls
   230  	// and to manage the writer routine - see `Writer.ctx`.
   231  	// Required.
   232  	ctx context.Context
   233  	// chunkSize - see `Writer.ChunkSize`.
   234  	// Optional.
   235  	chunkSize int
   236  	// chunkRetryDeadline - see `Writer.ChunkRetryDeadline`.
   237  	// Optional.
   238  	chunkRetryDeadline time.Duration
   239  
   240  	// Object/request properties
   241  
   242  	// bucket - see `Writer.o.bucket`.
   243  	// Required.
   244  	bucket string
   245  	// attrs - see `Writer.ObjectAttrs`.
   246  	// Required.
   247  	attrs *ObjectAttrs
   248  	// forceEmptyContentType - Disables auto-detect of Content-Type
   249  	// Optional.
   250  	forceEmptyContentType bool
   251  	// conds - see `Writer.o.conds`.
   252  	// Optional.
   253  	conds *Conditions
   254  	// encryptionKey - see `Writer.o.encryptionKey`
   255  	// Optional.
   256  	encryptionKey []byte
   257  	// sendCRC32C - see `Writer.SendCRC32C`.
   258  	// Optional.
   259  	sendCRC32C bool
   260  
   261  	// Writer callbacks
   262  
   263  	// donec - see `Writer.donec`.
   264  	// Required.
   265  	donec chan struct{}
   266  	// setError callback for reporting errors - see `Writer.error`.
   267  	// Required.
   268  	setError func(error)
   269  	// progress callback for reporting upload progress - see `Writer.progress`.
   270  	// Required.
   271  	progress func(int64)
   272  	// setObj callback for reporting the resulting object - see `Writer.obj`.
   273  	// Required.
   274  	setObj func(*ObjectAttrs)
   275  }
   276  
   277  type newRangeReaderParams struct {
   278  	bucket         string
   279  	conds          *Conditions
   280  	encryptionKey  []byte
   281  	gen            int64
   282  	length         int64
   283  	object         string
   284  	offset         int64
   285  	readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
   286  }
   287  
   288  type getObjectParams struct {
   289  	bucket, object string
   290  	gen            int64
   291  	encryptionKey  []byte
   292  	conds          *Conditions
   293  	softDeleted    bool
   294  }
   295  
   296  type updateObjectParams struct {
   297  	bucket, object    string
   298  	uattrs            *ObjectAttrsToUpdate
   299  	gen               int64
   300  	encryptionKey     []byte
   301  	conds             *Conditions
   302  	overrideRetention *bool
   303  }
   304  
   305  type restoreObjectParams struct {
   306  	bucket, object string
   307  	gen            int64
   308  	encryptionKey  []byte
   309  	conds          *Conditions
   310  	copySourceACL  bool
   311  }
   312  
   313  type composeObjectRequest struct {
   314  	dstBucket     string
   315  	dstObject     destinationObject
   316  	srcs          []sourceObject
   317  	predefinedACL string
   318  	sendCRC32C    bool
   319  }
   320  
   321  type sourceObject struct {
   322  	name          string
   323  	bucket        string
   324  	gen           int64
   325  	conds         *Conditions
   326  	encryptionKey []byte
   327  }
   328  
   329  type destinationObject struct {
   330  	name          string
   331  	bucket        string
   332  	conds         *Conditions
   333  	attrs         *ObjectAttrs // attrs to set on the destination object.
   334  	encryptionKey []byte
   335  	keyName       string
   336  }
   337  
   338  type rewriteObjectRequest struct {
   339  	srcObject                sourceObject
   340  	dstObject                destinationObject
   341  	predefinedACL            string
   342  	token                    string
   343  	maxBytesRewrittenPerCall int64
   344  }
   345  
   346  type rewriteObjectResponse struct {
   347  	resource *ObjectAttrs
   348  	done     bool
   349  	written  int64
   350  	size     int64
   351  	token    string
   352  }
   353
View as plain text