invoke_test.go

Documentation: cloud.google.com/go/storage

     1  // Copyright 2020 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package storage
    16  
    17  import (
    18  	"context"
    19  	"errors"
    20  	"fmt"
    21  	"io"
    22  	"net"
    23  	"net/http"
    24  	"net/url"
    25  	"regexp"
    26  	"strings"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/googleapis/gax-go/v2"
    31  	"github.com/googleapis/gax-go/v2/callctx"
    32  	"golang.org/x/xerrors"
    33  	"google.golang.org/api/googleapi"
    34  	"google.golang.org/grpc/codes"
    35  	"google.golang.org/grpc/status"
    36  )
    37  
    38  func TestInvoke(t *testing.T) {
    39  	t.Parallel()
    40  	ctx := context.Background()
    41  	// Time-based tests are flaky. We just make sure that invoke eventually
    42  	// returns with the right error.
    43  
    44  	for _, test := range []struct {
    45  		desc              string
    46  		count             int   // Number of times to return retryable error.
    47  		initialErr        error // Error to return initially.
    48  		finalErr          error // Error to return after count returns of retryCode.
    49  		retry             *retryConfig
    50  		isIdempotentValue bool
    51  		expectFinalErr    bool
    52  	}{
    53  		{
    54  			desc:              "test fn never returns initial error with count=0",
    55  			count:             0,
    56  			initialErr:        &googleapi.Error{Code: 0}, //non-retryable
    57  			finalErr:          nil,
    58  			isIdempotentValue: true,
    59  			expectFinalErr:    true,
    60  		},
    61  		{
    62  			desc:              "non-retryable error is returned without retrying",
    63  			count:             1,
    64  			initialErr:        &googleapi.Error{Code: 0},
    65  			finalErr:          nil,
    66  			isIdempotentValue: true,
    67  			expectFinalErr:    false,
    68  		},
    69  		{
    70  			desc:              "retryable error is retried",
    71  			count:             1,
    72  			initialErr:        &googleapi.Error{Code: 429},
    73  			finalErr:          nil,
    74  			isIdempotentValue: true,
    75  			expectFinalErr:    true,
    76  		},
    77  		{
    78  			desc:              "retryable gRPC error is retried",
    79  			count:             1,
    80  			initialErr:        status.Error(codes.ResourceExhausted, "rate limit"),
    81  			finalErr:          nil,
    82  			isIdempotentValue: true,
    83  			expectFinalErr:    true,
    84  		},
    85  		{
    86  			desc:              "returns non-retryable error after retryable error",
    87  			count:             1,
    88  			initialErr:        &googleapi.Error{Code: 429},
    89  			finalErr:          errors.New("bar"),
    90  			isIdempotentValue: true,
    91  			expectFinalErr:    true,
    92  		},
    93  		{
    94  			desc:              "retryable 5xx error is retried",
    95  			count:             2,
    96  			initialErr:        &googleapi.Error{Code: 518},
    97  			finalErr:          nil,
    98  			isIdempotentValue: true,
    99  			expectFinalErr:    true,
   100  		},
   101  		{
   102  			desc:              "retriable error not retried when non-idempotent",
   103  			count:             2,
   104  			initialErr:        &googleapi.Error{Code: 599},
   105  			finalErr:          nil,
   106  			isIdempotentValue: false,
   107  			expectFinalErr:    false,
   108  		},
   109  		{
   110  			desc:              "non-idempotent retriable error retried when policy is RetryAlways",
   111  			count:             2,
   112  			initialErr:        &googleapi.Error{Code: 500},
   113  			finalErr:          nil,
   114  			isIdempotentValue: false,
   115  			retry:             &retryConfig{policy: RetryAlways},
   116  			expectFinalErr:    true,
   117  		},
   118  		{
   119  			desc:              "retriable error not retried when policy is RetryNever",
   120  			count:             2,
   121  			initialErr:        &url.Error{Op: "blah", URL: "blah", Err: errors.New("connection refused")},
   122  			finalErr:          nil,
   123  			isIdempotentValue: true,
   124  			retry:             &retryConfig{policy: RetryNever},
   125  			expectFinalErr:    false,
   126  		},
   127  		{
   128  			desc:              "non-retriable error not retried when policy is RetryAlways",
   129  			count:             2,
   130  			initialErr:        xerrors.Errorf("non-retriable error: %w", &googleapi.Error{Code: 400}),
   131  			finalErr:          nil,
   132  			isIdempotentValue: true,
   133  			retry:             &retryConfig{policy: RetryAlways},
   134  			expectFinalErr:    false,
   135  		},
   136  		{
   137  			desc:              "non-retriable error retried with custom fn",
   138  			count:             2,
   139  			initialErr:        io.ErrNoProgress,
   140  			finalErr:          nil,
   141  			isIdempotentValue: true,
   142  			retry: &retryConfig{
   143  				shouldRetry: func(err error) bool {
   144  					return err == io.ErrNoProgress
   145  				},
   146  			},
   147  			expectFinalErr: true,
   148  		},
   149  		{
   150  			desc:              "retriable error not retried with custom fn",
   151  			count:             2,
   152  			initialErr:        io.ErrUnexpectedEOF,
   153  			finalErr:          nil,
   154  			isIdempotentValue: true,
   155  			retry: &retryConfig{
   156  				shouldRetry: func(err error) bool {
   157  					return err == io.ErrNoProgress
   158  				},
   159  			},
   160  			expectFinalErr: false,
   161  		},
   162  		{
   163  			desc:              "error not retried when policy is RetryNever despite custom fn",
   164  			count:             2,
   165  			initialErr:        io.ErrUnexpectedEOF,
   166  			finalErr:          nil,
   167  			isIdempotentValue: true,
   168  			retry: &retryConfig{
   169  				shouldRetry: func(err error) bool {
   170  					return err == io.ErrUnexpectedEOF
   171  				},
   172  				policy: RetryNever,
   173  			},
   174  			expectFinalErr: false,
   175  		},
   176  		{
   177  			desc:              "non-idempotent retriable error retried when policy is RetryAlways till maxAttempts",
   178  			count:             4,
   179  			initialErr:        &googleapi.Error{Code: 500},
   180  			finalErr:          nil,
   181  			isIdempotentValue: false,
   182  			retry:             &retryConfig{policy: RetryAlways, maxAttempts: expectedAttempts(2)},
   183  			expectFinalErr:    false,
   184  		},
   185  		{
   186  			desc:              "non-idempotent retriable error not retried when policy is RetryNever with maxAttempts set",
   187  			count:             4,
   188  			initialErr:        &googleapi.Error{Code: 500},
   189  			finalErr:          nil,
   190  			isIdempotentValue: false,
   191  			retry:             &retryConfig{policy: RetryNever, maxAttempts: expectedAttempts(2)},
   192  			expectFinalErr:    false,
   193  		},
   194  		{
   195  			desc:              "non-retriable error retried with custom fn till maxAttempts",
   196  			count:             4,
   197  			initialErr:        io.ErrNoProgress,
   198  			finalErr:          nil,
   199  			isIdempotentValue: true,
   200  			retry: &retryConfig{
   201  				shouldRetry: func(err error) bool {
   202  					return err == io.ErrNoProgress
   203  				},
   204  				maxAttempts: expectedAttempts(2),
   205  			},
   206  			expectFinalErr: false,
   207  		},
   208  		{
   209  			desc:              "non-idempotent retriable error retried when policy is RetryAlways till maxAttempts where count equals to maxAttempts-1",
   210  			count:             3,
   211  			initialErr:        &googleapi.Error{Code: 500},
   212  			finalErr:          nil,
   213  			isIdempotentValue: false,
   214  			retry:             &retryConfig{policy: RetryAlways, maxAttempts: expectedAttempts(4)},
   215  			expectFinalErr:    true,
   216  		},
   217  		{
   218  			desc:              "non-idempotent retriable error retried when policy is RetryAlways till maxAttempts where count equals to maxAttempts",
   219  			count:             4,
   220  			initialErr:        &googleapi.Error{Code: 500},
   221  			finalErr:          nil,
   222  			isIdempotentValue: true,
   223  			retry:             &retryConfig{policy: RetryAlways, maxAttempts: expectedAttempts(4)},
   224  			expectFinalErr:    false,
   225  		},
   226  		{
   227  			desc:              "non-idempotent retriable error not retried when policy is RetryAlways with maxAttempts equals to zero",
   228  			count:             4,
   229  			initialErr:        &googleapi.Error{Code: 500},
   230  			finalErr:          nil,
   231  			isIdempotentValue: true,
   232  			retry:             &retryConfig{maxAttempts: expectedAttempts(0), policy: RetryAlways},
   233  			expectFinalErr:    false,
   234  		},
   235  	} {
   236  		t.Run(test.desc, func(s *testing.T) {
   237  			counter := 0
   238  			var initialClientHeader, initialIdempotencyHeader string
   239  			var gotClientHeader, gotIdempotencyHeader string
   240  			call := func(ctx context.Context) error {
   241  				if counter == 0 {
   242  					headers := callctx.HeadersFromContext(ctx)
   243  					initialClientHeader = headers["x-goog-api-client"][0]
   244  					initialIdempotencyHeader = headers["x-goog-gcs-idempotency-token"][0]
   245  				}
   246  				counter++
   247  				headers := callctx.HeadersFromContext(ctx)
   248  				gotClientHeader = headers["x-goog-api-client"][0]
   249  				gotIdempotencyHeader = headers["x-goog-gcs-idempotency-token"][0]
   250  				if counter <= test.count {
   251  					return test.initialErr
   252  				}
   253  				return test.finalErr
   254  			}
   255  			// Use a short backoff to speed up the test.
   256  			if test.retry == nil {
   257  				test.retry = defaultRetry.clone()
   258  			}
   259  			test.retry.backoff = &gax.Backoff{Initial: time.Millisecond}
   260  			got := run(ctx, call, test.retry, test.isIdempotentValue)
   261  			if test.expectFinalErr && !errors.Is(got, test.finalErr) {
   262  				s.Errorf("got %v, want %v", got, test.finalErr)
   263  			} else if !test.expectFinalErr && !errors.Is(got, test.initialErr) {
   264  				s.Errorf("got %v, want %v", got, test.initialErr)
   265  			}
   266  			wantAttempts := 1 + test.count
   267  			if !test.expectFinalErr {
   268  				wantAttempts = 1
   269  			}
   270  			if test.retry != nil && test.retry.maxAttempts != nil && *test.retry.maxAttempts != 0 && test.retry.policy != RetryNever {
   271  				wantAttempts = *test.retry.maxAttempts
   272  			}
   273  
   274  			wantClientHeader := strings.ReplaceAll(initialClientHeader, "gccl-attempt-count/1", fmt.Sprintf("gccl-attempt-count/%v", wantAttempts))
   275  			if gotClientHeader != wantClientHeader {
   276  				t.Errorf("case %q, retry header:\ngot %v\nwant %v", test.desc, gotClientHeader, wantClientHeader)
   277  			}
   278  			wantClientHeaderFormat := "gccl-invocation-id/.{36} gccl-attempt-count/[0-9]+ gl-go/.* gccl/"
   279  			match, err := regexp.MatchString(wantClientHeaderFormat, gotClientHeader)
   280  			if err != nil {
   281  				s.Fatalf("compiling regexp: %v", err)
   282  			}
   283  			if !match {
   284  				s.Errorf("X-Goog-Api-Client header has wrong format\ngot %v\nwant regex matching %v", gotClientHeader, wantClientHeaderFormat)
   285  			}
   286  			if gotIdempotencyHeader != initialIdempotencyHeader {
   287  				t.Errorf("case %q, idempotency header:\ngot %v\nwant %v", test.desc, gotIdempotencyHeader, initialIdempotencyHeader)
   288  			}
   289  		})
   290  	}
   291  }
   292  
   293  type fakeApiaryRequest struct {
   294  	header http.Header
   295  }
   296  
   297  func (f *fakeApiaryRequest) Header() http.Header {
   298  	return f.header
   299  }
   300  
   301  func TestShouldRetry(t *testing.T) {
   302  	t.Parallel()
   303  
   304  	for _, test := range []struct {
   305  		desc        string
   306  		inputErr    error
   307  		shouldRetry bool
   308  	}{
   309  		{
   310  			desc:        "googleapi.Error{Code: 0}",
   311  			inputErr:    &googleapi.Error{Code: 0},
   312  			shouldRetry: false,
   313  		},
   314  		{
   315  			desc:        "googleapi.Error{Code: 429}",
   316  			inputErr:    &googleapi.Error{Code: 429},
   317  			shouldRetry: true,
   318  		},
   319  		{
   320  			desc:        "errors.New(foo)",
   321  			inputErr:    errors.New("foo"),
   322  			shouldRetry: false,
   323  		},
   324  		{
   325  			desc:        "googleapi.Error{Code: 518}",
   326  			inputErr:    &googleapi.Error{Code: 518},
   327  			shouldRetry: true,
   328  		},
   329  		{
   330  			desc:        "googleapi.Error{Code: 599}",
   331  			inputErr:    &googleapi.Error{Code: 599},
   332  			shouldRetry: true,
   333  		},
   334  		{
   335  			desc:        "googleapi.Error{Code: 428}",
   336  			inputErr:    &googleapi.Error{Code: 428},
   337  			shouldRetry: false,
   338  		},
   339  		{
   340  			desc:        "googleapi.Error{Code: 518}",
   341  			inputErr:    &googleapi.Error{Code: 518},
   342  			shouldRetry: true,
   343  		},
   344  		{
   345  			desc:        "url.Error{Err: errors.New(\"connection refused\")}",
   346  			inputErr:    &url.Error{Op: "blah", URL: "blah", Err: errors.New("connection refused")},
   347  			shouldRetry: true,
   348  		},
   349  		{
   350  			desc:        "net.OpError{Err: errors.New(\"connection reset by peer\")}",
   351  			inputErr:    &net.OpError{Op: "blah", Net: "tcp", Err: errors.New("connection reset by peer")},
   352  			shouldRetry: true,
   353  		},
   354  		{
   355  			desc:        "io.ErrUnexpectedEOF",
   356  			inputErr:    io.ErrUnexpectedEOF,
   357  			shouldRetry: true,
   358  		},
   359  		{
   360  			desc:        "wrapped retryable error",
   361  			inputErr:    xerrors.Errorf("Test unwrapping of a temporary error: %w", &googleapi.Error{Code: 500}),
   362  			shouldRetry: true,
   363  		},
   364  		{
   365  			desc:        "wrapped non-retryable error",
   366  			inputErr:    xerrors.Errorf("Test unwrapping of a non-retriable error: %w", &googleapi.Error{Code: 400}),
   367  			shouldRetry: false,
   368  		},
   369  		{
   370  			desc:        "googleapi.Error{Code: 400}",
   371  			inputErr:    &googleapi.Error{Code: 400},
   372  			shouldRetry: false,
   373  		},
   374  		{
   375  			desc:        "googleapi.Error{Code: 408}",
   376  			inputErr:    &googleapi.Error{Code: 408},
   377  			shouldRetry: true,
   378  		},
   379  		{
   380  			desc:        "retryable gRPC error",
   381  			inputErr:    status.Error(codes.Unavailable, "retryable gRPC error"),
   382  			shouldRetry: true,
   383  		},
   384  		{
   385  			desc:        "non-retryable gRPC error",
   386  			inputErr:    status.Error(codes.PermissionDenied, "non-retryable gRPC error"),
   387  			shouldRetry: false,
   388  		},
   389  		{
   390  			desc:        "wrapped net.ErrClosed",
   391  			inputErr:    &net.OpError{Err: net.ErrClosed},
   392  			shouldRetry: true,
   393  		},
   394  	} {
   395  		t.Run(test.desc, func(s *testing.T) {
   396  			got := ShouldRetry(test.inputErr)
   397  
   398  			if got != test.shouldRetry {
   399  				s.Errorf("got %v, want %v", got, test.shouldRetry)
   400  			}
   401  		})
   402  	}
   403  }
   404
View as plain text