1 // Copyright 2016, Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above 11 // copyright notice, this list of conditions and the following disclaimer 12 // in the documentation and/or other materials provided with the 13 // distribution. 14 // * Neither the name of Google Inc. nor the names of its 15 // contributors may be used to endorse or promote products derived from 16 // this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 package gax 31 32 import ( 33 "errors" 34 "math/rand" 35 "time" 36 37 "google.golang.org/api/googleapi" 38 "google.golang.org/grpc" 39 "google.golang.org/grpc/codes" 40 "google.golang.org/grpc/status" 41 ) 42 43 // CallOption is an option used by Invoke to control behaviors of RPC calls. 44 // CallOption works by modifying relevant fields of CallSettings. 45 type CallOption interface { 46 // Resolve applies the option by modifying cs. 47 Resolve(cs *CallSettings) 48 } 49 50 // Retryer is used by Invoke to determine retry behavior. 51 type Retryer interface { 52 // Retry reports whether a request should be retried and how long to pause before retrying 53 // if the previous attempt returned with err. Invoke never calls Retry with nil error. 54 Retry(err error) (pause time.Duration, shouldRetry bool) 55 } 56 57 type retryerOption func() Retryer 58 59 func (o retryerOption) Resolve(s *CallSettings) { 60 s.Retry = o 61 } 62 63 // WithRetry sets CallSettings.Retry to fn. 64 func WithRetry(fn func() Retryer) CallOption { 65 return retryerOption(fn) 66 } 67 68 // OnErrorFunc returns a Retryer that retries if and only if the previous attempt 69 // returns an error that satisfies shouldRetry. 70 // 71 // Pause times between retries are specified by bo. bo is only used for its 72 // parameters; each Retryer has its own copy. 73 func OnErrorFunc(bo Backoff, shouldRetry func(err error) bool) Retryer { 74 return &errorRetryer{ 75 shouldRetry: shouldRetry, 76 backoff: bo, 77 } 78 } 79 80 type errorRetryer struct { 81 backoff Backoff 82 shouldRetry func(err error) bool 83 } 84 85 func (r *errorRetryer) Retry(err error) (time.Duration, bool) { 86 if r.shouldRetry(err) { 87 return r.backoff.Pause(), true 88 } 89 90 return 0, false 91 } 92 93 // OnCodes returns a Retryer that retries if and only if 94 // the previous attempt returns a GRPC error whose error code is stored in cc. 95 // Pause times between retries are specified by bo. 96 // 97 // bo is only used for its parameters; each Retryer has its own copy. 98 func OnCodes(cc []codes.Code, bo Backoff) Retryer { 99 return &boRetryer{ 100 backoff: bo, 101 codes: append([]codes.Code(nil), cc...), 102 } 103 } 104 105 type boRetryer struct { 106 backoff Backoff 107 codes []codes.Code 108 } 109 110 func (r *boRetryer) Retry(err error) (time.Duration, bool) { 111 st, ok := status.FromError(err) 112 if !ok { 113 return 0, false 114 } 115 c := st.Code() 116 for _, rc := range r.codes { 117 if c == rc { 118 return r.backoff.Pause(), true 119 } 120 } 121 return 0, false 122 } 123 124 // OnHTTPCodes returns a Retryer that retries if and only if 125 // the previous attempt returns a googleapi.Error whose status code is stored in 126 // cc. Pause times between retries are specified by bo. 127 // 128 // bo is only used for its parameters; each Retryer has its own copy. 129 func OnHTTPCodes(bo Backoff, cc ...int) Retryer { 130 codes := make(map[int]bool, len(cc)) 131 for _, c := range cc { 132 codes[c] = true 133 } 134 135 return &httpRetryer{ 136 backoff: bo, 137 codes: codes, 138 } 139 } 140 141 type httpRetryer struct { 142 backoff Backoff 143 codes map[int]bool 144 } 145 146 func (r *httpRetryer) Retry(err error) (time.Duration, bool) { 147 var gerr *googleapi.Error 148 if !errors.As(err, &gerr) { 149 return 0, false 150 } 151 152 if r.codes[gerr.Code] { 153 return r.backoff.Pause(), true 154 } 155 156 return 0, false 157 } 158 159 // Backoff implements exponential backoff. The wait time between retries is a 160 // random value between 0 and the "retry period" - the time between retries. The 161 // retry period starts at Initial and increases by the factor of Multiplier 162 // every retry, but is capped at Max. 163 // 164 // Note: MaxNumRetries / RPCDeadline is specifically not provided. These should 165 // be built on top of Backoff. 166 type Backoff struct { 167 // Initial is the initial value of the retry period, defaults to 1 second. 168 Initial time.Duration 169 170 // Max is the maximum value of the retry period, defaults to 30 seconds. 171 Max time.Duration 172 173 // Multiplier is the factor by which the retry period increases. 174 // It should be greater than 1 and defaults to 2. 175 Multiplier float64 176 177 // cur is the current retry period. 178 cur time.Duration 179 } 180 181 // Pause returns the next time.Duration that the caller should use to backoff. 182 func (bo *Backoff) Pause() time.Duration { 183 if bo.Initial == 0 { 184 bo.Initial = time.Second 185 } 186 if bo.cur == 0 { 187 bo.cur = bo.Initial 188 } 189 if bo.Max == 0 { 190 bo.Max = 30 * time.Second 191 } 192 if bo.Multiplier < 1 { 193 bo.Multiplier = 2 194 } 195 // Select a duration between 1ns and the current max. It might seem 196 // counterintuitive to have so much jitter, but 197 // https://www.awsarchitectureblog.com/2015/03/backoff.html argues that 198 // that is the best strategy. 199 d := time.Duration(1 + rand.Int63n(int64(bo.cur))) 200 bo.cur = time.Duration(float64(bo.cur) * bo.Multiplier) 201 if bo.cur > bo.Max { 202 bo.cur = bo.Max 203 } 204 return d 205 } 206 207 type grpcOpt []grpc.CallOption 208 209 func (o grpcOpt) Resolve(s *CallSettings) { 210 s.GRPC = o 211 } 212 213 type pathOpt struct { 214 p string 215 } 216 217 func (p pathOpt) Resolve(s *CallSettings) { 218 s.Path = p.p 219 } 220 221 type timeoutOpt struct { 222 t time.Duration 223 } 224 225 func (t timeoutOpt) Resolve(s *CallSettings) { 226 s.timeout = t.t 227 } 228 229 // WithPath applies a Path override to the HTTP-based APICall. 230 // 231 // This is for internal use only. 232 func WithPath(p string) CallOption { 233 return &pathOpt{p: p} 234 } 235 236 // WithGRPCOptions allows passing gRPC call options during client creation. 237 func WithGRPCOptions(opt ...grpc.CallOption) CallOption { 238 return grpcOpt(append([]grpc.CallOption(nil), opt...)) 239 } 240 241 // WithTimeout is a convenience option for setting a context.WithTimeout on the 242 // singular context.Context used for **all** APICall attempts. Calculated from 243 // the start of the first APICall attempt. 244 // If the context.Context provided to Invoke already has a Deadline set, that 245 // will always be respected over the deadline calculated using this option. 246 func WithTimeout(t time.Duration) CallOption { 247 return &timeoutOpt{t: t} 248 } 249 250 // CallSettings allow fine-grained control over how calls are made. 251 type CallSettings struct { 252 // Retry returns a Retryer to be used to control retry logic of a method call. 253 // If Retry is nil or the returned Retryer is nil, the call will not be retried. 254 Retry func() Retryer 255 256 // CallOptions to be forwarded to GRPC. 257 GRPC []grpc.CallOption 258 259 // Path is an HTTP override for an APICall. 260 Path string 261 262 // Timeout defines the amount of time that Invoke has to complete. 263 // Unexported so it cannot be changed by the code in an APICall. 264 timeout time.Duration 265 } 266