1 // Copyright 2016 Michal Witkowski. All Rights Reserved. 2 // See LICENSE for licensing terms. 3 4 package grpc_retry 5 6 import ( 7 "context" 8 "time" 9 10 "google.golang.org/grpc" 11 "google.golang.org/grpc/codes" 12 ) 13 14 var ( 15 // DefaultRetriableCodes is a set of well known types gRPC codes that should be retri-able. 16 // 17 // `ResourceExhausted` means that the user quota, e.g. per-RPC limits, have been reached. 18 // `Unavailable` means that system is currently unavailable and the client should retry again. 19 DefaultRetriableCodes = []codes.Code{codes.ResourceExhausted, codes.Unavailable} 20 21 defaultOptions = &options{ 22 max: 0, // disabled 23 perCallTimeout: 0, // disabled 24 includeHeader: true, 25 codes: DefaultRetriableCodes, 26 backoffFunc: BackoffFuncContext(func(ctx context.Context, attempt uint) time.Duration { 27 return BackoffLinearWithJitter(50*time.Millisecond /*jitter*/, 0.10)(attempt) 28 }), 29 } 30 ) 31 32 // BackoffFunc denotes a family of functions that control the backoff duration between call retries. 33 // 34 // They are called with an identifier of the attempt, and should return a time the system client should 35 // hold off for. If the time returned is longer than the `context.Context.Deadline` of the request 36 // the deadline of the request takes precedence and the wait will be interrupted before proceeding 37 // with the next iteration. 38 type BackoffFunc func(attempt uint) time.Duration 39 40 // BackoffFuncContext denotes a family of functions that control the backoff duration between call retries. 41 // 42 // They are called with an identifier of the attempt, and should return a time the system client should 43 // hold off for. If the time returned is longer than the `context.Context.Deadline` of the request 44 // the deadline of the request takes precedence and the wait will be interrupted before proceeding 45 // with the next iteration. The context can be used to extract request scoped metadata and context values. 46 type BackoffFuncContext func(ctx context.Context, attempt uint) time.Duration 47 48 // Disable disables the retry behaviour on this call, or this interceptor. 49 // 50 // Its semantically the same to `WithMax` 51 func Disable() CallOption { 52 return WithMax(0) 53 } 54 55 // WithMax sets the maximum number of retries on this call, or this interceptor. 56 func WithMax(maxRetries uint) CallOption { 57 return CallOption{applyFunc: func(o *options) { 58 o.max = maxRetries 59 }} 60 } 61 62 // WithBackoff sets the `BackoffFunc` used to control time between retries. 63 func WithBackoff(bf BackoffFunc) CallOption { 64 return CallOption{applyFunc: func(o *options) { 65 o.backoffFunc = BackoffFuncContext(func(ctx context.Context, attempt uint) time.Duration { 66 return bf(attempt) 67 }) 68 }} 69 } 70 71 // WithBackoffContext sets the `BackoffFuncContext` used to control time between retries. 72 func WithBackoffContext(bf BackoffFuncContext) CallOption { 73 return CallOption{applyFunc: func(o *options) { 74 o.backoffFunc = bf 75 }} 76 } 77 78 // WithCodes sets which codes should be retried. 79 // 80 // Please *use with care*, as you may be retrying non-idempotent calls. 81 // 82 // You cannot automatically retry on Cancelled and Deadline, please use `WithPerRetryTimeout` for these. 83 func WithCodes(retryCodes ...codes.Code) CallOption { 84 return CallOption{applyFunc: func(o *options) { 85 o.codes = retryCodes 86 }} 87 } 88 89 // WithPerRetryTimeout sets the RPC timeout per call (including initial call) on this call, or this interceptor. 90 // 91 // The context.Deadline of the call takes precedence and sets the maximum time the whole invocation 92 // will take, but WithPerRetryTimeout can be used to limit the RPC time per each call. 93 // 94 // For example, with context.Deadline = now + 10s, and WithPerRetryTimeout(3 * time.Seconds), each 95 // of the retry calls (including the initial one) will have a deadline of now + 3s. 96 // 97 // A value of 0 disables the timeout overrides completely and returns to each retry call using the 98 // parent `context.Deadline`. 99 // 100 // Note that when this is enabled, any DeadlineExceeded errors that are propagated up will be retried. 101 func WithPerRetryTimeout(timeout time.Duration) CallOption { 102 return CallOption{applyFunc: func(o *options) { 103 o.perCallTimeout = timeout 104 }} 105 } 106 107 type options struct { 108 max uint 109 perCallTimeout time.Duration 110 includeHeader bool 111 codes []codes.Code 112 backoffFunc BackoffFuncContext 113 } 114 115 // CallOption is a grpc.CallOption that is local to grpc_retry. 116 type CallOption struct { 117 grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic. 118 applyFunc func(opt *options) 119 } 120 121 func reuseOrNewWithCallOptions(opt *options, callOptions []CallOption) *options { 122 if len(callOptions) == 0 { 123 return opt 124 } 125 optCopy := &options{} 126 *optCopy = *opt 127 for _, f := range callOptions { 128 f.applyFunc(optCopy) 129 } 130 return optCopy 131 } 132 133 func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []CallOption) { 134 for _, opt := range callOptions { 135 if co, ok := opt.(CallOption); ok { 136 retryOptions = append(retryOptions, co) 137 } else { 138 grpcOptions = append(grpcOptions, opt) 139 } 140 } 141 return grpcOptions, retryOptions 142 } 143