Source file src/github.com/hashicorp/go-retryablehttp/client.go

Documentation: github.com/hashicorp/go-retryablehttp

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  // Package retryablehttp provides a familiar HTTP client interface with
     5  // automatic retries and exponential backoff. It is a thin wrapper over the
     6  // standard net/http client library and exposes nearly the same public API.
     7  // This makes retryablehttp very easy to drop into existing programs.
     8  //
     9  // retryablehttp performs automatic retries under certain conditions. Mainly, if
    10  // an error is returned by the client (connection errors etc), or if a 500-range
    11  // response is received, then a retry is invoked. Otherwise, the response is
    12  // returned and left to the caller to interpret.
    13  //
    14  // Requests which take a request body should provide a non-nil function
    15  // parameter. The best choice is to provide either a function satisfying
    16  // ReaderFunc which provides multiple io.Readers in an efficient manner, a
    17  // *bytes.Buffer (the underlying raw byte slice will be used) or a raw byte
    18  // slice. As it is a reference type, and we will wrap it as needed by readers,
    19  // we can efficiently re-use the request body without needing to copy it. If an
    20  // io.Reader (such as a *bytes.Reader) is provided, the full body will be read
    21  // prior to the first request, and will be efficiently re-used for any retries.
    22  // ReadSeeker can be used, but some users have observed occasional data races
    23  // between the net/http library and the Seek functionality of some
    24  // implementations of ReadSeeker, so should be avoided if possible.
    25  package retryablehttp
    26  
    27  import (
    28  	"bytes"
    29  	"context"
    30  	"crypto/x509"
    31  	"fmt"
    32  	"io"
    33  	"io/ioutil"
    34  	"log"
    35  	"math"
    36  	"math/rand"
    37  	"net/http"
    38  	"net/url"
    39  	"os"
    40  	"regexp"
    41  	"strconv"
    42  	"strings"
    43  	"sync"
    44  	"time"
    45  
    46  	cleanhttp "github.com/hashicorp/go-cleanhttp"
    47  )
    48  
    49  var (
    50  	// Default retry configuration
    51  	defaultRetryWaitMin = 1 * time.Second
    52  	defaultRetryWaitMax = 30 * time.Second
    53  	defaultRetryMax     = 4
    54  
    55  	// defaultLogger is the logger provided with defaultClient
    56  	defaultLogger = log.New(os.Stderr, "", log.LstdFlags)
    57  
    58  	// defaultClient is used for performing requests without explicitly making
    59  	// a new client. It is purposely private to avoid modifications.
    60  	defaultClient = NewClient()
    61  
    62  	// We need to consume response bodies to maintain http connections, but
    63  	// limit the size we consume to respReadLimit.
    64  	respReadLimit = int64(4096)
    65  
    66  	// A regular expression to match the error returned by net/http when the
    67  	// configured number of redirects is exhausted. This error isn't typed
    68  	// specifically so we resort to matching on the error string.
    69  	redirectsErrorRe = regexp.MustCompile(`stopped after \d+ redirects\z`)
    70  
    71  	// A regular expression to match the error returned by net/http when the
    72  	// scheme specified in the URL is invalid. This error isn't typed
    73  	// specifically so we resort to matching on the error string.
    74  	schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`)
    75  
    76  	// A regular expression to match the error returned by net/http when the
    77  	// TLS certificate is not trusted. This error isn't typed
    78  	// specifically so we resort to matching on the error string.
    79  	notTrustedErrorRe = regexp.MustCompile(`certificate is not trusted`)
    80  )
    81  
    82  // ReaderFunc is the type of function that can be given natively to NewRequest
    83  type ReaderFunc func() (io.Reader, error)
    84  
    85  // ResponseHandlerFunc is a type of function that takes in a Response, and does something with it.
    86  // The ResponseHandlerFunc is called when the HTTP client successfully receives a response and the
    87  // CheckRetry function indicates that a retry of the base request is not necessary.
    88  // If an error is returned from this function, the CheckRetry policy will be used to determine
    89  // whether to retry the whole request (including this handler).
    90  //
    91  // Make sure to check status codes! Even if the request was completed it may have a non-2xx status code.
    92  //
    93  // The response body is not automatically closed. It must be closed either by the ResponseHandlerFunc or
    94  // by the caller out-of-band. Failure to do so will result in a memory leak.
    95  type ResponseHandlerFunc func(*http.Response) error
    96  
    97  // LenReader is an interface implemented by many in-memory io.Reader's. Used
    98  // for automatically sending the right Content-Length header when possible.
    99  type LenReader interface {
   100  	Len() int
   101  }
   102  
   103  // Request wraps the metadata needed to create HTTP requests.
   104  type Request struct {
   105  	// body is a seekable reader over the request body payload. This is
   106  	// used to rewind the request data in between retries.
   107  	body ReaderFunc
   108  
   109  	responseHandler ResponseHandlerFunc
   110  
   111  	// Embed an HTTP request directly. This makes a *Request act exactly
   112  	// like an *http.Request so that all meta methods are supported.
   113  	*http.Request
   114  }
   115  
   116  // WithContext returns wrapped Request with a shallow copy of underlying *http.Request
   117  // with its context changed to ctx. The provided ctx must be non-nil.
   118  func (r *Request) WithContext(ctx context.Context) *Request {
   119  	return &Request{
   120  		body:            r.body,
   121  		responseHandler: r.responseHandler,
   122  		Request:         r.Request.WithContext(ctx),
   123  	}
   124  }
   125  
   126  // SetResponseHandler allows setting the response handler.
   127  func (r *Request) SetResponseHandler(fn ResponseHandlerFunc) {
   128  	r.responseHandler = fn
   129  }
   130  
   131  // BodyBytes allows accessing the request body. It is an analogue to
   132  // http.Request's Body variable, but it returns a copy of the underlying data
   133  // rather than consuming it.
   134  //
   135  // This function is not thread-safe; do not call it at the same time as another
   136  // call, or at the same time this request is being used with Client.Do.
   137  func (r *Request) BodyBytes() ([]byte, error) {
   138  	if r.body == nil {
   139  		return nil, nil
   140  	}
   141  	body, err := r.body()
   142  	if err != nil {
   143  		return nil, err
   144  	}
   145  	buf := new(bytes.Buffer)
   146  	_, err = buf.ReadFrom(body)
   147  	if err != nil {
   148  		return nil, err
   149  	}
   150  	return buf.Bytes(), nil
   151  }
   152  
   153  // SetBody allows setting the request body.
   154  //
   155  // It is useful if a new body needs to be set without constructing a new Request.
   156  func (r *Request) SetBody(rawBody interface{}) error {
   157  	bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody)
   158  	if err != nil {
   159  		return err
   160  	}
   161  	r.body = bodyReader
   162  	r.ContentLength = contentLength
   163  	if bodyReader != nil {
   164  		r.GetBody = func() (io.ReadCloser, error) {
   165  			body, err := bodyReader()
   166  			if err != nil {
   167  				return nil, err
   168  			}
   169  			if rc, ok := body.(io.ReadCloser); ok {
   170  				return rc, nil
   171  			}
   172  			return io.NopCloser(body), nil
   173  		}
   174  	} else {
   175  		r.GetBody = func() (io.ReadCloser, error) { return http.NoBody, nil }
   176  	}
   177  	return nil
   178  }
   179  
   180  // WriteTo allows copying the request body into a writer.
   181  //
   182  // It writes data to w until there's no more data to write or
   183  // when an error occurs. The return int64 value is the number of bytes
   184  // written. Any error encountered during the write is also returned.
   185  // The signature matches io.WriterTo interface.
   186  func (r *Request) WriteTo(w io.Writer) (int64, error) {
   187  	body, err := r.body()
   188  	if err != nil {
   189  		return 0, err
   190  	}
   191  	if c, ok := body.(io.Closer); ok {
   192  		defer c.Close()
   193  	}
   194  	return io.Copy(w, body)
   195  }
   196  
   197  func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, error) {
   198  	var bodyReader ReaderFunc
   199  	var contentLength int64
   200  
   201  	switch body := rawBody.(type) {
   202  	// If they gave us a function already, great! Use it.
   203  	case ReaderFunc:
   204  		bodyReader = body
   205  		tmp, err := body()
   206  		if err != nil {
   207  			return nil, 0, err
   208  		}
   209  		if lr, ok := tmp.(LenReader); ok {
   210  			contentLength = int64(lr.Len())
   211  		}
   212  		if c, ok := tmp.(io.Closer); ok {
   213  			c.Close()
   214  		}
   215  
   216  	case func() (io.Reader, error):
   217  		bodyReader = body
   218  		tmp, err := body()
   219  		if err != nil {
   220  			return nil, 0, err
   221  		}
   222  		if lr, ok := tmp.(LenReader); ok {
   223  			contentLength = int64(lr.Len())
   224  		}
   225  		if c, ok := tmp.(io.Closer); ok {
   226  			c.Close()
   227  		}
   228  
   229  	// If a regular byte slice, we can read it over and over via new
   230  	// readers
   231  	case []byte:
   232  		buf := body
   233  		bodyReader = func() (io.Reader, error) {
   234  			return bytes.NewReader(buf), nil
   235  		}
   236  		contentLength = int64(len(buf))
   237  
   238  	// If a bytes.Buffer we can read the underlying byte slice over and
   239  	// over
   240  	case *bytes.Buffer:
   241  		buf := body
   242  		bodyReader = func() (io.Reader, error) {
   243  			return bytes.NewReader(buf.Bytes()), nil
   244  		}
   245  		contentLength = int64(buf.Len())
   246  
   247  	// We prioritize *bytes.Reader here because we don't really want to
   248  	// deal with it seeking so want it to match here instead of the
   249  	// io.ReadSeeker case.
   250  	case *bytes.Reader:
   251  		buf, err := ioutil.ReadAll(body)
   252  		if err != nil {
   253  			return nil, 0, err
   254  		}
   255  		bodyReader = func() (io.Reader, error) {
   256  			return bytes.NewReader(buf), nil
   257  		}
   258  		contentLength = int64(len(buf))
   259  
   260  	// Compat case
   261  	case io.ReadSeeker:
   262  		raw := body
   263  		bodyReader = func() (io.Reader, error) {
   264  			_, err := raw.Seek(0, 0)
   265  			return ioutil.NopCloser(raw), err
   266  		}
   267  		if lr, ok := raw.(LenReader); ok {
   268  			contentLength = int64(lr.Len())
   269  		}
   270  
   271  	// Read all in so we can reset
   272  	case io.Reader:
   273  		buf, err := ioutil.ReadAll(body)
   274  		if err != nil {
   275  			return nil, 0, err
   276  		}
   277  		if len(buf) == 0 {
   278  			bodyReader = func() (io.Reader, error) {
   279  				return http.NoBody, nil
   280  			}
   281  			contentLength = 0
   282  		} else {
   283  			bodyReader = func() (io.Reader, error) {
   284  				return bytes.NewReader(buf), nil
   285  			}
   286  			contentLength = int64(len(buf))
   287  		}
   288  
   289  	// No body provided, nothing to do
   290  	case nil:
   291  
   292  	// Unrecognized type
   293  	default:
   294  		return nil, 0, fmt.Errorf("cannot handle type %T", rawBody)
   295  	}
   296  	return bodyReader, contentLength, nil
   297  }
   298  
   299  // FromRequest wraps an http.Request in a retryablehttp.Request
   300  func FromRequest(r *http.Request) (*Request, error) {
   301  	bodyReader, _, err := getBodyReaderAndContentLength(r.Body)
   302  	if err != nil {
   303  		return nil, err
   304  	}
   305  	// Could assert contentLength == r.ContentLength
   306  	return &Request{body: bodyReader, Request: r}, nil
   307  }
   308  
   309  // NewRequest creates a new wrapped request.
   310  func NewRequest(method, url string, rawBody interface{}) (*Request, error) {
   311  	return NewRequestWithContext(context.Background(), method, url, rawBody)
   312  }
   313  
   314  // NewRequestWithContext creates a new wrapped request with the provided context.
   315  //
   316  // The context controls the entire lifetime of a request and its response:
   317  // obtaining a connection, sending the request, and reading the response headers and body.
   318  func NewRequestWithContext(ctx context.Context, method, url string, rawBody interface{}) (*Request, error) {
   319  	httpReq, err := http.NewRequestWithContext(ctx, method, url, nil)
   320  	if err != nil {
   321  		return nil, err
   322  	}
   323  
   324  	req := &Request{
   325  		Request: httpReq,
   326  	}
   327  	if err := req.SetBody(rawBody); err != nil {
   328  		return nil, err
   329  	}
   330  
   331  	return req, nil
   332  }
   333  
   334  // Logger interface allows to use other loggers than
   335  // standard log.Logger.
   336  type Logger interface {
   337  	Printf(string, ...interface{})
   338  }
   339  
   340  // LeveledLogger is an interface that can be implemented by any logger or a
   341  // logger wrapper to provide leveled logging. The methods accept a message
   342  // string and a variadic number of key-value pairs. For log.Printf style
   343  // formatting where message string contains a format specifier, use Logger
   344  // interface.
   345  type LeveledLogger interface {
   346  	Error(msg string, keysAndValues ...interface{})
   347  	Info(msg string, keysAndValues ...interface{})
   348  	Debug(msg string, keysAndValues ...interface{})
   349  	Warn(msg string, keysAndValues ...interface{})
   350  }
   351  
   352  // hookLogger adapts an LeveledLogger to Logger for use by the existing hook functions
   353  // without changing the API.
   354  type hookLogger struct {
   355  	LeveledLogger
   356  }
   357  
   358  func (h hookLogger) Printf(s string, args ...interface{}) {
   359  	h.Info(fmt.Sprintf(s, args...))
   360  }
   361  
   362  // RequestLogHook allows a function to run before each retry. The HTTP
   363  // request which will be made, and the retry number (0 for the initial
   364  // request) are available to users. The internal logger is exposed to
   365  // consumers.
   366  type RequestLogHook func(Logger, *http.Request, int)
   367  
   368  // ResponseLogHook is like RequestLogHook, but allows running a function
   369  // on each HTTP response. This function will be invoked at the end of
   370  // every HTTP request executed, regardless of whether a subsequent retry
   371  // needs to be performed or not. If the response body is read or closed
   372  // from this method, this will affect the response returned from Do().
   373  type ResponseLogHook func(Logger, *http.Response)
   374  
   375  // CheckRetry specifies a policy for handling retries. It is called
   376  // following each request with the response and error values returned by
   377  // the http.Client. If CheckRetry returns false, the Client stops retrying
   378  // and returns the response to the caller. If CheckRetry returns an error,
   379  // that error value is returned in lieu of the error from the request. The
   380  // Client will close any response body when retrying, but if the retry is
   381  // aborted it is up to the CheckRetry callback to properly close any
   382  // response body before returning.
   383  type CheckRetry func(ctx context.Context, resp *http.Response, err error) (bool, error)
   384  
   385  // Backoff specifies a policy for how long to wait between retries.
   386  // It is called after a failing request to determine the amount of time
   387  // that should pass before trying again.
   388  type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration
   389  
   390  // ErrorHandler is called if retries are expired, containing the last status
   391  // from the http library. If not specified, default behavior for the library is
   392  // to close the body and return an error indicating how many tries were
   393  // attempted. If overriding this, be sure to close the body if needed.
   394  type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error)
   395  
   396  // Client is used to make HTTP requests. It adds additional functionality
   397  // like automatic retries to tolerate minor outages.
   398  type Client struct {
   399  	HTTPClient *http.Client // Internal HTTP client.
   400  	Logger     interface{}  // Customer logger instance. Can be either Logger or LeveledLogger
   401  
   402  	RetryWaitMin time.Duration // Minimum time to wait
   403  	RetryWaitMax time.Duration // Maximum time to wait
   404  	RetryMax     int           // Maximum number of retries
   405  
   406  	// RequestLogHook allows a user-supplied function to be called
   407  	// before each retry.
   408  	RequestLogHook RequestLogHook
   409  
   410  	// ResponseLogHook allows a user-supplied function to be called
   411  	// with the response from each HTTP request executed.
   412  	ResponseLogHook ResponseLogHook
   413  
   414  	// CheckRetry specifies the policy for handling retries, and is called
   415  	// after each request. The default policy is DefaultRetryPolicy.
   416  	CheckRetry CheckRetry
   417  
   418  	// Backoff specifies the policy for how long to wait between retries
   419  	Backoff Backoff
   420  
   421  	// ErrorHandler specifies the custom error handler to use, if any
   422  	ErrorHandler ErrorHandler
   423  
   424  	loggerInit sync.Once
   425  	clientInit sync.Once
   426  }
   427  
   428  // NewClient creates a new Client with default settings.
   429  func NewClient() *Client {
   430  	return &Client{
   431  		HTTPClient:   cleanhttp.DefaultPooledClient(),
   432  		Logger:       defaultLogger,
   433  		RetryWaitMin: defaultRetryWaitMin,
   434  		RetryWaitMax: defaultRetryWaitMax,
   435  		RetryMax:     defaultRetryMax,
   436  		CheckRetry:   DefaultRetryPolicy,
   437  		Backoff:      DefaultBackoff,
   438  	}
   439  }
   440  
   441  func (c *Client) logger() interface{} {
   442  	c.loggerInit.Do(func() {
   443  		if c.Logger == nil {
   444  			return
   445  		}
   446  
   447  		switch c.Logger.(type) {
   448  		case Logger, LeveledLogger:
   449  			// ok
   450  		default:
   451  			// This should happen in dev when they are setting Logger and work on code, not in prod.
   452  			panic(fmt.Sprintf("invalid logger type passed, must be Logger or LeveledLogger, was %T", c.Logger))
   453  		}
   454  	})
   455  
   456  	return c.Logger
   457  }
   458  
   459  // DefaultRetryPolicy provides a default callback for Client.CheckRetry, which
   460  // will retry on connection errors and server errors.
   461  func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
   462  	// do not retry on context.Canceled or context.DeadlineExceeded
   463  	if ctx.Err() != nil {
   464  		return false, ctx.Err()
   465  	}
   466  
   467  	// don't propagate other errors
   468  	shouldRetry, _ := baseRetryPolicy(resp, err)
   469  	return shouldRetry, nil
   470  }
   471  
   472  // ErrorPropagatedRetryPolicy is the same as DefaultRetryPolicy, except it
   473  // propagates errors back instead of returning nil. This allows you to inspect
   474  // why it decided to retry or not.
   475  func ErrorPropagatedRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
   476  	// do not retry on context.Canceled or context.DeadlineExceeded
   477  	if ctx.Err() != nil {
   478  		return false, ctx.Err()
   479  	}
   480  
   481  	return baseRetryPolicy(resp, err)
   482  }
   483  
   484  func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
   485  	if err != nil {
   486  		if v, ok := err.(*url.Error); ok {
   487  			// Don't retry if the error was due to too many redirects.
   488  			if redirectsErrorRe.MatchString(v.Error()) {
   489  				return false, v
   490  			}
   491  
   492  			// Don't retry if the error was due to an invalid protocol scheme.
   493  			if schemeErrorRe.MatchString(v.Error()) {
   494  				return false, v
   495  			}
   496  
   497  			// Don't retry if the error was due to TLS cert verification failure.
   498  			if notTrustedErrorRe.MatchString(v.Error()) {
   499  				return false, v
   500  			}
   501  			if _, ok := v.Err.(x509.UnknownAuthorityError); ok {
   502  				return false, v
   503  			}
   504  		}
   505  
   506  		// The error is likely recoverable so retry.
   507  		return true, nil
   508  	}
   509  
   510  	// 429 Too Many Requests is recoverable. Sometimes the server puts
   511  	// a Retry-After response header to indicate when the server is
   512  	// available to start processing request from client.
   513  	if resp.StatusCode == http.StatusTooManyRequests {
   514  		return true, nil
   515  	}
   516  
   517  	// Check the response code. We retry on 500-range responses to allow
   518  	// the server time to recover, as 500's are typically not permanent
   519  	// errors and may relate to outages on the server side. This will catch
   520  	// invalid response codes as well, like 0 and 999.
   521  	if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented) {
   522  		return true, fmt.Errorf("unexpected HTTP status %s", resp.Status)
   523  	}
   524  
   525  	return false, nil
   526  }
   527  
   528  // DefaultBackoff provides a default callback for Client.Backoff which
   529  // will perform exponential backoff based on the attempt number and limited
   530  // by the provided minimum and maximum durations.
   531  //
   532  // It also tries to parse Retry-After response header when a http.StatusTooManyRequests
   533  // (HTTP Code 429) is found in the resp parameter. Hence it will return the number of
   534  // seconds the server states it may be ready to process more requests from this client.
   535  func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
   536  	if resp != nil {
   537  		if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
   538  			if s, ok := resp.Header["Retry-After"]; ok {
   539  				if sleep, err := strconv.ParseInt(s[0], 10, 64); err == nil {
   540  					return time.Second * time.Duration(sleep)
   541  				}
   542  			}
   543  		}
   544  	}
   545  
   546  	mult := math.Pow(2, float64(attemptNum)) * float64(min)
   547  	sleep := time.Duration(mult)
   548  	if float64(sleep) != mult || sleep > max {
   549  		sleep = max
   550  	}
   551  	return sleep
   552  }
   553  
   554  // LinearJitterBackoff provides a callback for Client.Backoff which will
   555  // perform linear backoff based on the attempt number and with jitter to
   556  // prevent a thundering herd.
   557  //
   558  // min and max here are *not* absolute values. The number to be multiplied by
   559  // the attempt number will be chosen at random from between them, thus they are
   560  // bounding the jitter.
   561  //
   562  // For instance:
   563  // * To get strictly linear backoff of one second increasing each retry, set
   564  // both to one second (1s, 2s, 3s, 4s, ...)
   565  // * To get a small amount of jitter centered around one second increasing each
   566  // retry, set to around one second, such as a min of 800ms and max of 1200ms
   567  // (892ms, 2102ms, 2945ms, 4312ms, ...)
   568  // * To get extreme jitter, set to a very wide spread, such as a min of 100ms
   569  // and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...)
   570  func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
   571  	// attemptNum always starts at zero but we want to start at 1 for multiplication
   572  	attemptNum++
   573  
   574  	if max <= min {
   575  		// Unclear what to do here, or they are the same, so return min *
   576  		// attemptNum
   577  		return min * time.Duration(attemptNum)
   578  	}
   579  
   580  	// Seed rand; doing this every time is fine
   581  	rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
   582  
   583  	// Pick a random number that lies somewhere between the min and max and
   584  	// multiply by the attemptNum. attemptNum starts at zero so we always
   585  	// increment here. We first get a random percentage, then apply that to the
   586  	// difference between min and max, and add to min.
   587  	jitter := rand.Float64() * float64(max-min)
   588  	jitterMin := int64(jitter) + int64(min)
   589  	return time.Duration(jitterMin * int64(attemptNum))
   590  }
   591  
   592  // PassthroughErrorHandler is an ErrorHandler that directly passes through the
   593  // values from the net/http library for the final request. The body is not
   594  // closed.
   595  func PassthroughErrorHandler(resp *http.Response, err error, _ int) (*http.Response, error) {
   596  	return resp, err
   597  }
   598  
   599  // Do wraps calling an HTTP method with retries.
   600  func (c *Client) Do(req *Request) (*http.Response, error) {
   601  	c.clientInit.Do(func() {
   602  		if c.HTTPClient == nil {
   603  			c.HTTPClient = cleanhttp.DefaultPooledClient()
   604  		}
   605  	})
   606  
   607  	logger := c.logger()
   608  
   609  	if logger != nil {
   610  		switch v := logger.(type) {
   611  		case LeveledLogger:
   612  			v.Debug("performing request", "method", req.Method, "url", req.URL)
   613  		case Logger:
   614  			v.Printf("[DEBUG] %s %s", req.Method, req.URL)
   615  		}
   616  	}
   617  
   618  	var resp *http.Response
   619  	var attempt int
   620  	var shouldRetry bool
   621  	var doErr, respErr, checkErr error
   622  
   623  	for i := 0; ; i++ {
   624  		doErr, respErr = nil, nil
   625  		attempt++
   626  
   627  		// Always rewind the request body when non-nil.
   628  		if req.body != nil {
   629  			body, err := req.body()
   630  			if err != nil {
   631  				c.HTTPClient.CloseIdleConnections()
   632  				return resp, err
   633  			}
   634  			if c, ok := body.(io.ReadCloser); ok {
   635  				req.Body = c
   636  			} else {
   637  				req.Body = ioutil.NopCloser(body)
   638  			}
   639  		}
   640  
   641  		if c.RequestLogHook != nil {
   642  			switch v := logger.(type) {
   643  			case LeveledLogger:
   644  				c.RequestLogHook(hookLogger{v}, req.Request, i)
   645  			case Logger:
   646  				c.RequestLogHook(v, req.Request, i)
   647  			default:
   648  				c.RequestLogHook(nil, req.Request, i)
   649  			}
   650  		}
   651  
   652  		// Attempt the request
   653  		resp, doErr = c.HTTPClient.Do(req.Request)
   654  
   655  		// Check if we should continue with retries.
   656  		shouldRetry, checkErr = c.CheckRetry(req.Context(), resp, doErr)
   657  		if !shouldRetry && doErr == nil && req.responseHandler != nil {
   658  			respErr = req.responseHandler(resp)
   659  			shouldRetry, checkErr = c.CheckRetry(req.Context(), resp, respErr)
   660  		}
   661  
   662  		err := doErr
   663  		if respErr != nil {
   664  			err = respErr
   665  		}
   666  		if err != nil {
   667  			switch v := logger.(type) {
   668  			case LeveledLogger:
   669  				v.Error("request failed", "error", err, "method", req.Method, "url", req.URL)
   670  			case Logger:
   671  				v.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err)
   672  			}
   673  		} else {
   674  			// Call this here to maintain the behavior of logging all requests,
   675  			// even if CheckRetry signals to stop.
   676  			if c.ResponseLogHook != nil {
   677  				// Call the response logger function if provided.
   678  				switch v := logger.(type) {
   679  				case LeveledLogger:
   680  					c.ResponseLogHook(hookLogger{v}, resp)
   681  				case Logger:
   682  					c.ResponseLogHook(v, resp)
   683  				default:
   684  					c.ResponseLogHook(nil, resp)
   685  				}
   686  			}
   687  		}
   688  
   689  		if !shouldRetry {
   690  			break
   691  		}
   692  
   693  		// We do this before drainBody because there's no need for the I/O if
   694  		// we're breaking out
   695  		remain := c.RetryMax - i
   696  		if remain <= 0 {
   697  			break
   698  		}
   699  
   700  		// We're going to retry, consume any response to reuse the connection.
   701  		if doErr == nil {
   702  			c.drainBody(resp.Body)
   703  		}
   704  
   705  		wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp)
   706  		if logger != nil {
   707  			desc := fmt.Sprintf("%s %s", req.Method, req.URL)
   708  			if resp != nil {
   709  				desc = fmt.Sprintf("%s (status: %d)", desc, resp.StatusCode)
   710  			}
   711  			switch v := logger.(type) {
   712  			case LeveledLogger:
   713  				v.Debug("retrying request", "request", desc, "timeout", wait, "remaining", remain)
   714  			case Logger:
   715  				v.Printf("[DEBUG] %s: retrying in %s (%d left)", desc, wait, remain)
   716  			}
   717  		}
   718  		timer := time.NewTimer(wait)
   719  		select {
   720  		case <-req.Context().Done():
   721  			timer.Stop()
   722  			c.HTTPClient.CloseIdleConnections()
   723  			return nil, req.Context().Err()
   724  		case <-timer.C:
   725  		}
   726  
   727  		// Make shallow copy of http Request so that we can modify its body
   728  		// without racing against the closeBody call in persistConn.writeLoop.
   729  		httpreq := *req.Request
   730  		req.Request = &httpreq
   731  	}
   732  
   733  	// this is the closest we have to success criteria
   734  	if doErr == nil && respErr == nil && checkErr == nil && !shouldRetry {
   735  		return resp, nil
   736  	}
   737  
   738  	defer c.HTTPClient.CloseIdleConnections()
   739  
   740  	var err error
   741  	if checkErr != nil {
   742  		err = checkErr
   743  	} else if respErr != nil {
   744  		err = respErr
   745  	} else {
   746  		err = doErr
   747  	}
   748  
   749  	if c.ErrorHandler != nil {
   750  		return c.ErrorHandler(resp, err, attempt)
   751  	}
   752  
   753  	// By default, we close the response body and return an error without
   754  	// returning the response
   755  	if resp != nil {
   756  		c.drainBody(resp.Body)
   757  	}
   758  
   759  	// this means CheckRetry thought the request was a failure, but didn't
   760  	// communicate why
   761  	if err == nil {
   762  		return nil, fmt.Errorf("%s %s giving up after %d attempt(s)",
   763  			req.Method, req.URL, attempt)
   764  	}
   765  
   766  	return nil, fmt.Errorf("%s %s giving up after %d attempt(s): %w",
   767  		req.Method, req.URL, attempt, err)
   768  }
   769  
   770  // Try to read the response body so we can reuse this connection.
   771  func (c *Client) drainBody(body io.ReadCloser) {
   772  	defer body.Close()
   773  	_, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit))
   774  	if err != nil {
   775  		if c.logger() != nil {
   776  			switch v := c.logger().(type) {
   777  			case LeveledLogger:
   778  				v.Error("error reading response body", "error", err)
   779  			case Logger:
   780  				v.Printf("[ERR] error reading response body: %v", err)
   781  			}
   782  		}
   783  	}
   784  }
   785  
   786  // Get is a shortcut for doing a GET request without making a new client.
   787  func Get(url string) (*http.Response, error) {
   788  	return defaultClient.Get(url)
   789  }
   790  
   791  // Get is a convenience helper for doing simple GET requests.
   792  func (c *Client) Get(url string) (*http.Response, error) {
   793  	req, err := NewRequest("GET", url, nil)
   794  	if err != nil {
   795  		return nil, err
   796  	}
   797  	return c.Do(req)
   798  }
   799  
   800  // Head is a shortcut for doing a HEAD request without making a new client.
   801  func Head(url string) (*http.Response, error) {
   802  	return defaultClient.Head(url)
   803  }
   804  
   805  // Head is a convenience method for doing simple HEAD requests.
   806  func (c *Client) Head(url string) (*http.Response, error) {
   807  	req, err := NewRequest("HEAD", url, nil)
   808  	if err != nil {
   809  		return nil, err
   810  	}
   811  	return c.Do(req)
   812  }
   813  
   814  // Post is a shortcut for doing a POST request without making a new client.
   815  func Post(url, bodyType string, body interface{}) (*http.Response, error) {
   816  	return defaultClient.Post(url, bodyType, body)
   817  }
   818  
   819  // Post is a convenience method for doing simple POST requests.
   820  func (c *Client) Post(url, bodyType string, body interface{}) (*http.Response, error) {
   821  	req, err := NewRequest("POST", url, body)
   822  	if err != nil {
   823  		return nil, err
   824  	}
   825  	req.Header.Set("Content-Type", bodyType)
   826  	return c.Do(req)
   827  }
   828  
   829  // PostForm is a shortcut to perform a POST with form data without creating
   830  // a new client.
   831  func PostForm(url string, data url.Values) (*http.Response, error) {
   832  	return defaultClient.PostForm(url, data)
   833  }
   834  
   835  // PostForm is a convenience method for doing simple POST operations using
   836  // pre-filled url.Values form data.
   837  func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) {
   838  	return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   839  }
   840  
   841  // StandardClient returns a stdlib *http.Client with a custom Transport, which
   842  // shims in a *retryablehttp.Client for added retries.
   843  func (c *Client) StandardClient() *http.Client {
   844  	return &http.Client{
   845  		Transport: &RoundTripper{Client: c},
   846  	}
   847  }
   848  

View as plain text