sender.go

Documentation: github.com/Azure/go-autorest/autorest

     1  package autorest
     2  
     3  // Copyright 2017 Microsoft Corporation
     4  //
     5  //  Licensed under the Apache License, Version 2.0 (the "License");
     6  //  you may not use this file except in compliance with the License.
     7  //  You may obtain a copy of the License at
     8  //
     9  //      http://www.apache.org/licenses/LICENSE-2.0
    10  //
    11  //  Unless required by applicable law or agreed to in writing, software
    12  //  distributed under the License is distributed on an "AS IS" BASIS,
    13  //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  //  See the License for the specific language governing permissions and
    15  //  limitations under the License.
    16  
    17  import (
    18  	"context"
    19  	"crypto/tls"
    20  	"fmt"
    21  	"log"
    22  	"math"
    23  	"net"
    24  	"net/http"
    25  	"net/http/cookiejar"
    26  	"strconv"
    27  	"sync"
    28  	"time"
    29  
    30  	"github.com/Azure/go-autorest/logger"
    31  	"github.com/Azure/go-autorest/tracing"
    32  )
    33  
    34  // there is one sender per TLS renegotiation type, i.e. count of tls.RenegotiationSupport enums
    35  const defaultSendersCount = 3
    36  
    37  type defaultSender struct {
    38  	sender Sender
    39  	init   *sync.Once
    40  }
    41  
    42  // each type of sender will be created on demand in sender()
    43  var defaultSenders [defaultSendersCount]defaultSender
    44  
    45  func init() {
    46  	for i := 0; i < defaultSendersCount; i++ {
    47  		defaultSenders[i].init = &sync.Once{}
    48  	}
    49  }
    50  
    51  // used as a key type in context.WithValue()
    52  type ctxSendDecorators struct{}
    53  
    54  // WithSendDecorators adds the specified SendDecorators to the provided context.
    55  // If no SendDecorators are provided the context is unchanged.
    56  func WithSendDecorators(ctx context.Context, sendDecorator []SendDecorator) context.Context {
    57  	if len(sendDecorator) == 0 {
    58  		return ctx
    59  	}
    60  	return context.WithValue(ctx, ctxSendDecorators{}, sendDecorator)
    61  }
    62  
    63  // GetSendDecorators returns the SendDecorators in the provided context or the provided default SendDecorators.
    64  func GetSendDecorators(ctx context.Context, defaultSendDecorators ...SendDecorator) []SendDecorator {
    65  	inCtx := ctx.Value(ctxSendDecorators{})
    66  	if sd, ok := inCtx.([]SendDecorator); ok {
    67  		return sd
    68  	}
    69  	return defaultSendDecorators
    70  }
    71  
    72  // Sender is the interface that wraps the Do method to send HTTP requests.
    73  //
    74  // The standard http.Client conforms to this interface.
    75  type Sender interface {
    76  	Do(*http.Request) (*http.Response, error)
    77  }
    78  
    79  // SenderFunc is a method that implements the Sender interface.
    80  type SenderFunc func(*http.Request) (*http.Response, error)
    81  
    82  // Do implements the Sender interface on SenderFunc.
    83  func (sf SenderFunc) Do(r *http.Request) (*http.Response, error) {
    84  	return sf(r)
    85  }
    86  
    87  // SendDecorator takes and possibly decorates, by wrapping, a Sender. Decorators may affect the
    88  // http.Request and pass it along or, first, pass the http.Request along then react to the
    89  // http.Response result.
    90  type SendDecorator func(Sender) Sender
    91  
    92  // CreateSender creates, decorates, and returns, as a Sender, the default http.Client.
    93  func CreateSender(decorators ...SendDecorator) Sender {
    94  	return DecorateSender(sender(tls.RenegotiateNever), decorators...)
    95  }
    96  
    97  // DecorateSender accepts a Sender and a, possibly empty, set of SendDecorators, which is applies to
    98  // the Sender. Decorators are applied in the order received, but their affect upon the request
    99  // depends on whether they are a pre-decorator (change the http.Request and then pass it along) or a
   100  // post-decorator (pass the http.Request along and react to the results in http.Response).
   101  func DecorateSender(s Sender, decorators ...SendDecorator) Sender {
   102  	for _, decorate := range decorators {
   103  		s = decorate(s)
   104  	}
   105  	return s
   106  }
   107  
   108  // Send sends, by means of the default http.Client, the passed http.Request, returning the
   109  // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
   110  // it will apply the http.Client before invoking the Do method.
   111  //
   112  // Send is a convenience method and not recommended for production. Advanced users should use
   113  // SendWithSender, passing and sharing their own Sender (e.g., instance of http.Client).
   114  //
   115  // Send will not poll or retry requests.
   116  func Send(r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
   117  	return SendWithSender(sender(tls.RenegotiateNever), r, decorators...)
   118  }
   119  
   120  // SendWithSender sends the passed http.Request, through the provided Sender, returning the
   121  // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
   122  // it will apply the http.Client before invoking the Do method.
   123  //
   124  // SendWithSender will not poll or retry requests.
   125  func SendWithSender(s Sender, r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
   126  	return DecorateSender(s, decorators...).Do(r)
   127  }
   128  
   129  func sender(renengotiation tls.RenegotiationSupport) Sender {
   130  	// note that we can't init defaultSenders in init() since it will
   131  	// execute before calling code has had a chance to enable tracing
   132  	defaultSenders[renengotiation].init.Do(func() {
   133  		// copied from http.DefaultTransport with a TLS minimum version.
   134  		transport := &http.Transport{
   135  			Proxy: http.ProxyFromEnvironment,
   136  			DialContext: (&net.Dialer{
   137  				Timeout:   30 * time.Second,
   138  				KeepAlive: 30 * time.Second,
   139  			}).DialContext,
   140  			ForceAttemptHTTP2:     true,
   141  			MaxIdleConns:          100,
   142  			IdleConnTimeout:       90 * time.Second,
   143  			TLSHandshakeTimeout:   10 * time.Second,
   144  			ExpectContinueTimeout: 1 * time.Second,
   145  			TLSClientConfig: &tls.Config{
   146  				MinVersion:    tls.VersionTLS12,
   147  				Renegotiation: renengotiation,
   148  			},
   149  		}
   150  		var roundTripper http.RoundTripper = transport
   151  		if tracing.IsEnabled() {
   152  			roundTripper = tracing.NewTransport(transport)
   153  		}
   154  		j, _ := cookiejar.New(nil)
   155  		defaultSenders[renengotiation].sender = &http.Client{Jar: j, Transport: roundTripper}
   156  	})
   157  	return defaultSenders[renengotiation].sender
   158  }
   159  
   160  // AfterDelay returns a SendDecorator that delays for the passed time.Duration before
   161  // invoking the Sender. The delay may be terminated by closing the optional channel on the
   162  // http.Request. If canceled, no further Senders are invoked.
   163  func AfterDelay(d time.Duration) SendDecorator {
   164  	return func(s Sender) Sender {
   165  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   166  			if !DelayForBackoff(d, 0, r.Context().Done()) {
   167  				return nil, fmt.Errorf("autorest: AfterDelay canceled before full delay")
   168  			}
   169  			return s.Do(r)
   170  		})
   171  	}
   172  }
   173  
   174  // AsIs returns a SendDecorator that invokes the passed Sender without modifying the http.Request.
   175  func AsIs() SendDecorator {
   176  	return func(s Sender) Sender {
   177  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   178  			return s.Do(r)
   179  		})
   180  	}
   181  }
   182  
   183  // DoCloseIfError returns a SendDecorator that first invokes the passed Sender after which
   184  // it closes the response if the passed Sender returns an error and the response body exists.
   185  func DoCloseIfError() SendDecorator {
   186  	return func(s Sender) Sender {
   187  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   188  			resp, err := s.Do(r)
   189  			if err != nil {
   190  				Respond(resp, ByDiscardingBody(), ByClosing())
   191  			}
   192  			return resp, err
   193  		})
   194  	}
   195  }
   196  
   197  // DoErrorIfStatusCode returns a SendDecorator that emits an error if the response StatusCode is
   198  // among the set passed. Since these are artificial errors, the response body may still require
   199  // closing.
   200  func DoErrorIfStatusCode(codes ...int) SendDecorator {
   201  	return func(s Sender) Sender {
   202  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   203  			resp, err := s.Do(r)
   204  			if err == nil && ResponseHasStatusCode(resp, codes...) {
   205  				err = NewErrorWithResponse("autorest", "DoErrorIfStatusCode", resp, "%v %v failed with %s",
   206  					resp.Request.Method,
   207  					resp.Request.URL,
   208  					resp.Status)
   209  			}
   210  			return resp, err
   211  		})
   212  	}
   213  }
   214  
   215  // DoErrorUnlessStatusCode returns a SendDecorator that emits an error unless the response
   216  // StatusCode is among the set passed. Since these are artificial errors, the response body
   217  // may still require closing.
   218  func DoErrorUnlessStatusCode(codes ...int) SendDecorator {
   219  	return func(s Sender) Sender {
   220  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   221  			resp, err := s.Do(r)
   222  			if err == nil && !ResponseHasStatusCode(resp, codes...) {
   223  				err = NewErrorWithResponse("autorest", "DoErrorUnlessStatusCode", resp, "%v %v failed with %s",
   224  					resp.Request.Method,
   225  					resp.Request.URL,
   226  					resp.Status)
   227  			}
   228  			return resp, err
   229  		})
   230  	}
   231  }
   232  
   233  // DoPollForStatusCodes returns a SendDecorator that polls if the http.Response contains one of the
   234  // passed status codes. It expects the http.Response to contain a Location header providing the
   235  // URL at which to poll (using GET) and will poll until the time passed is equal to or greater than
   236  // the supplied duration. It will delay between requests for the duration specified in the
   237  // RetryAfter header or, if the header is absent, the passed delay. Polling may be canceled by
   238  // closing the optional channel on the http.Request.
   239  func DoPollForStatusCodes(duration time.Duration, delay time.Duration, codes ...int) SendDecorator {
   240  	return func(s Sender) Sender {
   241  		return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
   242  			resp, err = s.Do(r)
   243  
   244  			if err == nil && ResponseHasStatusCode(resp, codes...) {
   245  				r, err = NewPollingRequestWithContext(r.Context(), resp)
   246  
   247  				for err == nil && ResponseHasStatusCode(resp, codes...) {
   248  					Respond(resp,
   249  						ByDiscardingBody(),
   250  						ByClosing())
   251  					resp, err = SendWithSender(s, r,
   252  						AfterDelay(GetRetryAfter(resp, delay)))
   253  				}
   254  			}
   255  
   256  			return resp, err
   257  		})
   258  	}
   259  }
   260  
   261  // DoRetryForAttempts returns a SendDecorator that retries a failed request for up to the specified
   262  // number of attempts, exponentially backing off between requests using the supplied backoff
   263  // time.Duration (which may be zero). Retrying may be canceled by closing the optional channel on
   264  // the http.Request.
   265  func DoRetryForAttempts(attempts int, backoff time.Duration) SendDecorator {
   266  	return func(s Sender) Sender {
   267  		return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
   268  			rr := NewRetriableRequest(r)
   269  			for attempt := 0; attempt < attempts; attempt++ {
   270  				err = rr.Prepare()
   271  				if err != nil {
   272  					return resp, err
   273  				}
   274  				DrainResponseBody(resp)
   275  				resp, err = s.Do(rr.Request())
   276  				if err == nil {
   277  					return resp, err
   278  				}
   279  				logger.Instance.Writef(logger.LogError, "DoRetryForAttempts: received error for attempt %d: %v\n", attempt+1, err)
   280  				if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
   281  					return nil, r.Context().Err()
   282  				}
   283  			}
   284  			return resp, err
   285  		})
   286  	}
   287  }
   288  
   289  // Count429AsRetry indicates that a 429 response should be included as a retry attempt.
   290  var Count429AsRetry = true
   291  
   292  // Max429Delay is the maximum duration to wait between retries on a 429 if no Retry-After header was received.
   293  var Max429Delay time.Duration
   294  
   295  // DoRetryForStatusCodes returns a SendDecorator that retries for specified statusCodes for up to the specified
   296  // number of attempts, exponentially backing off between requests using the supplied backoff
   297  // time.Duration (which may be zero). Retrying may be canceled by cancelling the context on the http.Request.
   298  // NOTE: Code http.StatusTooManyRequests (429) will *not* be counted against the number of attempts.
   299  func DoRetryForStatusCodes(attempts int, backoff time.Duration, codes ...int) SendDecorator {
   300  	return func(s Sender) Sender {
   301  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   302  			return doRetryForStatusCodesImpl(s, r, Count429AsRetry, attempts, backoff, 0, codes...)
   303  		})
   304  	}
   305  }
   306  
   307  // DoRetryForStatusCodesWithCap returns a SendDecorator that retries for specified statusCodes for up to the
   308  // specified number of attempts, exponentially backing off between requests using the supplied backoff
   309  // time.Duration (which may be zero). To cap the maximum possible delay between iterations specify a value greater
   310  // than zero for cap. Retrying may be canceled by cancelling the context on the http.Request.
   311  func DoRetryForStatusCodesWithCap(attempts int, backoff, cap time.Duration, codes ...int) SendDecorator {
   312  	return func(s Sender) Sender {
   313  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   314  			return doRetryForStatusCodesImpl(s, r, Count429AsRetry, attempts, backoff, cap, codes...)
   315  		})
   316  	}
   317  }
   318  
   319  func doRetryForStatusCodesImpl(s Sender, r *http.Request, count429 bool, attempts int, backoff, cap time.Duration, codes ...int) (resp *http.Response, err error) {
   320  	rr := NewRetriableRequest(r)
   321  	// Increment to add the first call (attempts denotes number of retries)
   322  	for attempt, delayCount := 0, 0; attempt < attempts+1; {
   323  		err = rr.Prepare()
   324  		if err != nil {
   325  			return
   326  		}
   327  		DrainResponseBody(resp)
   328  		resp, err = s.Do(rr.Request())
   329  		// we want to retry if err is not nil (e.g. transient network failure).  note that for failed authentication
   330  		// resp and err will both have a value, so in this case we don't want to retry as it will never succeed.
   331  		if err == nil && !ResponseHasStatusCode(resp, codes...) || IsTokenRefreshError(err) {
   332  			return resp, err
   333  		}
   334  		if err != nil {
   335  			logger.Instance.Writef(logger.LogError, "DoRetryForStatusCodes: received error for attempt %d: %v\n", attempt+1, err)
   336  		}
   337  		delayed := DelayWithRetryAfter(resp, r.Context().Done())
   338  		// if this was a 429 set the delay cap as specified.
   339  		// applicable only in the absence of a retry-after header.
   340  		if resp != nil && resp.StatusCode == http.StatusTooManyRequests {
   341  			cap = Max429Delay
   342  		}
   343  		if !delayed && !DelayForBackoffWithCap(backoff, cap, delayCount, r.Context().Done()) {
   344  			return resp, r.Context().Err()
   345  		}
   346  		// when count429 == false don't count a 429 against the number
   347  		// of attempts so that we continue to retry until it succeeds
   348  		if count429 || (resp == nil || resp.StatusCode != http.StatusTooManyRequests) {
   349  			attempt++
   350  		}
   351  		// delay count is tracked separately from attempts to
   352  		// ensure that 429 participates in exponential back-off
   353  		delayCount++
   354  	}
   355  	return resp, err
   356  }
   357  
   358  // DelayWithRetryAfter invokes time.After for the duration specified in the "Retry-After" header.
   359  // The value of Retry-After can be either the number of seconds or a date in RFC1123 format.
   360  // The function returns true after successfully waiting for the specified duration.  If there is
   361  // no Retry-After header or the wait is cancelled the return value is false.
   362  func DelayWithRetryAfter(resp *http.Response, cancel <-chan struct{}) bool {
   363  	if resp == nil {
   364  		return false
   365  	}
   366  	var dur time.Duration
   367  	ra := resp.Header.Get("Retry-After")
   368  	if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 {
   369  		dur = time.Duration(retryAfter) * time.Second
   370  	} else if t, err := time.Parse(time.RFC1123, ra); err == nil {
   371  		dur = t.Sub(time.Now())
   372  	}
   373  	if dur > 0 {
   374  		select {
   375  		case <-time.After(dur):
   376  			return true
   377  		case <-cancel:
   378  			return false
   379  		}
   380  	}
   381  	return false
   382  }
   383  
   384  // DoRetryForDuration returns a SendDecorator that retries the request until the total time is equal
   385  // to or greater than the specified duration, exponentially backing off between requests using the
   386  // supplied backoff time.Duration (which may be zero). Retrying may be canceled by closing the
   387  // optional channel on the http.Request.
   388  func DoRetryForDuration(d time.Duration, backoff time.Duration) SendDecorator {
   389  	return func(s Sender) Sender {
   390  		return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
   391  			rr := NewRetriableRequest(r)
   392  			end := time.Now().Add(d)
   393  			for attempt := 0; time.Now().Before(end); attempt++ {
   394  				err = rr.Prepare()
   395  				if err != nil {
   396  					return resp, err
   397  				}
   398  				DrainResponseBody(resp)
   399  				resp, err = s.Do(rr.Request())
   400  				if err == nil {
   401  					return resp, err
   402  				}
   403  				logger.Instance.Writef(logger.LogError, "DoRetryForDuration: received error for attempt %d: %v\n", attempt+1, err)
   404  				if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
   405  					return nil, r.Context().Err()
   406  				}
   407  			}
   408  			return resp, err
   409  		})
   410  	}
   411  }
   412  
   413  // WithLogging returns a SendDecorator that implements simple before and after logging of the
   414  // request.
   415  func WithLogging(logger *log.Logger) SendDecorator {
   416  	return func(s Sender) Sender {
   417  		return SenderFunc(func(r *http.Request) (*http.Response, error) {
   418  			logger.Printf("Sending %s %s", r.Method, r.URL)
   419  			resp, err := s.Do(r)
   420  			if err != nil {
   421  				logger.Printf("%s %s received error '%v'", r.Method, r.URL, err)
   422  			} else {
   423  				logger.Printf("%s %s received %s", r.Method, r.URL, resp.Status)
   424  			}
   425  			return resp, err
   426  		})
   427  	}
   428  }
   429  
   430  // DelayForBackoff invokes time.After for the supplied backoff duration raised to the power of
   431  // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
   432  // to zero for no delay. The delay may be canceled by closing the passed channel. If terminated early,
   433  // returns false.
   434  // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
   435  // count.
   436  func DelayForBackoff(backoff time.Duration, attempt int, cancel <-chan struct{}) bool {
   437  	return DelayForBackoffWithCap(backoff, 0, attempt, cancel)
   438  }
   439  
   440  // DelayForBackoffWithCap invokes time.After for the supplied backoff duration raised to the power of
   441  // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
   442  // to zero for no delay. To cap the maximum possible delay specify a value greater than zero for cap.
   443  // The delay may be canceled by closing the passed channel. If terminated early, returns false.
   444  // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
   445  // count.
   446  func DelayForBackoffWithCap(backoff, cap time.Duration, attempt int, cancel <-chan struct{}) bool {
   447  	d := time.Duration(backoff.Seconds()*math.Pow(2, float64(attempt))) * time.Second
   448  	if cap > 0 && d > cap {
   449  		d = cap
   450  	}
   451  	logger.Instance.Writef(logger.LogInfo, "DelayForBackoffWithCap: sleeping for %s\n", d)
   452  	select {
   453  	case <-time.After(d):
   454  		return true
   455  	case <-cancel:
   456  		return false
   457  	}
   458  }
   459
View as plain text