client.go

Documentation: github.com/google/certificate-transparency-go/jsonclient

     1  // Copyright 2016 Google LLC. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package jsonclient provides a simple client for fetching and parsing
    16  // JSON CT structures from a log.
    17  package jsonclient
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto"
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"io"
    27  	"log"
    28  	"math/rand"
    29  	"net/http"
    30  	"net/url"
    31  	"strconv"
    32  	"strings"
    33  	"time"
    34  
    35  	ct "github.com/google/certificate-transparency-go"
    36  	"github.com/google/certificate-transparency-go/x509"
    37  	"golang.org/x/net/context/ctxhttp"
    38  	"k8s.io/klog/v2"
    39  )
    40  
    41  const maxJitter = 250 * time.Millisecond
    42  
    43  type backoffer interface {
    44  	// set adjusts/increases the current backoff interval (typically on retryable failure);
    45  	// if the optional parameter is provided, this will be used as the interval if it is greater
    46  	// than the currently set interval.  Returns the current wait period so that it can be
    47  	// logged along with any error message.
    48  	set(*time.Duration) time.Duration
    49  	// decreaseMultiplier reduces the current backoff multiplier, typically on success.
    50  	decreaseMultiplier()
    51  	// until returns the time until which the client should wait before making a request,
    52  	// it may be in the past in which case it should be ignored.
    53  	until() time.Time
    54  }
    55  
    56  // JSONClient provides common functionality for interacting with a JSON server
    57  // that uses cryptographic signatures.
    58  type JSONClient struct {
    59  	uri        string                // the base URI of the server. e.g. https://ct.googleapis/pilot
    60  	httpClient *http.Client          // used to interact with the server via HTTP
    61  	Verifier   *ct.SignatureVerifier // nil for no verification (e.g. no public key available)
    62  	logger     Logger                // interface to use for logging warnings and errors
    63  	backoff    backoffer             // object used to store and calculate backoff information
    64  	userAgent  string                // If set, this is sent as the UserAgent header.
    65  }
    66  
    67  // Logger is a simple logging interface used to log internal errors and warnings
    68  type Logger interface {
    69  	// Printf formats and logs a message
    70  	Printf(string, ...interface{})
    71  }
    72  
    73  // Options are the options for creating a new JSONClient.
    74  type Options struct {
    75  	// Interface to use for logging warnings and errors, if nil the
    76  	// standard library log package will be used.
    77  	Logger Logger
    78  	// PEM format public key to use for signature verification.
    79  	PublicKey string
    80  	// DER format public key to use for signature verification.
    81  	PublicKeyDER []byte
    82  	// UserAgent, if set, will be sent as the User-Agent header with each request.
    83  	UserAgent string
    84  }
    85  
    86  // ParsePublicKey parses and returns the public key contained in opts.
    87  // If both opts.PublicKey and opts.PublicKeyDER are set, PublicKeyDER is used.
    88  // If neither is set, nil will be returned.
    89  func (opts *Options) ParsePublicKey() (crypto.PublicKey, error) {
    90  	if len(opts.PublicKeyDER) > 0 {
    91  		return x509.ParsePKIXPublicKey(opts.PublicKeyDER)
    92  	}
    93  
    94  	if opts.PublicKey != "" {
    95  		pubkey, _ /* keyhash */, rest, err := ct.PublicKeyFromPEM([]byte(opts.PublicKey))
    96  		if err != nil {
    97  			return nil, err
    98  		}
    99  		if len(rest) > 0 {
   100  			return nil, errors.New("extra data found after PEM key decoded")
   101  		}
   102  		return pubkey, nil
   103  	}
   104  
   105  	return nil, nil
   106  }
   107  
   108  type basicLogger struct{}
   109  
   110  func (bl *basicLogger) Printf(msg string, args ...interface{}) {
   111  	log.Printf(msg, args...)
   112  }
   113  
   114  // RspError represents an error that occurred when processing a response from a server,
   115  // and also includes key details from the http.Response that triggered the error.
   116  type RspError struct {
   117  	Err        error
   118  	StatusCode int
   119  	Body       []byte
   120  }
   121  
   122  // Error formats the RspError instance, focusing on the error.
   123  func (e RspError) Error() string {
   124  	return e.Err.Error()
   125  }
   126  
   127  // New constructs a new JSONClient instance, for the given base URI, using the
   128  // given http.Client object (if provided) and the Options object.
   129  // If opts does not specify a public key, signatures will not be verified.
   130  func New(uri string, hc *http.Client, opts Options) (*JSONClient, error) {
   131  	pubkey, err := opts.ParsePublicKey()
   132  	if err != nil {
   133  		return nil, fmt.Errorf("invalid public key: %v", err)
   134  	}
   135  
   136  	var verifier *ct.SignatureVerifier
   137  	if pubkey != nil {
   138  		var err error
   139  		verifier, err = ct.NewSignatureVerifier(pubkey)
   140  		if err != nil {
   141  			return nil, err
   142  		}
   143  	}
   144  
   145  	if hc == nil {
   146  		hc = new(http.Client)
   147  	}
   148  	logger := opts.Logger
   149  	if logger == nil {
   150  		logger = &basicLogger{}
   151  	}
   152  	return &JSONClient{
   153  		uri:        strings.TrimRight(uri, "/"),
   154  		httpClient: hc,
   155  		Verifier:   verifier,
   156  		logger:     logger,
   157  		backoff:    &backoff{},
   158  		userAgent:  opts.UserAgent,
   159  	}, nil
   160  }
   161  
   162  // BaseURI returns the base URI that the JSONClient makes queries to.
   163  func (c *JSONClient) BaseURI() string {
   164  	return c.uri
   165  }
   166  
   167  // GetAndParse makes a HTTP GET call to the given path, and attempts to parse
   168  // the response as a JSON representation of the rsp structure.  Returns the
   169  // http.Response, the body of the response, and an error (which may be of
   170  // type RspError if the HTTP response was available).
   171  func (c *JSONClient) GetAndParse(ctx context.Context, path string, params map[string]string, rsp interface{}) (*http.Response, []byte, error) {
   172  	if ctx == nil {
   173  		return nil, nil, errors.New("context.Context required")
   174  	}
   175  	// Build a GET request with URL-encoded parameters.
   176  	vals := url.Values{}
   177  	for k, v := range params {
   178  		vals.Add(k, v)
   179  	}
   180  	fullURI := fmt.Sprintf("%s%s?%s", c.uri, path, vals.Encode())
   181  	klog.V(2).Infof("GET %s", fullURI)
   182  	httpReq, err := http.NewRequest(http.MethodGet, fullURI, nil)
   183  	if err != nil {
   184  		return nil, nil, err
   185  	}
   186  	if len(c.userAgent) != 0 {
   187  		httpReq.Header.Set("User-Agent", c.userAgent)
   188  	}
   189  
   190  	httpRsp, err := ctxhttp.Do(ctx, c.httpClient, httpReq)
   191  	if err != nil {
   192  		return nil, nil, err
   193  	}
   194  
   195  	// Read everything now so http.Client can reuse the connection.
   196  	body, err := io.ReadAll(httpRsp.Body)
   197  	httpRsp.Body.Close()
   198  	if err != nil {
   199  		return nil, nil, RspError{Err: fmt.Errorf("failed to read response body: %v", err), StatusCode: httpRsp.StatusCode, Body: body}
   200  	}
   201  
   202  	if httpRsp.StatusCode != http.StatusOK {
   203  		return nil, nil, RspError{Err: fmt.Errorf("got HTTP Status %q", httpRsp.Status), StatusCode: httpRsp.StatusCode, Body: body}
   204  	}
   205  
   206  	if err := json.NewDecoder(bytes.NewReader(body)).Decode(rsp); err != nil {
   207  		return nil, nil, RspError{Err: err, StatusCode: httpRsp.StatusCode, Body: body}
   208  	}
   209  
   210  	return httpRsp, body, nil
   211  }
   212  
   213  // PostAndParse makes a HTTP POST call to the given path, including the request
   214  // parameters, and attempts to parse the response as a JSON representation of
   215  // the rsp structure. Returns the http.Response, the body of the response, and
   216  // an error (which may be of type RspError if the HTTP response was available).
   217  func (c *JSONClient) PostAndParse(ctx context.Context, path string, req, rsp interface{}) (*http.Response, []byte, error) {
   218  	if ctx == nil {
   219  		return nil, nil, errors.New("context.Context required")
   220  	}
   221  	// Build a POST request with JSON body.
   222  	postBody, err := json.Marshal(req)
   223  	if err != nil {
   224  		return nil, nil, err
   225  	}
   226  	fullURI := fmt.Sprintf("%s%s", c.uri, path)
   227  	klog.V(2).Infof("POST %s", fullURI)
   228  	httpReq, err := http.NewRequest(http.MethodPost, fullURI, bytes.NewReader(postBody))
   229  	if err != nil {
   230  		return nil, nil, err
   231  	}
   232  	if len(c.userAgent) != 0 {
   233  		httpReq.Header.Set("User-Agent", c.userAgent)
   234  	}
   235  	httpReq.Header.Set("Content-Type", "application/json")
   236  
   237  	httpRsp, err := ctxhttp.Do(ctx, c.httpClient, httpReq)
   238  
   239  	// Read all of the body, if there is one, so that the http.Client can do Keep-Alive.
   240  	var body []byte
   241  	if httpRsp != nil {
   242  		body, err = io.ReadAll(httpRsp.Body)
   243  		httpRsp.Body.Close()
   244  	}
   245  	if err != nil {
   246  		if httpRsp != nil {
   247  			return nil, nil, RspError{StatusCode: httpRsp.StatusCode, Body: body, Err: err}
   248  		}
   249  		return nil, nil, err
   250  	}
   251  
   252  	if httpRsp.StatusCode == http.StatusOK {
   253  		if err = json.Unmarshal(body, &rsp); err != nil {
   254  			return nil, nil, RspError{StatusCode: httpRsp.StatusCode, Body: body, Err: err}
   255  		}
   256  	}
   257  	return httpRsp, body, nil
   258  }
   259  
   260  // waitForBackoff blocks until the defined backoff interval or context has expired, if the returned
   261  // not before time is in the past it returns immediately.
   262  func (c *JSONClient) waitForBackoff(ctx context.Context) error {
   263  	dur := time.Until(c.backoff.until().Add(time.Millisecond * time.Duration(rand.Intn(int(maxJitter.Seconds()*1000)))))
   264  	if dur < 0 {
   265  		dur = 0
   266  	}
   267  	backoffTimer := time.NewTimer(dur)
   268  	select {
   269  	case <-ctx.Done():
   270  		return ctx.Err()
   271  	case <-backoffTimer.C:
   272  	}
   273  	return nil
   274  }
   275  
   276  // PostAndParseWithRetry makes a HTTP POST call, but retries (with backoff) on
   277  // retryable errors; the caller should set a deadline on the provided context
   278  // to prevent infinite retries.  Return values are as for PostAndParse.
   279  func (c *JSONClient) PostAndParseWithRetry(ctx context.Context, path string, req, rsp interface{}) (*http.Response, []byte, error) {
   280  	if ctx == nil {
   281  		return nil, nil, errors.New("context.Context required")
   282  	}
   283  	for {
   284  		httpRsp, body, err := c.PostAndParse(ctx, path, req, rsp)
   285  		if err != nil {
   286  			// Don't retry context errors.
   287  			if err == context.Canceled || err == context.DeadlineExceeded {
   288  				return nil, nil, err
   289  			}
   290  			wait := c.backoff.set(nil)
   291  			c.logger.Printf("Request to %s failed, backing-off %s: %s", c.uri, wait, err)
   292  		} else {
   293  			switch {
   294  			case httpRsp.StatusCode == http.StatusOK:
   295  				return httpRsp, body, nil
   296  			case httpRsp.StatusCode == http.StatusRequestTimeout:
   297  				// Request timeout, retry immediately
   298  				c.logger.Printf("Request to %s timed out, retrying immediately", c.uri)
   299  			case httpRsp.StatusCode == http.StatusServiceUnavailable:
   300  				fallthrough
   301  			case httpRsp.StatusCode == http.StatusTooManyRequests:
   302  				var backoff *time.Duration
   303  				// Retry-After may be either a number of seconds as a int or a RFC 1123
   304  				// date string (RFC 7231 Section 7.1.3)
   305  				if retryAfter := httpRsp.Header.Get("Retry-After"); retryAfter != "" {
   306  					if seconds, err := strconv.Atoi(retryAfter); err == nil {
   307  						b := time.Duration(seconds) * time.Second
   308  						backoff = &b
   309  					} else if date, err := time.Parse(time.RFC1123, retryAfter); err == nil {
   310  						b := time.Until(date)
   311  						backoff = &b
   312  					}
   313  				}
   314  				wait := c.backoff.set(backoff)
   315  				c.logger.Printf("Request to %s failed, backing-off for %s: got HTTP status %s", c.uri, wait, httpRsp.Status)
   316  			default:
   317  				return nil, nil, RspError{
   318  					StatusCode: httpRsp.StatusCode,
   319  					Body:       body,
   320  					Err:        fmt.Errorf("got HTTP status %q", httpRsp.Status)}
   321  			}
   322  		}
   323  		if err := c.waitForBackoff(ctx); err != nil {
   324  			return nil, nil, err
   325  		}
   326  	}
   327  }
   328
View as plain text