record.go

Documentation: cloud.google.com/go/httpreplay/internal/proxy

     1  // Copyright 2018 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package proxy provides a record/replay HTTP proxy. It is designed to support
    16  // both an in-memory API (cloud.google.com/go/httpreplay) and a standalone server
    17  // (cloud.google.com/go/httpreplay/cmd/httpr).
    18  package proxy
    19  
    20  // See github.com/google/martian/cmd/proxy/main.go for the origin of much of this.
    21  
    22  import (
    23  	"crypto/tls"
    24  	"crypto/x509"
    25  	"encoding/json"
    26  	"fmt"
    27  	"net"
    28  	"net/http"
    29  	"net/url"
    30  	"os"
    31  	"strings"
    32  	"sync"
    33  	"time"
    34  
    35  	"github.com/google/martian/v3"
    36  	"github.com/google/martian/v3/fifo"
    37  	"github.com/google/martian/v3/httpspec"
    38  	"github.com/google/martian/v3/martianlog"
    39  	"github.com/google/martian/v3/mitm"
    40  )
    41  
    42  // A Proxy is an HTTP proxy that supports recording or replaying requests.
    43  type Proxy struct {
    44  	// The certificate that the proxy uses to participate in TLS.
    45  	CACert *x509.Certificate
    46  
    47  	// The URL of the proxy.
    48  	URL *url.URL
    49  
    50  	// Initial state of the client.
    51  	Initial []byte
    52  
    53  	mproxy        *martian.Proxy
    54  	filename      string          // for log
    55  	logger        *Logger         // for recording only
    56  	ignoreHeaders map[string]bool // headers the user has asked to ignore
    57  }
    58  
    59  // ForRecording returns a Proxy configured to record.
    60  func ForRecording(filename string, port int) (*Proxy, error) {
    61  	p, err := newProxy(filename)
    62  	if err != nil {
    63  		return nil, err
    64  	}
    65  
    66  	// Construct a group that performs the standard proxy stack of request/response
    67  	// modifications.
    68  	stack, _ := httpspec.NewStack("httpr") // second arg is an internal group that we don't need
    69  	p.mproxy.SetRequestModifier(stack)
    70  	p.mproxy.SetResponseModifier(stack)
    71  
    72  	// Make a group for logging requests and responses.
    73  	logGroup := fifo.NewGroup()
    74  	skipAuth := skipLoggingByHost("accounts.google.com")
    75  	logGroup.AddRequestModifier(skipAuth)
    76  	logGroup.AddResponseModifier(skipAuth)
    77  	p.logger = newLogger()
    78  	logGroup.AddRequestModifier(p.logger)
    79  	logGroup.AddResponseModifier(p.logger)
    80  
    81  	stack.AddRequestModifier(logGroup)
    82  	stack.AddResponseModifier(logGroup)
    83  
    84  	// Ordinary debug logging.
    85  	logger := martianlog.NewLogger()
    86  	logger.SetDecode(true)
    87  	stack.AddRequestModifier(logger)
    88  	stack.AddResponseModifier(logger)
    89  
    90  	if err := p.start(port); err != nil {
    91  		return nil, err
    92  	}
    93  	return p, nil
    94  }
    95  
    96  var (
    97  	configOnce sync.Once
    98  	cert       *x509.Certificate
    99  	config     *mitm.Config
   100  	configErr  error
   101  )
   102  
   103  func newProxy(filename string) (*Proxy, error) {
   104  	configOnce.Do(func() {
   105  		// Set up a man-in-the-middle configuration with a CA certificate so the proxy can
   106  		// participate in TLS.
   107  		x509c, priv, err := mitm.NewAuthority("cloud.google.com/go/httpreplay", "HTTPReplay Authority", 100*time.Hour)
   108  		if err != nil {
   109  			configErr = err
   110  			return
   111  		}
   112  		cert = x509c
   113  		config, configErr = mitm.NewConfig(x509c, priv)
   114  		if config != nil {
   115  			config.SetValidity(100 * time.Hour)
   116  			config.SetOrganization("cloud.google.com/go/httpreplay")
   117  			config.SkipTLSVerify(false)
   118  		}
   119  	})
   120  	if configErr != nil {
   121  		return nil, configErr
   122  	}
   123  	mproxy := martian.NewProxy()
   124  	mproxy.SetMITM(config)
   125  	return &Proxy{
   126  		mproxy:        mproxy,
   127  		CACert:        cert,
   128  		filename:      filename,
   129  		ignoreHeaders: map[string]bool{},
   130  	}, nil
   131  }
   132  
   133  func (p *Proxy) start(port int) error {
   134  	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
   135  	if err != nil {
   136  		return err
   137  	}
   138  	p.URL = &url.URL{Scheme: "http", Host: l.Addr().String()}
   139  	go p.mproxy.Serve(l)
   140  	return nil
   141  }
   142  
   143  // Transport returns an http.Transport for clients who want to talk to the proxy.
   144  func (p *Proxy) Transport() *http.Transport {
   145  	caCertPool := x509.NewCertPool()
   146  	caCertPool.AddCert(p.CACert)
   147  	return &http.Transport{
   148  		TLSClientConfig: &tls.Config{RootCAs: caCertPool},
   149  		Proxy:           func(*http.Request) (*url.URL, error) { return p.URL, nil },
   150  	}
   151  }
   152  
   153  // RemoveRequestHeaders will remove request headers matching patterns from the log,
   154  // and skip matching them. Pattern is taken literally except for *, which matches any
   155  // sequence of characters.
   156  //
   157  // This only needs to be called during recording; the patterns will be saved to the
   158  // log for replay.
   159  func (p *Proxy) RemoveRequestHeaders(patterns []string) {
   160  	for _, pat := range patterns {
   161  		p.logger.log.Converter.registerRemoveRequestHeaders(pat)
   162  	}
   163  }
   164  
   165  // ClearHeaders will replace matching headers with CLEARED.
   166  //
   167  // This only needs to be called during recording; the patterns will be saved to the
   168  // log for replay.
   169  func (p *Proxy) ClearHeaders(patterns []string) {
   170  	for _, pat := range patterns {
   171  		p.logger.log.Converter.registerClearHeaders(pat)
   172  	}
   173  }
   174  
   175  // RemoveQueryParams will remove query parameters matching patterns from the request
   176  // URL before logging, and skip matching them. Pattern is taken literally except for
   177  // *, which matches any sequence of characters.
   178  //
   179  // This only needs to be called during recording; the patterns will be saved to the
   180  // log for replay.
   181  func (p *Proxy) RemoveQueryParams(patterns []string) {
   182  	for _, pat := range patterns {
   183  		p.logger.log.Converter.registerRemoveParams(pat)
   184  	}
   185  }
   186  
   187  // ClearQueryParams will replace matching query params in the request URL with CLEARED.
   188  //
   189  // This only needs to be called during recording; the patterns will be saved to the
   190  // log for replay.
   191  func (p *Proxy) ClearQueryParams(patterns []string) {
   192  	for _, pat := range patterns {
   193  		p.logger.log.Converter.registerClearParams(pat)
   194  	}
   195  }
   196  
   197  // IgnoreHeader will cause h to be ignored during matching on replay.
   198  // Deprecated: use RemoveRequestHeaders instead.
   199  func (p *Proxy) IgnoreHeader(h string) {
   200  	p.ignoreHeaders[http.CanonicalHeaderKey(h)] = true
   201  }
   202  
   203  // Close closes the proxy. If the proxy is recording, it also writes the log.
   204  func (p *Proxy) Close() error {
   205  	p.mproxy.Close()
   206  	if p.logger != nil {
   207  		return p.writeLog()
   208  	}
   209  	return nil
   210  }
   211  
   212  func (p *Proxy) writeLog() error {
   213  	lg := p.logger.Extract()
   214  	lg.Initial = p.Initial
   215  	bytes, err := json.MarshalIndent(lg, "", "  ")
   216  	if err != nil {
   217  		return err
   218  	}
   219  	return os.WriteFile(p.filename, bytes, 0600) // only accessible by owner
   220  }
   221  
   222  // skipLoggingByHost disables logging for traffic to a particular host.
   223  type skipLoggingByHost string
   224  
   225  func (s skipLoggingByHost) ModifyRequest(req *http.Request) error {
   226  	if strings.HasPrefix(req.Host, string(s)) {
   227  		martian.NewContext(req).SkipLogging()
   228  	}
   229  	return nil
   230  }
   231  
   232  func (s skipLoggingByHost) ModifyResponse(res *http.Response) error {
   233  	return s.ModifyRequest(res.Request)
   234  }
   235
View as plain text