httpreplay_test.go

Documentation: cloud.google.com/go/httpreplay

     1  // Copyright 2018 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package httpreplay_test
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"encoding/json"
    21  	"fmt"
    22  	"io"
    23  	"log"
    24  	"net/http"
    25  	"net/http/httptest"
    26  	"os"
    27  	"path/filepath"
    28  	"testing"
    29  	"time"
    30  
    31  	"cloud.google.com/go/httpreplay"
    32  	"cloud.google.com/go/internal/testutil"
    33  	"cloud.google.com/go/storage"
    34  	"google.golang.org/api/option"
    35  )
    36  
    37  const (
    38  	compressedFile   = "httpreplay_compressed.txt"
    39  	uncompressedFile = "httpreplay_uncompressed.txt"
    40  )
    41  
    42  func TestIntegration_RecordAndReplay(t *testing.T) {
    43  	httpreplay.DebugHeaders()
    44  	if testing.Short() {
    45  		t.Skip("Integration tests skipped in short mode")
    46  	}
    47  	replayFilename := tempFilename(t, "RecordAndReplay*.replay")
    48  	defer os.Remove(replayFilename)
    49  	projectID := testutil.ProjID()
    50  	if projectID == "" {
    51  		t.Skip("Need project ID. See CONTRIBUTING.md for details.")
    52  	}
    53  	ctx := context.Background()
    54  	cleanup, err := setup(ctx)
    55  	if err != nil {
    56  		t.Fatal(err)
    57  	}
    58  	defer cleanup()
    59  
    60  	// Record.
    61  	initial := time.Now()
    62  	ibytes, err := json.Marshal(initial)
    63  	if err != nil {
    64  		t.Fatal(err)
    65  	}
    66  	rec, err := httpreplay.NewRecorder(replayFilename, ibytes)
    67  	if err != nil {
    68  		t.Fatal(err)
    69  	}
    70  
    71  	hc, err := rec.Client(ctx, option.WithTokenSource(
    72  		testutil.TokenSource(ctx, storage.ScopeFullControl)))
    73  	if err != nil {
    74  		t.Fatal(err)
    75  	}
    76  	wanta, wantc := run(t, hc)
    77  	testReadCRC(t, hc, "recording")
    78  	if err := rec.Close(); err != nil {
    79  		t.Fatalf("rec.Close: %v", err)
    80  	}
    81  
    82  	// Replay.
    83  	rep, err := httpreplay.NewReplayer(replayFilename)
    84  	if err != nil {
    85  		t.Fatal(err)
    86  	}
    87  	defer rep.Close()
    88  	hc, err = rep.Client(ctx)
    89  	if err != nil {
    90  		t.Fatal(err)
    91  	}
    92  	gota, gotc := run(t, hc)
    93  	testReadCRC(t, hc, "replaying")
    94  
    95  	if diff := testutil.Diff(gota, wanta); diff != "" {
    96  		t.Error(diff)
    97  	}
    98  	if !bytes.Equal(gotc, wantc) {
    99  		t.Errorf("got %q, want %q", gotc, wantc)
   100  	}
   101  	var gotInitial time.Time
   102  	if err := json.Unmarshal(rep.Initial(), &gotInitial); err != nil {
   103  		t.Fatal(err)
   104  	}
   105  	if !gotInitial.Equal(initial) {
   106  		t.Errorf("initial: got %v, want %v", gotInitial, initial)
   107  	}
   108  }
   109  
   110  func setup(ctx context.Context) (cleanup func(), err error) {
   111  	ts := testutil.TokenSource(ctx, storage.ScopeFullControl)
   112  	client, err := storage.NewClient(ctx, option.WithTokenSource(ts))
   113  	if err != nil {
   114  		return nil, err
   115  	}
   116  	bucket := testutil.ProjID()
   117  
   118  	// upload compressed object
   119  	f1, err := os.Open(filepath.Join("internal", "testdata", "compressed.txt"))
   120  	if err != nil {
   121  		return nil, err
   122  	}
   123  	defer f1.Close()
   124  	w := client.Bucket(bucket).Object(compressedFile).NewWriter(ctx)
   125  	w.ContentEncoding = "gzip"
   126  	w.ContentType = "text/plain"
   127  	if _, err = io.Copy(w, f1); err != nil {
   128  		return nil, err
   129  	}
   130  	if err := w.Close(); err != nil {
   131  		return nil, err
   132  	}
   133  	// upload uncompressed object
   134  	f2, err := os.Open(filepath.Join("internal", "testdata", "uncompressed.txt"))
   135  	if err != nil {
   136  		return nil, err
   137  	}
   138  	defer f2.Close()
   139  	w = client.Bucket(testutil.ProjID()).Object(uncompressedFile).NewWriter(ctx)
   140  	if _, err = io.Copy(w, f2); err != nil {
   141  		return nil, err
   142  	}
   143  	if err := w.Close(); err != nil {
   144  		return nil, err
   145  	}
   146  	return func() {
   147  		client.Bucket(bucket).Object(compressedFile).Delete(ctx)
   148  		client.Bucket(bucket).Object(uncompressedFile).Delete(ctx)
   149  		client.Close()
   150  	}, nil
   151  }
   152  
   153  // TODO(jba): test errors
   154  
   155  func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
   156  	ctx, cc := context.WithTimeout(context.Background(), 10*time.Second)
   157  	defer cc()
   158  
   159  	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
   160  	if err != nil {
   161  		t.Fatal(err)
   162  	}
   163  	defer client.Close()
   164  	b := client.Bucket(testutil.ProjID())
   165  	attrs, err := b.Attrs(ctx)
   166  	if err != nil {
   167  		t.Fatal(err)
   168  	}
   169  	obj := b.Object("replay-test")
   170  	w := obj.NewWriter(ctx)
   171  	data := []byte{150, 151, 152}
   172  	if _, err := w.Write(data); err != nil {
   173  		t.Fatal(err)
   174  	}
   175  	if err := w.Close(); err != nil {
   176  		t.Fatal(err)
   177  	}
   178  
   179  	r, err := obj.NewReader(ctx)
   180  	if err != nil {
   181  		t.Fatal(err)
   182  	}
   183  	defer r.Close()
   184  	contents, err := io.ReadAll(r)
   185  	if err != nil {
   186  		t.Fatal(err)
   187  	}
   188  
   189  	return attrs, contents
   190  }
   191  
   192  func testReadCRC(t *testing.T, hc *http.Client, mode string) {
   193  	ctx, cc := context.WithTimeout(context.Background(), 10*time.Second)
   194  	defer cc()
   195  
   196  	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
   197  	if err != nil {
   198  		t.Fatalf("%s: %v", mode, err)
   199  	}
   200  	defer client.Close()
   201  
   202  	bucket := testutil.ProjID()
   203  	uncompressedObj := client.Bucket(bucket).Object(uncompressedFile)
   204  	gzippedObj := client.Bucket(bucket).Object(compressedFile)
   205  
   206  	for _, test := range []struct {
   207  		desc           string
   208  		obj            *storage.ObjectHandle
   209  		offset, length int64
   210  		readCompressed bool // don't decompress a gzipped file
   211  
   212  		wantErr bool
   213  		wantLen int // length of contents
   214  	}{
   215  		{
   216  			desc:           "uncompressed, entire file",
   217  			obj:            uncompressedObj,
   218  			offset:         0,
   219  			length:         -1,
   220  			readCompressed: false,
   221  			wantLen:        179,
   222  		},
   223  		{
   224  			desc:           "uncompressed, entire file, don't decompress",
   225  			obj:            uncompressedObj,
   226  			offset:         0,
   227  			length:         -1,
   228  			readCompressed: true,
   229  			wantLen:        179,
   230  		},
   231  		{
   232  			desc:           "uncompressed, suffix",
   233  			obj:            uncompressedObj,
   234  			offset:         9,
   235  			length:         -1,
   236  			readCompressed: false,
   237  			wantLen:        170,
   238  		},
   239  		{
   240  			desc:           "uncompressed, prefix",
   241  			obj:            uncompressedObj,
   242  			offset:         0,
   243  			length:         18,
   244  			readCompressed: false,
   245  			wantLen:        18,
   246  		},
   247  		{
   248  			// When a gzipped file is unzipped by GCS, we can't verify the checksum
   249  			// because it was computed against the zipped contents. There is no
   250  			// header that indicates that a gzipped file is being served unzipped.
   251  			// But our CRC check only happens if there is a Content-Length header,
   252  			// and that header is absent for this read.
   253  			desc:           "compressed, entire file, server unzips",
   254  			obj:            gzippedObj,
   255  			offset:         0,
   256  			length:         -1,
   257  			readCompressed: false,
   258  			wantLen:        179,
   259  		},
   260  		{
   261  			// When we read a gzipped file uncompressed, it's like reading a regular file:
   262  			// the served content and the CRC match.
   263  			desc:           "compressed, entire file, read compressed",
   264  			obj:            gzippedObj,
   265  			offset:         0,
   266  			length:         -1,
   267  			readCompressed: true,
   268  			wantLen:        128,
   269  		},
   270  		{
   271  			desc:           "compressed, partial, read compressed",
   272  			obj:            gzippedObj,
   273  			offset:         1,
   274  			length:         8,
   275  			readCompressed: true,
   276  			wantLen:        8,
   277  		},
   278  		{
   279  			desc:    "uncompressed, HEAD",
   280  			obj:     uncompressedObj,
   281  			offset:  0,
   282  			length:  0,
   283  			wantLen: 0,
   284  		},
   285  		{
   286  			desc:    "compressed, HEAD",
   287  			obj:     gzippedObj,
   288  			offset:  0,
   289  			length:  0,
   290  			wantLen: 0,
   291  		},
   292  	} {
   293  		obj := test.obj.ReadCompressed(test.readCompressed)
   294  		r, err := obj.NewRangeReader(ctx, test.offset, test.length)
   295  		if err != nil {
   296  			if test.wantErr {
   297  				continue
   298  			}
   299  			t.Errorf("%s: %s: %v", mode, test.desc, err)
   300  			continue
   301  		}
   302  		data, err := io.ReadAll(r)
   303  		_ = r.Close()
   304  		if err != nil {
   305  			t.Errorf("%s: %s: %v", mode, test.desc, err)
   306  			continue
   307  		}
   308  		if got, want := len(data), test.wantLen; got != want {
   309  			t.Errorf("%s: %s: len: got %d, want %d", mode, test.desc, got, want)
   310  		}
   311  	}
   312  }
   313  
   314  func TestRemoveAndClear(t *testing.T) {
   315  	// Disable logging for this test, since it generates a lot.
   316  	log.SetOutput(io.Discard)
   317  	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
   318  		fmt.Fprintln(w, "LGTM")
   319  	}))
   320  	defer srv.Close()
   321  
   322  	replayFilename := tempFilename(t, "TestRemoveAndClear*.replay")
   323  	defer os.Remove(replayFilename)
   324  
   325  	ctx := context.Background()
   326  	// Record
   327  	rec, err := httpreplay.NewRecorder(replayFilename, nil)
   328  	if err != nil {
   329  		t.Fatal(err)
   330  	}
   331  	rec.ClearHeaders("Clear")
   332  	rec.RemoveRequestHeaders("Rem*")
   333  	rec.ClearQueryParams("c")
   334  	rec.RemoveQueryParams("r")
   335  	hc, err := rec.Client(ctx, option.WithoutAuthentication())
   336  	if err != nil {
   337  		t.Fatal(err)
   338  	}
   339  	query := "k=1&r=2&c=3"
   340  	req, err := http.NewRequest("GET", srv.URL+"?"+query, nil)
   341  	if err != nil {
   342  		t.Fatal(err)
   343  	}
   344  	headers := map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "bye"}
   345  	for k, v := range headers {
   346  		req.Header.Set(k, v)
   347  	}
   348  	if _, err := hc.Do(req); err != nil {
   349  		t.Fatal(err)
   350  	}
   351  	if err := rec.Close(); err != nil {
   352  		t.Fatal(err)
   353  	}
   354  
   355  	// Replay
   356  	// For both headers and query param:
   357  	// - k or Keep must be present and identical
   358  	// - c or Clear must be present, but can be different
   359  	// - r or Remove can be anything
   360  	for _, test := range []struct {
   361  		query       string
   362  		headers     map[string]string
   363  		wantSuccess bool
   364  	}{
   365  		{query, headers, true}, // same query string and headers
   366  		{query,
   367  			map[string]string{"Keep": "oops", "Clear": "secret", "Remove": "bye"},
   368  			false, // different Keep
   369  		},
   370  		{query, map[string]string{}, false},                               // missing Keep and Clear
   371  		{query, map[string]string{"Keep": "ok"}, false},                   // missing Clear
   372  		{query, map[string]string{"Keep": "ok", "Clear": "secret"}, true}, // missing Remove is OK
   373  		{
   374  			query,
   375  			map[string]string{"Keep": "ok", "Clear": "secret", "Remove": "whatev"},
   376  			true,
   377  		}, // different Remove is OK
   378  		{query, map[string]string{"Keep": "ok", "Clear": "diff"}, true}, // different Clear is OK
   379  		{"", headers, false},            // no query string
   380  		{"k=x&r=2&c=3", headers, false}, // different k
   381  		{"r=2", headers, false},         // missing k and c
   382  		{"k=1&r=2", headers, false},     // missing c
   383  		{"k=1&c=3", headers, true},      // missing r is OK
   384  		{"k=1&r=x&c=3", headers, true},  // different r is OK,
   385  		{"k=1&r=2&c=x", headers, true},  // different clear is OK
   386  	} {
   387  		rep, err := httpreplay.NewReplayer(replayFilename)
   388  		if err != nil {
   389  			t.Fatal(err)
   390  		}
   391  		hc, err = rep.Client(ctx)
   392  		if err != nil {
   393  			t.Fatal(err)
   394  		}
   395  		url := srv.URL
   396  		if test.query != "" {
   397  			url += "?" + test.query
   398  		}
   399  		req, err = http.NewRequest("GET", url, nil)
   400  		if err != nil {
   401  			t.Fatal(err)
   402  		}
   403  		for k, v := range test.headers {
   404  			req.Header.Set(k, v)
   405  		}
   406  		resp, err := hc.Do(req)
   407  		if err != nil {
   408  			t.Fatal(err)
   409  		}
   410  		rep.Close()
   411  		if (resp.StatusCode == 200) != test.wantSuccess {
   412  			t.Errorf("%q, %v: got %d, wanted success=%t",
   413  				test.query, test.headers, resp.StatusCode, test.wantSuccess)
   414  		}
   415  	}
   416  }
   417  
   418  func tempFilename(t *testing.T, pattern string) string {
   419  	f, err := os.CreateTemp("", pattern)
   420  	if err != nil {
   421  		t.Fatal(err)
   422  	}
   423  	filename := f.Name()
   424  	if err := f.Close(); err != nil {
   425  		t.Fatal(err)
   426  	}
   427  	return filename
   428  }
   429
View as plain text