fakegcsserver_test.go

Documentation: edge-infra.dev/pkg/edge/chariot

     1  package chariot
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"crypto/md5" //nolint:gosec // md5 hashes are provided by GoogleCloudStorage
     7  	"fmt"
     8  	"io"
     9  	"strings"
    10  	"testing"
    11  	"time"
    12  
    13  	"cloud.google.com/go/storage"
    14  	"github.com/fsouza/fake-gcs-server/fakestorage"
    15  	"google.golang.org/api/iterator"
    16  )
    17  
    18  func startFakeGcsServer(objs ...fakestorage.Object) (client *storage.Client, stop func(), err error) {
    19  	var server *fakestorage.Server
    20  	if server, err = fakestorage.NewServerWithOptions(fakestorage.Options{
    21  		// Note: Buckets are created for InitialObjects[].ObjectAttrs.BucketName values.
    22  		InitialObjects: objs,
    23  	}); err != nil {
    24  		return nil, nil, err
    25  	}
    26  	client = server.Client()
    27  	stop = server.Stop
    28  
    29  	// The following is a stupid fix to force fake-gcs-server to calculate MD5 hashes.
    30  	// If you want to contribute to open source, you can get fake-gcs-server to do this
    31  	// when you provide InitialObjects.  Right now, if you provide InitialObjects with the
    32  	// MD5 hash manually set, it still returns an empty string for the hash, or an incorrect
    33  	// MD5 hash value that isn't even 16 bytes long. LOLWUT. So to remedy this flaw, I write
    34  	// the objects again once the fake-gcs-server has been created.  I can't believe I have to
    35  	// do this, but here we are.  I hope this comment saves you/me some time in the future.
    36  	for _, obj := range objs {
    37  		bname := obj.ObjectAttrs.BucketName
    38  		oname := obj.ObjectAttrs.Name
    39  		w := client.Bucket(bname).Object(oname).NewWriter(context.Background())
    40  		if _, err = io.Copy(w, bytes.NewReader(obj.Content)); err != nil {
    41  			stop()
    42  			break
    43  		} else if err = w.Close(); err != nil {
    44  			stop()
    45  			break
    46  		}
    47  	}
    48  	return client, stop, err
    49  }
    50  
    51  func TestFakeGcsServerSetsMD5InObjectAttrs(t *testing.T) {
    52  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
    53  	defer cancel()
    54  
    55  	const (
    56  		contentStr = "content of the file\nto check for md5 correctness"
    57  		bucketName = "chariot-unit-testing"
    58  		objectName = "path/to/hello-world.txt"
    59  	)
    60  
    61  	var rawMD5Hash = md5.Sum([]byte(contentStr)) //nolint:gosec // md5 hashes are provided by GoogleCloudStorage
    62  	var wantMD5HashStr = fmt.Sprintf("%x", rawMD5Hash[:])
    63  	t.Logf("Want MD5 hash: %q", wantMD5HashStr)
    64  
    65  	var obj = fakestorage.Object{
    66  		ObjectAttrs: fakestorage.ObjectAttrs{
    67  			BucketName: bucketName,
    68  			Name:       objectName,
    69  		},
    70  		Content: []byte(contentStr),
    71  	}
    72  
    73  	c, stop, err := startFakeGcsServer(obj)
    74  	if err != nil {
    75  		t.Fatalf("Got error starting fake gcs server: %v", err)
    76  	}
    77  	defer stop()
    78  
    79  	// Check the object attrs MD5 using the Object.Attrs() function (sans 's')
    80  	attrs, err := c.Bucket(bucketName).Object(objectName).Attrs(ctx)
    81  	if err != nil {
    82  		t.Fatal(err)
    83  	} else if len(attrs.MD5) == 0 {
    84  		t.Fatalf("Object(%q).Attrs(ctx) returned empty MD5 value", objectName)
    85  	}
    86  	// validate the md5 hash
    87  	gotMD5HashStr1 := fmt.Sprintf("%x", attrs.MD5)
    88  	t.Logf("Got MD5 hash %q for Object().Attrs() function", gotMD5HashStr1)
    89  	if wantMD5HashStr != gotMD5HashStr1 {
    90  		t.Fatalf("fake-gcs-server returned incorrect MD5 hash value for Object().Attrs() function")
    91  	}
    92  
    93  	// Check the object attrs MD5 using the Objects() function iterator.
    94  	attrs, err = c.Bucket(bucketName).Objects(ctx, nil).Next()
    95  	if err == iterator.Done {
    96  		t.Fatalf("Got iterator.Done from Objects call: %v", err)
    97  	} else if err != nil {
    98  		t.Fatal(err)
    99  	} else if attrs.Name != objectName {
   100  		t.Fatalf("Object name not correct. want %q got %q", objectName, attrs.Name)
   101  	} else if len(attrs.MD5) == 0 {
   102  		t.Fail()
   103  		t.Logf("Ugh why fake-gcs-server why! Got nil MD5 hash.")
   104  		t.Logf("Trying .Object(%q).Attrs(ctx) now", attrs.Name)
   105  		attrs, err = c.Bucket(bucketName).Object(attrs.Name).Attrs(ctx)
   106  		if err != nil {
   107  			t.Fatal(err)
   108  		}
   109  		t.Logf("Attrs %v", attrs)
   110  	}
   111  	// validate the md5 hash
   112  	gotMD5HashStr2 := fmt.Sprintf("%x", attrs.MD5)
   113  	t.Logf("Got MD5 hash %q for Objects().Next() function", gotMD5HashStr2)
   114  	if wantMD5HashStr != gotMD5HashStr2 {
   115  		t.Fatalf("fake-gcs-server returned incorrect MD5 hash value for Objects().Next() function")
   116  	}
   117  	t.Logf("Got correct MD5 hash values from fake-gcs-server")
   118  }
   119  
   120  func TestStartFakeGcsServerWithFakeStorageObjects(t *testing.T) {
   121  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   122  	defer cancel()
   123  
   124  	const (
   125  		bucketName = "chariot-unit-testing"
   126  		objectName = "path/to/hello-world.txt"
   127  	)
   128  	var obj = fakestorage.Object{
   129  		ObjectAttrs: fakestorage.ObjectAttrs{
   130  			BucketName: bucketName,
   131  			Name:       objectName,
   132  		},
   133  		Content: []byte("hello world"),
   134  	}
   135  
   136  	c, stop, err := startFakeGcsServer(obj)
   137  	if err != nil {
   138  		t.Fatalf("Got error starting fake gcs server: %v", err)
   139  	}
   140  	defer stop()
   141  
   142  	var gotBytes []byte
   143  	r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   144  	if err != nil {
   145  		t.Fatal(err)
   146  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   147  		t.Fatal(err)
   148  	}
   149  
   150  	if string(gotBytes) != string(obj.Content) {
   151  		t.Logf("Want bytes: %q", obj.Content)
   152  		t.Logf("Got bytes: %q", gotBytes)
   153  		t.Fatalf("Downloaded content does not equal input content.")
   154  	}
   155  	t.Logf("Success, bytes downloaded equal bytes input.")
   156  }
   157  
   158  func TestFakeGcsServerWriterOverwritingFilesWorks(t *testing.T) {
   159  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   160  	defer cancel()
   161  
   162  	const (
   163  		bucketName = "chariot-unit-testing"
   164  		objectName = "path/to/hello-world.txt"
   165  	)
   166  	var obj = fakestorage.Object{
   167  		ObjectAttrs: fakestorage.ObjectAttrs{
   168  			BucketName: bucketName,
   169  			Name:       objectName,
   170  		},
   171  		Content: []byte("hello world"),
   172  	}
   173  
   174  	c, stop, err := startFakeGcsServer(obj)
   175  	if err != nil {
   176  		t.Fatalf("Got error starting fake gcs server: %v", err)
   177  	}
   178  	defer stop()
   179  
   180  	var gotBytes []byte
   181  	r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   182  	if err != nil {
   183  		t.Fatal(err)
   184  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   185  		t.Fatal(err)
   186  	} else if err = r.Close(); err != nil {
   187  		t.Fatal(err)
   188  	}
   189  
   190  	if string(gotBytes) != string(obj.Content) {
   191  		t.Logf("Want bytes: %q", obj.Content)
   192  		t.Logf("Got bytes: %q", gotBytes)
   193  		t.Fatalf("Downloaded content does not equal input content.")
   194  	}
   195  
   196  	// Now test overwriting the file
   197  	const hnbc = "how now brown cow"
   198  	sr := strings.NewReader(hnbc)
   199  	w := c.Bucket(bucketName).Object(objectName).NewWriter(ctx)
   200  	if n, err := io.Copy(w, sr); err != nil {
   201  		t.Fatal(err)
   202  	} else if n != int64(len(hnbc)) {
   203  		t.Fatalf("Did not write len(%q)=%d bytes. Wrote %d bytes.", hnbc, len(hnbc), n)
   204  	} else if err = w.Close(); err != nil {
   205  		t.Fatal(err)
   206  	}
   207  
   208  	r, err = c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   209  	if err != nil {
   210  		t.Fatal(err)
   211  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   212  		t.Fatal(err)
   213  	}
   214  
   215  	if string(gotBytes) != hnbc {
   216  		t.Logf("Want bytes: %q", hnbc)
   217  		t.Logf("Got bytes: %q", gotBytes)
   218  		t.Fatalf("Downloaded content does not equal input content.")
   219  	}
   220  	t.Logf("Successfully overwrote file")
   221  }
   222  
   223  func TestFakeGcsServerDeleteFilesWorks(t *testing.T) {
   224  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   225  	defer cancel()
   226  
   227  	const (
   228  		bucketName = "chariot-unit-testing"
   229  		objectName = "path/to/hello-world.txt"
   230  	)
   231  	var obj = fakestorage.Object{
   232  		ObjectAttrs: fakestorage.ObjectAttrs{
   233  			BucketName: bucketName,
   234  			Name:       objectName,
   235  		},
   236  		Content: []byte("hello world"),
   237  	}
   238  
   239  	c, stop, err := startFakeGcsServer(obj)
   240  	if err != nil {
   241  		t.Fatalf("Got error starting fake gcs server: %v", err)
   242  	}
   243  	defer stop()
   244  
   245  	var gotBytes []byte
   246  	r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   247  	if err != nil {
   248  		t.Fatal(err)
   249  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   250  		t.Fatal(err)
   251  	} else if err = r.Close(); err != nil {
   252  		t.Fatal(err)
   253  	}
   254  
   255  	if string(gotBytes) != string(obj.Content) {
   256  		t.Logf("Want bytes: %q", obj.Content)
   257  		t.Logf("Got bytes: %q", gotBytes)
   258  		t.Fatalf("Downloaded content does not equal input content.")
   259  	}
   260  
   261  	// Now test Deleting the file
   262  	if err = c.Bucket(bucketName).Object(objectName).Delete(ctx); err != nil {
   263  		t.Logf("Deletion failed!")
   264  		t.Fatal(err)
   265  	}
   266  
   267  	_, err = c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   268  	if err == nil {
   269  		t.Fatalf("Did not delete object")
   270  	}
   271  
   272  	t.Logf("Successfully deleted file. NewReader gave error: %q", err.Error())
   273  }
   274  
   275  func TestFakeGcsServerWriterWritingFilesWorks(t *testing.T) {
   276  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   277  	defer cancel()
   278  
   279  	const (
   280  		bucketName = "chariot-unit-testing"
   281  		objectName = "path/to/hello-world.txt"
   282  	)
   283  	var obj = fakestorage.Object{
   284  		ObjectAttrs: fakestorage.ObjectAttrs{
   285  			BucketName: bucketName,
   286  			Name:       objectName,
   287  		},
   288  		Content: []byte("hello world"),
   289  	}
   290  
   291  	c, stop, err := startFakeGcsServer(obj)
   292  	if err != nil {
   293  		t.Fatalf("Got error starting fake gcs server: %v", err)
   294  	}
   295  	defer stop()
   296  
   297  	var gotBytes []byte
   298  	r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx)
   299  	if err != nil {
   300  		t.Fatal(err)
   301  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   302  		t.Fatal(err)
   303  	} else if err = r.Close(); err != nil {
   304  		t.Fatal(err)
   305  	}
   306  
   307  	if string(gotBytes) != string(obj.Content) {
   308  		t.Logf("Want bytes: %q", obj.Content)
   309  		t.Logf("Got bytes: %q", gotBytes)
   310  		t.Fatalf("Downloaded content does not equal input content.")
   311  	}
   312  
   313  	// Now test creating the new file
   314  	const hnbc = "how now brown cow"
   315  	newObjectName := "fake/path/prefix/" + objectName
   316  	sr := strings.NewReader(hnbc)
   317  	w := c.Bucket(bucketName).Object(newObjectName).NewWriter(ctx)
   318  	if n, err := io.Copy(w, sr); err != nil {
   319  		t.Fatal(err)
   320  	} else if n != int64(len(hnbc)) {
   321  		t.Fatalf("Did not write len(%q)=%d bytes. Wrote %d bytes.", hnbc, len(hnbc), n)
   322  	} else if err = w.Close(); err != nil {
   323  		t.Fatal(err)
   324  	}
   325  
   326  	r, err = c.Bucket(bucketName).Object(newObjectName).NewReader(ctx)
   327  	if err != nil {
   328  		t.Fatal(err)
   329  	} else if gotBytes, err = io.ReadAll(r); err != nil {
   330  		t.Fatal(err)
   331  	}
   332  
   333  	if string(gotBytes) != hnbc {
   334  		t.Logf("Want bytes: %q", hnbc)
   335  		t.Logf("Got bytes: %q", gotBytes)
   336  		t.Fatalf("Downloaded content does not equal input content.")
   337  	}
   338  	t.Logf("Successfully created new file object")
   339  }
   340
View as plain text