package chariot import ( "bytes" "context" "crypto/md5" //nolint:gosec // md5 hashes are provided by GoogleCloudStorage "fmt" "io" "strings" "testing" "time" "cloud.google.com/go/storage" "github.com/fsouza/fake-gcs-server/fakestorage" "google.golang.org/api/iterator" ) func startFakeGcsServer(objs ...fakestorage.Object) (client *storage.Client, stop func(), err error) { var server *fakestorage.Server if server, err = fakestorage.NewServerWithOptions(fakestorage.Options{ // Note: Buckets are created for InitialObjects[].ObjectAttrs.BucketName values. InitialObjects: objs, }); err != nil { return nil, nil, err } client = server.Client() stop = server.Stop // The following is a stupid fix to force fake-gcs-server to calculate MD5 hashes. // If you want to contribute to open source, you can get fake-gcs-server to do this // when you provide InitialObjects. Right now, if you provide InitialObjects with the // MD5 hash manually set, it still returns an empty string for the hash, or an incorrect // MD5 hash value that isn't even 16 bytes long. LOLWUT. So to remedy this flaw, I write // the objects again once the fake-gcs-server has been created. I can't believe I have to // do this, but here we are. I hope this comment saves you/me some time in the future. for _, obj := range objs { bname := obj.ObjectAttrs.BucketName oname := obj.ObjectAttrs.Name w := client.Bucket(bname).Object(oname).NewWriter(context.Background()) if _, err = io.Copy(w, bytes.NewReader(obj.Content)); err != nil { stop() break } else if err = w.Close(); err != nil { stop() break } } return client, stop, err } func TestFakeGcsServerSetsMD5InObjectAttrs(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() const ( contentStr = "content of the file\nto check for md5 correctness" bucketName = "chariot-unit-testing" objectName = "path/to/hello-world.txt" ) var rawMD5Hash = md5.Sum([]byte(contentStr)) //nolint:gosec // md5 hashes are provided by GoogleCloudStorage var wantMD5HashStr = fmt.Sprintf("%x", rawMD5Hash[:]) t.Logf("Want MD5 hash: %q", wantMD5HashStr) var obj = fakestorage.Object{ ObjectAttrs: fakestorage.ObjectAttrs{ BucketName: bucketName, Name: objectName, }, Content: []byte(contentStr), } c, stop, err := startFakeGcsServer(obj) if err != nil { t.Fatalf("Got error starting fake gcs server: %v", err) } defer stop() // Check the object attrs MD5 using the Object.Attrs() function (sans 's') attrs, err := c.Bucket(bucketName).Object(objectName).Attrs(ctx) if err != nil { t.Fatal(err) } else if len(attrs.MD5) == 0 { t.Fatalf("Object(%q).Attrs(ctx) returned empty MD5 value", objectName) } // validate the md5 hash gotMD5HashStr1 := fmt.Sprintf("%x", attrs.MD5) t.Logf("Got MD5 hash %q for Object().Attrs() function", gotMD5HashStr1) if wantMD5HashStr != gotMD5HashStr1 { t.Fatalf("fake-gcs-server returned incorrect MD5 hash value for Object().Attrs() function") } // Check the object attrs MD5 using the Objects() function iterator. attrs, err = c.Bucket(bucketName).Objects(ctx, nil).Next() if err == iterator.Done { t.Fatalf("Got iterator.Done from Objects call: %v", err) } else if err != nil { t.Fatal(err) } else if attrs.Name != objectName { t.Fatalf("Object name not correct. want %q got %q", objectName, attrs.Name) } else if len(attrs.MD5) == 0 { t.Fail() t.Logf("Ugh why fake-gcs-server why! Got nil MD5 hash.") t.Logf("Trying .Object(%q).Attrs(ctx) now", attrs.Name) attrs, err = c.Bucket(bucketName).Object(attrs.Name).Attrs(ctx) if err != nil { t.Fatal(err) } t.Logf("Attrs %v", attrs) } // validate the md5 hash gotMD5HashStr2 := fmt.Sprintf("%x", attrs.MD5) t.Logf("Got MD5 hash %q for Objects().Next() function", gotMD5HashStr2) if wantMD5HashStr != gotMD5HashStr2 { t.Fatalf("fake-gcs-server returned incorrect MD5 hash value for Objects().Next() function") } t.Logf("Got correct MD5 hash values from fake-gcs-server") } func TestStartFakeGcsServerWithFakeStorageObjects(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() const ( bucketName = "chariot-unit-testing" objectName = "path/to/hello-world.txt" ) var obj = fakestorage.Object{ ObjectAttrs: fakestorage.ObjectAttrs{ BucketName: bucketName, Name: objectName, }, Content: []byte("hello world"), } c, stop, err := startFakeGcsServer(obj) if err != nil { t.Fatalf("Got error starting fake gcs server: %v", err) } defer stop() var gotBytes []byte r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } if string(gotBytes) != string(obj.Content) { t.Logf("Want bytes: %q", obj.Content) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } t.Logf("Success, bytes downloaded equal bytes input.") } func TestFakeGcsServerWriterOverwritingFilesWorks(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() const ( bucketName = "chariot-unit-testing" objectName = "path/to/hello-world.txt" ) var obj = fakestorage.Object{ ObjectAttrs: fakestorage.ObjectAttrs{ BucketName: bucketName, Name: objectName, }, Content: []byte("hello world"), } c, stop, err := startFakeGcsServer(obj) if err != nil { t.Fatalf("Got error starting fake gcs server: %v", err) } defer stop() var gotBytes []byte r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } else if err = r.Close(); err != nil { t.Fatal(err) } if string(gotBytes) != string(obj.Content) { t.Logf("Want bytes: %q", obj.Content) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } // Now test overwriting the file const hnbc = "how now brown cow" sr := strings.NewReader(hnbc) w := c.Bucket(bucketName).Object(objectName).NewWriter(ctx) if n, err := io.Copy(w, sr); err != nil { t.Fatal(err) } else if n != int64(len(hnbc)) { t.Fatalf("Did not write len(%q)=%d bytes. Wrote %d bytes.", hnbc, len(hnbc), n) } else if err = w.Close(); err != nil { t.Fatal(err) } r, err = c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } if string(gotBytes) != hnbc { t.Logf("Want bytes: %q", hnbc) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } t.Logf("Successfully overwrote file") } func TestFakeGcsServerDeleteFilesWorks(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() const ( bucketName = "chariot-unit-testing" objectName = "path/to/hello-world.txt" ) var obj = fakestorage.Object{ ObjectAttrs: fakestorage.ObjectAttrs{ BucketName: bucketName, Name: objectName, }, Content: []byte("hello world"), } c, stop, err := startFakeGcsServer(obj) if err != nil { t.Fatalf("Got error starting fake gcs server: %v", err) } defer stop() var gotBytes []byte r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } else if err = r.Close(); err != nil { t.Fatal(err) } if string(gotBytes) != string(obj.Content) { t.Logf("Want bytes: %q", obj.Content) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } // Now test Deleting the file if err = c.Bucket(bucketName).Object(objectName).Delete(ctx); err != nil { t.Logf("Deletion failed!") t.Fatal(err) } _, err = c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err == nil { t.Fatalf("Did not delete object") } t.Logf("Successfully deleted file. NewReader gave error: %q", err.Error()) } func TestFakeGcsServerWriterWritingFilesWorks(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() const ( bucketName = "chariot-unit-testing" objectName = "path/to/hello-world.txt" ) var obj = fakestorage.Object{ ObjectAttrs: fakestorage.ObjectAttrs{ BucketName: bucketName, Name: objectName, }, Content: []byte("hello world"), } c, stop, err := startFakeGcsServer(obj) if err != nil { t.Fatalf("Got error starting fake gcs server: %v", err) } defer stop() var gotBytes []byte r, err := c.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } else if err = r.Close(); err != nil { t.Fatal(err) } if string(gotBytes) != string(obj.Content) { t.Logf("Want bytes: %q", obj.Content) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } // Now test creating the new file const hnbc = "how now brown cow" newObjectName := "fake/path/prefix/" + objectName sr := strings.NewReader(hnbc) w := c.Bucket(bucketName).Object(newObjectName).NewWriter(ctx) if n, err := io.Copy(w, sr); err != nil { t.Fatal(err) } else if n != int64(len(hnbc)) { t.Fatalf("Did not write len(%q)=%d bytes. Wrote %d bytes.", hnbc, len(hnbc), n) } else if err = w.Close(); err != nil { t.Fatal(err) } r, err = c.Bucket(bucketName).Object(newObjectName).NewReader(ctx) if err != nil { t.Fatal(err) } else if gotBytes, err = io.ReadAll(r); err != nil { t.Fatal(err) } if string(gotBytes) != hnbc { t.Logf("Want bytes: %q", hnbc) t.Logf("Got bytes: %q", gotBytes) t.Fatalf("Downloaded content does not equal input content.") } t.Logf("Successfully created new file object") }