blob_test.go

Documentation: github.com/docker/distribution/registry/storage

     1  package storage
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"crypto/sha256"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"path"
    11  	"reflect"
    12  	"testing"
    13  
    14  	"github.com/distribution/reference"
    15  	"github.com/docker/distribution"
    16  	"github.com/docker/distribution/registry/storage/cache/memory"
    17  	"github.com/docker/distribution/registry/storage/driver/testdriver"
    18  	"github.com/docker/distribution/testutil"
    19  	"github.com/opencontainers/go-digest"
    20  )
    21  
    22  // TestWriteSeek tests that the current file size can be
    23  // obtained using Seek
    24  func TestWriteSeek(t *testing.T) {
    25  	ctx := context.Background()
    26  	imageName, _ := reference.WithName("foo/bar")
    27  	driver := testdriver.New()
    28  	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
    29  	if err != nil {
    30  		t.Fatalf("error creating registry: %v", err)
    31  	}
    32  	repository, err := registry.Repository(ctx, imageName)
    33  	if err != nil {
    34  		t.Fatalf("unexpected error getting repo: %v", err)
    35  	}
    36  	bs := repository.Blobs(ctx)
    37  
    38  	blobUpload, err := bs.Create(ctx)
    39  
    40  	if err != nil {
    41  		t.Fatalf("unexpected error starting layer upload: %s", err)
    42  	}
    43  	contents := []byte{1, 2, 3}
    44  	blobUpload.Write(contents)
    45  	blobUpload.Close()
    46  	offset := blobUpload.Size()
    47  	if offset != int64(len(contents)) {
    48  		t.Fatalf("unexpected value for blobUpload offset:  %v != %v", offset, len(contents))
    49  	}
    50  
    51  }
    52  
    53  // TestSimpleBlobUpload covers the blob upload process, exercising common
    54  // error paths that might be seen during an upload.
    55  func TestSimpleBlobUpload(t *testing.T) {
    56  	randomDataReader, dgst, err := testutil.CreateRandomTarFile()
    57  	if err != nil {
    58  		t.Fatalf("error creating random reader: %v", err)
    59  	}
    60  
    61  	ctx := context.Background()
    62  	imageName, _ := reference.WithName("foo/bar")
    63  	driver := testdriver.New()
    64  	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
    65  	if err != nil {
    66  		t.Fatalf("error creating registry: %v", err)
    67  	}
    68  	repository, err := registry.Repository(ctx, imageName)
    69  	if err != nil {
    70  		t.Fatalf("unexpected error getting repo: %v", err)
    71  	}
    72  	bs := repository.Blobs(ctx)
    73  
    74  	h := sha256.New()
    75  	rd := io.TeeReader(randomDataReader, h)
    76  
    77  	blobUpload, err := bs.Create(ctx)
    78  
    79  	if err != nil {
    80  		t.Fatalf("unexpected error starting layer upload: %s", err)
    81  	}
    82  
    83  	// Cancel the upload then restart it
    84  	if err := blobUpload.Cancel(ctx); err != nil {
    85  		t.Fatalf("unexpected error during upload cancellation: %v", err)
    86  	}
    87  
    88  	// get the enclosing directory
    89  	uploadPath := path.Dir(blobUpload.(*blobWriter).path)
    90  
    91  	// ensure state was cleaned up
    92  	_, err = driver.List(ctx, uploadPath)
    93  	if err == nil {
    94  		t.Fatal("files in upload path after cleanup")
    95  	}
    96  
    97  	// Do a resume, get unknown upload
    98  	_, err = bs.Resume(ctx, blobUpload.ID())
    99  	if err != distribution.ErrBlobUploadUnknown {
   100  		t.Fatalf("unexpected error resuming upload, should be unknown: %v", err)
   101  	}
   102  
   103  	// Restart!
   104  	blobUpload, err = bs.Create(ctx)
   105  	if err != nil {
   106  		t.Fatalf("unexpected error starting layer upload: %s", err)
   107  	}
   108  
   109  	// Get the size of our random tarfile
   110  	randomDataSize, err := seekerSize(randomDataReader)
   111  	if err != nil {
   112  		t.Fatalf("error getting seeker size of random data: %v", err)
   113  	}
   114  
   115  	nn, err := io.Copy(blobUpload, rd)
   116  	if err != nil {
   117  		t.Fatalf("unexpected error uploading layer data: %v", err)
   118  	}
   119  
   120  	if nn != randomDataSize {
   121  		t.Fatalf("layer data write incomplete")
   122  	}
   123  
   124  	blobUpload.Close()
   125  
   126  	offset := blobUpload.Size()
   127  	if offset != nn {
   128  		t.Fatalf("blobUpload not updated with correct offset: %v != %v", offset, nn)
   129  	}
   130  
   131  	// Do a resume, for good fun
   132  	blobUpload, err = bs.Resume(ctx, blobUpload.ID())
   133  	if err != nil {
   134  		t.Fatalf("unexpected error resuming upload: %v", err)
   135  	}
   136  
   137  	sha256Digest := digest.NewDigest("sha256", h)
   138  	desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst})
   139  	if err != nil {
   140  		t.Fatalf("unexpected error finishing layer upload: %v", err)
   141  	}
   142  
   143  	// ensure state was cleaned up
   144  	uploadPath = path.Dir(blobUpload.(*blobWriter).path)
   145  	_, err = driver.List(ctx, uploadPath)
   146  	if err == nil {
   147  		t.Fatal("files in upload path after commit")
   148  	}
   149  
   150  	// After finishing an upload, it should no longer exist.
   151  	if _, err := bs.Resume(ctx, blobUpload.ID()); err != distribution.ErrBlobUploadUnknown {
   152  		t.Fatalf("expected layer upload to be unknown, got %v", err)
   153  	}
   154  
   155  	// Test for existence.
   156  	statDesc, err := bs.Stat(ctx, desc.Digest)
   157  	if err != nil {
   158  		t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
   159  	}
   160  
   161  	if !reflect.DeepEqual(statDesc, desc) {
   162  		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
   163  	}
   164  
   165  	rc, err := bs.Open(ctx, desc.Digest)
   166  	if err != nil {
   167  		t.Fatalf("unexpected error opening blob for read: %v", err)
   168  	}
   169  	defer rc.Close()
   170  
   171  	h.Reset()
   172  	nn, err = io.Copy(h, rc)
   173  	if err != nil {
   174  		t.Fatalf("error reading layer: %v", err)
   175  	}
   176  
   177  	if nn != randomDataSize {
   178  		t.Fatalf("incorrect read length")
   179  	}
   180  
   181  	if digest.NewDigest("sha256", h) != sha256Digest {
   182  		t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest)
   183  	}
   184  
   185  	// Delete a blob
   186  	err = bs.Delete(ctx, desc.Digest)
   187  	if err != nil {
   188  		t.Fatalf("Unexpected error deleting blob")
   189  	}
   190  
   191  	d, err := bs.Stat(ctx, desc.Digest)
   192  	if err == nil {
   193  		t.Fatalf("unexpected non-error stating deleted blob: %v", d)
   194  	}
   195  
   196  	switch err {
   197  	case distribution.ErrBlobUnknown:
   198  		break
   199  	default:
   200  		t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
   201  	}
   202  
   203  	_, err = bs.Open(ctx, desc.Digest)
   204  	if err == nil {
   205  		t.Fatalf("unexpected success opening deleted blob for read")
   206  	}
   207  
   208  	switch err {
   209  	case distribution.ErrBlobUnknown:
   210  		break
   211  	default:
   212  		t.Errorf("Unexpected error type getting deleted manifest: %#v", err)
   213  	}
   214  
   215  	// Re-upload the blob
   216  	randomBlob, err := ioutil.ReadAll(randomDataReader)
   217  	if err != nil {
   218  		t.Fatalf("Error reading all of blob %s", err.Error())
   219  	}
   220  	expectedDigest := digest.FromBytes(randomBlob)
   221  	simpleUpload(t, bs, randomBlob, expectedDigest)
   222  
   223  	d, err = bs.Stat(ctx, expectedDigest)
   224  	if err != nil {
   225  		t.Errorf("unexpected error stat-ing blob")
   226  	}
   227  	if d.Digest != expectedDigest {
   228  		t.Errorf("Mismatching digest with restored blob")
   229  	}
   230  
   231  	_, err = bs.Open(ctx, expectedDigest)
   232  	if err != nil {
   233  		t.Errorf("Unexpected error opening blob")
   234  	}
   235  
   236  	// Reuse state to test delete with a delete-disabled registry
   237  	registry, err = NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
   238  	if err != nil {
   239  		t.Fatalf("error creating registry: %v", err)
   240  	}
   241  	repository, err = registry.Repository(ctx, imageName)
   242  	if err != nil {
   243  		t.Fatalf("unexpected error getting repo: %v", err)
   244  	}
   245  	bs = repository.Blobs(ctx)
   246  	err = bs.Delete(ctx, desc.Digest)
   247  	if err == nil {
   248  		t.Errorf("Unexpected success deleting while disabled")
   249  	}
   250  }
   251  
   252  // TestSimpleBlobRead just creates a simple blob file and ensures that basic
   253  // open, read, seek, read works. More specific edge cases should be covered in
   254  // other tests.
   255  func TestSimpleBlobRead(t *testing.T) {
   256  	ctx := context.Background()
   257  	imageName, _ := reference.WithName("foo/bar")
   258  	driver := testdriver.New()
   259  	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
   260  	if err != nil {
   261  		t.Fatalf("error creating registry: %v", err)
   262  	}
   263  	repository, err := registry.Repository(ctx, imageName)
   264  	if err != nil {
   265  		t.Fatalf("unexpected error getting repo: %v", err)
   266  	}
   267  	bs := repository.Blobs(ctx)
   268  
   269  	randomLayerReader, dgst, err := testutil.CreateRandomTarFile() // TODO(stevvooe): Consider using just a random string.
   270  	if err != nil {
   271  		t.Fatalf("error creating random data: %v", err)
   272  	}
   273  
   274  	// Test for existence.
   275  	desc, err := bs.Stat(ctx, dgst)
   276  	if err != distribution.ErrBlobUnknown {
   277  		t.Fatalf("expected not found error when testing for existence: %v", err)
   278  	}
   279  
   280  	_, err = bs.Open(ctx, dgst)
   281  	if err != distribution.ErrBlobUnknown {
   282  		t.Fatalf("expected not found error when opening non-existent blob: %v", err)
   283  	}
   284  
   285  	randomLayerSize, err := seekerSize(randomLayerReader)
   286  	if err != nil {
   287  		t.Fatalf("error getting seeker size for random layer: %v", err)
   288  	}
   289  
   290  	descBefore := distribution.Descriptor{Digest: dgst, MediaType: "application/octet-stream", Size: randomLayerSize}
   291  	t.Logf("desc: %v", descBefore)
   292  
   293  	desc, err = addBlob(ctx, bs, descBefore, randomLayerReader)
   294  	if err != nil {
   295  		t.Fatalf("error adding blob to blobservice: %v", err)
   296  	}
   297  
   298  	if desc.Size != randomLayerSize {
   299  		t.Fatalf("committed blob has incorrect length: %v != %v", desc.Size, randomLayerSize)
   300  	}
   301  
   302  	rc, err := bs.Open(ctx, desc.Digest) // note that we are opening with original digest.
   303  	if err != nil {
   304  		t.Fatalf("error opening blob with %v: %v", dgst, err)
   305  	}
   306  	defer rc.Close()
   307  
   308  	// Now check the sha digest and ensure its the same
   309  	h := sha256.New()
   310  	nn, err := io.Copy(h, rc)
   311  	if err != nil {
   312  		t.Fatalf("unexpected error copying to hash: %v", err)
   313  	}
   314  
   315  	if nn != randomLayerSize {
   316  		t.Fatalf("stored incorrect number of bytes in blob: %d != %d", nn, randomLayerSize)
   317  	}
   318  
   319  	sha256Digest := digest.NewDigest("sha256", h)
   320  	if sha256Digest != desc.Digest {
   321  		t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, desc.Digest)
   322  	}
   323  
   324  	// Now seek back the blob, read the whole thing and check against randomLayerData
   325  	offset, err := rc.Seek(0, io.SeekStart)
   326  	if err != nil {
   327  		t.Fatalf("error seeking blob: %v", err)
   328  	}
   329  
   330  	if offset != 0 {
   331  		t.Fatalf("seek failed: expected 0 offset, got %d", offset)
   332  	}
   333  
   334  	p, err := ioutil.ReadAll(rc)
   335  	if err != nil {
   336  		t.Fatalf("error reading all of blob: %v", err)
   337  	}
   338  
   339  	if len(p) != int(randomLayerSize) {
   340  		t.Fatalf("blob data read has different length: %v != %v", len(p), randomLayerSize)
   341  	}
   342  
   343  	// Reset the randomLayerReader and read back the buffer
   344  	_, err = randomLayerReader.Seek(0, io.SeekStart)
   345  	if err != nil {
   346  		t.Fatalf("error resetting layer reader: %v", err)
   347  	}
   348  
   349  	randomLayerData, err := ioutil.ReadAll(randomLayerReader)
   350  	if err != nil {
   351  		t.Fatalf("random layer read failed: %v", err)
   352  	}
   353  
   354  	if !bytes.Equal(p, randomLayerData) {
   355  		t.Fatalf("layer data not equal")
   356  	}
   357  }
   358  
   359  // TestBlobMount covers the blob mount process, exercising common
   360  // error paths that might be seen during a mount.
   361  func TestBlobMount(t *testing.T) {
   362  	randomDataReader, dgst, err := testutil.CreateRandomTarFile()
   363  	if err != nil {
   364  		t.Fatalf("error creating random reader: %v", err)
   365  	}
   366  
   367  	ctx := context.Background()
   368  	imageName, _ := reference.WithName("foo/bar")
   369  	sourceImageName, _ := reference.WithName("foo/source")
   370  	driver := testdriver.New()
   371  	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
   372  	if err != nil {
   373  		t.Fatalf("error creating registry: %v", err)
   374  	}
   375  
   376  	repository, err := registry.Repository(ctx, imageName)
   377  	if err != nil {
   378  		t.Fatalf("unexpected error getting repo: %v", err)
   379  	}
   380  	sourceRepository, err := registry.Repository(ctx, sourceImageName)
   381  	if err != nil {
   382  		t.Fatalf("unexpected error getting repo: %v", err)
   383  	}
   384  
   385  	sbs := sourceRepository.Blobs(ctx)
   386  
   387  	blobUpload, err := sbs.Create(ctx)
   388  
   389  	if err != nil {
   390  		t.Fatalf("unexpected error starting layer upload: %s", err)
   391  	}
   392  
   393  	// Get the size of our random tarfile
   394  	randomDataSize, err := seekerSize(randomDataReader)
   395  	if err != nil {
   396  		t.Fatalf("error getting seeker size of random data: %v", err)
   397  	}
   398  
   399  	_, err = io.Copy(blobUpload, randomDataReader)
   400  	if err != nil {
   401  		t.Fatalf("unexpected error uploading layer data: %v", err)
   402  	}
   403  
   404  	desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst})
   405  	if err != nil {
   406  		t.Fatalf("unexpected error finishing layer upload: %v", err)
   407  	}
   408  
   409  	// Test for existence.
   410  	statDesc, err := sbs.Stat(ctx, desc.Digest)
   411  	if err != nil {
   412  		t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs)
   413  	}
   414  
   415  	if !reflect.DeepEqual(statDesc, desc) {
   416  		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
   417  	}
   418  
   419  	bs := repository.Blobs(ctx)
   420  	// Test destination for existence.
   421  	_, err = bs.Stat(ctx, desc.Digest)
   422  	if err == nil {
   423  		t.Fatalf("unexpected non-error stating unmounted blob: %v", desc)
   424  	}
   425  
   426  	canonicalRef, err := reference.WithDigest(sourceRepository.Named(), desc.Digest)
   427  	if err != nil {
   428  		t.Fatal(err)
   429  	}
   430  
   431  	bw, err := bs.Create(ctx, WithMountFrom(canonicalRef))
   432  	if bw != nil {
   433  		t.Fatal("unexpected blobwriter returned from Create call, should mount instead")
   434  	}
   435  
   436  	ebm, ok := err.(distribution.ErrBlobMounted)
   437  	if !ok {
   438  		t.Fatalf("unexpected error mounting layer: %v", err)
   439  	}
   440  
   441  	if !reflect.DeepEqual(ebm.Descriptor, desc) {
   442  		t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc)
   443  	}
   444  
   445  	// Test for existence.
   446  	statDesc, err = bs.Stat(ctx, desc.Digest)
   447  	if err != nil {
   448  		t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
   449  	}
   450  
   451  	if !reflect.DeepEqual(statDesc, desc) {
   452  		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
   453  	}
   454  
   455  	rc, err := bs.Open(ctx, desc.Digest)
   456  	if err != nil {
   457  		t.Fatalf("unexpected error opening blob for read: %v", err)
   458  	}
   459  	defer rc.Close()
   460  
   461  	h := sha256.New()
   462  	nn, err := io.Copy(h, rc)
   463  	if err != nil {
   464  		t.Fatalf("error reading layer: %v", err)
   465  	}
   466  
   467  	if nn != randomDataSize {
   468  		t.Fatalf("incorrect read length")
   469  	}
   470  
   471  	if digest.NewDigest("sha256", h) != dgst {
   472  		t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), dgst)
   473  	}
   474  
   475  	// Delete the blob from the source repo
   476  	err = sbs.Delete(ctx, desc.Digest)
   477  	if err != nil {
   478  		t.Fatalf("Unexpected error deleting blob")
   479  	}
   480  
   481  	_, err = bs.Stat(ctx, desc.Digest)
   482  	if err != nil {
   483  		t.Fatalf("unexpected error stating blob deleted from source repository: %v", err)
   484  	}
   485  
   486  	d, err := sbs.Stat(ctx, desc.Digest)
   487  	if err == nil {
   488  		t.Fatalf("unexpected non-error stating deleted blob: %v", d)
   489  	}
   490  
   491  	switch err {
   492  	case distribution.ErrBlobUnknown:
   493  		break
   494  	default:
   495  		t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
   496  	}
   497  
   498  	// Delete the blob from the dest repo
   499  	err = bs.Delete(ctx, desc.Digest)
   500  	if err != nil {
   501  		t.Fatalf("Unexpected error deleting blob")
   502  	}
   503  
   504  	d, err = bs.Stat(ctx, desc.Digest)
   505  	if err == nil {
   506  		t.Fatalf("unexpected non-error stating deleted blob: %v", d)
   507  	}
   508  
   509  	switch err {
   510  	case distribution.ErrBlobUnknown:
   511  		break
   512  	default:
   513  		t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
   514  	}
   515  }
   516  
   517  // TestLayerUploadZeroLength uploads zero-length
   518  func TestLayerUploadZeroLength(t *testing.T) {
   519  	ctx := context.Background()
   520  	imageName, _ := reference.WithName("foo/bar")
   521  	driver := testdriver.New()
   522  	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
   523  	if err != nil {
   524  		t.Fatalf("error creating registry: %v", err)
   525  	}
   526  	repository, err := registry.Repository(ctx, imageName)
   527  	if err != nil {
   528  		t.Fatalf("unexpected error getting repo: %v", err)
   529  	}
   530  	bs := repository.Blobs(ctx)
   531  
   532  	simpleUpload(t, bs, []byte{}, digestSha256Empty)
   533  }
   534  
   535  func simpleUpload(t *testing.T, bs distribution.BlobIngester, blob []byte, expectedDigest digest.Digest) {
   536  	ctx := context.Background()
   537  	wr, err := bs.Create(ctx)
   538  	if err != nil {
   539  		t.Fatalf("unexpected error starting upload: %v", err)
   540  	}
   541  
   542  	nn, err := io.Copy(wr, bytes.NewReader(blob))
   543  	if err != nil {
   544  		t.Fatalf("error copying into blob writer: %v", err)
   545  	}
   546  
   547  	if nn != 0 {
   548  		t.Fatalf("unexpected number of bytes copied: %v > 0", nn)
   549  	}
   550  
   551  	dgst, err := digest.FromReader(bytes.NewReader(blob))
   552  	if err != nil {
   553  		t.Fatalf("error getting digest: %v", err)
   554  	}
   555  
   556  	if dgst != expectedDigest {
   557  		// sanity check on zero digest
   558  		t.Fatalf("digest not as expected: %v != %v", dgst, expectedDigest)
   559  	}
   560  
   561  	desc, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst})
   562  	if err != nil {
   563  		t.Fatalf("unexpected error committing write: %v", err)
   564  	}
   565  
   566  	if desc.Digest != dgst {
   567  		t.Fatalf("unexpected digest: %v != %v", desc.Digest, dgst)
   568  	}
   569  }
   570  
   571  // seekerSize seeks to the end of seeker, checks the size and returns it to
   572  // the original state, returning the size. The state of the seeker should be
   573  // treated as unknown if an error is returned.
   574  func seekerSize(seeker io.ReadSeeker) (int64, error) {
   575  	current, err := seeker.Seek(0, io.SeekCurrent)
   576  	if err != nil {
   577  		return 0, err
   578  	}
   579  
   580  	end, err := seeker.Seek(0, io.SeekEnd)
   581  	if err != nil {
   582  		return 0, err
   583  	}
   584  
   585  	resumed, err := seeker.Seek(current, io.SeekStart)
   586  	if err != nil {
   587  		return 0, err
   588  	}
   589  
   590  	if resumed != current {
   591  		return 0, fmt.Errorf("error returning seeker to original state, could not seek back to original location")
   592  	}
   593  
   594  	return end, nil
   595  }
   596  
   597  // addBlob simply consumes the reader and inserts into the blob service,
   598  // returning a descriptor on success.
   599  func addBlob(ctx context.Context, bs distribution.BlobIngester, desc distribution.Descriptor, rd io.Reader) (distribution.Descriptor, error) {
   600  	wr, err := bs.Create(ctx)
   601  	if err != nil {
   602  		return distribution.Descriptor{}, err
   603  	}
   604  	defer wr.Cancel(ctx)
   605  
   606  	if nn, err := io.Copy(wr, rd); err != nil {
   607  		return distribution.Descriptor{}, err
   608  	} else if nn != desc.Size {
   609  		return distribution.Descriptor{}, fmt.Errorf("incorrect number of bytes copied: %v != %v", nn, desc.Size)
   610  	}
   611  
   612  	return wr.Commit(ctx, desc)
   613  }
   614
View as plain text