testsuites.go

Documentation: github.com/docker/distribution/registry/storage/driver/testsuites

     1  package testsuites
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	crand "crypto/rand"
     7  	"crypto/sha1"
     8  	"io"
     9  	"io/ioutil"
    10  	"math/rand"
    11  	"net/http"
    12  	"os"
    13  	"path"
    14  	"sort"
    15  	"strings"
    16  	"sync"
    17  	"testing"
    18  	"time"
    19  
    20  	storagedriver "github.com/docker/distribution/registry/storage/driver"
    21  	"gopkg.in/check.v1"
    22  )
    23  
    24  // Test hooks up gocheck into the "go test" runner.
    25  func Test(t *testing.T) { check.TestingT(t) }
    26  
    27  // RegisterSuite registers an in-process storage driver test suite with
    28  // the go test runner.
    29  func RegisterSuite(driverConstructor DriverConstructor, skipCheck SkipCheck) {
    30  	check.Suite(&DriverSuite{
    31  		Constructor: driverConstructor,
    32  		SkipCheck:   skipCheck,
    33  		ctx:         context.Background(),
    34  	})
    35  }
    36  
    37  // SkipCheck is a function used to determine if a test suite should be skipped.
    38  // If a SkipCheck returns a non-empty skip reason, the suite is skipped with
    39  // the given reason.
    40  type SkipCheck func() (reason string)
    41  
    42  // NeverSkip is a default SkipCheck which never skips the suite.
    43  var NeverSkip SkipCheck = func() string { return "" }
    44  
    45  // DriverConstructor is a function which returns a new
    46  // storagedriver.StorageDriver.
    47  type DriverConstructor func() (storagedriver.StorageDriver, error)
    48  
    49  // DriverTeardown is a function which cleans up a suite's
    50  // storagedriver.StorageDriver.
    51  type DriverTeardown func() error
    52  
    53  // DriverSuite is a gocheck test suite designed to test a
    54  // storagedriver.StorageDriver. The intended way to create a DriverSuite is
    55  // with RegisterSuite.
    56  type DriverSuite struct {
    57  	Constructor DriverConstructor
    58  	Teardown    DriverTeardown
    59  	SkipCheck
    60  	storagedriver.StorageDriver
    61  	ctx context.Context
    62  }
    63  
    64  // SetUpSuite sets up the gocheck test suite.
    65  func (suite *DriverSuite) SetUpSuite(c *check.C) {
    66  	if reason := suite.SkipCheck(); reason != "" {
    67  		c.Skip(reason)
    68  	}
    69  	d, err := suite.Constructor()
    70  	c.Assert(err, check.IsNil)
    71  	suite.StorageDriver = d
    72  }
    73  
    74  // TearDownSuite tears down the gocheck test suite.
    75  func (suite *DriverSuite) TearDownSuite(c *check.C) {
    76  	if suite.Teardown != nil {
    77  		err := suite.Teardown()
    78  		c.Assert(err, check.IsNil)
    79  	}
    80  }
    81  
    82  // TearDownTest tears down the gocheck test.
    83  // This causes the suite to abort if any files are left around in the storage
    84  // driver.
    85  func (suite *DriverSuite) TearDownTest(c *check.C) {
    86  	files, _ := suite.StorageDriver.List(suite.ctx, "/")
    87  	if len(files) > 0 {
    88  		c.Fatalf("Storage driver did not clean up properly. Offending files: %#v", files)
    89  	}
    90  }
    91  
    92  // TestRootExists ensures that all storage drivers have a root path by default.
    93  func (suite *DriverSuite) TestRootExists(c *check.C) {
    94  	_, err := suite.StorageDriver.List(suite.ctx, "/")
    95  	if err != nil {
    96  		c.Fatalf(`the root path "/" should always exist: %v`, err)
    97  	}
    98  }
    99  
   100  // TestValidPaths checks that various valid file paths are accepted by the
   101  // storage driver.
   102  func (suite *DriverSuite) TestValidPaths(c *check.C) {
   103  	contents := randomContents(64)
   104  	validFiles := []string{
   105  		"/a",
   106  		"/2",
   107  		"/aa",
   108  		"/a.a",
   109  		"/0-9/abcdefg",
   110  		"/abcdefg/z.75",
   111  		"/abc/1.2.3.4.5-6_zyx/123.z/4",
   112  		"/docker/docker-registry",
   113  		"/123.abc",
   114  		"/abc./abc",
   115  		"/.abc",
   116  		"/a--b",
   117  		"/a-.b",
   118  		"/_.abc",
   119  		"/Docker/docker-registry",
   120  		"/Abc/Cba"}
   121  
   122  	for _, filename := range validFiles {
   123  		err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   124  		defer suite.deletePath(c, firstPart(filename))
   125  		c.Assert(err, check.IsNil)
   126  
   127  		received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
   128  		c.Assert(err, check.IsNil)
   129  		c.Assert(received, check.DeepEquals, contents)
   130  	}
   131  }
   132  
   133  func (suite *DriverSuite) deletePath(c *check.C, path string) {
   134  	for tries := 2; tries > 0; tries-- {
   135  		err := suite.StorageDriver.Delete(suite.ctx, path)
   136  		if _, ok := err.(storagedriver.PathNotFoundError); ok {
   137  			err = nil
   138  		}
   139  		c.Assert(err, check.IsNil)
   140  		paths, _ := suite.StorageDriver.List(suite.ctx, path)
   141  		if len(paths) == 0 {
   142  			break
   143  		}
   144  		time.Sleep(time.Second * 2)
   145  	}
   146  }
   147  
   148  // TestInvalidPaths checks that various invalid file paths are rejected by the
   149  // storage driver.
   150  func (suite *DriverSuite) TestInvalidPaths(c *check.C) {
   151  	contents := randomContents(64)
   152  	invalidFiles := []string{
   153  		"",
   154  		"/",
   155  		"abc",
   156  		"123.abc",
   157  		"//bcd",
   158  		"/abc_123/"}
   159  
   160  	for _, filename := range invalidFiles {
   161  		err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   162  		// only delete if file was successfully written
   163  		if err == nil {
   164  			defer suite.deletePath(c, firstPart(filename))
   165  		}
   166  		c.Assert(err, check.NotNil)
   167  		c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
   168  		c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   169  
   170  		_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
   171  		c.Assert(err, check.NotNil)
   172  		c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
   173  		c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   174  	}
   175  }
   176  
   177  // TestWriteRead1 tests a simple write-read workflow.
   178  func (suite *DriverSuite) TestWriteRead1(c *check.C) {
   179  	filename := randomPath(32)
   180  	contents := []byte("a")
   181  	suite.writeReadCompare(c, filename, contents)
   182  }
   183  
   184  // TestWriteRead2 tests a simple write-read workflow with unicode data.
   185  func (suite *DriverSuite) TestWriteRead2(c *check.C) {
   186  	filename := randomPath(32)
   187  	contents := []byte("\xc3\x9f")
   188  	suite.writeReadCompare(c, filename, contents)
   189  }
   190  
   191  // TestWriteRead3 tests a simple write-read workflow with a small string.
   192  func (suite *DriverSuite) TestWriteRead3(c *check.C) {
   193  	filename := randomPath(32)
   194  	contents := randomContents(32)
   195  	suite.writeReadCompare(c, filename, contents)
   196  }
   197  
   198  // TestWriteRead4 tests a simple write-read workflow with 1MB of data.
   199  func (suite *DriverSuite) TestWriteRead4(c *check.C) {
   200  	filename := randomPath(32)
   201  	contents := randomContents(1024 * 1024)
   202  	suite.writeReadCompare(c, filename, contents)
   203  }
   204  
   205  // TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage
   206  // driver safely.
   207  func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) {
   208  	filename := randomPath(32)
   209  	contents := []byte{0x80, 0x80, 0x80, 0x80}
   210  	suite.writeReadCompare(c, filename, contents)
   211  }
   212  
   213  // TestTruncate tests that putting smaller contents than an original file does
   214  // remove the excess contents.
   215  func (suite *DriverSuite) TestTruncate(c *check.C) {
   216  	filename := randomPath(32)
   217  	contents := randomContents(1024 * 1024)
   218  	suite.writeReadCompare(c, filename, contents)
   219  
   220  	contents = randomContents(1024)
   221  	suite.writeReadCompare(c, filename, contents)
   222  }
   223  
   224  // TestReadNonexistent tests reading content from an empty path.
   225  func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
   226  	filename := randomPath(32)
   227  	_, err := suite.StorageDriver.GetContent(suite.ctx, filename)
   228  	c.Assert(err, check.NotNil)
   229  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   230  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   231  }
   232  
   233  // TestWriteReadStreams1 tests a simple write-read streaming workflow.
   234  func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
   235  	filename := randomPath(32)
   236  	contents := []byte("a")
   237  	suite.writeReadCompareStreams(c, filename, contents)
   238  }
   239  
   240  // TestWriteReadStreams2 tests a simple write-read streaming workflow with
   241  // unicode data.
   242  func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
   243  	filename := randomPath(32)
   244  	contents := []byte("\xc3\x9f")
   245  	suite.writeReadCompareStreams(c, filename, contents)
   246  }
   247  
   248  // TestWriteReadStreams3 tests a simple write-read streaming workflow with a
   249  // small amount of data.
   250  func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) {
   251  	filename := randomPath(32)
   252  	contents := randomContents(32)
   253  	suite.writeReadCompareStreams(c, filename, contents)
   254  }
   255  
   256  // TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB
   257  // of data.
   258  func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) {
   259  	filename := randomPath(32)
   260  	contents := randomContents(1024 * 1024)
   261  	suite.writeReadCompareStreams(c, filename, contents)
   262  }
   263  
   264  // TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the
   265  // storage driver safely.
   266  func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) {
   267  	filename := randomPath(32)
   268  	contents := []byte{0x80, 0x80, 0x80, 0x80}
   269  	suite.writeReadCompareStreams(c, filename, contents)
   270  }
   271  
   272  // TestWriteReadLargeStreams tests that a 5GB file may be written to the storage
   273  // driver safely.
   274  func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) {
   275  	if testing.Short() {
   276  		c.Skip("Skipping test in short mode")
   277  	}
   278  
   279  	filename := randomPath(32)
   280  	defer suite.deletePath(c, firstPart(filename))
   281  
   282  	checksum := sha1.New()
   283  	var fileSize int64 = 5 * 1024 * 1024 * 1024
   284  
   285  	contents := newRandReader(fileSize)
   286  
   287  	writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
   288  	c.Assert(err, check.IsNil)
   289  	written, err := io.Copy(writer, io.TeeReader(contents, checksum))
   290  	c.Assert(err, check.IsNil)
   291  	c.Assert(written, check.Equals, fileSize)
   292  
   293  	err = writer.Commit()
   294  	c.Assert(err, check.IsNil)
   295  	err = writer.Close()
   296  	c.Assert(err, check.IsNil)
   297  
   298  	reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
   299  	c.Assert(err, check.IsNil)
   300  	defer reader.Close()
   301  
   302  	writtenChecksum := sha1.New()
   303  	io.Copy(writtenChecksum, reader)
   304  
   305  	c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil))
   306  }
   307  
   308  // TestReaderWithOffset tests that the appropriate data is streamed when
   309  // reading with a given offset.
   310  func (suite *DriverSuite) TestReaderWithOffset(c *check.C) {
   311  	filename := randomPath(32)
   312  	defer suite.deletePath(c, firstPart(filename))
   313  
   314  	chunkSize := int64(32)
   315  
   316  	contentsChunk1 := randomContents(chunkSize)
   317  	contentsChunk2 := randomContents(chunkSize)
   318  	contentsChunk3 := randomContents(chunkSize)
   319  
   320  	err := suite.StorageDriver.PutContent(suite.ctx, filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
   321  	c.Assert(err, check.IsNil)
   322  
   323  	reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
   324  	c.Assert(err, check.IsNil)
   325  	defer reader.Close()
   326  
   327  	readContents, err := ioutil.ReadAll(reader)
   328  	c.Assert(err, check.IsNil)
   329  
   330  	c.Assert(readContents, check.DeepEquals, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
   331  
   332  	reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize)
   333  	c.Assert(err, check.IsNil)
   334  	defer reader.Close()
   335  
   336  	readContents, err = ioutil.ReadAll(reader)
   337  	c.Assert(err, check.IsNil)
   338  
   339  	c.Assert(readContents, check.DeepEquals, append(contentsChunk2, contentsChunk3...))
   340  
   341  	reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*2)
   342  	c.Assert(err, check.IsNil)
   343  	defer reader.Close()
   344  
   345  	readContents, err = ioutil.ReadAll(reader)
   346  	c.Assert(err, check.IsNil)
   347  	c.Assert(readContents, check.DeepEquals, contentsChunk3)
   348  
   349  	// Ensure we get invalid offest for negative offsets.
   350  	reader, err = suite.StorageDriver.Reader(suite.ctx, filename, -1)
   351  	c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{})
   352  	c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1))
   353  	c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename)
   354  	c.Assert(reader, check.IsNil)
   355  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   356  
   357  	// Read past the end of the content and make sure we get a reader that
   358  	// returns 0 bytes and io.EOF
   359  	reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3)
   360  	c.Assert(err, check.IsNil)
   361  	defer reader.Close()
   362  
   363  	buf := make([]byte, chunkSize)
   364  	n, err := reader.Read(buf)
   365  	c.Assert(err, check.Equals, io.EOF)
   366  	c.Assert(n, check.Equals, 0)
   367  
   368  	// Check the N-1 boundary condition, ensuring we get 1 byte then io.EOF.
   369  	reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3-1)
   370  	c.Assert(err, check.IsNil)
   371  	defer reader.Close()
   372  
   373  	n, err = reader.Read(buf)
   374  	c.Assert(n, check.Equals, 1)
   375  
   376  	// We don't care whether the io.EOF comes on the this read or the first
   377  	// zero read, but the only error acceptable here is io.EOF.
   378  	if err != nil {
   379  		c.Assert(err, check.Equals, io.EOF)
   380  	}
   381  
   382  	// Any more reads should result in zero bytes and io.EOF
   383  	n, err = reader.Read(buf)
   384  	c.Assert(n, check.Equals, 0)
   385  	c.Assert(err, check.Equals, io.EOF)
   386  }
   387  
   388  // TestContinueStreamAppendLarge tests that a stream write can be appended to without
   389  // corrupting the data with a large chunk size.
   390  func (suite *DriverSuite) TestContinueStreamAppendLarge(c *check.C) {
   391  	suite.testContinueStreamAppend(c, int64(10*1024*1024))
   392  }
   393  
   394  // TestContinueStreamAppendSmall is the same as TestContinueStreamAppendLarge, but only
   395  // with a tiny chunk size in order to test corner cases for some cloud storage drivers.
   396  func (suite *DriverSuite) TestContinueStreamAppendSmall(c *check.C) {
   397  	suite.testContinueStreamAppend(c, int64(32))
   398  }
   399  
   400  func (suite *DriverSuite) testContinueStreamAppend(c *check.C, chunkSize int64) {
   401  	filename := randomPath(32)
   402  	defer suite.deletePath(c, firstPart(filename))
   403  
   404  	contentsChunk1 := randomContents(chunkSize)
   405  	contentsChunk2 := randomContents(chunkSize)
   406  	contentsChunk3 := randomContents(chunkSize)
   407  
   408  	fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)
   409  
   410  	writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
   411  	c.Assert(err, check.IsNil)
   412  	nn, err := io.Copy(writer, bytes.NewReader(contentsChunk1))
   413  	c.Assert(err, check.IsNil)
   414  	c.Assert(nn, check.Equals, int64(len(contentsChunk1)))
   415  
   416  	err = writer.Close()
   417  	c.Assert(err, check.IsNil)
   418  
   419  	curSize := writer.Size()
   420  	c.Assert(curSize, check.Equals, int64(len(contentsChunk1)))
   421  
   422  	writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
   423  	c.Assert(err, check.IsNil)
   424  	c.Assert(writer.Size(), check.Equals, curSize)
   425  
   426  	nn, err = io.Copy(writer, bytes.NewReader(contentsChunk2))
   427  	c.Assert(err, check.IsNil)
   428  	c.Assert(nn, check.Equals, int64(len(contentsChunk2)))
   429  
   430  	err = writer.Close()
   431  	c.Assert(err, check.IsNil)
   432  
   433  	curSize = writer.Size()
   434  	c.Assert(curSize, check.Equals, 2*chunkSize)
   435  
   436  	writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
   437  	c.Assert(err, check.IsNil)
   438  	c.Assert(writer.Size(), check.Equals, curSize)
   439  
   440  	nn, err = io.Copy(writer, bytes.NewReader(fullContents[curSize:]))
   441  	c.Assert(err, check.IsNil)
   442  	c.Assert(nn, check.Equals, int64(len(fullContents[curSize:])))
   443  
   444  	err = writer.Commit()
   445  	c.Assert(err, check.IsNil)
   446  	err = writer.Close()
   447  	c.Assert(err, check.IsNil)
   448  
   449  	received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
   450  	c.Assert(err, check.IsNil)
   451  	c.Assert(received, check.DeepEquals, fullContents)
   452  }
   453  
   454  // TestReadNonexistentStream tests that reading a stream for a nonexistent path
   455  // fails.
   456  func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
   457  	filename := randomPath(32)
   458  
   459  	_, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
   460  	c.Assert(err, check.NotNil)
   461  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   462  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   463  
   464  	_, err = suite.StorageDriver.Reader(suite.ctx, filename, 64)
   465  	c.Assert(err, check.NotNil)
   466  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   467  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   468  }
   469  
   470  // TestList checks the returned list of keys after populating a directory tree.
   471  func (suite *DriverSuite) TestList(c *check.C) {
   472  	rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8)))
   473  	defer suite.deletePath(c, rootDirectory)
   474  
   475  	doesnotexist := path.Join(rootDirectory, "nonexistent")
   476  	_, err := suite.StorageDriver.List(suite.ctx, doesnotexist)
   477  	c.Assert(err, check.Equals, storagedriver.PathNotFoundError{
   478  		Path:       doesnotexist,
   479  		DriverName: suite.StorageDriver.Name(),
   480  	})
   481  
   482  	parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
   483  	childFiles := make([]string, 50)
   484  	for i := 0; i < len(childFiles); i++ {
   485  		childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
   486  		childFiles[i] = childFile
   487  		err := suite.StorageDriver.PutContent(suite.ctx, childFile, randomContents(32))
   488  		c.Assert(err, check.IsNil)
   489  	}
   490  	sort.Strings(childFiles)
   491  
   492  	keys, err := suite.StorageDriver.List(suite.ctx, "/")
   493  	c.Assert(err, check.IsNil)
   494  	c.Assert(keys, check.DeepEquals, []string{rootDirectory})
   495  
   496  	keys, err = suite.StorageDriver.List(suite.ctx, rootDirectory)
   497  	c.Assert(err, check.IsNil)
   498  	c.Assert(keys, check.DeepEquals, []string{parentDirectory})
   499  
   500  	keys, err = suite.StorageDriver.List(suite.ctx, parentDirectory)
   501  	c.Assert(err, check.IsNil)
   502  
   503  	sort.Strings(keys)
   504  	c.Assert(keys, check.DeepEquals, childFiles)
   505  
   506  	// A few checks to add here (check out #819 for more discussion on this):
   507  	// 1. Ensure that all paths are absolute.
   508  	// 2. Ensure that listings only include direct children.
   509  	// 3. Ensure that we only respond to directory listings that end with a slash (maybe?).
   510  }
   511  
   512  // TestMove checks that a moved object no longer exists at the source path and
   513  // does exist at the destination.
   514  func (suite *DriverSuite) TestMove(c *check.C) {
   515  	contents := randomContents(32)
   516  	sourcePath := randomPath(32)
   517  	destPath := randomPath(32)
   518  
   519  	defer suite.deletePath(c, firstPart(sourcePath))
   520  	defer suite.deletePath(c, firstPart(destPath))
   521  
   522  	err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, contents)
   523  	c.Assert(err, check.IsNil)
   524  
   525  	err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
   526  	c.Assert(err, check.IsNil)
   527  
   528  	received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
   529  	c.Assert(err, check.IsNil)
   530  	c.Assert(received, check.DeepEquals, contents)
   531  
   532  	_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
   533  	c.Assert(err, check.NotNil)
   534  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   535  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   536  }
   537  
   538  // TestMoveOverwrite checks that a moved object no longer exists at the source
   539  // path and overwrites the contents at the destination.
   540  func (suite *DriverSuite) TestMoveOverwrite(c *check.C) {
   541  	sourcePath := randomPath(32)
   542  	destPath := randomPath(32)
   543  	sourceContents := randomContents(32)
   544  	destContents := randomContents(64)
   545  
   546  	defer suite.deletePath(c, firstPart(sourcePath))
   547  	defer suite.deletePath(c, firstPart(destPath))
   548  
   549  	err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, sourceContents)
   550  	c.Assert(err, check.IsNil)
   551  
   552  	err = suite.StorageDriver.PutContent(suite.ctx, destPath, destContents)
   553  	c.Assert(err, check.IsNil)
   554  
   555  	err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
   556  	c.Assert(err, check.IsNil)
   557  
   558  	received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
   559  	c.Assert(err, check.IsNil)
   560  	c.Assert(received, check.DeepEquals, sourceContents)
   561  
   562  	_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
   563  	c.Assert(err, check.NotNil)
   564  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   565  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   566  }
   567  
   568  // TestMoveNonexistent checks that moving a nonexistent key fails and does not
   569  // delete the data at the destination path.
   570  func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
   571  	contents := randomContents(32)
   572  	sourcePath := randomPath(32)
   573  	destPath := randomPath(32)
   574  
   575  	defer suite.deletePath(c, firstPart(destPath))
   576  
   577  	err := suite.StorageDriver.PutContent(suite.ctx, destPath, contents)
   578  	c.Assert(err, check.IsNil)
   579  
   580  	err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
   581  	c.Assert(err, check.NotNil)
   582  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   583  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   584  
   585  	received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
   586  	c.Assert(err, check.IsNil)
   587  	c.Assert(received, check.DeepEquals, contents)
   588  }
   589  
   590  // TestMoveInvalid provides various checks for invalid moves.
   591  func (suite *DriverSuite) TestMoveInvalid(c *check.C) {
   592  	contents := randomContents(32)
   593  
   594  	// Create a regular file.
   595  	err := suite.StorageDriver.PutContent(suite.ctx, "/notadir", contents)
   596  	c.Assert(err, check.IsNil)
   597  	defer suite.deletePath(c, "/notadir")
   598  
   599  	// Now try to move a non-existent file under it.
   600  	err = suite.StorageDriver.Move(suite.ctx, "/notadir/foo", "/notadir/bar")
   601  	c.Assert(err, check.NotNil) // non-nil error
   602  }
   603  
   604  // TestDelete checks that the delete operation removes data from the storage
   605  // driver
   606  func (suite *DriverSuite) TestDelete(c *check.C) {
   607  	filename := randomPath(32)
   608  	contents := randomContents(32)
   609  
   610  	defer suite.deletePath(c, firstPart(filename))
   611  
   612  	err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   613  	c.Assert(err, check.IsNil)
   614  
   615  	err = suite.StorageDriver.Delete(suite.ctx, filename)
   616  	c.Assert(err, check.IsNil)
   617  
   618  	_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
   619  	c.Assert(err, check.NotNil)
   620  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   621  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   622  }
   623  
   624  // TestURLFor checks that the URLFor method functions properly, but only if it
   625  // is implemented
   626  func (suite *DriverSuite) TestURLFor(c *check.C) {
   627  	filename := randomPath(32)
   628  	contents := randomContents(32)
   629  
   630  	defer suite.deletePath(c, firstPart(filename))
   631  
   632  	err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   633  	c.Assert(err, check.IsNil)
   634  
   635  	url, err := suite.StorageDriver.URLFor(suite.ctx, filename, nil)
   636  	if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
   637  		return
   638  	}
   639  	c.Assert(err, check.IsNil)
   640  
   641  	response, err := http.Get(url)
   642  	c.Assert(err, check.IsNil)
   643  	defer response.Body.Close()
   644  
   645  	read, err := ioutil.ReadAll(response.Body)
   646  	c.Assert(err, check.IsNil)
   647  	c.Assert(read, check.DeepEquals, contents)
   648  
   649  	url, err = suite.StorageDriver.URLFor(suite.ctx, filename, map[string]interface{}{"method": "HEAD"})
   650  	if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
   651  		return
   652  	}
   653  	c.Assert(err, check.IsNil)
   654  
   655  	response, _ = http.Head(url)
   656  	c.Assert(response.StatusCode, check.Equals, 200)
   657  	c.Assert(response.ContentLength, check.Equals, int64(32))
   658  }
   659  
   660  // TestDeleteNonexistent checks that removing a nonexistent key fails.
   661  func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
   662  	filename := randomPath(32)
   663  	err := suite.StorageDriver.Delete(suite.ctx, filename)
   664  	c.Assert(err, check.NotNil)
   665  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   666  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   667  }
   668  
   669  // TestDeleteFolder checks that deleting a folder removes all child elements.
   670  func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
   671  	dirname := randomPath(32)
   672  	filename1 := randomPath(32)
   673  	filename2 := randomPath(32)
   674  	filename3 := randomPath(32)
   675  	contents := randomContents(32)
   676  
   677  	defer suite.deletePath(c, firstPart(dirname))
   678  
   679  	err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename1), contents)
   680  	c.Assert(err, check.IsNil)
   681  
   682  	err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename2), contents)
   683  	c.Assert(err, check.IsNil)
   684  
   685  	err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename3), contents)
   686  	c.Assert(err, check.IsNil)
   687  
   688  	err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename1))
   689  	c.Assert(err, check.IsNil)
   690  
   691  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
   692  	c.Assert(err, check.NotNil)
   693  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   694  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   695  
   696  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
   697  	c.Assert(err, check.IsNil)
   698  
   699  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
   700  	c.Assert(err, check.IsNil)
   701  
   702  	err = suite.StorageDriver.Delete(suite.ctx, dirname)
   703  	c.Assert(err, check.IsNil)
   704  
   705  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
   706  	c.Assert(err, check.NotNil)
   707  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   708  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   709  
   710  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
   711  	c.Assert(err, check.NotNil)
   712  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   713  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   714  
   715  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
   716  	c.Assert(err, check.NotNil)
   717  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   718  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   719  }
   720  
   721  // TestDeleteOnlyDeletesSubpaths checks that deleting path A does not
   722  // delete path B when A is a prefix of B but B is not a subpath of A (so that
   723  // deleting "/a" does not delete "/ab").  This matters for services like S3 that
   724  // do not implement directories.
   725  func (suite *DriverSuite) TestDeleteOnlyDeletesSubpaths(c *check.C) {
   726  	dirname := randomPath(32)
   727  	filename := randomPath(32)
   728  	contents := randomContents(32)
   729  
   730  	defer suite.deletePath(c, firstPart(dirname))
   731  
   732  	err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename), contents)
   733  	c.Assert(err, check.IsNil)
   734  
   735  	err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename+"suffix"), contents)
   736  	c.Assert(err, check.IsNil)
   737  
   738  	err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname, filename), contents)
   739  	c.Assert(err, check.IsNil)
   740  
   741  	err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename), contents)
   742  	c.Assert(err, check.IsNil)
   743  
   744  	err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename))
   745  	c.Assert(err, check.IsNil)
   746  
   747  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename))
   748  	c.Assert(err, check.NotNil)
   749  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   750  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   751  
   752  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename+"suffix"))
   753  	c.Assert(err, check.IsNil)
   754  
   755  	err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, dirname))
   756  	c.Assert(err, check.IsNil)
   757  
   758  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname, filename))
   759  	c.Assert(err, check.NotNil)
   760  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   761  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   762  
   763  	_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename))
   764  	c.Assert(err, check.IsNil)
   765  }
   766  
   767  // TestStatCall runs verifies the implementation of the storagedriver's Stat call.
   768  func (suite *DriverSuite) TestStatCall(c *check.C) {
   769  	content := randomContents(4096)
   770  	dirPath := randomPath(32)
   771  	fileName := randomFilename(32)
   772  	filePath := path.Join(dirPath, fileName)
   773  
   774  	defer suite.deletePath(c, firstPart(dirPath))
   775  
   776  	// Call on non-existent file/dir, check error.
   777  	fi, err := suite.StorageDriver.Stat(suite.ctx, dirPath)
   778  	c.Assert(err, check.NotNil)
   779  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   780  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   781  	c.Assert(fi, check.IsNil)
   782  
   783  	fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
   784  	c.Assert(err, check.NotNil)
   785  	c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
   786  	c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
   787  	c.Assert(fi, check.IsNil)
   788  
   789  	err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
   790  	c.Assert(err, check.IsNil)
   791  
   792  	// Call on regular file, check results
   793  	fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
   794  	c.Assert(err, check.IsNil)
   795  	c.Assert(fi, check.NotNil)
   796  	c.Assert(fi.Path(), check.Equals, filePath)
   797  	c.Assert(fi.Size(), check.Equals, int64(len(content)))
   798  	c.Assert(fi.IsDir(), check.Equals, false)
   799  	createdTime := fi.ModTime()
   800  
   801  	// Sleep and modify the file
   802  	time.Sleep(time.Second * 10)
   803  	content = randomContents(4096)
   804  	err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
   805  	c.Assert(err, check.IsNil)
   806  	fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
   807  	c.Assert(err, check.IsNil)
   808  	c.Assert(fi, check.NotNil)
   809  	time.Sleep(time.Second * 5) // allow changes to propagate (eventual consistency)
   810  
   811  	// Check if the modification time is after the creation time.
   812  	// In case of cloud storage services, storage frontend nodes might have
   813  	// time drift between them, however that should be solved with sleeping
   814  	// before update.
   815  	modTime := fi.ModTime()
   816  	if !modTime.After(createdTime) {
   817  		c.Errorf("modtime (%s) is before the creation time (%s)", modTime, createdTime)
   818  	}
   819  
   820  	// Call on directory (do not check ModTime as dirs don't need to support it)
   821  	fi, err = suite.StorageDriver.Stat(suite.ctx, dirPath)
   822  	c.Assert(err, check.IsNil)
   823  	c.Assert(fi, check.NotNil)
   824  	c.Assert(fi.Path(), check.Equals, dirPath)
   825  	c.Assert(fi.Size(), check.Equals, int64(0))
   826  	c.Assert(fi.IsDir(), check.Equals, true)
   827  }
   828  
   829  // TestPutContentMultipleTimes checks that if storage driver can overwrite the content
   830  // in the subsequent puts. Validates that PutContent does not have to work
   831  // with an offset like Writer does and overwrites the file entirely
   832  // rather than writing the data to the [0,len(data)) of the file.
   833  func (suite *DriverSuite) TestPutContentMultipleTimes(c *check.C) {
   834  	filename := randomPath(32)
   835  	contents := randomContents(4096)
   836  
   837  	defer suite.deletePath(c, firstPart(filename))
   838  	err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   839  	c.Assert(err, check.IsNil)
   840  
   841  	contents = randomContents(2048) // upload a different, smaller file
   842  	err = suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   843  	c.Assert(err, check.IsNil)
   844  
   845  	readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
   846  	c.Assert(err, check.IsNil)
   847  	c.Assert(readContents, check.DeepEquals, contents)
   848  }
   849  
   850  // TestConcurrentStreamReads checks that multiple clients can safely read from
   851  // the same file simultaneously with various offsets.
   852  func (suite *DriverSuite) TestConcurrentStreamReads(c *check.C) {
   853  	var filesize int64 = 128 * 1024 * 1024
   854  
   855  	if testing.Short() {
   856  		filesize = 10 * 1024 * 1024
   857  		c.Log("Reducing file size to 10MB for short mode")
   858  	}
   859  
   860  	filename := randomPath(32)
   861  	contents := randomContents(filesize)
   862  
   863  	defer suite.deletePath(c, firstPart(filename))
   864  
   865  	err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
   866  	c.Assert(err, check.IsNil)
   867  
   868  	var wg sync.WaitGroup
   869  
   870  	readContents := func() {
   871  		defer wg.Done()
   872  		offset := rand.Int63n(int64(len(contents)))
   873  		reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
   874  		c.Assert(err, check.IsNil)
   875  
   876  		readContents, err := ioutil.ReadAll(reader)
   877  		c.Assert(err, check.IsNil)
   878  		c.Assert(readContents, check.DeepEquals, contents[offset:])
   879  	}
   880  
   881  	wg.Add(10)
   882  	for i := 0; i < 10; i++ {
   883  		go readContents()
   884  	}
   885  	wg.Wait()
   886  }
   887  
   888  // TestConcurrentFileStreams checks that multiple *os.File objects can be passed
   889  // in to Writer concurrently without hanging.
   890  func (suite *DriverSuite) TestConcurrentFileStreams(c *check.C) {
   891  	numStreams := 32
   892  
   893  	if testing.Short() {
   894  		numStreams = 8
   895  		c.Log("Reducing number of streams to 8 for short mode")
   896  	}
   897  
   898  	var wg sync.WaitGroup
   899  
   900  	testStream := func(size int64) {
   901  		defer wg.Done()
   902  		suite.testFileStreams(c, size)
   903  	}
   904  
   905  	wg.Add(numStreams)
   906  	for i := numStreams; i > 0; i-- {
   907  		go testStream(int64(numStreams) * 1024 * 1024)
   908  	}
   909  
   910  	wg.Wait()
   911  }
   912  
   913  // TODO (brianbland): evaluate the relevancy of this test
   914  // TestEventualConsistency checks that if stat says that a file is a certain size, then
   915  // you can freely read from the file (this is the only guarantee that the driver needs to provide)
   916  // func (suite *DriverSuite) TestEventualConsistency(c *check.C) {
   917  // 	if testing.Short() {
   918  // 		c.Skip("Skipping test in short mode")
   919  // 	}
   920  //
   921  // 	filename := randomPath(32)
   922  // 	defer suite.deletePath(c, firstPart(filename))
   923  //
   924  // 	var offset int64
   925  // 	var misswrites int
   926  // 	var chunkSize int64 = 32
   927  //
   928  // 	for i := 0; i < 1024; i++ {
   929  // 		contents := randomContents(chunkSize)
   930  // 		read, err := suite.StorageDriver.Writer(suite.ctx, filename, offset, bytes.NewReader(contents))
   931  // 		c.Assert(err, check.IsNil)
   932  //
   933  // 		fi, err := suite.StorageDriver.Stat(suite.ctx, filename)
   934  // 		c.Assert(err, check.IsNil)
   935  //
   936  // 		// We are most concerned with being able to read data as soon as Stat declares
   937  // 		// it is uploaded. This is the strongest guarantee that some drivers (that guarantee
   938  // 		// at best eventual consistency) absolutely need to provide.
   939  // 		if fi.Size() == offset+chunkSize {
   940  // 			reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
   941  // 			c.Assert(err, check.IsNil)
   942  //
   943  // 			readContents, err := ioutil.ReadAll(reader)
   944  // 			c.Assert(err, check.IsNil)
   945  //
   946  // 			c.Assert(readContents, check.DeepEquals, contents)
   947  //
   948  // 			reader.Close()
   949  // 			offset += read
   950  // 		} else {
   951  // 			misswrites++
   952  // 		}
   953  // 	}
   954  //
   955  // 	if misswrites > 0 {
   956  //		c.Log("There were " + string(misswrites) + " occurrences of a write not being instantly available.")
   957  // 	}
   958  //
   959  // 	c.Assert(misswrites, check.Not(check.Equals), 1024)
   960  // }
   961  
   962  // BenchmarkPutGetEmptyFiles benchmarks PutContent/GetContent for 0B files
   963  func (suite *DriverSuite) BenchmarkPutGetEmptyFiles(c *check.C) {
   964  	suite.benchmarkPutGetFiles(c, 0)
   965  }
   966  
   967  // BenchmarkPutGet1KBFiles benchmarks PutContent/GetContent for 1KB files
   968  func (suite *DriverSuite) BenchmarkPutGet1KBFiles(c *check.C) {
   969  	suite.benchmarkPutGetFiles(c, 1024)
   970  }
   971  
   972  // BenchmarkPutGet1MBFiles benchmarks PutContent/GetContent for 1MB files
   973  func (suite *DriverSuite) BenchmarkPutGet1MBFiles(c *check.C) {
   974  	suite.benchmarkPutGetFiles(c, 1024*1024)
   975  }
   976  
   977  // BenchmarkPutGet1GBFiles benchmarks PutContent/GetContent for 1GB files
   978  func (suite *DriverSuite) BenchmarkPutGet1GBFiles(c *check.C) {
   979  	suite.benchmarkPutGetFiles(c, 1024*1024*1024)
   980  }
   981  
   982  func (suite *DriverSuite) benchmarkPutGetFiles(c *check.C, size int64) {
   983  	c.SetBytes(size)
   984  	parentDir := randomPath(8)
   985  	defer func() {
   986  		c.StopTimer()
   987  		suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
   988  	}()
   989  
   990  	for i := 0; i < c.N; i++ {
   991  		filename := path.Join(parentDir, randomPath(32))
   992  		err := suite.StorageDriver.PutContent(suite.ctx, filename, randomContents(size))
   993  		c.Assert(err, check.IsNil)
   994  
   995  		_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
   996  		c.Assert(err, check.IsNil)
   997  	}
   998  }
   999  
  1000  // BenchmarkStreamEmptyFiles benchmarks Writer/Reader for 0B files
  1001  func (suite *DriverSuite) BenchmarkStreamEmptyFiles(c *check.C) {
  1002  	suite.benchmarkStreamFiles(c, 0)
  1003  }
  1004  
  1005  // BenchmarkStream1KBFiles benchmarks Writer/Reader for 1KB files
  1006  func (suite *DriverSuite) BenchmarkStream1KBFiles(c *check.C) {
  1007  	suite.benchmarkStreamFiles(c, 1024)
  1008  }
  1009  
  1010  // BenchmarkStream1MBFiles benchmarks Writer/Reader for 1MB files
  1011  func (suite *DriverSuite) BenchmarkStream1MBFiles(c *check.C) {
  1012  	suite.benchmarkStreamFiles(c, 1024*1024)
  1013  }
  1014  
  1015  // BenchmarkStream1GBFiles benchmarks Writer/Reader for 1GB files
  1016  func (suite *DriverSuite) BenchmarkStream1GBFiles(c *check.C) {
  1017  	suite.benchmarkStreamFiles(c, 1024*1024*1024)
  1018  }
  1019  
  1020  func (suite *DriverSuite) benchmarkStreamFiles(c *check.C, size int64) {
  1021  	c.SetBytes(size)
  1022  	parentDir := randomPath(8)
  1023  	defer func() {
  1024  		c.StopTimer()
  1025  		suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
  1026  	}()
  1027  
  1028  	for i := 0; i < c.N; i++ {
  1029  		filename := path.Join(parentDir, randomPath(32))
  1030  		writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
  1031  		c.Assert(err, check.IsNil)
  1032  		written, err := io.Copy(writer, bytes.NewReader(randomContents(size)))
  1033  		c.Assert(err, check.IsNil)
  1034  		c.Assert(written, check.Equals, size)
  1035  
  1036  		err = writer.Commit()
  1037  		c.Assert(err, check.IsNil)
  1038  		err = writer.Close()
  1039  		c.Assert(err, check.IsNil)
  1040  
  1041  		rc, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
  1042  		c.Assert(err, check.IsNil)
  1043  		rc.Close()
  1044  	}
  1045  }
  1046  
  1047  // BenchmarkList5Files benchmarks List for 5 small files
  1048  func (suite *DriverSuite) BenchmarkList5Files(c *check.C) {
  1049  	suite.benchmarkListFiles(c, 5)
  1050  }
  1051  
  1052  // BenchmarkList50Files benchmarks List for 50 small files
  1053  func (suite *DriverSuite) BenchmarkList50Files(c *check.C) {
  1054  	suite.benchmarkListFiles(c, 50)
  1055  }
  1056  
  1057  func (suite *DriverSuite) benchmarkListFiles(c *check.C, numFiles int64) {
  1058  	parentDir := randomPath(8)
  1059  	defer func() {
  1060  		c.StopTimer()
  1061  		suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
  1062  	}()
  1063  
  1064  	for i := int64(0); i < numFiles; i++ {
  1065  		err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
  1066  		c.Assert(err, check.IsNil)
  1067  	}
  1068  
  1069  	c.ResetTimer()
  1070  	for i := 0; i < c.N; i++ {
  1071  		files, err := suite.StorageDriver.List(suite.ctx, parentDir)
  1072  		c.Assert(err, check.IsNil)
  1073  		c.Assert(int64(len(files)), check.Equals, numFiles)
  1074  	}
  1075  }
  1076  
  1077  // BenchmarkDelete5Files benchmarks Delete for 5 small files
  1078  func (suite *DriverSuite) BenchmarkDelete5Files(c *check.C) {
  1079  	suite.benchmarkDeleteFiles(c, 5)
  1080  }
  1081  
  1082  // BenchmarkDelete50Files benchmarks Delete for 50 small files
  1083  func (suite *DriverSuite) BenchmarkDelete50Files(c *check.C) {
  1084  	suite.benchmarkDeleteFiles(c, 50)
  1085  }
  1086  
  1087  func (suite *DriverSuite) benchmarkDeleteFiles(c *check.C, numFiles int64) {
  1088  	for i := 0; i < c.N; i++ {
  1089  		parentDir := randomPath(8)
  1090  		defer suite.deletePath(c, firstPart(parentDir))
  1091  
  1092  		c.StopTimer()
  1093  		for j := int64(0); j < numFiles; j++ {
  1094  			err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
  1095  			c.Assert(err, check.IsNil)
  1096  		}
  1097  		c.StartTimer()
  1098  
  1099  		// This is the operation we're benchmarking
  1100  		err := suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
  1101  		c.Assert(err, check.IsNil)
  1102  	}
  1103  }
  1104  
  1105  func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
  1106  	tf, err := ioutil.TempFile("", "tf")
  1107  	c.Assert(err, check.IsNil)
  1108  	defer os.Remove(tf.Name())
  1109  	defer tf.Close()
  1110  
  1111  	filename := randomPath(32)
  1112  	defer suite.deletePath(c, firstPart(filename))
  1113  
  1114  	contents := randomContents(size)
  1115  
  1116  	_, err = tf.Write(contents)
  1117  	c.Assert(err, check.IsNil)
  1118  
  1119  	tf.Sync()
  1120  	tf.Seek(0, io.SeekStart)
  1121  
  1122  	writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
  1123  	c.Assert(err, check.IsNil)
  1124  	nn, err := io.Copy(writer, tf)
  1125  	c.Assert(err, check.IsNil)
  1126  	c.Assert(nn, check.Equals, size)
  1127  
  1128  	err = writer.Commit()
  1129  	c.Assert(err, check.IsNil)
  1130  	err = writer.Close()
  1131  	c.Assert(err, check.IsNil)
  1132  
  1133  	reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
  1134  	c.Assert(err, check.IsNil)
  1135  	defer reader.Close()
  1136  
  1137  	readContents, err := ioutil.ReadAll(reader)
  1138  	c.Assert(err, check.IsNil)
  1139  
  1140  	c.Assert(readContents, check.DeepEquals, contents)
  1141  }
  1142  
  1143  func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) {
  1144  	defer suite.deletePath(c, firstPart(filename))
  1145  
  1146  	err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
  1147  	c.Assert(err, check.IsNil)
  1148  
  1149  	readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
  1150  	c.Assert(err, check.IsNil)
  1151  
  1152  	c.Assert(readContents, check.DeepEquals, contents)
  1153  }
  1154  
  1155  func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) {
  1156  	defer suite.deletePath(c, firstPart(filename))
  1157  
  1158  	writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
  1159  	c.Assert(err, check.IsNil)
  1160  	nn, err := io.Copy(writer, bytes.NewReader(contents))
  1161  	c.Assert(err, check.IsNil)
  1162  	c.Assert(nn, check.Equals, int64(len(contents)))
  1163  
  1164  	err = writer.Commit()
  1165  	c.Assert(err, check.IsNil)
  1166  	err = writer.Close()
  1167  	c.Assert(err, check.IsNil)
  1168  
  1169  	reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
  1170  	c.Assert(err, check.IsNil)
  1171  	defer reader.Close()
  1172  
  1173  	readContents, err := ioutil.ReadAll(reader)
  1174  	c.Assert(err, check.IsNil)
  1175  
  1176  	c.Assert(readContents, check.DeepEquals, contents)
  1177  }
  1178  
  1179  var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
  1180  var separatorChars = []byte("._-")
  1181  
  1182  func randomPath(length int64) string {
  1183  	path := "/"
  1184  	for int64(len(path)) < length {
  1185  		chunkLength := rand.Int63n(length-int64(len(path))) + 1
  1186  		chunk := randomFilename(chunkLength)
  1187  		path += chunk
  1188  		remaining := length - int64(len(path))
  1189  		if remaining == 1 {
  1190  			path += randomFilename(1)
  1191  		} else if remaining > 1 {
  1192  			path += "/"
  1193  		}
  1194  	}
  1195  	return path
  1196  }
  1197  
  1198  func randomFilename(length int64) string {
  1199  	b := make([]byte, length)
  1200  	wasSeparator := true
  1201  	for i := range b {
  1202  		if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 {
  1203  			b[i] = separatorChars[rand.Intn(len(separatorChars))]
  1204  			wasSeparator = true
  1205  		} else {
  1206  			b[i] = filenameChars[rand.Intn(len(filenameChars))]
  1207  			wasSeparator = false
  1208  		}
  1209  	}
  1210  	return string(b)
  1211  }
  1212  
  1213  // randomBytes pre-allocates all of the memory sizes needed for the test. If
  1214  // anything panics while accessing randomBytes, just make this number bigger.
  1215  var randomBytes = make([]byte, 128<<20)
  1216  
  1217  func init() {
  1218  	_, _ = crand.Read(randomBytes) // always returns len(randomBytes) and nil error
  1219  }
  1220  
  1221  func randomContents(length int64) []byte {
  1222  	return randomBytes[:length]
  1223  }
  1224  
  1225  type randReader struct {
  1226  	r int64
  1227  	m sync.Mutex
  1228  }
  1229  
  1230  func (rr *randReader) Read(p []byte) (n int, err error) {
  1231  	rr.m.Lock()
  1232  	defer rr.m.Unlock()
  1233  
  1234  	toread := int64(len(p))
  1235  	if toread > rr.r {
  1236  		toread = rr.r
  1237  	}
  1238  	n = copy(p, randomContents(toread))
  1239  	rr.r -= int64(n)
  1240  
  1241  	if rr.r <= 0 {
  1242  		err = io.EOF
  1243  	}
  1244  
  1245  	return
  1246  }
  1247  
  1248  func newRandReader(n int64) *randReader {
  1249  	return &randReader{r: n}
  1250  }
  1251  
  1252  func firstPart(filePath string) string {
  1253  	if filePath == "" {
  1254  		return "/"
  1255  	}
  1256  	for {
  1257  		if filePath[len(filePath)-1] == '/' {
  1258  			filePath = filePath[:len(filePath)-1]
  1259  		}
  1260  
  1261  		dir, file := path.Split(filePath)
  1262  		if dir == "" && file == "" {
  1263  			return "/"
  1264  		}
  1265  		if dir == "/" || dir == "" {
  1266  			return "/" + file
  1267  		}
  1268  		if file == "" {
  1269  			return dir
  1270  		}
  1271  		filePath = dir
  1272  	}
  1273  }
  1274
View as plain text