1 package s3
2
3 import (
4 "bytes"
5 "crypto/rand"
6 "io/ioutil"
7 "os"
8 "strconv"
9 "testing"
10
11 "gopkg.in/check.v1"
12
13 "github.com/aws/aws-sdk-go/aws"
14 "github.com/aws/aws-sdk-go/service/s3"
15
16 "github.com/docker/distribution/context"
17 storagedriver "github.com/docker/distribution/registry/storage/driver"
18 "github.com/docker/distribution/registry/storage/driver/testsuites"
19 )
20
21
22 func Test(t *testing.T) { check.TestingT(t) }
23
24 var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
25 var skipS3 func() string
26
27 func init() {
28 accessKey := os.Getenv("AWS_ACCESS_KEY")
29 secretKey := os.Getenv("AWS_SECRET_KEY")
30 bucket := os.Getenv("S3_BUCKET")
31 encrypt := os.Getenv("S3_ENCRYPT")
32 keyID := os.Getenv("S3_KEY_ID")
33 secure := os.Getenv("S3_SECURE")
34 skipVerify := os.Getenv("S3_SKIP_VERIFY")
35 v4Auth := os.Getenv("S3_V4_AUTH")
36 region := os.Getenv("AWS_REGION")
37 objectACL := os.Getenv("S3_OBJECT_ACL")
38 root, err := ioutil.TempDir("", "driver-")
39 regionEndpoint := os.Getenv("REGION_ENDPOINT")
40 sessionToken := os.Getenv("AWS_SESSION_TOKEN")
41 if err != nil {
42 panic(err)
43 }
44 defer os.Remove(root)
45
46 s3DriverConstructor = func(rootDirectory, storageClass string) (*Driver, error) {
47 encryptBool := false
48 if encrypt != "" {
49 encryptBool, err = strconv.ParseBool(encrypt)
50 if err != nil {
51 return nil, err
52 }
53 }
54
55 secureBool := true
56 if secure != "" {
57 secureBool, err = strconv.ParseBool(secure)
58 if err != nil {
59 return nil, err
60 }
61 }
62
63 skipVerifyBool := false
64 if skipVerify != "" {
65 skipVerifyBool, err = strconv.ParseBool(skipVerify)
66 if err != nil {
67 return nil, err
68 }
69 }
70
71 v4Bool := true
72 if v4Auth != "" {
73 v4Bool, err = strconv.ParseBool(v4Auth)
74 if err != nil {
75 return nil, err
76 }
77 }
78
79 parameters := DriverParameters{
80 accessKey,
81 secretKey,
82 bucket,
83 region,
84 regionEndpoint,
85 encryptBool,
86 keyID,
87 secureBool,
88 skipVerifyBool,
89 v4Bool,
90 minChunkSize,
91 defaultMultipartCopyChunkSize,
92 defaultMultipartCopyMaxConcurrency,
93 defaultMultipartCopyThresholdSize,
94 rootDirectory,
95 storageClass,
96 driverName + "-test",
97 objectACL,
98 sessionToken,
99 }
100
101 return New(parameters)
102 }
103
104
105 skipS3 = func() string {
106 if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
107 return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
108 }
109 return ""
110 }
111
112 testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
113 return s3DriverConstructor(root, s3.StorageClassStandard)
114 }, skipS3)
115 }
116
117 func TestEmptyRootList(t *testing.T) {
118 if skipS3() != "" {
119 t.Skip(skipS3())
120 }
121
122 validRoot, err := ioutil.TempDir("", "driver-")
123 if err != nil {
124 t.Fatalf("unexpected error creating temporary directory: %v", err)
125 }
126 defer os.Remove(validRoot)
127
128 rootedDriver, err := s3DriverConstructor(validRoot, s3.StorageClassStandard)
129 if err != nil {
130 t.Fatalf("unexpected error creating rooted driver: %v", err)
131 }
132
133 emptyRootDriver, err := s3DriverConstructor("", s3.StorageClassStandard)
134 if err != nil {
135 t.Fatalf("unexpected error creating empty root driver: %v", err)
136 }
137
138 slashRootDriver, err := s3DriverConstructor("/", s3.StorageClassStandard)
139 if err != nil {
140 t.Fatalf("unexpected error creating slash root driver: %v", err)
141 }
142
143 filename := "/test"
144 contents := []byte("contents")
145 ctx := context.Background()
146 err = rootedDriver.PutContent(ctx, filename, contents)
147 if err != nil {
148 t.Fatalf("unexpected error creating content: %v", err)
149 }
150 defer rootedDriver.Delete(ctx, filename)
151
152 keys, _ := emptyRootDriver.List(ctx, "/")
153 for _, path := range keys {
154 if !storagedriver.PathRegexp.MatchString(path) {
155 t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
156 }
157 }
158
159 keys, _ = slashRootDriver.List(ctx, "/")
160 for _, path := range keys {
161 if !storagedriver.PathRegexp.MatchString(path) {
162 t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
163 }
164 }
165 }
166
167 func TestStorageClass(t *testing.T) {
168 if skipS3() != "" {
169 t.Skip(skipS3())
170 }
171
172 rootDir, err := ioutil.TempDir("", "driver-")
173 if err != nil {
174 t.Fatalf("unexpected error creating temporary directory: %v", err)
175 }
176 defer os.Remove(rootDir)
177
178 standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
179 if err != nil {
180 t.Fatalf("unexpected error creating driver with standard storage: %v", err)
181 }
182
183 rrDriver, err := s3DriverConstructor(rootDir, s3.StorageClassReducedRedundancy)
184 if err != nil {
185 t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
186 }
187
188 if _, err = s3DriverConstructor(rootDir, noStorageClass); err != nil {
189 t.Fatalf("unexpected error creating driver without storage class: %v", err)
190 }
191
192 standardFilename := "/test-standard"
193 rrFilename := "/test-rr"
194 contents := []byte("contents")
195 ctx := context.Background()
196
197 err = standardDriver.PutContent(ctx, standardFilename, contents)
198 if err != nil {
199 t.Fatalf("unexpected error creating content: %v", err)
200 }
201 defer standardDriver.Delete(ctx, standardFilename)
202
203 err = rrDriver.PutContent(ctx, rrFilename, contents)
204 if err != nil {
205 t.Fatalf("unexpected error creating content: %v", err)
206 }
207 defer rrDriver.Delete(ctx, rrFilename)
208
209 standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
210 resp, err := standardDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
211 Bucket: aws.String(standardDriverUnwrapped.Bucket),
212 Key: aws.String(standardDriverUnwrapped.s3Path(standardFilename)),
213 })
214 if err != nil {
215 t.Fatalf("unexpected error retrieving standard storage file: %v", err)
216 }
217 defer resp.Body.Close()
218
219 if resp.StorageClass != nil {
220 t.Fatalf("unexpected storage class for standard file: %v", resp.StorageClass)
221 }
222
223 rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
224 resp, err = rrDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
225 Bucket: aws.String(rrDriverUnwrapped.Bucket),
226 Key: aws.String(rrDriverUnwrapped.s3Path(rrFilename)),
227 })
228 if err != nil {
229 t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
230 }
231 defer resp.Body.Close()
232 if resp.StorageClass == nil {
233 t.Fatalf("unexpected storage class for reduced-redundancy file: %v", s3.StorageClassStandard)
234 } else if *resp.StorageClass != s3.StorageClassReducedRedundancy {
235 t.Fatalf("unexpected storage class for reduced-redundancy file: %v", *resp.StorageClass)
236 }
237
238 }
239
240 func TestOverThousandBlobs(t *testing.T) {
241 if skipS3() != "" {
242 t.Skip(skipS3())
243 }
244
245 rootDir, err := ioutil.TempDir("", "driver-")
246 if err != nil {
247 t.Fatalf("unexpected error creating temporary directory: %v", err)
248 }
249 defer os.Remove(rootDir)
250
251 standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
252 if err != nil {
253 t.Fatalf("unexpected error creating driver with standard storage: %v", err)
254 }
255
256 ctx := context.Background()
257 for i := 0; i < 1005; i++ {
258 filename := "/thousandfiletest/file" + strconv.Itoa(i)
259 contents := []byte("contents")
260 err = standardDriver.PutContent(ctx, filename, contents)
261 if err != nil {
262 t.Fatalf("unexpected error creating content: %v", err)
263 }
264 }
265
266
267 err = standardDriver.Delete(ctx, "/thousandfiletest")
268 if err != nil {
269 t.Fatalf("unexpected error deleting thousand files: %v", err)
270 }
271 }
272
273 func TestMoveWithMultipartCopy(t *testing.T) {
274 if skipS3() != "" {
275 t.Skip(skipS3())
276 }
277
278 rootDir, err := ioutil.TempDir("", "driver-")
279 if err != nil {
280 t.Fatalf("unexpected error creating temporary directory: %v", err)
281 }
282 defer os.Remove(rootDir)
283
284 d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
285 if err != nil {
286 t.Fatalf("unexpected error creating driver: %v", err)
287 }
288
289 ctx := context.Background()
290 sourcePath := "/source"
291 destPath := "/dest"
292
293 defer d.Delete(ctx, sourcePath)
294 defer d.Delete(ctx, destPath)
295
296
297 multipartCopyThresholdSize := d.baseEmbed.Base.StorageDriver.(*driver).MultipartCopyThresholdSize
298 contents := make([]byte, 2*multipartCopyThresholdSize)
299 rand.Read(contents)
300
301 err = d.PutContent(ctx, sourcePath, contents)
302 if err != nil {
303 t.Fatalf("unexpected error creating content: %v", err)
304 }
305
306 err = d.Move(ctx, sourcePath, destPath)
307 if err != nil {
308 t.Fatalf("unexpected error moving file: %v", err)
309 }
310
311 received, err := d.GetContent(ctx, destPath)
312 if err != nil {
313 t.Fatalf("unexpected error getting content: %v", err)
314 }
315 if !bytes.Equal(contents, received) {
316 t.Fatal("content differs")
317 }
318
319 _, err = d.GetContent(ctx, sourcePath)
320 switch err.(type) {
321 case storagedriver.PathNotFoundError:
322 default:
323 t.Fatalf("unexpected error getting content: %v", err)
324 }
325 }
326
View as plain text