1 package storage
2
3 import (
4 "io"
5 "path"
6 "testing"
7
8 "github.com/distribution/reference"
9 "github.com/docker/distribution"
10 "github.com/docker/distribution/context"
11 "github.com/docker/distribution/registry/storage/driver"
12 "github.com/docker/distribution/registry/storage/driver/inmemory"
13 "github.com/docker/distribution/testutil"
14 "github.com/docker/libtrust"
15 "github.com/opencontainers/go-digest"
16 )
17
18 type image struct {
19 manifest distribution.Manifest
20 manifestDigest digest.Digest
21 layers map[digest.Digest]io.ReadSeeker
22 }
23
24 func createRegistry(t *testing.T, driver driver.StorageDriver, options ...RegistryOption) distribution.Namespace {
25 ctx := context.Background()
26 k, err := libtrust.GenerateECP256PrivateKey()
27 if err != nil {
28 t.Fatal(err)
29 }
30 options = append([]RegistryOption{EnableDelete, Schema1SigningKey(k), EnableSchema1}, options...)
31 registry, err := NewRegistry(ctx, driver, options...)
32 if err != nil {
33 t.Fatalf("Failed to construct namespace")
34 }
35 return registry
36 }
37
38 func makeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository {
39 ctx := context.Background()
40
41
42 named, err := reference.WithName(name)
43 if err != nil {
44 t.Fatalf("Failed to parse name %s: %v", name, err)
45 }
46
47 repo, err := registry.Repository(ctx, named)
48 if err != nil {
49 t.Fatalf("Failed to construct repository: %v", err)
50 }
51 return repo
52 }
53
54 func makeManifestService(t *testing.T, repository distribution.Repository) distribution.ManifestService {
55 ctx := context.Background()
56
57 manifestService, err := repository.Manifests(ctx)
58 if err != nil {
59 t.Fatalf("Failed to construct manifest store: %v", err)
60 }
61 return manifestService
62 }
63
64 func allManifests(t *testing.T, manifestService distribution.ManifestService) map[digest.Digest]struct{} {
65 ctx := context.Background()
66 allManMap := make(map[digest.Digest]struct{})
67 manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
68 if !ok {
69 t.Fatalf("unable to convert ManifestService into ManifestEnumerator")
70 }
71 err := manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
72 allManMap[dgst] = struct{}{}
73 return nil
74 })
75 if err != nil {
76 t.Fatalf("Error getting all manifests: %v", err)
77 }
78 return allManMap
79 }
80
81 func allBlobs(t *testing.T, registry distribution.Namespace) map[digest.Digest]struct{} {
82 ctx := context.Background()
83 blobService := registry.Blobs()
84 allBlobsMap := make(map[digest.Digest]struct{})
85 err := blobService.Enumerate(ctx, func(dgst digest.Digest) error {
86 allBlobsMap[dgst] = struct{}{}
87 return nil
88 })
89 if err != nil {
90 t.Fatalf("Error getting all blobs: %v", err)
91 }
92 return allBlobsMap
93 }
94
95 func uploadImage(t *testing.T, repository distribution.Repository, im image) digest.Digest {
96
97 err := testutil.UploadBlobs(repository, im.layers)
98 if err != nil {
99 t.Fatalf("layer upload failed: %v", err)
100 }
101
102
103 ctx := context.Background()
104 manifestService := makeManifestService(t, repository)
105 manifestDigest, err := manifestService.Put(ctx, im.manifest)
106 if err != nil {
107 t.Fatalf("manifest upload failed: %v", err)
108 }
109
110 return manifestDigest
111 }
112
113 func uploadRandomSchema1Image(t *testing.T, repository distribution.Repository) image {
114 randomLayers, err := testutil.CreateRandomLayers(2)
115 if err != nil {
116 t.Fatalf("%v", err)
117 }
118
119 digests := []digest.Digest{}
120 for digest := range randomLayers {
121 digests = append(digests, digest)
122 }
123
124 manifest, err := testutil.MakeSchema1Manifest(digests)
125 if err != nil {
126 t.Fatalf("%v", err)
127 }
128
129 manifestDigest := uploadImage(t, repository, image{manifest: manifest, layers: randomLayers})
130 return image{
131 manifest: manifest,
132 manifestDigest: manifestDigest,
133 layers: randomLayers,
134 }
135 }
136
137 func uploadRandomSchema2Image(t *testing.T, repository distribution.Repository) image {
138 randomLayers, err := testutil.CreateRandomLayers(2)
139 if err != nil {
140 t.Fatalf("%v", err)
141 }
142
143 digests := []digest.Digest{}
144 for digest := range randomLayers {
145 digests = append(digests, digest)
146 }
147
148 manifest, err := testutil.MakeSchema2Manifest(repository, digests)
149 if err != nil {
150 t.Fatalf("%v", err)
151 }
152
153 manifestDigest := uploadImage(t, repository, image{manifest: manifest, layers: randomLayers})
154 return image{
155 manifest: manifest,
156 manifestDigest: manifestDigest,
157 layers: randomLayers,
158 }
159 }
160
161 func TestNoDeletionNoEffect(t *testing.T) {
162 ctx := context.Background()
163 inmemoryDriver := inmemory.New()
164
165 registry := createRegistry(t, inmemoryDriver)
166 repo := makeRepository(t, registry, "palailogos")
167 manifestService, _ := repo.Manifests(ctx)
168
169 image1 := uploadRandomSchema1Image(t, repo)
170 image2 := uploadRandomSchema1Image(t, repo)
171 uploadRandomSchema2Image(t, repo)
172
173
174 blobstatter := registry.BlobStatter()
175 manifestList, err := testutil.MakeManifestList(blobstatter, []digest.Digest{
176 image1.manifestDigest, image2.manifestDigest})
177 if err != nil {
178 t.Fatalf("Failed to make manifest list: %v", err)
179 }
180
181 _, err = manifestService.Put(ctx, manifestList)
182 if err != nil {
183 t.Fatalf("Failed to add manifest list: %v", err)
184 }
185
186 before := allBlobs(t, registry)
187
188
189 err = MarkAndSweep(context.Background(), inmemoryDriver, registry, GCOpts{
190 DryRun: false,
191 RemoveUntagged: false,
192 })
193 if err != nil {
194 t.Fatalf("Failed mark and sweep: %v", err)
195 }
196
197 after := allBlobs(t, registry)
198 if len(before) != len(after) {
199 t.Fatalf("Garbage collection affected storage: %d != %d", len(before), len(after))
200 }
201 }
202
203 func TestDeleteManifestIfTagNotFound(t *testing.T) {
204 ctx := context.Background()
205 inmemoryDriver := inmemory.New()
206
207 registry := createRegistry(t, inmemoryDriver)
208 repo := makeRepository(t, registry, "deletemanifests")
209 manifestService, _ := repo.Manifests(ctx)
210
211
212 randomLayers1, err := testutil.CreateRandomLayers(3)
213 if err != nil {
214 t.Fatalf("failed to make layers: %v", err)
215 }
216
217 randomLayers2, err := testutil.CreateRandomLayers(3)
218 if err != nil {
219 t.Fatalf("failed to make layers: %v", err)
220 }
221
222
223 err = testutil.UploadBlobs(repo, randomLayers1)
224 if err != nil {
225 t.Fatalf("failed to upload layers: %v", err)
226 }
227
228 err = testutil.UploadBlobs(repo, randomLayers2)
229 if err != nil {
230 t.Fatalf("failed to upload layers: %v", err)
231 }
232
233
234 manifest1, err := testutil.MakeSchema1Manifest(getKeys(randomLayers1))
235 if err != nil {
236 t.Fatalf("failed to make manifest: %v", err)
237 }
238
239 manifest2, err := testutil.MakeSchema1Manifest(getKeys(randomLayers2))
240 if err != nil {
241 t.Fatalf("failed to make manifest: %v", err)
242 }
243
244 _, err = manifestService.Put(ctx, manifest1)
245 if err != nil {
246 t.Fatalf("manifest upload failed: %v", err)
247 }
248
249 _, err = manifestService.Put(ctx, manifest2)
250 if err != nil {
251 t.Fatalf("manifest upload failed: %v", err)
252 }
253
254 manifestEnumerator, _ := manifestService.(distribution.ManifestEnumerator)
255 manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
256 repo.Tags(ctx).Tag(ctx, "test", distribution.Descriptor{Digest: dgst})
257 return nil
258 })
259
260 before1 := allBlobs(t, registry)
261 before2 := allManifests(t, manifestService)
262
263
264 err = MarkAndSweep(context.Background(), inmemoryDriver, registry, GCOpts{
265 DryRun: true,
266 RemoveUntagged: true,
267 })
268 if err != nil {
269 t.Fatalf("Failed mark and sweep: %v", err)
270 }
271 afterDry1 := allBlobs(t, registry)
272 afterDry2 := allManifests(t, manifestService)
273 if len(before1) != len(afterDry1) {
274 t.Fatalf("Garbage collection affected blobs storage: %d != %d", len(before1), len(afterDry1))
275 }
276 if len(before2) != len(afterDry2) {
277 t.Fatalf("Garbage collection affected manifest storage: %d != %d", len(before2), len(afterDry2))
278 }
279
280
281 err = MarkAndSweep(context.Background(), inmemoryDriver, registry, GCOpts{
282 DryRun: false,
283 RemoveUntagged: true,
284 })
285 if err != nil {
286 t.Fatalf("Failed mark and sweep: %v", err)
287 }
288
289 after1 := allBlobs(t, registry)
290 after2 := allManifests(t, manifestService)
291 if len(before1) == len(after1) {
292 t.Fatalf("Garbage collection affected blobs storage: %d == %d", len(before1), len(after1))
293 }
294 if len(before2) == len(after2) {
295 t.Fatalf("Garbage collection affected manifest storage: %d == %d", len(before2), len(after2))
296 }
297 }
298
299 func TestGCWithMissingManifests(t *testing.T) {
300 ctx := context.Background()
301 d := inmemory.New()
302
303 registry := createRegistry(t, d)
304 repo := makeRepository(t, registry, "testrepo")
305 uploadRandomSchema1Image(t, repo)
306
307
308 revPath, err := pathFor(manifestRevisionsPathSpec{"testrepo"})
309 if err != nil {
310 t.Fatal(err)
311 }
312
313 _manifestsPath := path.Dir(revPath)
314 err = d.Delete(ctx, _manifestsPath)
315 if err != nil {
316 t.Fatal(err)
317 }
318
319 err = MarkAndSweep(context.Background(), d, registry, GCOpts{
320 DryRun: false,
321 RemoveUntagged: false,
322 })
323 if err != nil {
324 t.Fatalf("Failed mark and sweep: %v", err)
325 }
326
327 blobs := allBlobs(t, registry)
328 if len(blobs) > 0 {
329 t.Errorf("unexpected blobs after gc")
330 }
331 }
332
333 func TestDeletionHasEffect(t *testing.T) {
334 ctx := context.Background()
335 inmemoryDriver := inmemory.New()
336
337 registry := createRegistry(t, inmemoryDriver)
338 repo := makeRepository(t, registry, "komnenos")
339 manifests, _ := repo.Manifests(ctx)
340
341 image1 := uploadRandomSchema1Image(t, repo)
342 image2 := uploadRandomSchema1Image(t, repo)
343 image3 := uploadRandomSchema2Image(t, repo)
344
345 manifests.Delete(ctx, image2.manifestDigest)
346 manifests.Delete(ctx, image3.manifestDigest)
347
348
349 err := MarkAndSweep(context.Background(), inmemoryDriver, registry, GCOpts{
350 DryRun: false,
351 RemoveUntagged: false,
352 })
353 if err != nil {
354 t.Fatalf("Failed mark and sweep: %v", err)
355 }
356
357 blobs := allBlobs(t, registry)
358
359
360 if _, ok := blobs[image1.manifestDigest]; !ok {
361 t.Fatalf("First manifest is missing")
362 }
363
364 for layer := range image1.layers {
365 if _, ok := blobs[layer]; !ok {
366 t.Fatalf("manifest 1 layer is missing: %v", layer)
367 }
368 }
369
370
371 for layer := range image2.layers {
372 if _, ok := blobs[layer]; ok {
373 t.Fatalf("manifest 2 layer is present: %v", layer)
374 }
375 }
376
377 for layer := range image3.layers {
378 if _, ok := blobs[layer]; ok {
379 t.Fatalf("manifest 3 layer is present: %v", layer)
380 }
381 }
382 }
383
384 func getAnyKey(digests map[digest.Digest]io.ReadSeeker) (d digest.Digest) {
385 for d = range digests {
386 break
387 }
388 return
389 }
390
391 func getKeys(digests map[digest.Digest]io.ReadSeeker) (ds []digest.Digest) {
392 for d := range digests {
393 ds = append(ds, d)
394 }
395 return
396 }
397
398 func TestDeletionWithSharedLayer(t *testing.T) {
399 ctx := context.Background()
400 inmemoryDriver := inmemory.New()
401
402 registry := createRegistry(t, inmemoryDriver)
403 repo := makeRepository(t, registry, "tzimiskes")
404
405
406 randomLayers1, err := testutil.CreateRandomLayers(3)
407 if err != nil {
408 t.Fatalf("failed to make layers: %v", err)
409 }
410
411 randomLayers2, err := testutil.CreateRandomLayers(3)
412 if err != nil {
413 t.Fatalf("failed to make layers: %v", err)
414 }
415
416
417 err = testutil.UploadBlobs(repo, randomLayers1)
418 if err != nil {
419 t.Fatalf("failed to upload layers: %v", err)
420 }
421
422 err = testutil.UploadBlobs(repo, randomLayers2)
423 if err != nil {
424 t.Fatalf("failed to upload layers: %v", err)
425 }
426
427
428 manifest1, err := testutil.MakeSchema1Manifest(getKeys(randomLayers1))
429 if err != nil {
430 t.Fatalf("failed to make manifest: %v", err)
431 }
432
433 sharedKey := getAnyKey(randomLayers1)
434 manifest2, err := testutil.MakeSchema2Manifest(repo, append(getKeys(randomLayers2), sharedKey))
435 if err != nil {
436 t.Fatalf("failed to make manifest: %v", err)
437 }
438
439 manifestService := makeManifestService(t, repo)
440
441
442 _, err = manifestService.Put(ctx, manifest1)
443 if err != nil {
444 t.Fatalf("manifest upload failed: %v", err)
445 }
446
447 manifestDigest2, err := manifestService.Put(ctx, manifest2)
448 if err != nil {
449 t.Fatalf("manifest upload failed: %v", err)
450 }
451
452
453 err = manifestService.Delete(ctx, manifestDigest2)
454 if err != nil {
455 t.Fatalf("manifest deletion failed: %v", err)
456 }
457
458
459 blobs := allBlobs(t, registry)
460 for dgst := range randomLayers1 {
461 if _, ok := blobs[dgst]; !ok {
462 t.Fatalf("random layer 1 blob missing: %v", dgst)
463 }
464 }
465 }
466
467 func TestOrphanBlobDeleted(t *testing.T) {
468 inmemoryDriver := inmemory.New()
469
470 registry := createRegistry(t, inmemoryDriver)
471 repo := makeRepository(t, registry, "michael_z_doukas")
472
473 digests, err := testutil.CreateRandomLayers(1)
474 if err != nil {
475 t.Fatalf("Failed to create random digest: %v", err)
476 }
477
478 if err = testutil.UploadBlobs(repo, digests); err != nil {
479 t.Fatalf("Failed to upload blob: %v", err)
480 }
481
482
483 uploadRandomSchema2Image(t, repo)
484
485
486 err = MarkAndSweep(context.Background(), inmemoryDriver, registry, GCOpts{
487 DryRun: false,
488 RemoveUntagged: false,
489 })
490 if err != nil {
491 t.Fatalf("Failed mark and sweep: %v", err)
492 }
493
494 blobs := allBlobs(t, registry)
495
496
497 for dgst := range digests {
498 if _, ok := blobs[dgst]; ok {
499 t.Fatalf("Orphan layer is present: %v", dgst)
500 }
501 }
502 }
503
View as plain text