garbagecollect.go

Documentation: github.com/docker/distribution/registry/storage

     1  package storage
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	"github.com/distribution/reference"
     8  	"github.com/docker/distribution"
     9  	"github.com/docker/distribution/registry/storage/driver"
    10  	"github.com/opencontainers/go-digest"
    11  )
    12  
    13  func emit(format string, a ...interface{}) {
    14  	fmt.Printf(format+"\n", a...)
    15  }
    16  
    17  // GCOpts contains options for garbage collector
    18  type GCOpts struct {
    19  	DryRun         bool
    20  	RemoveUntagged bool
    21  }
    22  
    23  // ManifestDel contains manifest structure which will be deleted
    24  type ManifestDel struct {
    25  	Name   string
    26  	Digest digest.Digest
    27  	Tags   []string
    28  }
    29  
    30  // MarkAndSweep performs a mark and sweep of registry data
    31  func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, opts GCOpts) error {
    32  	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
    33  	if !ok {
    34  		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
    35  	}
    36  
    37  	// mark
    38  	markSet := make(map[digest.Digest]struct{})
    39  	manifestArr := make([]ManifestDel, 0)
    40  	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
    41  		emit(repoName)
    42  
    43  		var err error
    44  		named, err := reference.WithName(repoName)
    45  		if err != nil {
    46  			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
    47  		}
    48  		repository, err := registry.Repository(ctx, named)
    49  		if err != nil {
    50  			return fmt.Errorf("failed to construct repository: %v", err)
    51  		}
    52  
    53  		manifestService, err := repository.Manifests(ctx)
    54  		if err != nil {
    55  			return fmt.Errorf("failed to construct manifest service: %v", err)
    56  		}
    57  
    58  		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
    59  		if !ok {
    60  			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
    61  		}
    62  
    63  		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
    64  			if opts.RemoveUntagged {
    65  				// fetch all tags where this manifest is the latest one
    66  				tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst})
    67  				if err != nil {
    68  					return fmt.Errorf("failed to retrieve tags for digest %v: %v", dgst, err)
    69  				}
    70  				if len(tags) == 0 {
    71  					emit("manifest eligible for deletion: %s", dgst)
    72  					// fetch all tags from repository
    73  					// all of these tags could contain manifest in history
    74  					// which means that we need check (and delete) those references when deleting manifest
    75  					allTags, err := repository.Tags(ctx).All(ctx)
    76  					if err != nil {
    77  						return fmt.Errorf("failed to retrieve tags %v", err)
    78  					}
    79  					manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags})
    80  					return nil
    81  				}
    82  			}
    83  			// Mark the manifest's blob
    84  			emit("%s: marking manifest %s ", repoName, dgst)
    85  			markSet[dgst] = struct{}{}
    86  
    87  			manifest, err := manifestService.Get(ctx, dgst)
    88  			if err != nil {
    89  				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
    90  			}
    91  
    92  			descriptors := manifest.References()
    93  			for _, descriptor := range descriptors {
    94  				markSet[descriptor.Digest] = struct{}{}
    95  				emit("%s: marking blob %s", repoName, descriptor.Digest)
    96  			}
    97  
    98  			return nil
    99  		})
   100  
   101  		// In certain situations such as unfinished uploads, deleting all
   102  		// tags in S3 or removing the _manifests folder manually, this
   103  		// error may be of type PathNotFound.
   104  		//
   105  		// In these cases we can continue marking other manifests safely.
   106  		if _, ok := err.(driver.PathNotFoundError); ok {
   107  			return nil
   108  		}
   109  
   110  		return err
   111  	})
   112  
   113  	if err != nil {
   114  		return fmt.Errorf("failed to mark: %v", err)
   115  	}
   116  
   117  	// sweep
   118  	vacuum := NewVacuum(ctx, storageDriver)
   119  	if !opts.DryRun {
   120  		for _, obj := range manifestArr {
   121  			err = vacuum.RemoveManifest(obj.Name, obj.Digest, obj.Tags)
   122  			if err != nil {
   123  				return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err)
   124  			}
   125  		}
   126  	}
   127  	blobService := registry.Blobs()
   128  	deleteSet := make(map[digest.Digest]struct{})
   129  	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
   130  		// check if digest is in markSet. If not, delete it!
   131  		if _, ok := markSet[dgst]; !ok {
   132  			deleteSet[dgst] = struct{}{}
   133  		}
   134  		return nil
   135  	})
   136  	if err != nil {
   137  		return fmt.Errorf("error enumerating blobs: %v", err)
   138  	}
   139  	emit("\n%d blobs marked, %d blobs and %d manifests eligible for deletion", len(markSet), len(deleteSet), len(manifestArr))
   140  	for dgst := range deleteSet {
   141  		emit("blob eligible for deletion: %s", dgst)
   142  		if opts.DryRun {
   143  			continue
   144  		}
   145  		err = vacuum.RemoveBlob(string(dgst))
   146  		if err != nil {
   147  			return fmt.Errorf("failed to delete blob %s: %v", dgst, err)
   148  		}
   149  	}
   150  
   151  	return err
   152  }
   153
View as plain text