purgeuploads.go

Documentation: github.com/docker/distribution/registry/storage

     1  package storage
     2  
     3  import (
     4  	"context"
     5  	"path"
     6  	"strings"
     7  	"time"
     8  
     9  	storageDriver "github.com/docker/distribution/registry/storage/driver"
    10  	"github.com/docker/distribution/uuid"
    11  	"github.com/sirupsen/logrus"
    12  )
    13  
    14  // uploadData stored the location of temporary files created during a layer upload
    15  // along with the date the upload was started
    16  type uploadData struct {
    17  	containingDir string
    18  	startedAt     time.Time
    19  }
    20  
    21  func newUploadData() uploadData {
    22  	return uploadData{
    23  		containingDir: "",
    24  		// default to far in future to protect against missing startedat
    25  		startedAt: time.Now().Add(10000 * time.Hour),
    26  	}
    27  }
    28  
    29  // PurgeUploads deletes files from the upload directory
    30  // created before olderThan.  The list of files deleted and errors
    31  // encountered are returned
    32  func PurgeUploads(ctx context.Context, driver storageDriver.StorageDriver, olderThan time.Time, actuallyDelete bool) ([]string, []error) {
    33  	logrus.Infof("PurgeUploads starting: olderThan=%s, actuallyDelete=%t", olderThan, actuallyDelete)
    34  	uploadData, errors := getOutstandingUploads(ctx, driver)
    35  	var deleted []string
    36  	for _, uploadData := range uploadData {
    37  		if uploadData.startedAt.Before(olderThan) {
    38  			var err error
    39  			logrus.Infof("Upload files in %s have older date (%s) than purge date (%s).  Removing upload directory.",
    40  				uploadData.containingDir, uploadData.startedAt, olderThan)
    41  			if actuallyDelete {
    42  				err = driver.Delete(ctx, uploadData.containingDir)
    43  			}
    44  			if err == nil {
    45  				deleted = append(deleted, uploadData.containingDir)
    46  			} else {
    47  				errors = append(errors, err)
    48  			}
    49  		}
    50  	}
    51  
    52  	logrus.Infof("Purge uploads finished.  Num deleted=%d, num errors=%d", len(deleted), len(errors))
    53  	return deleted, errors
    54  }
    55  
    56  // getOutstandingUploads walks the upload directory, collecting files
    57  // which could be eligible for deletion.  The only reliable way to
    58  // classify the age of a file is with the date stored in the startedAt
    59  // file, so gather files by UUID with a date from startedAt.
    60  func getOutstandingUploads(ctx context.Context, driver storageDriver.StorageDriver) (map[string]uploadData, []error) {
    61  	var errors []error
    62  	uploads := make(map[string]uploadData)
    63  
    64  	inUploadDir := false
    65  	root, err := pathFor(repositoriesRootPathSpec{})
    66  	if err != nil {
    67  		return uploads, append(errors, err)
    68  	}
    69  
    70  	err = driver.Walk(ctx, root, func(fileInfo storageDriver.FileInfo) error {
    71  		filePath := fileInfo.Path()
    72  		_, file := path.Split(filePath)
    73  		if file[0] == '_' {
    74  			// Reserved directory
    75  			inUploadDir = (file == "_uploads")
    76  
    77  			if fileInfo.IsDir() && !inUploadDir {
    78  				return storageDriver.ErrSkipDir
    79  			}
    80  
    81  		}
    82  
    83  		uuid, isContainingDir := uuidFromPath(filePath)
    84  		if uuid == "" {
    85  			// Cannot reliably delete
    86  			return nil
    87  		}
    88  		ud, ok := uploads[uuid]
    89  		if !ok {
    90  			ud = newUploadData()
    91  		}
    92  		if isContainingDir {
    93  			ud.containingDir = filePath
    94  		}
    95  		if file == "startedat" {
    96  			if t, err := readStartedAtFile(driver, filePath); err == nil {
    97  				ud.startedAt = t
    98  			} else {
    99  				errors = pushError(errors, filePath, err)
   100  			}
   101  
   102  		}
   103  
   104  		uploads[uuid] = ud
   105  		return nil
   106  	})
   107  
   108  	if err != nil {
   109  		errors = pushError(errors, root, err)
   110  	}
   111  	return uploads, errors
   112  }
   113  
   114  // uuidFromPath extracts the upload UUID from a given path
   115  // If the UUID is the last path component, this is the containing
   116  // directory for all upload files
   117  func uuidFromPath(path string) (string, bool) {
   118  	components := strings.Split(path, "/")
   119  	for i := len(components) - 1; i >= 0; i-- {
   120  		if u, err := uuid.Parse(components[i]); err == nil {
   121  			return u.String(), i == len(components)-1
   122  		}
   123  	}
   124  	return "", false
   125  }
   126  
   127  // readStartedAtFile reads the date from an upload's startedAtFile
   128  func readStartedAtFile(driver storageDriver.StorageDriver, path string) (time.Time, error) {
   129  	// todo:(richardscothern) - pass in a context
   130  	startedAtBytes, err := driver.GetContent(context.Background(), path)
   131  	if err != nil {
   132  		return time.Now(), err
   133  	}
   134  	startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes))
   135  	if err != nil {
   136  		return time.Now(), err
   137  	}
   138  	return startedAt, nil
   139  }
   140
View as plain text