layout.go

Documentation: edge-infra.dev/pkg/f8n/warehouse/oci/layout

     1  // Package layout defines utilities for working with a Pallet OCI Image Layout.
     2  //
     3  // All artifacts written to the layout must have a name annotation representing
     4  // the tag for that artifact, e.g., "shoot:latest".
     5  package layout
     6  
     7  import (
     8  	"encoding/json"
     9  	"fmt"
    10  	"io/fs"
    11  	"os"
    12  	"path/filepath"
    13  	"sort"
    14  
    15  	"github.com/google/go-containerregistry/pkg/name"
    16  	v1 "github.com/google/go-containerregistry/pkg/v1"
    17  	"github.com/google/go-containerregistry/pkg/v1/empty"
    18  	"github.com/google/go-containerregistry/pkg/v1/layout"
    19  	ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
    20  
    21  	wh "edge-infra.dev/pkg/f8n/warehouse"
    22  	"edge-infra.dev/pkg/f8n/warehouse/oci"
    23  	"edge-infra.dev/pkg/f8n/warehouse/oci/cmp"
    24  	"edge-infra.dev/pkg/f8n/warehouse/oci/match"
    25  	whname "edge-infra.dev/pkg/f8n/warehouse/oci/name"
    26  )
    27  
    28  // TODO: implement functions for tagging images in the layout and retrieving
    29  // images by tag in the layout? current impl loses name information (since
    30  // name is combined with tag to make a valid ref for the descriptor annotation)
    31  // and doesn't allow for multiple tags of the same artifact. it would also
    32  // be nice to know when something was written to the warehouse
    33  
    34  // Path represents an OCI Image Layout on disk. It is used for storing and accessing
    35  // packages locally.
    36  type Path struct {
    37  	layout.Path
    38  }
    39  
    40  // New creates a new Path at the specified filepath. If an Image Layout doesn't
    41  // already exist at that path, a new one is created.
    42  func New(path string) (*Path, error) {
    43  	p, err := resolvePath(path)
    44  	if err != nil {
    45  		return nil, err
    46  	}
    47  	return &Path{p}, nil
    48  }
    49  
    50  func Clear(path string) (*Path, error) {
    51  	p, err := layout.Write(path, empty.Index)
    52  	if err != nil {
    53  		return nil, fmt.Errorf("failed to create new index at %s: %w", path, err)
    54  	}
    55  	return &Path{p}, nil
    56  }
    57  
    58  // Get retrieves an OCI artifact from the layout for the ref, based on what kind
    59  // of ref it is. If the ref is a name.Tag, it is looked up using the OCI reference
    60  // annotation defined in the OCI Image spec. If it is a name.Digest, the digest
    61  // is searched for directly.
    62  func (p *Path) Get(ref name.Reference) (oci.Artifact, error) {
    63  	idx, err := p.ImageIndex()
    64  	if err != nil {
    65  		return nil, fmt.Errorf("failed to load layout index: %w", err)
    66  	}
    67  
    68  	if ref, ok := ref.(name.Digest); ok {
    69  		hash, err := v1.NewHash(ref.DigestStr())
    70  		if err != nil {
    71  			return nil, err
    72  		}
    73  		desc, err := match.FindManifest(idx, match.Digests(hash))
    74  		if err != nil {
    75  			return nil, fmt.Errorf("failed to locate %s in warehouse cache: %w",
    76  				ref, err)
    77  		}
    78  		return oci.ArtifactFromIdx(idx, desc)
    79  	}
    80  
    81  	desc, err := match.FindManifest(idx, RefMatcher(ref))
    82  	if err != nil {
    83  		return nil, fmt.Errorf("failed to locate %s in warehouse cache: %w",
    84  			ref, err)
    85  	}
    86  	return oci.ArtifactFromIdx(idx, desc)
    87  }
    88  
    89  // Append adds the artifact to the image layout with correct warehouse annotations
    90  // for fetching based on the name and tag.
    91  func (p *Path) Append(ref name.Reference, a oci.Artifact) error {
    92  	if a, ok := a.(oci.Unwrapper); ok {
    93  		// Short-circuit recursing with unwrapped artifact if type implements Unwrapper
    94  		return p.Append(ref, a.Unwrap())
    95  	}
    96  
    97  	pkgName, err := whname.FromArtifact(a)
    98  	if err != nil {
    99  		return fmt.Errorf("failed to read package name from artifact: %w", err)
   100  	}
   101  
   102  	annos := map[string]string{wh.AnnotationRefName: pkgName}
   103  	if t, ok := ref.(name.Tag); ok {
   104  		annos[ociv1.AnnotationRefName] = t.String()
   105  	}
   106  
   107  	matcher := RefMatcher(ref)
   108  	if ref, ok := ref.(name.Digest); ok {
   109  		hash, err := v1.NewHash(ref.DigestStr())
   110  		if err != nil {
   111  			return err
   112  		}
   113  		matcher = match.Digests(hash)
   114  	}
   115  
   116  	switch a := a.(type) {
   117  	case v1.Image:
   118  		return p.appendImage(a, matcher, layout.WithAnnotations(annos))
   119  	case v1.ImageIndex:
   120  		return p.appendIndex(a, matcher, layout.WithAnnotations(annos))
   121  	default:
   122  		return fmt.Errorf("layout.Append: %w", oci.ErrInvalidArtifact)
   123  	}
   124  }
   125  
   126  // Sort improves the reproducibility of the layout's IndexManifest (index.json)
   127  // by sorting the descriptors by digest. The initial index is read in as a GGCR
   128  // v1.IndexManifest. Its descriptors are then sorted by digest. The modified
   129  // index is written back to the same location.
   130  func (p *Path) Sort() error {
   131  	layoutIndex, err := p.ImageIndex()
   132  	if err != nil {
   133  		return fmt.Errorf("failed to retrieve cache index: %w", err)
   134  	}
   135  
   136  	idx, err := layoutIndex.IndexManifest()
   137  	if err != nil {
   138  		return fmt.Errorf("failed to unmarshal cache index: %w", err)
   139  	}
   140  
   141  	// Sort descriptors by digest
   142  	sort.Slice(idx.Manifests, func(i, j int) bool { return idx.Manifests[i].Digest.String() < idx.Manifests[j].Digest.String() })
   143  
   144  	rawIndex, err := json.MarshalIndent(idx, "", "   ")
   145  	if err != nil {
   146  		return fmt.Errorf("failed to marshal sorted cache index: %w", err)
   147  	}
   148  
   149  	return p.WriteFile("index.json", rawIndex, os.ModePerm)
   150  }
   151  
   152  func (p *Path) appendImage(i v1.Image, m match.Matcher, opts ...layout.Option) error {
   153  	return p.ReplaceImage(i, m, opts...)
   154  }
   155  
   156  func (p *Path) appendIndex(ii v1.ImageIndex, m match.Matcher, opts ...layout.Option) error {
   157  	return p.ReplaceIndex(ii, m, opts...)
   158  }
   159  
   160  // resolvePath resolves the provided path to a layout.Path which can be
   161  // written to. If an existing Image Layout does not exist at the path, an
   162  // empty one is created and returned.
   163  func resolvePath(path string) (layout.Path, error) {
   164  	p, err := layout.FromPath(path)
   165  	if err != nil {
   166  		p, err = layout.Write(path, empty.Index)
   167  		if err != nil {
   168  			return "", fmt.Errorf("failed to create new index at %s: %w", path, err)
   169  		}
   170  	}
   171  	return p, nil
   172  }
   173  
   174  // RefMatcher returns a matcher for looking up the OCI v1 annotation used to
   175  // look up tagged artifacts
   176  func RefMatcher(ref name.Reference) match.Matcher {
   177  	return match.Annotation(ociv1.AnnotationRefName, ref.String())
   178  }
   179  
   180  // FromFS reads an Image Layout from the root of fsys. If a layout exists at
   181  // another location in the file system, [fs.Sub] can be used to create an
   182  // appropriate FS:
   183  //
   184  //	f, err := fs.Sub(embeddedFSVar, "layout/root")
   185  //	if err != nil { return err }
   186  //	p, err := layout.FromFS(f)
   187  func FromFS(fsys fs.FS, dst string) (*Path, error) {
   188  	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
   189  		if err != nil {
   190  			return err
   191  		}
   192  		if d.IsDir() {
   193  			return nil
   194  		}
   195  		err = os.MkdirAll(filepath.Join(dst, filepath.Dir(path)), 0777)
   196  		if err != nil {
   197  			return err
   198  		}
   199  
   200  		data, err := fs.ReadFile(fsys, path)
   201  		if err != nil {
   202  			return fmt.Errorf("failed to read %s from FS: %w", path, err)
   203  		}
   204  
   205  		if err := os.WriteFile(filepath.Join(dst, path), data, 0644); err != nil {
   206  			return fmt.Errorf("failed to write %s: %w", filepath.Join(dst, path), err)
   207  		}
   208  
   209  		return nil
   210  	}); err != nil {
   211  		return nil, fmt.Errorf("failed to write layout to %s: %w", dst, err)
   212  	}
   213  
   214  	p, err := New(dst)
   215  	if err != nil {
   216  		return nil, fmt.Errorf("failed to create Image Layout from %s: %w", dst, err)
   217  	}
   218  	return p, nil
   219  }
   220  
   221  // Walk path back to fetch the cached blobs from the given directory
   222  // If error is found in path we default to the home warehouse cache
   223  func (p *Path) cachedBlobs() ([]string, error) {
   224  	dir, err := fs.ReadDir(os.DirFS(filepath.Join(string(p.Path), "blobs/sha256")), ".")
   225  	if err != nil {
   226  		return nil, err
   227  	}
   228  	hashes := make([]string, 0, len(dir))
   229  	for _, d := range dir {
   230  		hashes = append(hashes, d.Name())
   231  	}
   232  	return hashes, nil
   233  }
   234  
   235  func (p *Path) Prune() ([]string, error) {
   236  	var pruned []string
   237  	idx, err := p.ImageIndex()
   238  	if err != nil {
   239  		return nil, err
   240  	}
   241  	cached, err := p.cachedBlobs()
   242  	if err != nil {
   243  		return nil, err
   244  	}
   245  	hashes, err := cmp.Hashes(idx)
   246  	if err != nil {
   247  		return nil, err
   248  	}
   249  	blobs := make(map[string]bool, len(hashes))
   250  	for _, i := range hashes {
   251  		blobs[i.Hex] = true
   252  	}
   253  	for _, b := range cached {
   254  		// All blobs that are in the cache that aren't in the image index are pruned
   255  		if !blobs[b] {
   256  			pruned = append(pruned, b)
   257  			b = "sha256:" + b
   258  			hash, err := v1.NewHash(b)
   259  			if err != nil {
   260  				continue
   261  			}
   262  			err = p.RemoveBlob(hash)
   263  			if err != nil {
   264  				return nil, err
   265  			}
   266  		}
   267  	}
   268  	return pruned, nil
   269  }
   270
View as plain text