write.go

Documentation: github.com/google/go-containerregistry/pkg/v1/layout

     1  // Copyright 2018 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package layout
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/json"
    20  	"errors"
    21  	"fmt"
    22  	"io"
    23  	"os"
    24  	"path/filepath"
    25  
    26  	"github.com/google/go-containerregistry/pkg/logs"
    27  	v1 "github.com/google/go-containerregistry/pkg/v1"
    28  	"github.com/google/go-containerregistry/pkg/v1/match"
    29  	"github.com/google/go-containerregistry/pkg/v1/mutate"
    30  	"github.com/google/go-containerregistry/pkg/v1/partial"
    31  	"github.com/google/go-containerregistry/pkg/v1/stream"
    32  	"github.com/google/go-containerregistry/pkg/v1/types"
    33  	"golang.org/x/sync/errgroup"
    34  )
    35  
    36  var layoutFile = `{
    37      "imageLayoutVersion": "1.0.0"
    38  }`
    39  
    40  // AppendImage writes a v1.Image to the Path and updates
    41  // the index.json to reference it.
    42  func (l Path) AppendImage(img v1.Image, options ...Option) error {
    43  	if err := l.WriteImage(img); err != nil {
    44  		return err
    45  	}
    46  
    47  	desc, err := partial.Descriptor(img)
    48  	if err != nil {
    49  		return err
    50  	}
    51  
    52  	o := makeOptions(options...)
    53  	for _, opt := range o.descOpts {
    54  		opt(desc)
    55  	}
    56  
    57  	return l.AppendDescriptor(*desc)
    58  }
    59  
    60  // AppendIndex writes a v1.ImageIndex to the Path and updates
    61  // the index.json to reference it.
    62  func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error {
    63  	if err := l.WriteIndex(ii); err != nil {
    64  		return err
    65  	}
    66  
    67  	desc, err := partial.Descriptor(ii)
    68  	if err != nil {
    69  		return err
    70  	}
    71  
    72  	o := makeOptions(options...)
    73  	for _, opt := range o.descOpts {
    74  		opt(desc)
    75  	}
    76  
    77  	return l.AppendDescriptor(*desc)
    78  }
    79  
    80  // AppendDescriptor adds a descriptor to the index.json of the Path.
    81  func (l Path) AppendDescriptor(desc v1.Descriptor) error {
    82  	ii, err := l.ImageIndex()
    83  	if err != nil {
    84  		return err
    85  	}
    86  
    87  	index, err := ii.IndexManifest()
    88  	if err != nil {
    89  		return err
    90  	}
    91  
    92  	index.Manifests = append(index.Manifests, desc)
    93  
    94  	rawIndex, err := json.MarshalIndent(index, "", "   ")
    95  	if err != nil {
    96  		return err
    97  	}
    98  
    99  	return l.WriteFile("index.json", rawIndex, os.ModePerm)
   100  }
   101  
   102  // ReplaceImage writes a v1.Image to the Path and updates
   103  // the index.json to reference it, replacing any existing one that matches matcher, if found.
   104  func (l Path) ReplaceImage(img v1.Image, matcher match.Matcher, options ...Option) error {
   105  	if err := l.WriteImage(img); err != nil {
   106  		return err
   107  	}
   108  
   109  	return l.replaceDescriptor(img, matcher, options...)
   110  }
   111  
   112  // ReplaceIndex writes a v1.ImageIndex to the Path and updates
   113  // the index.json to reference it, replacing any existing one that matches matcher, if found.
   114  func (l Path) ReplaceIndex(ii v1.ImageIndex, matcher match.Matcher, options ...Option) error {
   115  	if err := l.WriteIndex(ii); err != nil {
   116  		return err
   117  	}
   118  
   119  	return l.replaceDescriptor(ii, matcher, options...)
   120  }
   121  
   122  // replaceDescriptor adds a descriptor to the index.json of the Path, replacing
   123  // any one matching matcher, if found.
   124  func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, options ...Option) error {
   125  	ii, err := l.ImageIndex()
   126  	if err != nil {
   127  		return err
   128  	}
   129  
   130  	desc, err := partial.Descriptor(append)
   131  	if err != nil {
   132  		return err
   133  	}
   134  
   135  	o := makeOptions(options...)
   136  	for _, opt := range o.descOpts {
   137  		opt(desc)
   138  	}
   139  
   140  	add := mutate.IndexAddendum{
   141  		Add:        append,
   142  		Descriptor: *desc,
   143  	}
   144  	ii = mutate.AppendManifests(mutate.RemoveManifests(ii, matcher), add)
   145  
   146  	index, err := ii.IndexManifest()
   147  	if err != nil {
   148  		return err
   149  	}
   150  
   151  	rawIndex, err := json.MarshalIndent(index, "", "   ")
   152  	if err != nil {
   153  		return err
   154  	}
   155  
   156  	return l.WriteFile("index.json", rawIndex, os.ModePerm)
   157  }
   158  
   159  // RemoveDescriptors removes any descriptors that match the match.Matcher from the index.json of the Path.
   160  func (l Path) RemoveDescriptors(matcher match.Matcher) error {
   161  	ii, err := l.ImageIndex()
   162  	if err != nil {
   163  		return err
   164  	}
   165  	ii = mutate.RemoveManifests(ii, matcher)
   166  
   167  	index, err := ii.IndexManifest()
   168  	if err != nil {
   169  		return err
   170  	}
   171  
   172  	rawIndex, err := json.MarshalIndent(index, "", "   ")
   173  	if err != nil {
   174  		return err
   175  	}
   176  
   177  	return l.WriteFile("index.json", rawIndex, os.ModePerm)
   178  }
   179  
   180  // WriteFile write a file with arbitrary data at an arbitrary location in a v1
   181  // layout. Used mostly internally to write files like "oci-layout" and
   182  // "index.json", also can be used to write other arbitrary files. Do *not* use
   183  // this to write blobs. Use only WriteBlob() for that.
   184  func (l Path) WriteFile(name string, data []byte, perm os.FileMode) error {
   185  	if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) {
   186  		return err
   187  	}
   188  
   189  	return os.WriteFile(l.path(name), data, perm)
   190  }
   191  
   192  // WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at
   193  // blobs/{hash.Algorithm}/{hash.Hex}.
   194  func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error {
   195  	return l.writeBlob(hash, -1, r, nil)
   196  }
   197  
   198  func (l Path) writeBlob(hash v1.Hash, size int64, rc io.ReadCloser, renamer func() (v1.Hash, error)) error {
   199  	defer rc.Close()
   200  	if hash.Hex == "" && renamer == nil {
   201  		panic("writeBlob called an invalid hash and no renamer")
   202  	}
   203  
   204  	dir := l.path("blobs", hash.Algorithm)
   205  	if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
   206  		return err
   207  	}
   208  
   209  	// Check if blob already exists and is the correct size
   210  	file := filepath.Join(dir, hash.Hex)
   211  	if s, err := os.Stat(file); err == nil && !s.IsDir() && (s.Size() == size || size == -1) {
   212  		return nil
   213  	}
   214  
   215  	// If a renamer func was provided write to a temporary file
   216  	open := func() (*os.File, error) { return os.Create(file) }
   217  	if renamer != nil {
   218  		open = func() (*os.File, error) { return os.CreateTemp(dir, hash.Hex) }
   219  	}
   220  	w, err := open()
   221  	if err != nil {
   222  		return err
   223  	}
   224  	if renamer != nil {
   225  		// Delete temp file if an error is encountered before renaming
   226  		defer func() {
   227  			if err := os.Remove(w.Name()); err != nil && !errors.Is(err, os.ErrNotExist) {
   228  				logs.Warn.Printf("error removing temporary file after encountering an error while writing blob: %v", err)
   229  			}
   230  		}()
   231  	}
   232  	defer w.Close()
   233  
   234  	// Write to file and exit if not renaming
   235  	if n, err := io.Copy(w, rc); err != nil || renamer == nil {
   236  		return err
   237  	} else if size != -1 && n != size {
   238  		return fmt.Errorf("expected blob size %d, but only wrote %d", size, n)
   239  	}
   240  
   241  	// Always close reader before renaming, since Close computes the digest in
   242  	// the case of streaming layers. If Close is not called explicitly, it will
   243  	// occur in a goroutine that is not guaranteed to succeed before renamer is
   244  	// called. When renamer is the layer's Digest method, it can return
   245  	// ErrNotComputed.
   246  	if err := rc.Close(); err != nil {
   247  		return err
   248  	}
   249  
   250  	// Always close file before renaming
   251  	if err := w.Close(); err != nil {
   252  		return err
   253  	}
   254  
   255  	// Rename file based on the final hash
   256  	finalHash, err := renamer()
   257  	if err != nil {
   258  		return fmt.Errorf("error getting final digest of layer: %w", err)
   259  	}
   260  
   261  	renamePath := l.path("blobs", finalHash.Algorithm, finalHash.Hex)
   262  	return os.Rename(w.Name(), renamePath)
   263  }
   264  
   265  // writeLayer writes the compressed layer to a blob. Unlike WriteBlob it will
   266  // write to a temporary file (suffixed with .tmp) within the layout until the
   267  // compressed reader is fully consumed and written to disk. Also unlike
   268  // WriteBlob, it will not skip writing and exit without error when a blob file
   269  // exists, but does not have the correct size. (The blob hash is not
   270  // considered, because it may be expensive to compute.)
   271  func (l Path) writeLayer(layer v1.Layer) error {
   272  	d, err := layer.Digest()
   273  	if errors.Is(err, stream.ErrNotComputed) {
   274  		// Allow digest errors, since streams may not have calculated the hash
   275  		// yet. Instead, use an empty value, which will be transformed into a
   276  		// random file name with `os.CreateTemp` and the final digest will be
   277  		// calculated after writing to a temp file and before renaming to the
   278  		// final path.
   279  		d = v1.Hash{Algorithm: "sha256", Hex: ""}
   280  	} else if err != nil {
   281  		return err
   282  	}
   283  
   284  	s, err := layer.Size()
   285  	if errors.Is(err, stream.ErrNotComputed) {
   286  		// Allow size errors, since streams may not have calculated the size
   287  		// yet. Instead, use zero as a sentinel value meaning that no size
   288  		// comparison can be done and any sized blob file should be considered
   289  		// valid and not overwritten.
   290  		//
   291  		// TODO: Provide an option to always overwrite blobs.
   292  		s = -1
   293  	} else if err != nil {
   294  		return err
   295  	}
   296  
   297  	r, err := layer.Compressed()
   298  	if err != nil {
   299  		return err
   300  	}
   301  
   302  	if err := l.writeBlob(d, s, r, layer.Digest); err != nil {
   303  		return fmt.Errorf("error writing layer: %w", err)
   304  	}
   305  	return nil
   306  }
   307  
   308  // RemoveBlob removes a file from the blobs directory in the Path
   309  // at blobs/{hash.Algorithm}/{hash.Hex}
   310  // It does *not* remove any reference to it from other manifests or indexes, or
   311  // from the root index.json.
   312  func (l Path) RemoveBlob(hash v1.Hash) error {
   313  	dir := l.path("blobs", hash.Algorithm)
   314  	err := os.Remove(filepath.Join(dir, hash.Hex))
   315  	if err != nil && !os.IsNotExist(err) {
   316  		return err
   317  	}
   318  	return nil
   319  }
   320  
   321  // WriteImage writes an image, including its manifest, config and all of its
   322  // layers, to the blobs directory. If any blob already exists, as determined by
   323  // the hash filename, does not write it.
   324  // This function does *not* update the `index.json` file. If you want to write the
   325  // image and also update the `index.json`, call AppendImage(), which wraps this
   326  // and also updates the `index.json`.
   327  func (l Path) WriteImage(img v1.Image) error {
   328  	layers, err := img.Layers()
   329  	if err != nil {
   330  		return err
   331  	}
   332  
   333  	// Write the layers concurrently.
   334  	var g errgroup.Group
   335  	for _, layer := range layers {
   336  		layer := layer
   337  		g.Go(func() error {
   338  			return l.writeLayer(layer)
   339  		})
   340  	}
   341  	if err := g.Wait(); err != nil {
   342  		return err
   343  	}
   344  
   345  	// Write the config.
   346  	cfgName, err := img.ConfigName()
   347  	if err != nil {
   348  		return err
   349  	}
   350  	cfgBlob, err := img.RawConfigFile()
   351  	if err != nil {
   352  		return err
   353  	}
   354  	if err := l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
   355  		return err
   356  	}
   357  
   358  	// Write the img manifest.
   359  	d, err := img.Digest()
   360  	if err != nil {
   361  		return err
   362  	}
   363  	manifest, err := img.RawManifest()
   364  	if err != nil {
   365  		return err
   366  	}
   367  
   368  	return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest)))
   369  }
   370  
   371  type withLayer interface {
   372  	Layer(v1.Hash) (v1.Layer, error)
   373  }
   374  
   375  type withBlob interface {
   376  	Blob(v1.Hash) (io.ReadCloser, error)
   377  }
   378  
   379  func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error {
   380  	index, err := ii.IndexManifest()
   381  	if err != nil {
   382  		return err
   383  	}
   384  
   385  	// Walk the descriptors and write any v1.Image or v1.ImageIndex that we find.
   386  	// If we come across something we don't expect, just write it as a blob.
   387  	for _, desc := range index.Manifests {
   388  		switch desc.MediaType {
   389  		case types.OCIImageIndex, types.DockerManifestList:
   390  			ii, err := ii.ImageIndex(desc.Digest)
   391  			if err != nil {
   392  				return err
   393  			}
   394  			if err := l.WriteIndex(ii); err != nil {
   395  				return err
   396  			}
   397  		case types.OCIManifestSchema1, types.DockerManifestSchema2:
   398  			img, err := ii.Image(desc.Digest)
   399  			if err != nil {
   400  				return err
   401  			}
   402  			if err := l.WriteImage(img); err != nil {
   403  				return err
   404  			}
   405  		default:
   406  			// TODO: The layout could reference arbitrary things, which we should
   407  			// probably just pass through.
   408  
   409  			var blob io.ReadCloser
   410  			// Workaround for #819.
   411  			if wl, ok := ii.(withLayer); ok {
   412  				layer, lerr := wl.Layer(desc.Digest)
   413  				if lerr != nil {
   414  					return lerr
   415  				}
   416  				blob, err = layer.Compressed()
   417  			} else if wb, ok := ii.(withBlob); ok {
   418  				blob, err = wb.Blob(desc.Digest)
   419  			}
   420  			if err != nil {
   421  				return err
   422  			}
   423  			if err := l.WriteBlob(desc.Digest, blob); err != nil {
   424  				return err
   425  			}
   426  		}
   427  	}
   428  
   429  	rawIndex, err := ii.RawManifest()
   430  	if err != nil {
   431  		return err
   432  	}
   433  
   434  	return l.WriteFile(indexFile, rawIndex, os.ModePerm)
   435  }
   436  
   437  // WriteIndex writes an index to the blobs directory. Walks down the children,
   438  // including its children manifests and/or indexes, and down the tree until all of
   439  // config and all layers, have been written. If any blob already exists, as determined by
   440  // the hash filename, does not write it.
   441  // This function does *not* update the `index.json` file. If you want to write the
   442  // index and also update the `index.json`, call AppendIndex(), which wraps this
   443  // and also updates the `index.json`.
   444  func (l Path) WriteIndex(ii v1.ImageIndex) error {
   445  	// Always just write oci-layout file, since it's small.
   446  	if err := l.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil {
   447  		return err
   448  	}
   449  
   450  	h, err := ii.Digest()
   451  	if err != nil {
   452  		return err
   453  	}
   454  
   455  	indexFile := filepath.Join("blobs", h.Algorithm, h.Hex)
   456  	return l.writeIndexToFile(indexFile, ii)
   457  }
   458  
   459  // Write constructs a Path at path from an ImageIndex.
   460  //
   461  // The contents are written in the following format:
   462  // At the top level, there is:
   463  //
   464  //	One oci-layout file containing the version of this image-layout.
   465  //	One index.json file listing descriptors for the contained images.
   466  //
   467  // Under blobs/, there is, for each image:
   468  //
   469  //	One file for each layer, named after the layer's SHA.
   470  //	One file for each config blob, named after its SHA.
   471  //	One file for each manifest blob, named after its SHA.
   472  func Write(path string, ii v1.ImageIndex) (Path, error) {
   473  	lp := Path(path)
   474  	// Always just write oci-layout file, since it's small.
   475  	if err := lp.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil {
   476  		return "", err
   477  	}
   478  
   479  	// TODO create blobs/ in case there is a blobs file which would prevent the directory from being created
   480  
   481  	return lp, lp.writeIndexToFile("index.json", ii)
   482  }
   483
View as plain text