layer.go

Documentation: github.com/google/go-containerregistry/pkg/v1/tarball

     1  // Copyright 2018 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package tarball
    16  
    17  import (
    18  	"bytes"
    19  	"compress/gzip"
    20  	"fmt"
    21  	"io"
    22  	"os"
    23  	"sync"
    24  
    25  	"github.com/containerd/stargz-snapshotter/estargz"
    26  	"github.com/google/go-containerregistry/internal/and"
    27  	comp "github.com/google/go-containerregistry/internal/compression"
    28  	gestargz "github.com/google/go-containerregistry/internal/estargz"
    29  	ggzip "github.com/google/go-containerregistry/internal/gzip"
    30  	"github.com/google/go-containerregistry/internal/zstd"
    31  	"github.com/google/go-containerregistry/pkg/compression"
    32  	"github.com/google/go-containerregistry/pkg/logs"
    33  	v1 "github.com/google/go-containerregistry/pkg/v1"
    34  	"github.com/google/go-containerregistry/pkg/v1/types"
    35  )
    36  
    37  type layer struct {
    38  	digest             v1.Hash
    39  	diffID             v1.Hash
    40  	size               int64
    41  	compressedopener   Opener
    42  	uncompressedopener Opener
    43  	compression        compression.Compression
    44  	compressionLevel   int
    45  	annotations        map[string]string
    46  	estgzopts          []estargz.Option
    47  	mediaType          types.MediaType
    48  }
    49  
    50  // Descriptor implements partial.withDescriptor.
    51  func (l *layer) Descriptor() (*v1.Descriptor, error) {
    52  	digest, err := l.Digest()
    53  	if err != nil {
    54  		return nil, err
    55  	}
    56  	return &v1.Descriptor{
    57  		Size:        l.size,
    58  		Digest:      digest,
    59  		Annotations: l.annotations,
    60  		MediaType:   l.mediaType,
    61  	}, nil
    62  }
    63  
    64  // Digest implements v1.Layer
    65  func (l *layer) Digest() (v1.Hash, error) {
    66  	return l.digest, nil
    67  }
    68  
    69  // DiffID implements v1.Layer
    70  func (l *layer) DiffID() (v1.Hash, error) {
    71  	return l.diffID, nil
    72  }
    73  
    74  // Compressed implements v1.Layer
    75  func (l *layer) Compressed() (io.ReadCloser, error) {
    76  	return l.compressedopener()
    77  }
    78  
    79  // Uncompressed implements v1.Layer
    80  func (l *layer) Uncompressed() (io.ReadCloser, error) {
    81  	return l.uncompressedopener()
    82  }
    83  
    84  // Size implements v1.Layer
    85  func (l *layer) Size() (int64, error) {
    86  	return l.size, nil
    87  }
    88  
    89  // MediaType implements v1.Layer
    90  func (l *layer) MediaType() (types.MediaType, error) {
    91  	return l.mediaType, nil
    92  }
    93  
    94  // LayerOption applies options to layer
    95  type LayerOption func(*layer)
    96  
    97  // WithCompression is a functional option for overriding the default
    98  // compression algorithm used for compressing uncompressed tarballs.
    99  // Please note that WithCompression(compression.ZStd) should be used
   100  // in conjunction with WithMediaType(types.OCILayerZStd)
   101  func WithCompression(comp compression.Compression) LayerOption {
   102  	return func(l *layer) {
   103  		switch comp {
   104  		case compression.ZStd:
   105  			l.compression = compression.ZStd
   106  		case compression.GZip:
   107  			l.compression = compression.GZip
   108  		case compression.None:
   109  			logs.Warn.Printf("Compression type 'none' is not supported for tarball layers; using gzip compression.")
   110  			l.compression = compression.GZip
   111  		default:
   112  			logs.Warn.Printf("Unexpected compression type for WithCompression(): %s; using gzip compression instead.", comp)
   113  			l.compression = compression.GZip
   114  		}
   115  	}
   116  }
   117  
   118  // WithCompressionLevel is a functional option for overriding the default
   119  // compression level used for compressing uncompressed tarballs.
   120  func WithCompressionLevel(level int) LayerOption {
   121  	return func(l *layer) {
   122  		l.compressionLevel = level
   123  	}
   124  }
   125  
   126  // WithMediaType is a functional option for overriding the layer's media type.
   127  func WithMediaType(mt types.MediaType) LayerOption {
   128  	return func(l *layer) {
   129  		l.mediaType = mt
   130  	}
   131  }
   132  
   133  // WithCompressedCaching is a functional option that overrides the
   134  // logic for accessing the compressed bytes to memoize the result
   135  // and avoid expensive repeated gzips.
   136  func WithCompressedCaching(l *layer) {
   137  	var once sync.Once
   138  	var err error
   139  
   140  	buf := bytes.NewBuffer(nil)
   141  	og := l.compressedopener
   142  
   143  	l.compressedopener = func() (io.ReadCloser, error) {
   144  		once.Do(func() {
   145  			var rc io.ReadCloser
   146  			rc, err = og()
   147  			if err == nil {
   148  				defer rc.Close()
   149  				_, err = io.Copy(buf, rc)
   150  			}
   151  		})
   152  		if err != nil {
   153  			return nil, err
   154  		}
   155  
   156  		return io.NopCloser(bytes.NewBuffer(buf.Bytes())), nil
   157  	}
   158  }
   159  
   160  // WithEstargzOptions is a functional option that allow the caller to pass
   161  // through estargz.Options to the underlying compression layer.  This is
   162  // only meaningful when estargz is enabled.
   163  //
   164  // Deprecated: WithEstargz is deprecated, and will be removed in a future release.
   165  func WithEstargzOptions(opts ...estargz.Option) LayerOption {
   166  	return func(l *layer) {
   167  		l.estgzopts = opts
   168  	}
   169  }
   170  
   171  // WithEstargz is a functional option that explicitly enables estargz support.
   172  //
   173  // Deprecated: WithEstargz is deprecated, and will be removed in a future release.
   174  func WithEstargz(l *layer) {
   175  	oguncompressed := l.uncompressedopener
   176  	estargz := func() (io.ReadCloser, error) {
   177  		crc, err := oguncompressed()
   178  		if err != nil {
   179  			return nil, err
   180  		}
   181  		eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compressionLevel))
   182  		rc, h, err := gestargz.ReadCloser(crc, eopts...)
   183  		if err != nil {
   184  			return nil, err
   185  		}
   186  		l.annotations[estargz.TOCJSONDigestAnnotation] = h.String()
   187  		return &and.ReadCloser{
   188  			Reader: rc,
   189  			CloseFunc: func() error {
   190  				err := rc.Close()
   191  				if err != nil {
   192  					return err
   193  				}
   194  				// As an optimization, leverage the DiffID exposed by the estargz ReadCloser
   195  				l.diffID, err = v1.NewHash(rc.DiffID().String())
   196  				return err
   197  			},
   198  		}, nil
   199  	}
   200  	uncompressed := func() (io.ReadCloser, error) {
   201  		urc, err := estargz()
   202  		if err != nil {
   203  			return nil, err
   204  		}
   205  		return ggzip.UnzipReadCloser(urc)
   206  	}
   207  
   208  	l.compressedopener = estargz
   209  	l.uncompressedopener = uncompressed
   210  }
   211  
   212  // LayerFromFile returns a v1.Layer given a tarball
   213  func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) {
   214  	opener := func() (io.ReadCloser, error) {
   215  		return os.Open(path)
   216  	}
   217  	return LayerFromOpener(opener, opts...)
   218  }
   219  
   220  // LayerFromOpener returns a v1.Layer given an Opener function.
   221  // The Opener may return either an uncompressed tarball (common),
   222  // or a compressed tarball (uncommon).
   223  //
   224  // When using this in conjunction with something like remote.Write
   225  // the uncompressed path may end up gzipping things multiple times:
   226  //  1. Compute the layer SHA256
   227  //  2. Upload the compressed layer.
   228  //
   229  // Since gzip can be expensive, we support an option to memoize the
   230  // compression that can be passed here: tarball.WithCompressedCaching
   231  func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
   232  	comp, err := comp.GetCompression(opener)
   233  	if err != nil {
   234  		return nil, err
   235  	}
   236  
   237  	layer := &layer{
   238  		compression:      compression.GZip,
   239  		compressionLevel: gzip.BestSpeed,
   240  		annotations:      make(map[string]string, 1),
   241  		mediaType:        types.DockerLayer,
   242  	}
   243  
   244  	if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" {
   245  		logs.Warn.Println("GGCR_EXPERIMENT_ESTARGZ is deprecated, and will be removed in a future release.")
   246  		opts = append([]LayerOption{WithEstargz}, opts...)
   247  	}
   248  
   249  	switch comp {
   250  	case compression.GZip:
   251  		layer.compressedopener = opener
   252  		layer.uncompressedopener = func() (io.ReadCloser, error) {
   253  			urc, err := opener()
   254  			if err != nil {
   255  				return nil, err
   256  			}
   257  			return ggzip.UnzipReadCloser(urc)
   258  		}
   259  	case compression.ZStd:
   260  		layer.compressedopener = opener
   261  		layer.uncompressedopener = func() (io.ReadCloser, error) {
   262  			urc, err := opener()
   263  			if err != nil {
   264  				return nil, err
   265  			}
   266  			return zstd.UnzipReadCloser(urc)
   267  		}
   268  	default:
   269  		layer.uncompressedopener = opener
   270  		layer.compressedopener = func() (io.ReadCloser, error) {
   271  			crc, err := opener()
   272  			if err != nil {
   273  				return nil, err
   274  			}
   275  
   276  			if layer.compression == compression.ZStd {
   277  				return zstd.ReadCloserLevel(crc, layer.compressionLevel), nil
   278  			}
   279  
   280  			return ggzip.ReadCloserLevel(crc, layer.compressionLevel), nil
   281  		}
   282  	}
   283  
   284  	for _, opt := range opts {
   285  		opt(layer)
   286  	}
   287  
   288  	// Warn if media type does not match compression
   289  	var mediaTypeMismatch = false
   290  	switch layer.compression {
   291  	case compression.GZip:
   292  		mediaTypeMismatch =
   293  			layer.mediaType != types.OCILayer &&
   294  				layer.mediaType != types.OCIRestrictedLayer &&
   295  				layer.mediaType != types.DockerLayer
   296  
   297  	case compression.ZStd:
   298  		mediaTypeMismatch = layer.mediaType != types.OCILayerZStd
   299  	}
   300  
   301  	if mediaTypeMismatch {
   302  		logs.Warn.Printf("Unexpected mediaType (%s) for selected compression in %s in LayerFromOpener().", layer.mediaType, layer.compression)
   303  	}
   304  
   305  	if layer.digest, layer.size, err = computeDigest(layer.compressedopener); err != nil {
   306  		return nil, err
   307  	}
   308  
   309  	empty := v1.Hash{}
   310  	if layer.diffID == empty {
   311  		if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil {
   312  			return nil, err
   313  		}
   314  	}
   315  
   316  	return layer, nil
   317  }
   318  
   319  // LayerFromReader returns a v1.Layer given a io.Reader.
   320  //
   321  // The reader's contents are read and buffered to a temp file in the process.
   322  //
   323  // Deprecated: Use LayerFromOpener or stream.NewLayer instead, if possible.
   324  func LayerFromReader(reader io.Reader, opts ...LayerOption) (v1.Layer, error) {
   325  	tmp, err := os.CreateTemp("", "")
   326  	if err != nil {
   327  		return nil, fmt.Errorf("creating temp file to buffer reader: %w", err)
   328  	}
   329  	if _, err := io.Copy(tmp, reader); err != nil {
   330  		return nil, fmt.Errorf("writing temp file to buffer reader: %w", err)
   331  	}
   332  	return LayerFromFile(tmp.Name(), opts...)
   333  }
   334  
   335  func computeDigest(opener Opener) (v1.Hash, int64, error) {
   336  	rc, err := opener()
   337  	if err != nil {
   338  		return v1.Hash{}, 0, err
   339  	}
   340  	defer rc.Close()
   341  
   342  	return v1.SHA256(rc)
   343  }
   344  
   345  func computeDiffID(opener Opener) (v1.Hash, error) {
   346  	rc, err := opener()
   347  	if err != nil {
   348  		return v1.Hash{}, err
   349  	}
   350  	defer rc.Close()
   351  
   352  	digest, _, err := v1.SHA256(rc)
   353  	return digest, err
   354  }
   355
View as plain text