externaltoc.go

Documentation: github.com/containerd/stargz-snapshotter/estargz/externaltoc

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  /*
    18     Copyright 2019 The Go Authors. All rights reserved.
    19     Use of this source code is governed by a BSD-style
    20     license that can be found in the LICENSE file.
    21  */
    22  
    23  package externaltoc
    24  
    25  import (
    26  	"archive/tar"
    27  	"bytes"
    28  	"compress/gzip"
    29  	"encoding/binary"
    30  	"encoding/json"
    31  	"fmt"
    32  	"hash"
    33  	"io"
    34  	"sync"
    35  
    36  	"github.com/containerd/stargz-snapshotter/estargz"
    37  	digest "github.com/opencontainers/go-digest"
    38  )
    39  
    40  type GzipCompression struct {
    41  	*GzipCompressor
    42  	*GzipDecompressor
    43  }
    44  
    45  func NewGzipCompressionWithLevel(provideTOC func() ([]byte, error), level int) estargz.Compression {
    46  	return &GzipCompression{
    47  		NewGzipCompressorWithLevel(level),
    48  		NewGzipDecompressor(provideTOC),
    49  	}
    50  }
    51  
    52  func NewGzipCompressor() *GzipCompressor {
    53  	return &GzipCompressor{compressionLevel: gzip.BestCompression}
    54  }
    55  
    56  func NewGzipCompressorWithLevel(level int) *GzipCompressor {
    57  	return &GzipCompressor{compressionLevel: level}
    58  }
    59  
    60  type GzipCompressor struct {
    61  	compressionLevel int
    62  	buf              *bytes.Buffer
    63  }
    64  
    65  func (gc *GzipCompressor) WriteTOCTo(w io.Writer) (int, error) {
    66  	if len(gc.buf.Bytes()) == 0 {
    67  		return 0, fmt.Errorf("TOC hasn't been registered")
    68  	}
    69  	return w.Write(gc.buf.Bytes())
    70  }
    71  
    72  func (gc *GzipCompressor) Writer(w io.Writer) (estargz.WriteFlushCloser, error) {
    73  	return gzip.NewWriterLevel(w, gc.compressionLevel)
    74  }
    75  
    76  func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTOC, diffHash hash.Hash) (digest.Digest, error) {
    77  	tocJSON, err := json.MarshalIndent(toc, "", "\t")
    78  	if err != nil {
    79  		return "", err
    80  	}
    81  	buf := new(bytes.Buffer)
    82  	gz, _ := gzip.NewWriterLevel(buf, gc.compressionLevel)
    83  	// TOC isn't written to layer so no effect to diff ID
    84  	tw := tar.NewWriter(gz)
    85  	if err := tw.WriteHeader(&tar.Header{
    86  		Typeflag: tar.TypeReg,
    87  		Name:     estargz.TOCTarName,
    88  		Size:     int64(len(tocJSON)),
    89  	}); err != nil {
    90  		return "", err
    91  	}
    92  	if _, err := tw.Write(tocJSON); err != nil {
    93  		return "", err
    94  	}
    95  
    96  	if err := tw.Close(); err != nil {
    97  		return "", err
    98  	}
    99  	if err := gz.Close(); err != nil {
   100  		return "", err
   101  	}
   102  	gc.buf = buf
   103  	footerBytes, err := gzipFooterBytes()
   104  	if err != nil {
   105  		return "", err
   106  	}
   107  	if _, err := w.Write(footerBytes); err != nil {
   108  		return "", err
   109  	}
   110  	return digest.FromBytes(tocJSON), nil
   111  }
   112  
   113  // The footer is an empty gzip stream with no compression and an Extra header.
   114  //
   115  // 46 comes from:
   116  //
   117  // 10 bytes  gzip header
   118  // 2  bytes  XLEN (length of Extra field) = 21 (4 bytes header + len("STARGZEXTERNALTOC"))
   119  // 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
   120  // 2  bytes  Extra: LEN = 17 (len("STARGZEXTERNALTOC"))
   121  // 17 bytes  Extra: subfield = "STARGZEXTERNALTOC"
   122  // 5  bytes  flate header
   123  // 8  bytes  gzip footer
   124  // (End of the eStargz blob)
   125  const FooterSize = 46
   126  
   127  // gzipFooterBytes returns the 104 bytes footer.
   128  func gzipFooterBytes() ([]byte, error) {
   129  	buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
   130  	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
   131  
   132  	// Extra header indicating the offset of TOCJSON
   133  	// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
   134  	header := make([]byte, 4)
   135  	header[0], header[1] = 'S', 'G'
   136  	subfield := "STARGZEXTERNALTOC"                                   // len("STARGZEXTERNALTOC") = 17
   137  	binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
   138  	gz.Header.Extra = append(header, []byte(subfield)...)
   139  	if err := gz.Close(); err != nil {
   140  		return nil, err
   141  	}
   142  	if buf.Len() != FooterSize {
   143  		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
   144  	}
   145  	return buf.Bytes(), nil
   146  }
   147  
   148  func NewGzipDecompressor(provideTOCFunc func() ([]byte, error)) *GzipDecompressor {
   149  	return &GzipDecompressor{provideTOCFunc: provideTOCFunc}
   150  }
   151  
   152  type GzipDecompressor struct {
   153  	provideTOCFunc func() ([]byte, error)
   154  	rawTOC         []byte // Do not access this field directly. Get this through getTOC() method.
   155  	getTOCOnce     sync.Once
   156  }
   157  
   158  func (gz *GzipDecompressor) getTOC() ([]byte, error) {
   159  	if len(gz.rawTOC) == 0 {
   160  		var retErr error
   161  		gz.getTOCOnce.Do(func() {
   162  			if gz.provideTOCFunc == nil {
   163  				retErr = fmt.Errorf("TOC hasn't been provided")
   164  				return
   165  			}
   166  			rawTOC, err := gz.provideTOCFunc()
   167  			if err != nil {
   168  				retErr = err
   169  				return
   170  			}
   171  			gz.rawTOC = rawTOC
   172  		})
   173  		if retErr != nil {
   174  			return nil, retErr
   175  		}
   176  		if len(gz.rawTOC) == 0 {
   177  			return nil, fmt.Errorf("no TOC is provided")
   178  		}
   179  	}
   180  	return gz.rawTOC, nil
   181  }
   182  
   183  func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
   184  	return gzip.NewReader(r)
   185  }
   186  
   187  func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
   188  	if r != nil {
   189  		return nil, "", fmt.Errorf("TOC must be provided externally but got internal one")
   190  	}
   191  	rawTOC, err := gz.getTOC()
   192  	if err != nil {
   193  		return nil, "", fmt.Errorf("failed to get TOC: %v", err)
   194  	}
   195  	return parseTOCEStargz(bytes.NewReader(rawTOC))
   196  }
   197  
   198  func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
   199  	if len(p) != FooterSize {
   200  		return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
   201  	}
   202  	zr, err := gzip.NewReader(bytes.NewReader(p))
   203  	if err != nil {
   204  		return 0, 0, 0, err
   205  	}
   206  	defer zr.Close()
   207  	extra := zr.Header.Extra
   208  	si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
   209  	if si1 != 'S' || si2 != 'G' {
   210  		return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
   211  	}
   212  	if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(len("STARGZEXTERNALTOC")) {
   213  		return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
   214  	}
   215  	if string(subfield) != "STARGZEXTERNALTOC" {
   216  		return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
   217  	}
   218  	// tocOffset < 0 indicates external TOC.
   219  	// blobPayloadSize < 0 indicates the entire blob size.
   220  	return -1, -1, 0, nil
   221  }
   222  
   223  func (gz *GzipDecompressor) FooterSize() int64 {
   224  	return FooterSize
   225  }
   226  
   227  func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
   228  	if r != nil {
   229  		return nil, fmt.Errorf("TOC must be provided externally but got internal one")
   230  	}
   231  	rawTOC, err := gz.getTOC()
   232  	if err != nil {
   233  		return nil, fmt.Errorf("failed to get TOC: %v", err)
   234  	}
   235  	return decompressTOCEStargz(bytes.NewReader(rawTOC))
   236  }
   237  
   238  func parseTOCEStargz(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
   239  	tr, err := decompressTOCEStargz(r)
   240  	if err != nil {
   241  		return nil, "", err
   242  	}
   243  	dgstr := digest.Canonical.Digester()
   244  	toc = new(estargz.JTOC)
   245  	if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
   246  		return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
   247  	}
   248  	if err := tr.Close(); err != nil {
   249  		return nil, "", err
   250  	}
   251  	return toc, dgstr.Digest(), nil
   252  }
   253  
   254  func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) {
   255  	zr, err := gzip.NewReader(r)
   256  	if err != nil {
   257  		return nil, fmt.Errorf("malformed TOC gzip header: %v", err)
   258  	}
   259  	zr.Multistream(false)
   260  	tr := tar.NewReader(zr)
   261  	h, err := tr.Next()
   262  	if err != nil {
   263  		return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
   264  	}
   265  	if h.Name != estargz.TOCTarName {
   266  		return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, estargz.TOCTarName)
   267  	}
   268  	return readCloser{tr, zr.Close}, nil
   269  }
   270  
   271  type readCloser struct {
   272  	io.Reader
   273  	closeFunc func() error
   274  }
   275  
   276  func (rc readCloser) Close() error {
   277  	return rc.closeFunc()
   278  }
   279
View as plain text