Source file src/github.com/containerd/stargz-snapshotter/estargz/testutil.go

Documentation: github.com/containerd/stargz-snapshotter/estargz

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  /*
    18     Copyright 2019 The Go Authors. All rights reserved.
    19     Use of this source code is governed by a BSD-style
    20     license that can be found in the LICENSE file.
    21  */
    22  
    23  package estargz
    24  
    25  import (
    26  	"archive/tar"
    27  	"bytes"
    28  	"compress/gzip"
    29  	"crypto/sha256"
    30  	"encoding/json"
    31  	"errors"
    32  	"fmt"
    33  	"io"
    34  	"math/rand"
    35  	"os"
    36  	"path/filepath"
    37  	"reflect"
    38  	"sort"
    39  	"strings"
    40  	"testing"
    41  	"time"
    42  
    43  	"github.com/containerd/stargz-snapshotter/estargz/errorutil"
    44  	"github.com/klauspost/compress/zstd"
    45  	digest "github.com/opencontainers/go-digest"
    46  )
    47  
    48  func init() {
    49  	rand.Seed(time.Now().UnixNano())
    50  }
    51  
    52  // TestingController is Compression with some helper methods necessary for testing.
    53  type TestingController interface {
    54  	Compression
    55  	TestStreams(t *testing.T, b []byte, streams []int64)
    56  	DiffIDOf(*testing.T, []byte) string
    57  	String() string
    58  }
    59  
    60  // CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them.
    61  func CompressionTestSuite(t *testing.T, controllers ...TestingControllerFactory) {
    62  	t.Run("testBuild", func(t *testing.T) { t.Parallel(); testBuild(t, controllers...) })
    63  	t.Run("testDigestAndVerify", func(t *testing.T) { t.Parallel(); testDigestAndVerify(t, controllers...) })
    64  	t.Run("testWriteAndOpen", func(t *testing.T) { t.Parallel(); testWriteAndOpen(t, controllers...) })
    65  }
    66  
    67  type TestingControllerFactory func() TestingController
    68  
    69  const (
    70  	uncompressedType int = iota
    71  	gzipType
    72  	zstdType
    73  )
    74  
    75  var srcCompressions = []int{
    76  	uncompressedType,
    77  	gzipType,
    78  	zstdType,
    79  }
    80  
    81  var allowedPrefix = [4]string{"", "./", "/", "../"}
    82  
    83  // testBuild tests the resulting stargz blob built by this pkg has the same
    84  // contents as the normal stargz blob.
    85  func testBuild(t *testing.T, controllers ...TestingControllerFactory) {
    86  	tests := []struct {
    87  		name         string
    88  		chunkSize    int
    89  		minChunkSize []int
    90  		in           []tarEntry
    91  	}{
    92  		{
    93  			name:      "regfiles and directories",
    94  			chunkSize: 4,
    95  			in: tarOf(
    96  				file("foo", "test1"),
    97  				dir("foo2/"),
    98  				file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})),
    99  			),
   100  		},
   101  		{
   102  			name:      "empty files",
   103  			chunkSize: 4,
   104  			in: tarOf(
   105  				file("foo", "tttttt"),
   106  				file("foo_empty", ""),
   107  				file("foo2", "tttttt"),
   108  				file("foo_empty2", ""),
   109  				file("foo3", "tttttt"),
   110  				file("foo_empty3", ""),
   111  				file("foo4", "tttttt"),
   112  				file("foo_empty4", ""),
   113  				file("foo5", "tttttt"),
   114  				file("foo_empty5", ""),
   115  				file("foo6", "tttttt"),
   116  			),
   117  		},
   118  		{
   119  			name:         "various files",
   120  			chunkSize:    4,
   121  			minChunkSize: []int{0, 64000},
   122  			in: tarOf(
   123  				file("baz.txt", "bazbazbazbazbazbazbaz"),
   124  				file("foo1.txt", "a"),
   125  				file("bar/foo2.txt", "b"),
   126  				file("foo3.txt", "c"),
   127  				symlink("barlink", "test/bar.txt"),
   128  				dir("test/"),
   129  				dir("dev/"),
   130  				blockdev("dev/testblock", 3, 4),
   131  				fifo("dev/testfifo"),
   132  				chardev("dev/testchar1", 5, 6),
   133  				file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})),
   134  				dir("test2/"),
   135  				link("test2/bazlink", "baz.txt"),
   136  				chardev("dev/testchar2", 1, 2),
   137  			),
   138  		},
   139  		{
   140  			name:      "no contents",
   141  			chunkSize: 4,
   142  			in: tarOf(
   143  				file("baz.txt", ""),
   144  				symlink("barlink", "test/bar.txt"),
   145  				dir("test/"),
   146  				dir("dev/"),
   147  				blockdev("dev/testblock", 3, 4),
   148  				fifo("dev/testfifo"),
   149  				chardev("dev/testchar1", 5, 6),
   150  				file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})),
   151  				dir("test2/"),
   152  				link("test2/bazlink", "baz.txt"),
   153  				chardev("dev/testchar2", 1, 2),
   154  			),
   155  		},
   156  	}
   157  	for _, tt := range tests {
   158  		if len(tt.minChunkSize) == 0 {
   159  			tt.minChunkSize = []int{0}
   160  		}
   161  		for _, srcCompression := range srcCompressions {
   162  			srcCompression := srcCompression
   163  			for _, newCL := range controllers {
   164  				newCL := newCL
   165  				for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
   166  					srcTarFormat := srcTarFormat
   167  					for _, prefix := range allowedPrefix {
   168  						prefix := prefix
   169  						for _, minChunkSize := range tt.minChunkSize {
   170  							minChunkSize := minChunkSize
   171  							t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *testing.T) {
   172  								tarBlob := buildTar(t, tt.in, prefix, srcTarFormat)
   173  								// Test divideEntries()
   174  								entries, err := sortEntries(tarBlob, nil, nil) // identical order
   175  								if err != nil {
   176  									t.Fatalf("failed to parse tar: %v", err)
   177  								}
   178  								var merged []*entry
   179  								for _, part := range divideEntries(entries, 4) {
   180  									merged = append(merged, part...)
   181  								}
   182  								if !reflect.DeepEqual(entries, merged) {
   183  									for _, e := range entries {
   184  										t.Logf("Original: %v", e.header)
   185  									}
   186  									for _, e := range merged {
   187  										t.Logf("Merged: %v", e.header)
   188  									}
   189  									t.Errorf("divided entries couldn't be merged")
   190  									return
   191  								}
   192  
   193  								// Prepare sample data
   194  								cl1 := newCL()
   195  								wantBuf := new(bytes.Buffer)
   196  								sw := NewWriterWithCompressor(wantBuf, cl1)
   197  								sw.MinChunkSize = minChunkSize
   198  								sw.ChunkSize = tt.chunkSize
   199  								if err := sw.AppendTar(tarBlob); err != nil {
   200  									t.Fatalf("failed to append tar to want stargz: %v", err)
   201  								}
   202  								if _, err := sw.Close(); err != nil {
   203  									t.Fatalf("failed to prepare want stargz: %v", err)
   204  								}
   205  								wantData := wantBuf.Bytes()
   206  								want, err := Open(io.NewSectionReader(
   207  									bytes.NewReader(wantData), 0, int64(len(wantData))),
   208  									WithDecompressors(cl1),
   209  								)
   210  								if err != nil {
   211  									t.Fatalf("failed to parse the want stargz: %v", err)
   212  								}
   213  
   214  								// Prepare testing data
   215  								var opts []Option
   216  								if minChunkSize > 0 {
   217  									opts = append(opts, WithMinChunkSize(minChunkSize))
   218  								}
   219  								cl2 := newCL()
   220  								rc, err := Build(compressBlob(t, tarBlob, srcCompression),
   221  									append(opts, WithChunkSize(tt.chunkSize), WithCompression(cl2))...)
   222  								if err != nil {
   223  									t.Fatalf("failed to build stargz: %v", err)
   224  								}
   225  								defer rc.Close()
   226  								gotBuf := new(bytes.Buffer)
   227  								if _, err := io.Copy(gotBuf, rc); err != nil {
   228  									t.Fatalf("failed to copy built stargz blob: %v", err)
   229  								}
   230  								gotData := gotBuf.Bytes()
   231  								got, err := Open(io.NewSectionReader(
   232  									bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))),
   233  									WithDecompressors(cl2),
   234  								)
   235  								if err != nil {
   236  									t.Fatalf("failed to parse the got stargz: %v", err)
   237  								}
   238  
   239  								// Check DiffID is properly calculated
   240  								rc.Close()
   241  								diffID := rc.DiffID()
   242  								wantDiffID := cl2.DiffIDOf(t, gotData)
   243  								if diffID.String() != wantDiffID {
   244  									t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
   245  								}
   246  
   247  								// Compare as stargz
   248  								if !isSameVersion(t, cl1, wantData, cl2, gotData) {
   249  									t.Errorf("built stargz hasn't same json")
   250  									return
   251  								}
   252  								if !isSameEntries(t, want, got) {
   253  									t.Errorf("built stargz isn't same as the original")
   254  									return
   255  								}
   256  
   257  								// Compare as tar.gz
   258  								if !isSameTarGz(t, cl1, wantData, cl2, gotData) {
   259  									t.Errorf("built stargz isn't same tar.gz")
   260  									return
   261  								}
   262  							})
   263  						}
   264  					}
   265  				}
   266  			}
   267  		}
   268  	}
   269  }
   270  
   271  func isSameTarGz(t *testing.T, cla TestingController, a []byte, clb TestingController, b []byte) bool {
   272  	aGz, err := cla.Reader(bytes.NewReader(a))
   273  	if err != nil {
   274  		t.Fatalf("failed to read A")
   275  	}
   276  	defer aGz.Close()
   277  	bGz, err := clb.Reader(bytes.NewReader(b))
   278  	if err != nil {
   279  		t.Fatalf("failed to read B")
   280  	}
   281  	defer bGz.Close()
   282  
   283  	// Same as tar's Next() method but ignores landmarks and TOCJSON file
   284  	next := func(r *tar.Reader) (h *tar.Header, err error) {
   285  		for {
   286  			if h, err = r.Next(); err != nil {
   287  				return
   288  			}
   289  			if h.Name != PrefetchLandmark &&
   290  				h.Name != NoPrefetchLandmark &&
   291  				h.Name != TOCTarName {
   292  				return
   293  			}
   294  		}
   295  	}
   296  
   297  	aTar := tar.NewReader(aGz)
   298  	bTar := tar.NewReader(bGz)
   299  	for {
   300  		// Fetch and parse next header.
   301  		aH, aErr := next(aTar)
   302  		bH, bErr := next(bTar)
   303  		if aErr != nil || bErr != nil {
   304  			if aErr == io.EOF && bErr == io.EOF {
   305  				break
   306  			}
   307  			t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr)
   308  		}
   309  		if !reflect.DeepEqual(aH, bH) {
   310  			t.Logf("different header (A = %v; B = %v)", aH, bH)
   311  			return false
   312  
   313  		}
   314  		aFile, err := io.ReadAll(aTar)
   315  		if err != nil {
   316  			t.Fatal("failed to read tar payload of A")
   317  		}
   318  		bFile, err := io.ReadAll(bTar)
   319  		if err != nil {
   320  			t.Fatal("failed to read tar payload of B")
   321  		}
   322  		if !bytes.Equal(aFile, bFile) {
   323  			t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b))
   324  			return false
   325  		}
   326  	}
   327  
   328  	return true
   329  }
   330  
   331  func isSameVersion(t *testing.T, cla TestingController, a []byte, clb TestingController, b []byte) bool {
   332  	aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), cla)
   333  	if err != nil {
   334  		t.Fatalf("failed to parse A: %v", err)
   335  	}
   336  	bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), clb)
   337  	if err != nil {
   338  		t.Fatalf("failed to parse B: %v", err)
   339  	}
   340  	t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, aJTOC))
   341  	t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bJTOC))
   342  	return aJTOC.Version == bJTOC.Version
   343  }
   344  
   345  func isSameEntries(t *testing.T, a, b *Reader) bool {
   346  	aroot, ok := a.Lookup("")
   347  	if !ok {
   348  		t.Fatalf("failed to get root of A")
   349  	}
   350  	broot, ok := b.Lookup("")
   351  	if !ok {
   352  		t.Fatalf("failed to get root of B")
   353  	}
   354  	aEntry := stargzEntry{aroot, a}
   355  	bEntry := stargzEntry{broot, b}
   356  	return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry)
   357  }
   358  
   359  func compressBlob(t *testing.T, src *io.SectionReader, srcCompression int) *io.SectionReader {
   360  	buf := new(bytes.Buffer)
   361  	var w io.WriteCloser
   362  	var err error
   363  	if srcCompression == gzipType {
   364  		w = gzip.NewWriter(buf)
   365  	} else if srcCompression == zstdType {
   366  		w, err = zstd.NewWriter(buf)
   367  		if err != nil {
   368  			t.Fatalf("failed to init zstd writer: %v", err)
   369  		}
   370  	} else {
   371  		return src
   372  	}
   373  	src.Seek(0, io.SeekStart)
   374  	if _, err := io.Copy(w, src); err != nil {
   375  		t.Fatalf("failed to compress source")
   376  	}
   377  	if err := w.Close(); err != nil {
   378  		t.Fatalf("failed to finalize compress source")
   379  	}
   380  	data := buf.Bytes()
   381  	return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data)))
   382  
   383  }
   384  
   385  type stargzEntry struct {
   386  	e *TOCEntry
   387  	r *Reader
   388  }
   389  
   390  // contains checks if all child entries in "b" are also contained in "a".
   391  // This function also checks if the files/chunks contain the same contents among "a" and "b".
   392  func contains(t *testing.T, a, b stargzEntry) bool {
   393  	ae, ar := a.e, a.r
   394  	be, br := b.e, b.r
   395  	t.Logf("Comparing: %q vs %q", ae.Name, be.Name)
   396  	if !equalEntry(ae, be) {
   397  		t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be)
   398  		return false
   399  	}
   400  	if ae.Type == "dir" {
   401  		t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name,
   402  			allChildrenName(ae), allChildrenName(be))
   403  		iscontain := true
   404  		ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool {
   405  			// Walk through all files on this stargz file.
   406  
   407  			if aChild.Name == PrefetchLandmark ||
   408  				aChild.Name == NoPrefetchLandmark {
   409  				return true // Ignore landmarks
   410  			}
   411  
   412  			// Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory
   413  			// because this points to the root directory itself.
   414  			if aChild.Name == "" && ae.Name == "" {
   415  				return true
   416  			}
   417  
   418  			bChild, ok := be.LookupChild(aBaseName)
   419  			if !ok {
   420  				t.Logf("%q (base: %q): not found in b: %v",
   421  					ae.Name, aBaseName, allChildrenName(be))
   422  				iscontain = false
   423  				return false
   424  			}
   425  
   426  			childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r})
   427  			if !childcontain {
   428  				t.Logf("%q != %q: non-equal dir", ae.Name, be.Name)
   429  				iscontain = false
   430  				return false
   431  			}
   432  			return true
   433  		})
   434  		return iscontain
   435  	} else if ae.Type == "reg" {
   436  		af, err := ar.OpenFile(ae.Name)
   437  		if err != nil {
   438  			t.Fatalf("failed to open file %q on A: %v", ae.Name, err)
   439  		}
   440  		bf, err := br.OpenFile(be.Name)
   441  		if err != nil {
   442  			t.Fatalf("failed to open file %q on B: %v", be.Name, err)
   443  		}
   444  
   445  		var nr int64
   446  		for nr < ae.Size {
   447  			abytes, anext, aok := readOffset(t, af, nr, a)
   448  			bbytes, bnext, bok := readOffset(t, bf, nr, b)
   449  			if !aok && !bok {
   450  				break
   451  			} else if !(aok && bok) || anext != bnext {
   452  				t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v",
   453  					ae.Name, be.Name, nr, aok, bok, anext, bnext)
   454  				return false
   455  			}
   456  			nr = anext
   457  			if !bytes.Equal(abytes, bbytes) {
   458  				t.Logf("%q != %q: different contents %v vs %v",
   459  					ae.Name, be.Name, string(abytes), string(bbytes))
   460  				return false
   461  			}
   462  		}
   463  		return true
   464  	}
   465  
   466  	return true
   467  }
   468  
   469  func allChildrenName(e *TOCEntry) (children []string) {
   470  	e.ForeachChild(func(baseName string, _ *TOCEntry) bool {
   471  		children = append(children, baseName)
   472  		return true
   473  	})
   474  	return
   475  }
   476  
   477  func equalEntry(a, b *TOCEntry) bool {
   478  	// Here, we selectively compare fileds that we are interested in.
   479  	return a.Name == b.Name &&
   480  		a.Type == b.Type &&
   481  		a.Size == b.Size &&
   482  		a.ModTime3339 == b.ModTime3339 &&
   483  		a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime     time.Time
   484  		a.LinkName == b.LinkName &&
   485  		a.Mode == b.Mode &&
   486  		a.UID == b.UID &&
   487  		a.GID == b.GID &&
   488  		a.Uname == b.Uname &&
   489  		a.Gname == b.Gname &&
   490  		(a.Offset >= 0) == (b.Offset >= 0) &&
   491  		(a.NextOffset() > 0) == (b.NextOffset() > 0) &&
   492  		a.DevMajor == b.DevMajor &&
   493  		a.DevMinor == b.DevMinor &&
   494  		a.NumLink == b.NumLink &&
   495  		reflect.DeepEqual(a.Xattrs, b.Xattrs) &&
   496  		// chunk-related infomations aren't compared in this function.
   497  		// ChunkOffset int64 `json:"chunkOffset,omitempty"`
   498  		// ChunkSize   int64 `json:"chunkSize,omitempty"`
   499  		// children map[string]*TOCEntry
   500  		a.Digest == b.Digest
   501  }
   502  
   503  func readOffset(t *testing.T, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) {
   504  	ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset)
   505  	if !ok {
   506  		return nil, 0, false
   507  	}
   508  	data := make([]byte, ce.ChunkSize)
   509  	t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset())
   510  	n, err := r.ReadAt(data, ce.ChunkOffset)
   511  	if err != nil {
   512  		t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v",
   513  			e.e.Name, ce.ChunkOffset, ce.ChunkSize, err)
   514  	}
   515  	if int64(n) != ce.ChunkSize {
   516  		t.Fatalf("unexpected copied data size %d; want %d",
   517  			n, ce.ChunkSize)
   518  	}
   519  	return data[:n], offset + ce.ChunkSize, true
   520  }
   521  
   522  func dumpTOCJSON(t *testing.T, tocJSON *JTOC) string {
   523  	jtocData, err := json.Marshal(*tocJSON)
   524  	if err != nil {
   525  		t.Fatalf("failed to marshal TOC JSON: %v", err)
   526  	}
   527  	buf := new(bytes.Buffer)
   528  	if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil {
   529  		t.Fatalf("failed to read toc json blob: %v", err)
   530  	}
   531  	return buf.String()
   532  }
   533  
   534  const chunkSize = 3
   535  
   536  // type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int)
   537  type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory)
   538  
   539  // testDigestAndVerify runs specified checks against sample stargz blobs.
   540  func testDigestAndVerify(t *testing.T, controllers ...TestingControllerFactory) {
   541  	tests := []struct {
   542  		name         string
   543  		tarInit      func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry)
   544  		checks       []check
   545  		minChunkSize []int
   546  	}{
   547  		{
   548  			name: "no-regfile",
   549  			tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) {
   550  				return tarOf(
   551  					dir("test/"),
   552  				)
   553  			},
   554  			checks: []check{
   555  				checkStargzTOC,
   556  				checkVerifyTOC,
   557  				checkVerifyInvalidStargzFail(buildTar(t, tarOf(
   558  					dir("test2/"), // modified
   559  				), allowedPrefix[0])),
   560  			},
   561  		},
   562  		{
   563  			name: "small-files",
   564  			tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) {
   565  				return tarOf(
   566  					regDigest(t, "baz.txt", "", dgstMap),
   567  					regDigest(t, "foo.txt", "a", dgstMap),
   568  					dir("test/"),
   569  					regDigest(t, "test/bar.txt", "bbb", dgstMap),
   570  				)
   571  			},
   572  			minChunkSize: []int{0, 64000},
   573  			checks: []check{
   574  				checkStargzTOC,
   575  				checkVerifyTOC,
   576  				checkVerifyInvalidStargzFail(buildTar(t, tarOf(
   577  					file("baz.txt", ""),
   578  					file("foo.txt", "M"), // modified
   579  					dir("test/"),
   580  					file("test/bar.txt", "bbb"),
   581  				), allowedPrefix[0])),
   582  				// checkVerifyInvalidTOCEntryFail("foo.txt"), // TODO
   583  				checkVerifyBrokenContentFail("foo.txt"),
   584  			},
   585  		},
   586  		{
   587  			name: "big-files",
   588  			tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) {
   589  				return tarOf(
   590  					regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap),
   591  					regDigest(t, "foo.txt", "a", dgstMap),
   592  					dir("test/"),
   593  					regDigest(t, "test/bar.txt", "testbartestbar", dgstMap),
   594  				)
   595  			},
   596  			checks: []check{
   597  				checkStargzTOC,
   598  				checkVerifyTOC,
   599  				checkVerifyInvalidStargzFail(buildTar(t, tarOf(
   600  					file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified
   601  					file("foo.txt", "a"),
   602  					dir("test/"),
   603  					file("test/bar.txt", "testbartestbar"),
   604  				), allowedPrefix[0])),
   605  				checkVerifyInvalidTOCEntryFail("test/bar.txt"),
   606  				checkVerifyBrokenContentFail("test/bar.txt"),
   607  			},
   608  		},
   609  		{
   610  			name:         "with-non-regfiles",
   611  			minChunkSize: []int{0, 64000},
   612  			tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) {
   613  				return tarOf(
   614  					regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap),
   615  					regDigest(t, "foo.txt", "a", dgstMap),
   616  					regDigest(t, "bar/foo2.txt", "b", dgstMap),
   617  					regDigest(t, "foo3.txt", "c", dgstMap),
   618  					symlink("barlink", "test/bar.txt"),
   619  					dir("test/"),
   620  					regDigest(t, "test/bar.txt", "testbartestbar", dgstMap),
   621  					dir("test2/"),
   622  					link("test2/bazlink", "baz.txt"),
   623  				)
   624  			},
   625  			checks: []check{
   626  				checkStargzTOC,
   627  				checkVerifyTOC,
   628  				checkVerifyInvalidStargzFail(buildTar(t, tarOf(
   629  					file("baz.txt", "bazbazbazbazbazbazbaz"),
   630  					file("foo.txt", "a"),
   631  					file("bar/foo2.txt", "b"),
   632  					file("foo3.txt", "c"),
   633  					symlink("barlink", "test/bar.txt"),
   634  					dir("test/"),
   635  					file("test/bar.txt", "testbartestbar"),
   636  					dir("test2/"),
   637  					link("test2/bazlink", "foo.txt"), // modified
   638  				), allowedPrefix[0])),
   639  				checkVerifyInvalidTOCEntryFail("test/bar.txt"),
   640  				checkVerifyBrokenContentFail("test/bar.txt"),
   641  			},
   642  		},
   643  	}
   644  
   645  	for _, tt := range tests {
   646  		if len(tt.minChunkSize) == 0 {
   647  			tt.minChunkSize = []int{0}
   648  		}
   649  		for _, srcCompression := range srcCompressions {
   650  			srcCompression := srcCompression
   651  			for _, newCL := range controllers {
   652  				newCL := newCL
   653  				for _, prefix := range allowedPrefix {
   654  					prefix := prefix
   655  					for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
   656  						srcTarFormat := srcTarFormat
   657  						for _, minChunkSize := range tt.minChunkSize {
   658  							minChunkSize := minChunkSize
   659  							t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *testing.T) {
   660  								// Get original tar file and chunk digests
   661  								dgstMap := make(map[string]digest.Digest)
   662  								tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat)
   663  
   664  								cl := newCL()
   665  								rc, err := Build(compressBlob(t, tarBlob, srcCompression),
   666  									WithChunkSize(chunkSize), WithCompression(cl))
   667  								if err != nil {
   668  									t.Fatalf("failed to convert stargz: %v", err)
   669  								}
   670  								tocDigest := rc.TOCDigest()
   671  								defer rc.Close()
   672  								buf := new(bytes.Buffer)
   673  								if _, err := io.Copy(buf, rc); err != nil {
   674  									t.Fatalf("failed to copy built stargz blob: %v", err)
   675  								}
   676  								newStargz := buf.Bytes()
   677  								// NoPrefetchLandmark is added during `Bulid`, which is expected behaviour.
   678  								dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents})
   679  
   680  								for _, check := range tt.checks {
   681  									check(t, newStargz, tocDigest, dgstMap, cl, newCL)
   682  								}
   683  							})
   684  						}
   685  					}
   686  				}
   687  			}
   688  		}
   689  	}
   690  }
   691  
   692  // checkStargzTOC checks the TOC JSON of the passed stargz has the expected
   693  // digest and contains valid chunks. It walks all entries in the stargz and
   694  // checks all chunk digests stored to the TOC JSON match the actual contents.
   695  func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
   696  	sgz, err := Open(
   697  		io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
   698  		WithDecompressors(controller),
   699  	)
   700  	if err != nil {
   701  		t.Errorf("failed to parse converted stargz: %v", err)
   702  		return
   703  	}
   704  	digestMapTOC, err := listDigests(io.NewSectionReader(
   705  		bytes.NewReader(sgzData), 0, int64(len(sgzData))),
   706  		controller,
   707  	)
   708  	if err != nil {
   709  		t.Fatalf("failed to list digest: %v", err)
   710  	}
   711  	found := make(map[string]bool)
   712  	for id := range dgstMap {
   713  		found[id] = false
   714  	}
   715  	zr, err := controller.Reader(bytes.NewReader(sgzData))
   716  	if err != nil {
   717  		t.Fatalf("failed to decompress converted stargz: %v", err)
   718  	}
   719  	defer zr.Close()
   720  	tr := tar.NewReader(zr)
   721  	for {
   722  		h, err := tr.Next()
   723  		if err != nil {
   724  			if err != io.EOF {
   725  				t.Errorf("failed to read tar entry: %v", err)
   726  				return
   727  			}
   728  			break
   729  		}
   730  		if h.Name == TOCTarName {
   731  			// Check the digest of TOC JSON based on the actual contents
   732  			// It's sure that TOC JSON exists in this archive because
   733  			// Open succeeded.
   734  			dgstr := digest.Canonical.Digester()
   735  			if _, err := io.Copy(dgstr.Hash(), tr); err != nil {
   736  				t.Fatalf("failed to calculate digest of TOC JSON: %v",
   737  					err)
   738  			}
   739  			if dgstr.Digest() != tocDigest {
   740  				t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest())
   741  			}
   742  			continue
   743  		}
   744  		if _, ok := sgz.Lookup(h.Name); !ok {
   745  			t.Errorf("lost stargz entry %q in the converted TOC", h.Name)
   746  			return
   747  		}
   748  		var n int64
   749  		for n < h.Size {
   750  			ce, ok := sgz.ChunkEntryForOffset(h.Name, n)
   751  			if !ok {
   752  				t.Errorf("lost chunk %q(offset=%d) in the converted TOC",
   753  					h.Name, n)
   754  				return
   755  			}
   756  
   757  			// Get the original digest to make sure the file contents are kept unchanged
   758  			// from the original tar, during the whole conversion steps.
   759  			id := chunkID(h.Name, n, ce.ChunkSize)
   760  			want, ok := dgstMap[id]
   761  			if !ok {
   762  				t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v",
   763  					h.Name, n, ce.ChunkSize, dgstMap)
   764  				return
   765  			}
   766  			found[id] = true
   767  
   768  			// Check the file contents
   769  			dgstr := digest.Canonical.Digester()
   770  			if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil {
   771  				t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)",
   772  					h.Name, n, ce.ChunkSize)
   773  			}
   774  			if want != dgstr.Digest() {
   775  				t.Errorf("Invalid contents in converted stargz %q: %q; want %q",
   776  					h.Name, dgstr.Digest(), want)
   777  				return
   778  			}
   779  
   780  			// Check the digest stored in TOC JSON
   781  			dgstTOC, ok := digestMapTOC[ce.Offset]
   782  			if !ok {
   783  				t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered",
   784  					h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset)
   785  			}
   786  			if want != dgstTOC {
   787  				t.Errorf("Invalid digest in TOCEntry %q: %q; want %q",
   788  					h.Name, dgstTOC, want)
   789  				return
   790  			}
   791  
   792  			n += ce.ChunkSize
   793  		}
   794  	}
   795  
   796  	for id, ok := range found {
   797  		if !ok {
   798  			t.Errorf("required chunk %q not found in the converted stargz: %v", id, found)
   799  		}
   800  	}
   801  }
   802  
   803  // checkVerifyTOC checks the verification works for the TOC JSON of the passed
   804  // stargz. It walks all entries in the stargz and checks the verifications for
   805  // all chunks work.
   806  func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
   807  	sgz, err := Open(
   808  		io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
   809  		WithDecompressors(controller),
   810  	)
   811  	if err != nil {
   812  		t.Errorf("failed to parse converted stargz: %v", err)
   813  		return
   814  	}
   815  	ev, err := sgz.VerifyTOC(tocDigest)
   816  	if err != nil {
   817  		t.Errorf("failed to verify stargz: %v", err)
   818  		return
   819  	}
   820  
   821  	found := make(map[string]bool)
   822  	for id := range dgstMap {
   823  		found[id] = false
   824  	}
   825  	zr, err := controller.Reader(bytes.NewReader(sgzData))
   826  	if err != nil {
   827  		t.Fatalf("failed to decompress converted stargz: %v", err)
   828  	}
   829  	defer zr.Close()
   830  	tr := tar.NewReader(zr)
   831  	for {
   832  		h, err := tr.Next()
   833  		if err != nil {
   834  			if err != io.EOF {
   835  				t.Errorf("failed to read tar entry: %v", err)
   836  				return
   837  			}
   838  			break
   839  		}
   840  		if h.Name == TOCTarName {
   841  			continue
   842  		}
   843  		if _, ok := sgz.Lookup(h.Name); !ok {
   844  			t.Errorf("lost stargz entry %q in the converted TOC", h.Name)
   845  			return
   846  		}
   847  		var n int64
   848  		for n < h.Size {
   849  			ce, ok := sgz.ChunkEntryForOffset(h.Name, n)
   850  			if !ok {
   851  				t.Errorf("lost chunk %q(offset=%d) in the converted TOC",
   852  					h.Name, n)
   853  				return
   854  			}
   855  
   856  			v, err := ev.Verifier(ce)
   857  			if err != nil {
   858  				t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n)
   859  			}
   860  
   861  			found[chunkID(h.Name, n, ce.ChunkSize)] = true
   862  
   863  			// Check the file contents
   864  			if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil {
   865  				t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)",
   866  					h.Name, n, ce.ChunkSize)
   867  			}
   868  			if !v.Verified() {
   869  				t.Errorf("Invalid contents in converted stargz %q (should be succeeded)",
   870  					h.Name)
   871  				return
   872  			}
   873  			n += ce.ChunkSize
   874  		}
   875  	}
   876  
   877  	for id, ok := range found {
   878  		if !ok {
   879  			t.Errorf("required chunk %q not found in the converted stargz: %v", id, found)
   880  		}
   881  	}
   882  }
   883  
   884  // checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be
   885  // detected during the verification and the verification returns an error.
   886  func checkVerifyInvalidTOCEntryFail(filename string) check {
   887  	return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
   888  		funcs := map[string]rewriteFunc{
   889  			"lost digest in a entry": func(t *testing.T, toc *JTOC, sgz *io.SectionReader) {
   890  				var found bool
   891  				for _, e := range toc.Entries {
   892  					if cleanEntryName(e.Name) == filename {
   893  						if e.Type != "reg" && e.Type != "chunk" {
   894  							t.Fatalf("entry %q to break must be regfile or chunk", filename)
   895  						}
   896  						if e.ChunkDigest == "" {
   897  							t.Fatalf("entry %q is already invalid", filename)
   898  						}
   899  						e.ChunkDigest = ""
   900  						found = true
   901  					}
   902  				}
   903  				if !found {
   904  					t.Fatalf("rewrite target not found")
   905  				}
   906  			},
   907  			"duplicated entry offset": func(t *testing.T, toc *JTOC, sgz *io.SectionReader) {
   908  				var (
   909  					sampleEntry *TOCEntry
   910  					targetEntry *TOCEntry
   911  				)
   912  				for _, e := range toc.Entries {
   913  					if e.Type == "reg" || e.Type == "chunk" {
   914  						if cleanEntryName(e.Name) == filename {
   915  							targetEntry = e
   916  						} else {
   917  							sampleEntry = e
   918  						}
   919  					}
   920  				}
   921  				if sampleEntry == nil {
   922  					t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target")
   923  				}
   924  				if targetEntry == nil {
   925  					t.Fatalf("rewrite target not found")
   926  				}
   927  				targetEntry.Offset = sampleEntry.Offset
   928  			},
   929  		}
   930  
   931  		for name, rFunc := range funcs {
   932  			t.Run(name, func(t *testing.T) {
   933  				newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, controller)
   934  				buf := new(bytes.Buffer)
   935  				if _, err := io.Copy(buf, newSgz); err != nil {
   936  					t.Fatalf("failed to get converted stargz")
   937  				}
   938  				isgz := buf.Bytes()
   939  
   940  				sgz, err := Open(
   941  					io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz))),
   942  					WithDecompressors(controller),
   943  				)
   944  				if err != nil {
   945  					t.Fatalf("failed to parse converted stargz: %v", err)
   946  					return
   947  				}
   948  				_, err = sgz.VerifyTOC(newTocDigest)
   949  				if err == nil {
   950  					t.Errorf("must fail for invalid TOC")
   951  					return
   952  				}
   953  			})
   954  		}
   955  	}
   956  }
   957  
   958  // checkVerifyInvalidStargzFail checks if the verification detects that the
   959  // given stargz file doesn't match to the expected digest and returns error.
   960  func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check {
   961  	return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
   962  		cl := newController()
   963  		rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(cl))
   964  		if err != nil {
   965  			t.Fatalf("failed to convert stargz: %v", err)
   966  		}
   967  		defer rc.Close()
   968  		buf := new(bytes.Buffer)
   969  		if _, err := io.Copy(buf, rc); err != nil {
   970  			t.Fatalf("failed to copy built stargz blob: %v", err)
   971  		}
   972  		mStargz := buf.Bytes()
   973  
   974  		sgz, err := Open(
   975  			io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))),
   976  			WithDecompressors(cl),
   977  		)
   978  		if err != nil {
   979  			t.Fatalf("failed to parse converted stargz: %v", err)
   980  			return
   981  		}
   982  		_, err = sgz.VerifyTOC(tocDigest)
   983  		if err == nil {
   984  			t.Errorf("must fail for invalid TOC")
   985  			return
   986  		}
   987  	}
   988  }
   989  
   990  // checkVerifyBrokenContentFail checks if the verifier detects broken contents
   991  // that doesn't match to the expected digest and returns error.
   992  func checkVerifyBrokenContentFail(filename string) check {
   993  	return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
   994  		// Parse stargz file
   995  		sgz, err := Open(
   996  			io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
   997  			WithDecompressors(controller),
   998  		)
   999  		if err != nil {
  1000  			t.Fatalf("failed to parse converted stargz: %v", err)
  1001  			return
  1002  		}
  1003  		ev, err := sgz.VerifyTOC(tocDigest)
  1004  		if err != nil {
  1005  			t.Fatalf("failed to verify stargz: %v", err)
  1006  			return
  1007  		}
  1008  
  1009  		// Open the target file
  1010  		sr, err := sgz.OpenFile(filename)
  1011  		if err != nil {
  1012  			t.Fatalf("failed to open file %q", filename)
  1013  		}
  1014  		ce, ok := sgz.ChunkEntryForOffset(filename, 0)
  1015  		if !ok {
  1016  			t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0)
  1017  			return
  1018  		}
  1019  		if ce.ChunkSize == 0 {
  1020  			t.Fatalf("file mustn't be empty")
  1021  			return
  1022  		}
  1023  		data := make([]byte, ce.ChunkSize)
  1024  		if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil {
  1025  			t.Errorf("failed to get data of a chunk of %q(offset=%q)",
  1026  				filename, ce.ChunkOffset)
  1027  		}
  1028  
  1029  		// Check the broken chunk (must fail)
  1030  		v, err := ev.Verifier(ce)
  1031  		if err != nil {
  1032  			t.Fatalf("failed to get verifier for %q", filename)
  1033  		}
  1034  		broken := append([]byte{^data[0]}, data[1:]...)
  1035  		if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil {
  1036  			t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)",
  1037  				filename, ce.ChunkOffset, ce.ChunkSize)
  1038  		}
  1039  		if v.Verified() {
  1040  			t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)",
  1041  				filename, data, broken)
  1042  		}
  1043  	}
  1044  }
  1045  
  1046  func chunkID(name string, offset, size int64) string {
  1047  	return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size)
  1048  }
  1049  
  1050  type rewriteFunc func(t *testing.T, toc *JTOC, sgz *io.SectionReader)
  1051  
  1052  func rewriteTOCJSON(t *testing.T, sgz *io.SectionReader, rewrite rewriteFunc, controller TestingController) (newSgz io.Reader, tocDigest digest.Digest) {
  1053  	decodedJTOC, jtocOffset, err := parseStargz(sgz, controller)
  1054  	if err != nil {
  1055  		t.Fatalf("failed to extract TOC JSON: %v", err)
  1056  	}
  1057  
  1058  	rewrite(t, decodedJTOC, sgz)
  1059  
  1060  	tocFooter, tocDigest, err := tocAndFooter(controller, decodedJTOC, jtocOffset)
  1061  	if err != nil {
  1062  		t.Fatalf("failed to create toc and footer: %v", err)
  1063  	}
  1064  
  1065  	// Reconstruct stargz file with the modified TOC JSON
  1066  	if _, err := sgz.Seek(0, io.SeekStart); err != nil {
  1067  		t.Fatalf("failed to reset the seek position of stargz: %v", err)
  1068  	}
  1069  	return io.MultiReader(
  1070  		io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON)
  1071  		tocFooter,                       // Rewritten TOC and footer
  1072  	), tocDigest
  1073  }
  1074  
  1075  func listDigests(sgz *io.SectionReader, controller TestingController) (map[int64]digest.Digest, error) {
  1076  	decodedJTOC, _, err := parseStargz(sgz, controller)
  1077  	if err != nil {
  1078  		return nil, err
  1079  	}
  1080  	digestMap := make(map[int64]digest.Digest)
  1081  	for _, e := range decodedJTOC.Entries {
  1082  		if e.Type == "reg" || e.Type == "chunk" {
  1083  			if e.Type == "reg" && e.Size == 0 {
  1084  				continue // ignores empty file
  1085  			}
  1086  			if e.ChunkDigest == "" {
  1087  				return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON",
  1088  					e.Name, e.Offset)
  1089  			}
  1090  			d, err := digest.Parse(e.ChunkDigest)
  1091  			if err != nil {
  1092  				return nil, err
  1093  			}
  1094  			digestMap[e.Offset] = d
  1095  		}
  1096  	}
  1097  	return digestMap, nil
  1098  }
  1099  
  1100  func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJTOC *JTOC, jtocOffset int64, err error) {
  1101  	fSize := controller.FooterSize()
  1102  	footer := make([]byte, fSize)
  1103  	if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil {
  1104  		return nil, 0, fmt.Errorf("error reading footer: %w", err)
  1105  	}
  1106  	_, tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):])
  1107  	if err != nil {
  1108  		return nil, 0, fmt.Errorf("failed to parse footer: %w", err)
  1109  	}
  1110  
  1111  	// Decode the TOC JSON
  1112  	var tocReader io.Reader
  1113  	if tocOffset >= 0 {
  1114  		tocReader = io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize)
  1115  	}
  1116  	decodedJTOC, _, err = controller.ParseTOC(tocReader)
  1117  	if err != nil {
  1118  		return nil, 0, fmt.Errorf("failed to parse TOC: %w", err)
  1119  	}
  1120  	return decodedJTOC, tocOffset, nil
  1121  }
  1122  
  1123  func testWriteAndOpen(t *testing.T, controllers ...TestingControllerFactory) {
  1124  	const content = "Some contents"
  1125  	invalidUtf8 := "\xff\xfe\xfd"
  1126  
  1127  	xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8}
  1128  	sampleOwner := owner{uid: 50, gid: 100}
  1129  
  1130  	data64KB := randomContents(64000)
  1131  
  1132  	tests := []struct {
  1133  		name         string
  1134  		chunkSize    int
  1135  		minChunkSize int
  1136  		in           []tarEntry
  1137  		want         []stargzCheck
  1138  		wantNumGz    int // expected number of streams
  1139  
  1140  		wantNumGzLossLess  int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz
  1141  		wantFailOnLossLess bool
  1142  		wantTOCVersion     int // default = 1
  1143  	}{
  1144  		{
  1145  			name:      "empty",
  1146  			in:        tarOf(),
  1147  			wantNumGz: 2, // (empty tar) + TOC + footer
  1148  			want: checks(
  1149  				numTOCEntries(0),
  1150  			),
  1151  		},
  1152  		{
  1153  			name: "1dir_1empty_file",
  1154  			in: tarOf(
  1155  				dir("foo/"),
  1156  				file("foo/bar.txt", ""),
  1157  			),
  1158  			wantNumGz: 3, // dir, TOC, footer
  1159  			want: checks(
  1160  				numTOCEntries(2),
  1161  				hasDir("foo/"),
  1162  				hasFileLen("foo/bar.txt", 0),
  1163  				entryHasChildren("foo", "bar.txt"),
  1164  				hasFileDigest("foo/bar.txt", digestFor("")),
  1165  			),
  1166  		},
  1167  		{
  1168  			name: "1dir_1file",
  1169  			in: tarOf(
  1170  				dir("foo/"),
  1171  				file("foo/bar.txt", content, xAttrFile),
  1172  			),
  1173  			wantNumGz: 4, // var dir, foo.txt alone, TOC, footer
  1174  			want: checks(
  1175  				numTOCEntries(2),
  1176  				hasDir("foo/"),
  1177  				hasFileLen("foo/bar.txt", len(content)),
  1178  				hasFileDigest("foo/bar.txt", digestFor(content)),
  1179  				hasFileContentsRange("foo/bar.txt", 0, content),
  1180  				hasFileContentsRange("foo/bar.txt", 1, content[1:]),
  1181  				entryHasChildren("", "foo"),
  1182  				entryHasChildren("foo", "bar.txt"),
  1183  				hasFileXattrs("foo/bar.txt", "foo", "bar"),
  1184  				hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8),
  1185  			),
  1186  		},
  1187  		{
  1188  			name: "2meta_2file",
  1189  			in: tarOf(
  1190  				dir("bar/", sampleOwner),
  1191  				dir("foo/", sampleOwner),
  1192  				file("foo/bar.txt", content, sampleOwner),
  1193  			),
  1194  			wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer
  1195  			want: checks(
  1196  				numTOCEntries(3),
  1197  				hasDir("bar/"),
  1198  				hasDir("foo/"),
  1199  				hasFileLen("foo/bar.txt", len(content)),
  1200  				entryHasChildren("", "bar", "foo"),
  1201  				entryHasChildren("foo", "bar.txt"),
  1202  				hasChunkEntries("foo/bar.txt", 1),
  1203  				hasEntryOwner("bar/", sampleOwner),
  1204  				hasEntryOwner("foo/", sampleOwner),
  1205  				hasEntryOwner("foo/bar.txt", sampleOwner),
  1206  			),
  1207  		},
  1208  		{
  1209  			name: "3dir",
  1210  			in: tarOf(
  1211  				dir("bar/"),
  1212  				dir("foo/"),
  1213  				dir("foo/bar/"),
  1214  			),
  1215  			wantNumGz: 3, // 3 dirs, TOC, footer
  1216  			want: checks(
  1217  				hasDirLinkCount("bar/", 2),
  1218  				hasDirLinkCount("foo/", 3),
  1219  				hasDirLinkCount("foo/bar/", 2),
  1220  			),
  1221  		},
  1222  		{
  1223  			name: "symlink",
  1224  			in: tarOf(
  1225  				dir("foo/"),
  1226  				symlink("foo/bar", "../../x"),
  1227  			),
  1228  			wantNumGz: 3, // metas + TOC + footer
  1229  			want: checks(
  1230  				numTOCEntries(2),
  1231  				hasSymlink("foo/bar", "../../x"),
  1232  				entryHasChildren("", "foo"),
  1233  				entryHasChildren("foo", "bar"),
  1234  			),
  1235  		},
  1236  		{
  1237  			name:      "chunked_file",
  1238  			chunkSize: 4,
  1239  			in: tarOf(
  1240  				dir("foo/"),
  1241  				file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"),
  1242  			),
  1243  			wantNumGz: 9, // dir + big.txt(6 chunks) + TOC + footer
  1244  			want: checks(
  1245  				numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file
  1246  				hasDir("foo/"),
  1247  				hasFileLen("foo/big.txt", len("This is such a big file")),
  1248  				hasFileDigest("foo/big.txt", digestFor("This is such a big file")),
  1249  				hasFileContentsRange("foo/big.txt", 0, "This is such a big file"),
  1250  				hasFileContentsRange("foo/big.txt", 1, "his is such a big file"),
  1251  				hasFileContentsRange("foo/big.txt", 2, "is is such a big file"),
  1252  				hasFileContentsRange("foo/big.txt", 3, "s is such a big file"),
  1253  				hasFileContentsRange("foo/big.txt", 4, " is such a big file"),
  1254  				hasFileContentsRange("foo/big.txt", 5, "is such a big file"),
  1255  				hasFileContentsRange("foo/big.txt", 6, "s such a big file"),
  1256  				hasFileContentsRange("foo/big.txt", 7, " such a big file"),
  1257  				hasFileContentsRange("foo/big.txt", 8, "such a big file"),
  1258  				hasFileContentsRange("foo/big.txt", 9, "uch a big file"),
  1259  				hasFileContentsRange("foo/big.txt", 10, "ch a big file"),
  1260  				hasFileContentsRange("foo/big.txt", 11, "h a big file"),
  1261  				hasFileContentsRange("foo/big.txt", 12, " a big file"),
  1262  				hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""),
  1263  				hasChunkEntries("foo/big.txt", 6),
  1264  			),
  1265  		},
  1266  		{
  1267  			name: "recursive",
  1268  			in: tarOf(
  1269  				dir("/", sampleOwner),
  1270  				dir("bar/", sampleOwner),
  1271  				dir("foo/", sampleOwner),
  1272  				file("foo/bar.txt", content, sampleOwner),
  1273  			),
  1274  			wantNumGz: 4, // dirs, bar.txt alone, TOC, footer
  1275  			want: checks(
  1276  				maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt"
  1277  			),
  1278  		},
  1279  		{
  1280  			name: "block_char_fifo",
  1281  			in: tarOf(
  1282  				tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  1283  					return w.WriteHeader(&tar.Header{
  1284  						Name:     prefix + "b",
  1285  						Typeflag: tar.TypeBlock,
  1286  						Devmajor: 123,
  1287  						Devminor: 456,
  1288  						Format:   format,
  1289  					})
  1290  				}),
  1291  				tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  1292  					return w.WriteHeader(&tar.Header{
  1293  						Name:     prefix + "c",
  1294  						Typeflag: tar.TypeChar,
  1295  						Devmajor: 111,
  1296  						Devminor: 222,
  1297  						Format:   format,
  1298  					})
  1299  				}),
  1300  				tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  1301  					return w.WriteHeader(&tar.Header{
  1302  						Name:     prefix + "f",
  1303  						Typeflag: tar.TypeFifo,
  1304  						Format:   format,
  1305  					})
  1306  				}),
  1307  			),
  1308  			wantNumGz: 3,
  1309  			want: checks(
  1310  				lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}),
  1311  				lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}),
  1312  				lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}),
  1313  			),
  1314  		},
  1315  		{
  1316  			name: "modes",
  1317  			in: tarOf(
  1318  				dir("foo1/", 0755|os.ModeDir|os.ModeSetgid),
  1319  				file("foo1/bar1", content, 0700|os.ModeSetuid),
  1320  				file("foo1/bar2", content, 0755|os.ModeSetgid),
  1321  				dir("foo2/", 0755|os.ModeDir|os.ModeSticky),
  1322  				file("foo2/bar3", content, 0755|os.ModeSticky),
  1323  				dir("foo3/", 0755|os.ModeDir),
  1324  				file("foo3/bar4", content, os.FileMode(0700)),
  1325  				file("foo3/bar5", content, os.FileMode(0755)),
  1326  			),
  1327  			wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer
  1328  			want: checks(
  1329  				hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid),
  1330  				hasMode("foo1/bar1", 0700|os.ModeSetuid),
  1331  				hasMode("foo1/bar2", 0755|os.ModeSetgid),
  1332  				hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky),
  1333  				hasMode("foo2/bar3", 0755|os.ModeSticky),
  1334  				hasMode("foo3/", 0755|os.ModeDir),
  1335  				hasMode("foo3/bar4", os.FileMode(0700)),
  1336  				hasMode("foo3/bar5", os.FileMode(0755)),
  1337  			),
  1338  		},
  1339  		{
  1340  			name: "lossy",
  1341  			in: tarOf(
  1342  				dir("bar/", sampleOwner),
  1343  				dir("foo/", sampleOwner),
  1344  				file("foo/bar.txt", content, sampleOwner),
  1345  				file(TOCTarName, "dummy"), // ignored by the writer. (lossless write returns error)
  1346  			),
  1347  			wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer
  1348  			want: checks(
  1349  				numTOCEntries(3),
  1350  				hasDir("bar/"),
  1351  				hasDir("foo/"),
  1352  				hasFileLen("foo/bar.txt", len(content)),
  1353  				entryHasChildren("", "bar", "foo"),
  1354  				entryHasChildren("foo", "bar.txt"),
  1355  				hasChunkEntries("foo/bar.txt", 1),
  1356  				hasEntryOwner("bar/", sampleOwner),
  1357  				hasEntryOwner("foo/", sampleOwner),
  1358  				hasEntryOwner("foo/bar.txt", sampleOwner),
  1359  			),
  1360  			wantFailOnLossLess: true,
  1361  		},
  1362  		{
  1363  			name: "hardlink should be replaced to the destination entry",
  1364  			in: tarOf(
  1365  				dir("foo/"),
  1366  				file("foo/foo1", "test"),
  1367  				link("foolink", "foo/foo1"),
  1368  			),
  1369  			wantNumGz: 4, // dir, foo1 + link, TOC, footer
  1370  			want: checks(
  1371  				mustSameEntry("foo/foo1", "foolink"),
  1372  			),
  1373  		},
  1374  		{
  1375  			name:         "several_files_in_chunk",
  1376  			minChunkSize: 8000,
  1377  			in: tarOf(
  1378  				dir("foo/"),
  1379  				file("foo/foo1", data64KB),
  1380  				file("foo2", "bb"),
  1381  				file("foo22", "ccc"),
  1382  				dir("bar/"),
  1383  				file("bar/bar.txt", "aaa"),
  1384  				file("foo3", data64KB),
  1385  			),
  1386  			// NOTE: we assume that the compressed "data64KB" is still larger than 8KB
  1387  			wantNumGz: 4, // dir+foo1, foo2+foo22+dir+bar.txt+foo3, TOC, footer
  1388  			want: checks(
  1389  				numTOCEntries(7), // dir, foo1, foo2, foo22, dir, bar.txt, foo3
  1390  				hasDir("foo/"),
  1391  				hasDir("bar/"),
  1392  				hasFileLen("foo/foo1", len(data64KB)),
  1393  				hasFileLen("foo2", len("bb")),
  1394  				hasFileLen("foo22", len("ccc")),
  1395  				hasFileLen("bar/bar.txt", len("aaa")),
  1396  				hasFileLen("foo3", len(data64KB)),
  1397  				hasFileDigest("foo/foo1", digestFor(data64KB)),
  1398  				hasFileDigest("foo2", digestFor("bb")),
  1399  				hasFileDigest("foo22", digestFor("ccc")),
  1400  				hasFileDigest("bar/bar.txt", digestFor("aaa")),
  1401  				hasFileDigest("foo3", digestFor(data64KB)),
  1402  				hasFileContentsWithPreRead("foo22", 0, "ccc", chunkInfo{"foo2", "bb"}, chunkInfo{"bar/bar.txt", "aaa"}, chunkInfo{"foo3", data64KB}),
  1403  				hasFileContentsRange("foo/foo1", 0, data64KB),
  1404  				hasFileContentsRange("foo2", 0, "bb"),
  1405  				hasFileContentsRange("foo2", 1, "b"),
  1406  				hasFileContentsRange("foo22", 0, "ccc"),
  1407  				hasFileContentsRange("foo22", 1, "cc"),
  1408  				hasFileContentsRange("foo22", 2, "c"),
  1409  				hasFileContentsRange("bar/bar.txt", 0, "aaa"),
  1410  				hasFileContentsRange("bar/bar.txt", 1, "aa"),
  1411  				hasFileContentsRange("bar/bar.txt", 2, "a"),
  1412  				hasFileContentsRange("foo3", 0, data64KB),
  1413  				hasFileContentsRange("foo3", 1, data64KB[1:]),
  1414  				hasFileContentsRange("foo3", 2, data64KB[2:]),
  1415  				hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]),
  1416  				hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
  1417  			),
  1418  		},
  1419  		{
  1420  			name:         "several_files_in_chunk_chunked",
  1421  			minChunkSize: 8000,
  1422  			chunkSize:    32000,
  1423  			in: tarOf(
  1424  				dir("foo/"),
  1425  				file("foo/foo1", data64KB),
  1426  				file("foo2", "bb"),
  1427  				dir("bar/"),
  1428  				file("foo3", data64KB),
  1429  			),
  1430  			// NOTE: we assume that the compressed chunk of "data64KB" is still larger than 8KB
  1431  			wantNumGz: 6, // dir+foo1(1), foo1(2), foo2+dir+foo3(1), foo3(2), TOC, footer
  1432  			want: checks(
  1433  				numTOCEntries(7), // dir, foo1(2 chunks), foo2, dir, foo3(2 chunks)
  1434  				hasDir("foo/"),
  1435  				hasDir("bar/"),
  1436  				hasFileLen("foo/foo1", len(data64KB)),
  1437  				hasFileLen("foo2", len("bb")),
  1438  				hasFileLen("foo3", len(data64KB)),
  1439  				hasFileDigest("foo/foo1", digestFor(data64KB)),
  1440  				hasFileDigest("foo2", digestFor("bb")),
  1441  				hasFileDigest("foo3", digestFor(data64KB)),
  1442  				hasFileContentsWithPreRead("foo2", 0, "bb", chunkInfo{"foo3", data64KB[:32000]}),
  1443  				hasFileContentsRange("foo/foo1", 0, data64KB),
  1444  				hasFileContentsRange("foo/foo1", 1, data64KB[1:]),
  1445  				hasFileContentsRange("foo/foo1", 2, data64KB[2:]),
  1446  				hasFileContentsRange("foo/foo1", len(data64KB)/2, data64KB[len(data64KB)/2:]),
  1447  				hasFileContentsRange("foo/foo1", len(data64KB)-1, data64KB[len(data64KB)-1:]),
  1448  				hasFileContentsRange("foo2", 0, "bb"),
  1449  				hasFileContentsRange("foo2", 1, "b"),
  1450  				hasFileContentsRange("foo3", 0, data64KB),
  1451  				hasFileContentsRange("foo3", 1, data64KB[1:]),
  1452  				hasFileContentsRange("foo3", 2, data64KB[2:]),
  1453  				hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]),
  1454  				hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
  1455  			),
  1456  		},
  1457  	}
  1458  
  1459  	for _, tt := range tests {
  1460  		for _, newCL := range controllers {
  1461  			newCL := newCL
  1462  			for _, prefix := range allowedPrefix {
  1463  				prefix := prefix
  1464  				for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
  1465  					srcTarFormat := srcTarFormat
  1466  					for _, lossless := range []bool{true, false} {
  1467  						t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *testing.T) {
  1468  							var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat)
  1469  							origTarDgstr := digest.Canonical.Digester()
  1470  							tr = io.TeeReader(tr, origTarDgstr.Hash())
  1471  							var stargzBuf bytes.Buffer
  1472  							cl1 := newCL()
  1473  							w := NewWriterWithCompressor(&stargzBuf, cl1)
  1474  							w.ChunkSize = tt.chunkSize
  1475  							w.MinChunkSize = tt.minChunkSize
  1476  							if lossless {
  1477  								err := w.AppendTarLossLess(tr)
  1478  								if tt.wantFailOnLossLess {
  1479  									if err != nil {
  1480  										return // expected to fail
  1481  									}
  1482  									t.Fatalf("Append wanted to fail on lossless")
  1483  								}
  1484  								if err != nil {
  1485  									t.Fatalf("Append(lossless): %v", err)
  1486  								}
  1487  							} else {
  1488  								if err := w.AppendTar(tr); err != nil {
  1489  									t.Fatalf("Append: %v", err)
  1490  								}
  1491  							}
  1492  							if _, err := w.Close(); err != nil {
  1493  								t.Fatalf("Writer.Close: %v", err)
  1494  							}
  1495  							b := stargzBuf.Bytes()
  1496  
  1497  							if lossless {
  1498  								// Check if the result blob reserves original tar metadata
  1499  								rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl1)
  1500  								if err != nil {
  1501  									t.Errorf("failed to decompress blob: %v", err)
  1502  									return
  1503  								}
  1504  								defer rc.Close()
  1505  								resultDgstr := digest.Canonical.Digester()
  1506  								if _, err := io.Copy(resultDgstr.Hash(), rc); err != nil {
  1507  									t.Errorf("failed to read result decompressed blob: %v", err)
  1508  									return
  1509  								}
  1510  								if resultDgstr.Digest() != origTarDgstr.Digest() {
  1511  									t.Errorf("lossy compression occurred: digest=%v; want %v",
  1512  										resultDgstr.Digest(), origTarDgstr.Digest())
  1513  									return
  1514  								}
  1515  							}
  1516  
  1517  							diffID := w.DiffID()
  1518  							wantDiffID := cl1.DiffIDOf(t, b)
  1519  							if diffID != wantDiffID {
  1520  								t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
  1521  							}
  1522  
  1523  							telemetry, checkCalled := newCalledTelemetry()
  1524  							sr := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b)))
  1525  							r, err := Open(
  1526  								sr,
  1527  								WithDecompressors(cl1),
  1528  								WithTelemetry(telemetry),
  1529  							)
  1530  							if err != nil {
  1531  								t.Fatalf("stargz.Open: %v", err)
  1532  							}
  1533  							wantTOCVersion := 1
  1534  							if tt.wantTOCVersion > 0 {
  1535  								wantTOCVersion = tt.wantTOCVersion
  1536  							}
  1537  							if r.toc.Version != wantTOCVersion {
  1538  								t.Fatalf("invalid TOC Version %d; wanted %d", r.toc.Version, wantTOCVersion)
  1539  							}
  1540  
  1541  							footerSize := cl1.FooterSize()
  1542  							footerOffset := sr.Size() - footerSize
  1543  							footer := make([]byte, footerSize)
  1544  							if _, err := sr.ReadAt(footer, footerOffset); err != nil {
  1545  								t.Errorf("failed to read footer: %v", err)
  1546  							}
  1547  							_, tocOffset, _, err := cl1.ParseFooter(footer)
  1548  							if err != nil {
  1549  								t.Errorf("failed to parse footer: %v", err)
  1550  							}
  1551  							if err := checkCalled(tocOffset >= 0); err != nil {
  1552  								t.Errorf("telemetry failure: %v", err)
  1553  							}
  1554  
  1555  							wantNumGz := tt.wantNumGz
  1556  							if lossless && tt.wantNumGzLossLess > 0 {
  1557  								wantNumGz = tt.wantNumGzLossLess
  1558  							}
  1559  							streamOffsets := []int64{0}
  1560  							prevOffset := int64(-1)
  1561  							streams := 0
  1562  							for _, e := range r.toc.Entries {
  1563  								if e.Offset > prevOffset {
  1564  									streamOffsets = append(streamOffsets, e.Offset)
  1565  									prevOffset = e.Offset
  1566  									streams++
  1567  								}
  1568  							}
  1569  							streams++ // TOC
  1570  							if tocOffset >= 0 {
  1571  								// toc is in the blob
  1572  								streamOffsets = append(streamOffsets, tocOffset)
  1573  							}
  1574  							streams++ // footer
  1575  							streamOffsets = append(streamOffsets, footerOffset)
  1576  							if streams != wantNumGz {
  1577  								t.Errorf("number of streams in TOC = %d; want %d", streams, wantNumGz)
  1578  							}
  1579  
  1580  							t.Logf("testing streams: %+v", streamOffsets)
  1581  							cl1.TestStreams(t, b, streamOffsets)
  1582  
  1583  							for _, want := range tt.want {
  1584  								want.check(t, r)
  1585  							}
  1586  						})
  1587  					}
  1588  				}
  1589  			}
  1590  		}
  1591  	}
  1592  }
  1593  
  1594  type chunkInfo struct {
  1595  	name string
  1596  	data string
  1597  }
  1598  
  1599  func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) error) {
  1600  	var getFooterLatencyCalled bool
  1601  	var getTocLatencyCalled bool
  1602  	var deserializeTocLatencyCalled bool
  1603  	return &Telemetry{
  1604  			func(time.Time) { getFooterLatencyCalled = true },
  1605  			func(time.Time) { getTocLatencyCalled = true },
  1606  			func(time.Time) { deserializeTocLatencyCalled = true },
  1607  		}, func(needsGetTOC bool) error {
  1608  			var allErr []error
  1609  			if !getFooterLatencyCalled {
  1610  				allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
  1611  			}
  1612  			if needsGetTOC {
  1613  				if !getTocLatencyCalled {
  1614  					allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
  1615  				}
  1616  			}
  1617  			if !deserializeTocLatencyCalled {
  1618  				allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
  1619  			}
  1620  			return errorutil.Aggregate(allErr)
  1621  		}
  1622  }
  1623  
  1624  func digestFor(content string) string {
  1625  	sum := sha256.Sum256([]byte(content))
  1626  	return fmt.Sprintf("sha256:%x", sum)
  1627  }
  1628  
  1629  type numTOCEntries int
  1630  
  1631  func (n numTOCEntries) check(t *testing.T, r *Reader) {
  1632  	if r.toc == nil {
  1633  		t.Fatal("nil TOC")
  1634  	}
  1635  	if got, want := len(r.toc.Entries), int(n); got != want {
  1636  		t.Errorf("got %d TOC entries; want %d", got, want)
  1637  	}
  1638  	t.Logf("got TOC entries:")
  1639  	for i, ent := range r.toc.Entries {
  1640  		entj, _ := json.Marshal(ent)
  1641  		t.Logf("  [%d]: %s\n", i, entj)
  1642  	}
  1643  	if t.Failed() {
  1644  		t.FailNow()
  1645  	}
  1646  }
  1647  
  1648  func checks(s ...stargzCheck) []stargzCheck { return s }
  1649  
  1650  type stargzCheck interface {
  1651  	check(t *testing.T, r *Reader)
  1652  }
  1653  
  1654  type stargzCheckFn func(*testing.T, *Reader)
  1655  
  1656  func (f stargzCheckFn) check(t *testing.T, r *Reader) { f(t, r) }
  1657  
  1658  func maxDepth(max int) stargzCheck {
  1659  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1660  		e, ok := r.Lookup("")
  1661  		if !ok {
  1662  			t.Fatal("root directory not found")
  1663  		}
  1664  		d, err := getMaxDepth(t, e, 0, 10*max)
  1665  		if err != nil {
  1666  			t.Errorf("failed to get max depth (wanted %d): %v", max, err)
  1667  			return
  1668  		}
  1669  		if d != max {
  1670  			t.Errorf("invalid depth %d; want %d", d, max)
  1671  			return
  1672  		}
  1673  	})
  1674  }
  1675  
  1676  func getMaxDepth(t *testing.T, e *TOCEntry, current, limit int) (max int, rErr error) {
  1677  	if current > limit {
  1678  		return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d",
  1679  			current, limit)
  1680  	}
  1681  	max = current
  1682  	e.ForeachChild(func(baseName string, ent *TOCEntry) bool {
  1683  		t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name)
  1684  		d, err := getMaxDepth(t, ent, current+1, limit)
  1685  		if err != nil {
  1686  			rErr = err
  1687  			return false
  1688  		}
  1689  		if d > max {
  1690  			max = d
  1691  		}
  1692  		return true
  1693  	})
  1694  	return
  1695  }
  1696  
  1697  func hasFileLen(file string, wantLen int) stargzCheck {
  1698  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1699  		for _, ent := range r.toc.Entries {
  1700  			if ent.Name == file {
  1701  				if ent.Type != "reg" {
  1702  					t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type)
  1703  				} else if ent.Size != int64(wantLen) {
  1704  					t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen)
  1705  				}
  1706  				return
  1707  			}
  1708  		}
  1709  		t.Errorf("file %q not found", file)
  1710  	})
  1711  }
  1712  
  1713  func hasFileXattrs(file, name, value string) stargzCheck {
  1714  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1715  		for _, ent := range r.toc.Entries {
  1716  			if ent.Name == file {
  1717  				if ent.Type != "reg" {
  1718  					t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type)
  1719  				}
  1720  				if ent.Xattrs == nil {
  1721  					t.Errorf("file %q has no xattrs", file)
  1722  					return
  1723  				}
  1724  				valueFound, found := ent.Xattrs[name]
  1725  				if !found {
  1726  					t.Errorf("file %q has no xattr %q", file, name)
  1727  					return
  1728  				}
  1729  				if string(valueFound) != value {
  1730  					t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value)
  1731  				}
  1732  
  1733  				return
  1734  			}
  1735  		}
  1736  		t.Errorf("file %q not found", file)
  1737  	})
  1738  }
  1739  
  1740  func hasFileDigest(file string, digest string) stargzCheck {
  1741  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1742  		ent, ok := r.Lookup(file)
  1743  		if !ok {
  1744  			t.Fatalf("didn't find TOCEntry for file %q", file)
  1745  		}
  1746  		if ent.Digest != digest {
  1747  			t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest)
  1748  		}
  1749  	})
  1750  }
  1751  
  1752  func hasFileContentsWithPreRead(file string, offset int, want string, extra ...chunkInfo) stargzCheck {
  1753  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1754  		extraMap := make(map[string]chunkInfo)
  1755  		for _, e := range extra {
  1756  			extraMap[e.name] = e
  1757  		}
  1758  		var extraNames []string
  1759  		for n := range extraMap {
  1760  			extraNames = append(extraNames, n)
  1761  		}
  1762  		f, err := r.OpenFileWithPreReader(file, func(e *TOCEntry, cr io.Reader) error {
  1763  			t.Logf("On %q: got preread of %q", file, e.Name)
  1764  			ex, ok := extraMap[e.Name]
  1765  			if !ok {
  1766  				t.Fatalf("fail on %q: unexpected entry %q: %+v, %+v", file, e.Name, e, extraNames)
  1767  			}
  1768  			got, err := io.ReadAll(cr)
  1769  			if err != nil {
  1770  				t.Fatalf("fail on %q: failed to read %q: %v", file, e.Name, err)
  1771  			}
  1772  			if ex.data != string(got) {
  1773  				t.Fatalf("fail on %q: unexpected contents of %q: len=%d; want=%d", file, e.Name, len(got), len(ex.data))
  1774  			}
  1775  			delete(extraMap, e.Name)
  1776  			return nil
  1777  		})
  1778  		if err != nil {
  1779  			t.Fatal(err)
  1780  		}
  1781  		got := make([]byte, len(want))
  1782  		n, err := f.ReadAt(got, int64(offset))
  1783  		if err != nil {
  1784  			t.Fatalf("ReadAt(len %d, offset %d, size %d) = %v, %v", len(got), offset, f.Size(), n, err)
  1785  		}
  1786  		if string(got) != want {
  1787  			t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want)))
  1788  		}
  1789  		if len(extraMap) != 0 {
  1790  			var exNames []string
  1791  			for _, ex := range extraMap {
  1792  				exNames = append(exNames, ex.name)
  1793  			}
  1794  			t.Fatalf("fail on %q: some entries aren't read: %+v", file, exNames)
  1795  		}
  1796  	})
  1797  }
  1798  
  1799  func hasFileContentsRange(file string, offset int, want string) stargzCheck {
  1800  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1801  		f, err := r.OpenFile(file)
  1802  		if err != nil {
  1803  			t.Fatal(err)
  1804  		}
  1805  		got := make([]byte, len(want))
  1806  		n, err := f.ReadAt(got, int64(offset))
  1807  		if err != nil {
  1808  			t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err)
  1809  		}
  1810  		if string(got) != want {
  1811  			t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want)))
  1812  		}
  1813  	})
  1814  }
  1815  
  1816  func hasChunkEntries(file string, wantChunks int) stargzCheck {
  1817  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1818  		ent, ok := r.Lookup(file)
  1819  		if !ok {
  1820  			t.Fatalf("no file for %q", file)
  1821  		}
  1822  		if ent.Type != "reg" {
  1823  			t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type)
  1824  		}
  1825  		chunks := r.getChunks(ent)
  1826  		if len(chunks) != wantChunks {
  1827  			t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks)
  1828  			return
  1829  		}
  1830  		f := chunks[0]
  1831  
  1832  		var gotChunks []*TOCEntry
  1833  		var last *TOCEntry
  1834  		for off := int64(0); off < f.Size; off++ {
  1835  			e, ok := r.ChunkEntryForOffset(file, off)
  1836  			if !ok {
  1837  				t.Errorf("no ChunkEntryForOffset at %d", off)
  1838  				return
  1839  			}
  1840  			if last != e {
  1841  				gotChunks = append(gotChunks, e)
  1842  				last = e
  1843  			}
  1844  		}
  1845  		if !reflect.DeepEqual(chunks, gotChunks) {
  1846  			t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks)
  1847  		}
  1848  
  1849  		// And verify the NextOffset
  1850  		for i := 0; i < len(gotChunks)-1; i++ {
  1851  			ci := gotChunks[i]
  1852  			cnext := gotChunks[i+1]
  1853  			if ci.NextOffset() != cnext.Offset {
  1854  				t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset)
  1855  			}
  1856  		}
  1857  	})
  1858  }
  1859  
  1860  func entryHasChildren(dir string, want ...string) stargzCheck {
  1861  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1862  		want := append([]string(nil), want...)
  1863  		var got []string
  1864  		ent, ok := r.Lookup(dir)
  1865  		if !ok {
  1866  			t.Fatalf("didn't find TOCEntry for dir node %q", dir)
  1867  		}
  1868  		for baseName := range ent.children {
  1869  			got = append(got, baseName)
  1870  		}
  1871  		sort.Strings(got)
  1872  		sort.Strings(want)
  1873  		if !reflect.DeepEqual(got, want) {
  1874  			t.Errorf("children of %q = %q; want %q", dir, got, want)
  1875  		}
  1876  	})
  1877  }
  1878  
  1879  func hasDir(file string) stargzCheck {
  1880  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1881  		for _, ent := range r.toc.Entries {
  1882  			if ent.Name == cleanEntryName(file) {
  1883  				if ent.Type != "dir" {
  1884  					t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type)
  1885  				}
  1886  				return
  1887  			}
  1888  		}
  1889  		t.Errorf("directory %q not found", file)
  1890  	})
  1891  }
  1892  
  1893  func hasDirLinkCount(file string, count int) stargzCheck {
  1894  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1895  		for _, ent := range r.toc.Entries {
  1896  			if ent.Name == cleanEntryName(file) {
  1897  				if ent.Type != "dir" {
  1898  					t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type)
  1899  					return
  1900  				}
  1901  				if ent.NumLink != count {
  1902  					t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count)
  1903  				}
  1904  				return
  1905  			}
  1906  		}
  1907  		t.Errorf("directory %q not found", file)
  1908  	})
  1909  }
  1910  
  1911  func hasMode(file string, mode os.FileMode) stargzCheck {
  1912  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1913  		for _, ent := range r.toc.Entries {
  1914  			if ent.Name == cleanEntryName(file) {
  1915  				if ent.Stat().Mode() != mode {
  1916  					t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode)
  1917  					return
  1918  				}
  1919  				return
  1920  			}
  1921  		}
  1922  		t.Errorf("file %q not found", file)
  1923  	})
  1924  }
  1925  
  1926  func hasSymlink(file, target string) stargzCheck {
  1927  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1928  		for _, ent := range r.toc.Entries {
  1929  			if ent.Name == file {
  1930  				if ent.Type != "symlink" {
  1931  					t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type)
  1932  				} else if ent.LinkName != target {
  1933  					t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target)
  1934  				}
  1935  				return
  1936  			}
  1937  		}
  1938  		t.Errorf("symlink %q not found", file)
  1939  	})
  1940  }
  1941  
  1942  func lookupMatch(name string, want *TOCEntry) stargzCheck {
  1943  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1944  		e, ok := r.Lookup(name)
  1945  		if !ok {
  1946  			t.Fatalf("failed to Lookup entry %q", name)
  1947  		}
  1948  		if !reflect.DeepEqual(e, want) {
  1949  			t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want)
  1950  		}
  1951  
  1952  	})
  1953  }
  1954  
  1955  func hasEntryOwner(entry string, owner owner) stargzCheck {
  1956  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1957  		ent, ok := r.Lookup(strings.TrimSuffix(entry, "/"))
  1958  		if !ok {
  1959  			t.Errorf("entry %q not found", entry)
  1960  			return
  1961  		}
  1962  		if ent.UID != owner.uid || ent.GID != owner.gid {
  1963  			t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid)
  1964  			return
  1965  		}
  1966  	})
  1967  }
  1968  
  1969  func mustSameEntry(files ...string) stargzCheck {
  1970  	return stargzCheckFn(func(t *testing.T, r *Reader) {
  1971  		var first *TOCEntry
  1972  		for _, f := range files {
  1973  			if first == nil {
  1974  				var ok bool
  1975  				first, ok = r.Lookup(f)
  1976  				if !ok {
  1977  					t.Errorf("unknown first file on Lookup: %q", f)
  1978  					return
  1979  				}
  1980  			}
  1981  
  1982  			// Test Lookup
  1983  			e, ok := r.Lookup(f)
  1984  			if !ok {
  1985  				t.Errorf("unknown file on Lookup: %q", f)
  1986  				return
  1987  			}
  1988  			if e != first {
  1989  				t.Errorf("Lookup: %+v(%p) != %+v(%p)", e, e, first, first)
  1990  				return
  1991  			}
  1992  
  1993  			// Test LookupChild
  1994  			pe, ok := r.Lookup(filepath.Dir(filepath.Clean(f)))
  1995  			if !ok {
  1996  				t.Errorf("failed to get parent of %q", f)
  1997  				return
  1998  			}
  1999  			e, ok = pe.LookupChild(filepath.Base(filepath.Clean(f)))
  2000  			if !ok {
  2001  				t.Errorf("failed to get %q as the child of %+v", f, pe)
  2002  				return
  2003  			}
  2004  			if e != first {
  2005  				t.Errorf("LookupChild: %+v(%p) != %+v(%p)", e, e, first, first)
  2006  				return
  2007  			}
  2008  
  2009  			// Test ForeachChild
  2010  			pe.ForeachChild(func(baseName string, e *TOCEntry) bool {
  2011  				if baseName == filepath.Base(filepath.Clean(f)) {
  2012  					if e != first {
  2013  						t.Errorf("ForeachChild: %+v(%p) != %+v(%p)", e, e, first, first)
  2014  						return false
  2015  					}
  2016  				}
  2017  				return true
  2018  			})
  2019  		}
  2020  	})
  2021  }
  2022  
  2023  func viewContent(c []byte) string {
  2024  	if len(c) < 100 {
  2025  		return string(c)
  2026  	}
  2027  	return string(c[:50]) + "...(omit)..." + string(c[50:100])
  2028  }
  2029  
  2030  func tarOf(s ...tarEntry) []tarEntry { return s }
  2031  
  2032  type tarEntry interface {
  2033  	appendTar(tw *tar.Writer, prefix string, format tar.Format) error
  2034  }
  2035  
  2036  type tarEntryFunc func(*tar.Writer, string, tar.Format) error
  2037  
  2038  func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format) error {
  2039  	return f(tw, prefix, format)
  2040  }
  2041  
  2042  func buildTar(t *testing.T, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader {
  2043  	format := tar.FormatUnknown
  2044  	for _, opt := range opts {
  2045  		switch v := opt.(type) {
  2046  		case tar.Format:
  2047  			format = v
  2048  		default:
  2049  			panic(fmt.Errorf("unsupported opt for buildTar: %v", opt))
  2050  		}
  2051  	}
  2052  	buf := new(bytes.Buffer)
  2053  	tw := tar.NewWriter(buf)
  2054  	for _, ent := range ents {
  2055  		if err := ent.appendTar(tw, prefix, format); err != nil {
  2056  			t.Fatalf("building input tar: %v", err)
  2057  		}
  2058  	}
  2059  	if err := tw.Close(); err != nil {
  2060  		t.Errorf("closing write of input tar: %v", err)
  2061  	}
  2062  	data := append(buf.Bytes(), make([]byte, 100)...) // append empty bytes at the tail to see lossless works
  2063  	return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data)))
  2064  }
  2065  
  2066  func dir(name string, opts ...interface{}) tarEntry {
  2067  	return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
  2068  		var o owner
  2069  		mode := os.FileMode(0755)
  2070  		for _, opt := range opts {
  2071  			switch v := opt.(type) {
  2072  			case owner:
  2073  				o = v
  2074  			case os.FileMode:
  2075  				mode = v
  2076  			default:
  2077  				return errors.New("unsupported opt")
  2078  			}
  2079  		}
  2080  		if !strings.HasSuffix(name, "/") {
  2081  			panic(fmt.Sprintf("missing trailing slash in dir %q ", name))
  2082  		}
  2083  		tm, err := fileModeToTarMode(mode)
  2084  		if err != nil {
  2085  			return err
  2086  		}
  2087  		return tw.WriteHeader(&tar.Header{
  2088  			Typeflag: tar.TypeDir,
  2089  			Name:     prefix + name,
  2090  			Mode:     tm,
  2091  			Uid:      o.uid,
  2092  			Gid:      o.gid,
  2093  			Format:   format,
  2094  		})
  2095  	})
  2096  }
  2097  
  2098  // xAttr are extended attributes to set on test files created with the file func.
  2099  type xAttr map[string]string
  2100  
  2101  // owner is owner ot set on test files and directories with the file and dir functions.
  2102  type owner struct {
  2103  	uid int
  2104  	gid int
  2105  }
  2106  
  2107  func file(name, contents string, opts ...interface{}) tarEntry {
  2108  	return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
  2109  		var xattrs xAttr
  2110  		var o owner
  2111  		mode := os.FileMode(0644)
  2112  		for _, opt := range opts {
  2113  			switch v := opt.(type) {
  2114  			case xAttr:
  2115  				xattrs = v
  2116  			case owner:
  2117  				o = v
  2118  			case os.FileMode:
  2119  				mode = v
  2120  			default:
  2121  				return errors.New("unsupported opt")
  2122  			}
  2123  		}
  2124  		if strings.HasSuffix(name, "/") {
  2125  			return fmt.Errorf("bogus trailing slash in file %q", name)
  2126  		}
  2127  		tm, err := fileModeToTarMode(mode)
  2128  		if err != nil {
  2129  			return err
  2130  		}
  2131  		if len(xattrs) > 0 {
  2132  			format = tar.FormatPAX // only PAX supports xattrs
  2133  		}
  2134  		if err := tw.WriteHeader(&tar.Header{
  2135  			Typeflag: tar.TypeReg,
  2136  			Name:     prefix + name,
  2137  			Mode:     tm,
  2138  			Xattrs:   xattrs,
  2139  			Size:     int64(len(contents)),
  2140  			Uid:      o.uid,
  2141  			Gid:      o.gid,
  2142  			Format:   format,
  2143  		}); err != nil {
  2144  			return err
  2145  		}
  2146  		_, err = io.WriteString(tw, contents)
  2147  		return err
  2148  	})
  2149  }
  2150  
  2151  func symlink(name, target string) tarEntry {
  2152  	return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error {
  2153  		return tw.WriteHeader(&tar.Header{
  2154  			Typeflag: tar.TypeSymlink,
  2155  			Name:     prefix + name,
  2156  			Linkname: target,
  2157  			Mode:     0644,
  2158  			Format:   format,
  2159  		})
  2160  	})
  2161  }
  2162  
  2163  func link(name string, linkname string) tarEntry {
  2164  	now := time.Now()
  2165  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2166  		return w.WriteHeader(&tar.Header{
  2167  			Typeflag: tar.TypeLink,
  2168  			Name:     prefix + name,
  2169  			Linkname: linkname,
  2170  			ModTime:  now,
  2171  			Format:   format,
  2172  		})
  2173  	})
  2174  }
  2175  
  2176  func chardev(name string, major, minor int64) tarEntry {
  2177  	now := time.Now()
  2178  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2179  		return w.WriteHeader(&tar.Header{
  2180  			Typeflag: tar.TypeChar,
  2181  			Name:     prefix + name,
  2182  			Devmajor: major,
  2183  			Devminor: minor,
  2184  			ModTime:  now,
  2185  			Format:   format,
  2186  		})
  2187  	})
  2188  }
  2189  
  2190  func blockdev(name string, major, minor int64) tarEntry {
  2191  	now := time.Now()
  2192  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2193  		return w.WriteHeader(&tar.Header{
  2194  			Typeflag: tar.TypeBlock,
  2195  			Name:     prefix + name,
  2196  			Devmajor: major,
  2197  			Devminor: minor,
  2198  			ModTime:  now,
  2199  			Format:   format,
  2200  		})
  2201  	})
  2202  }
  2203  func fifo(name string) tarEntry {
  2204  	now := time.Now()
  2205  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2206  		return w.WriteHeader(&tar.Header{
  2207  			Typeflag: tar.TypeFifo,
  2208  			Name:     prefix + name,
  2209  			ModTime:  now,
  2210  			Format:   format,
  2211  		})
  2212  	})
  2213  }
  2214  
  2215  func prefetchLandmark() tarEntry {
  2216  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2217  		if err := w.WriteHeader(&tar.Header{
  2218  			Name:     PrefetchLandmark,
  2219  			Typeflag: tar.TypeReg,
  2220  			Size:     int64(len([]byte{landmarkContents})),
  2221  			Format:   format,
  2222  		}); err != nil {
  2223  			return err
  2224  		}
  2225  		contents := []byte{landmarkContents}
  2226  		if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil {
  2227  			return err
  2228  		}
  2229  		return nil
  2230  	})
  2231  }
  2232  
  2233  func noPrefetchLandmark() tarEntry {
  2234  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2235  		if err := w.WriteHeader(&tar.Header{
  2236  			Name:     NoPrefetchLandmark,
  2237  			Typeflag: tar.TypeReg,
  2238  			Size:     int64(len([]byte{landmarkContents})),
  2239  			Format:   format,
  2240  		}); err != nil {
  2241  			return err
  2242  		}
  2243  		contents := []byte{landmarkContents}
  2244  		if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil {
  2245  			return err
  2246  		}
  2247  		return nil
  2248  	})
  2249  }
  2250  
  2251  func regDigest(t *testing.T, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry {
  2252  	if digestMap == nil {
  2253  		t.Fatalf("digest map mustn't be nil")
  2254  	}
  2255  	content := []byte(contentStr)
  2256  
  2257  	var n int64
  2258  	for n < int64(len(content)) {
  2259  		size := int64(chunkSize)
  2260  		remain := int64(len(content)) - n
  2261  		if remain < size {
  2262  			size = remain
  2263  		}
  2264  		dgstr := digest.Canonical.Digester()
  2265  		if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil {
  2266  			t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)",
  2267  				string(content[n:n+size]), name, n, size)
  2268  		}
  2269  		digestMap[chunkID(name, n, size)] = dgstr.Digest()
  2270  		n += size
  2271  	}
  2272  
  2273  	return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error {
  2274  		if err := w.WriteHeader(&tar.Header{
  2275  			Typeflag: tar.TypeReg,
  2276  			Name:     prefix + name,
  2277  			Size:     int64(len(content)),
  2278  			Format:   format,
  2279  		}); err != nil {
  2280  			return err
  2281  		}
  2282  		if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil {
  2283  			return err
  2284  		}
  2285  		return nil
  2286  	})
  2287  }
  2288  
  2289  var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
  2290  
  2291  func randomContents(n int) string {
  2292  	b := make([]rune, n)
  2293  	for i := range b {
  2294  		b[i] = runes[rand.Intn(len(runes))]
  2295  	}
  2296  	return string(b)
  2297  }
  2298  
  2299  func fileModeToTarMode(mode os.FileMode) (int64, error) {
  2300  	h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "")
  2301  	if err != nil {
  2302  		return 0, err
  2303  	}
  2304  	return h.Mode, nil
  2305  }
  2306  
  2307  // fileInfoOnlyMode is os.FileMode that populates only file mode.
  2308  type fileInfoOnlyMode os.FileMode
  2309  
  2310  func (f fileInfoOnlyMode) Name() string       { return "" }
  2311  func (f fileInfoOnlyMode) Size() int64        { return 0 }
  2312  func (f fileInfoOnlyMode) Mode() os.FileMode  { return os.FileMode(f) }
  2313  func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() }
  2314  func (f fileInfoOnlyMode) IsDir() bool        { return os.FileMode(f).IsDir() }
  2315  func (f fileInfoOnlyMode) Sys() interface{}   { return nil }
  2316  
  2317  func CheckGzipHasStreams(t *testing.T, b []byte, streams []int64) {
  2318  	if len(streams) == 0 {
  2319  		return // nop
  2320  	}
  2321  
  2322  	wants := map[int64]struct{}{}
  2323  	for _, s := range streams {
  2324  		wants[s] = struct{}{}
  2325  	}
  2326  
  2327  	len0 := len(b)
  2328  	br := bytes.NewReader(b)
  2329  	zr := new(gzip.Reader)
  2330  	t.Logf("got gzip streams:")
  2331  	numStreams := 0
  2332  	for {
  2333  		zoff := len0 - br.Len()
  2334  		if err := zr.Reset(br); err != nil {
  2335  			if err == io.EOF {
  2336  				return
  2337  			}
  2338  			t.Fatalf("countStreams(gzip), Reset: %v", err)
  2339  		}
  2340  		zr.Multistream(false)
  2341  		n, err := io.Copy(io.Discard, zr)
  2342  		if err != nil {
  2343  			t.Fatalf("countStreams(gzip), Copy: %v", err)
  2344  		}
  2345  		var extra string
  2346  		if len(zr.Header.Extra) > 0 {
  2347  			extra = fmt.Sprintf("; extra=%q", zr.Header.Extra)
  2348  		}
  2349  		t.Logf("  [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra)
  2350  		delete(wants, int64(zoff))
  2351  		numStreams++
  2352  	}
  2353  }
  2354  
  2355  func GzipDiffIDOf(t *testing.T, b []byte) string {
  2356  	h := sha256.New()
  2357  	zr, err := gzip.NewReader(bytes.NewReader(b))
  2358  	if err != nil {
  2359  		t.Fatalf("diffIDOf(gzip): %v", err)
  2360  	}
  2361  	defer zr.Close()
  2362  	if _, err := io.Copy(h, zr); err != nil {
  2363  		t.Fatalf("diffIDOf(gzip).Copy: %v", err)
  2364  	}
  2365  	return fmt.Sprintf("sha256:%x", h.Sum(nil))
  2366  }
  2367  

View as plain text