manifest_test.go

Documentation: github.com/containerd/continuity

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5     Copyright The containerd Authors.
     6  
     7     Licensed under the Apache License, Version 2.0 (the "License");
     8     you may not use this file except in compliance with the License.
     9     You may obtain a copy of the License at
    10  
    11         http://www.apache.org/licenses/LICENSE-2.0
    12  
    13     Unless required by applicable law or agreed to in writing, software
    14     distributed under the License is distributed on an "AS IS" BASIS,
    15     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16     See the License for the specific language governing permissions and
    17     limitations under the License.
    18  */
    19  
    20  package continuity
    21  
    22  import (
    23  	"bytes"
    24  	_ "crypto/sha256"
    25  	"errors"
    26  	"fmt"
    27  	"math/rand"
    28  	"os"
    29  	"path/filepath"
    30  	"sort"
    31  	"syscall"
    32  	"testing"
    33  
    34  	"github.com/containerd/continuity/devices"
    35  	"github.com/opencontainers/go-digest"
    36  )
    37  
    38  // Hard things:
    39  //  1. Groups/gid - no standard library support.
    40  //  2. xattrs - must choose package to provide this.
    41  //  3. ADS - no clue where to start.
    42  
    43  func TestWalkFS(t *testing.T) {
    44  	rand.Seed(1)
    45  
    46  	// Testing:
    47  	// 1. Setup different files:
    48  	//		- links
    49  	//			- sibling directory - relative
    50  	//			- sibling directory - absolute
    51  	//			- parent directory - absolute
    52  	//			- parent directory - relative
    53  	//		- illegal links
    54  	//			- parent directory - relative, out of root
    55  	//			- parent directory - absolute, out of root
    56  	//		- regular files
    57  	//		- character devices
    58  	//		- what about sticky bits?
    59  	// 2. Build the manifest.
    60  	// 3. Verify expected result.
    61  	testResources := []dresource{
    62  		{
    63  			path: "a",
    64  			mode: 0o644,
    65  		},
    66  		{
    67  			kind:   rhardlink,
    68  			path:   "a-hardlink",
    69  			target: "a",
    70  		},
    71  		{
    72  			kind: rdirectory,
    73  			path: "b",
    74  			mode: 0o755,
    75  		},
    76  		{
    77  			kind:   rhardlink,
    78  			path:   "b/a-hardlink",
    79  			target: "a",
    80  		},
    81  		{
    82  			path: "b/a",
    83  			mode: 0o600 | os.ModeSticky,
    84  		},
    85  		{
    86  			kind: rdirectory,
    87  			path: "c",
    88  			mode: 0o755,
    89  		},
    90  		{
    91  			path: "c/a",
    92  			mode: 0o644,
    93  		},
    94  		{
    95  			kind:   rrelsymlink,
    96  			path:   "c/ca-relsymlink",
    97  			mode:   0o600,
    98  			target: "a",
    99  		},
   100  		{
   101  			kind:   rrelsymlink,
   102  			path:   "c/a-relsymlink",
   103  			mode:   0o600,
   104  			target: "../a",
   105  		},
   106  		{
   107  			kind:   rabssymlink,
   108  			path:   "c/a-abssymlink",
   109  			mode:   0o600,
   110  			target: "a",
   111  		},
   112  		// TODO(stevvooe): Make sure we can test this case and get proper
   113  		// errors when it is encountered.
   114  		// {
   115  		// 	// create a bad symlink and make sure we don't include it.
   116  		// 	kind:   relsymlink,
   117  		// 	path:   "c/a-badsymlink",
   118  		// 	mode:   0600,
   119  		// 	target: "../../..",
   120  		// },
   121  
   122  		// TODO(stevvooe): Must add tests for xattrs, with symlinks,
   123  		// directories and regular files.
   124  
   125  		{
   126  			kind: rnamedpipe,
   127  			path: "fifo",
   128  			mode: 0o666 | os.ModeNamedPipe,
   129  		},
   130  
   131  		{
   132  			kind: rdirectory,
   133  			path: "/dev",
   134  			mode: 0o755,
   135  		},
   136  
   137  		// NOTE(stevvooe): Below here, we add a few simple character devices.
   138  		// Block devices are untested but should be nearly the same as
   139  		// character devices.
   140  		// devNullResource,
   141  		// devZeroResource,
   142  	}
   143  
   144  	root := t.TempDir()
   145  	generateTestFiles(t, root, testResources)
   146  
   147  	ctx, err := NewContext(root)
   148  	if err != nil {
   149  		t.Fatalf("error getting context: %v", err)
   150  	}
   151  
   152  	m, err := BuildManifest(ctx)
   153  	if err != nil {
   154  		t.Fatalf("error building manifest: %v", err)
   155  	}
   156  
   157  	var b bytes.Buffer
   158  	MarshalText(&b, m)
   159  	t.Log(b.String())
   160  
   161  	// TODO(dmcgowan): always verify, currently hard links not supported
   162  	//if err := VerifyManifest(ctx, m); err != nil {
   163  	//	t.Fatalf("error verifying manifest: %v")
   164  	//}
   165  
   166  	expectedResources, err := expectedResourceList(root, testResources)
   167  	if err != nil {
   168  		// TODO(dmcgowan): update function to panic, this would mean test setup error
   169  		t.Fatalf("error creating resource list: %v", err)
   170  	}
   171  
   172  	// Diff resources
   173  	diff := diffResourceList(expectedResources, m.Resources)
   174  	if diff.HasDiff() {
   175  		t.Log("Resource list difference")
   176  		for _, a := range diff.Additions {
   177  			t.Logf("Unexpected resource: %#v", a)
   178  		}
   179  		for _, d := range diff.Deletions {
   180  			t.Logf("Missing resource: %#v", d)
   181  		}
   182  		for _, u := range diff.Updates {
   183  			t.Logf("Changed resource:\n\tExpected: %#v\n\tActual:   %#v", u.Original, u.Updated)
   184  		}
   185  
   186  		t.FailNow()
   187  	}
   188  }
   189  
   190  // TODO(stevvooe): At this time, we have a nice testing framework to define
   191  // and build resources. This will likely be a pre-cursor to the packages
   192  // public interface.
   193  type kind int
   194  
   195  func (k kind) String() string {
   196  	switch k {
   197  	case rfile:
   198  		return "file"
   199  	case rdirectory:
   200  		return "directory"
   201  	case rhardlink:
   202  		return "hardlink"
   203  	case rchardev:
   204  		return "chardev"
   205  	case rnamedpipe:
   206  		return "namedpipe"
   207  	}
   208  
   209  	panic(fmt.Sprintf("unknown kind: %v", int(k)))
   210  }
   211  
   212  const (
   213  	rfile kind = iota
   214  	rdirectory
   215  	rhardlink
   216  	rrelsymlink
   217  	rabssymlink
   218  	rchardev
   219  	rnamedpipe
   220  )
   221  
   222  type dresource struct {
   223  	kind         kind
   224  	path         string
   225  	mode         os.FileMode
   226  	target       string // hard/soft link target
   227  	digest       digest.Digest
   228  	size         int
   229  	uid          int64
   230  	gid          int64
   231  	major, minor int
   232  }
   233  
   234  func generateTestFiles(t *testing.T, root string, resources []dresource) {
   235  	for i, resource := range resources {
   236  		p := filepath.Join(root, resource.path)
   237  		switch resource.kind {
   238  		case rfile:
   239  			size := rand.Intn(4 << 20)
   240  			d := make([]byte, size)
   241  			randomBytes(d)
   242  			dgst := digest.FromBytes(d)
   243  			resources[i].digest = dgst
   244  			resources[i].size = size
   245  
   246  			// this relies on the proper directory parent being defined.
   247  			if err := os.WriteFile(p, d, resource.mode); err != nil {
   248  				t.Fatalf("error writing %q: %v", p, err)
   249  			}
   250  		case rdirectory:
   251  			if err := os.Mkdir(p, resource.mode); err != nil {
   252  				t.Fatalf("error creating directory %q: %v", p, err)
   253  			}
   254  		case rhardlink:
   255  			target := filepath.Join(root, resource.target)
   256  			if err := os.Link(target, p); err != nil {
   257  				t.Fatalf("error creating hardlink: %v", err)
   258  			}
   259  		case rrelsymlink:
   260  			if err := os.Symlink(resource.target, p); err != nil {
   261  				t.Fatalf("error creating symlink: %v", err)
   262  			}
   263  		case rabssymlink:
   264  			// for absolute links, we join with root.
   265  			target := filepath.Join(root, resource.target)
   266  
   267  			if err := os.Symlink(target, p); err != nil {
   268  				t.Fatalf("error creating symlink: %v", err)
   269  			}
   270  		case rchardev, rnamedpipe:
   271  			if err := devices.Mknod(p, resource.mode, resource.major, resource.minor); err != nil {
   272  				t.Fatalf("error creating device %q: %v", p, err)
   273  			}
   274  		default:
   275  			t.Fatalf("unknown resource type: %v", resource.kind)
   276  		}
   277  
   278  		st, err := os.Lstat(p)
   279  		if err != nil {
   280  			t.Fatalf("error statting after creation: %v", err)
   281  		}
   282  		resources[i].uid = int64(st.Sys().(*syscall.Stat_t).Uid)
   283  		resources[i].gid = int64(st.Sys().(*syscall.Stat_t).Gid)
   284  		resources[i].mode = st.Mode()
   285  
   286  		// TODO: Readback and join xattr
   287  	}
   288  
   289  	// log the test root for future debugging
   290  	if err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
   291  		if fi.Mode()&os.ModeSymlink != 0 {
   292  			target, err := os.Readlink(p)
   293  			if err != nil {
   294  				return err
   295  			}
   296  			t.Log(fi.Mode(), p, "->", target)
   297  		} else {
   298  			t.Log(fi.Mode(), p)
   299  		}
   300  
   301  		return nil
   302  	}); err != nil {
   303  		t.Fatalf("error walking created root: %v", err)
   304  	}
   305  
   306  	var b bytes.Buffer
   307  	if err := tree(&b, root); err != nil {
   308  		t.Fatalf("error running tree: %v", err)
   309  	}
   310  	t.Logf("\n%s", b.String())
   311  }
   312  
   313  func randomBytes(p []byte) {
   314  	for i := range p {
   315  		p[i] = byte(rand.Intn(1<<8 - 1))
   316  	}
   317  }
   318  
   319  // expectedResourceList sorts the set of resources into the order
   320  // expected in the manifest and collapses hardlinks
   321  func expectedResourceList(root string, resources []dresource) ([]Resource, error) {
   322  	resourceMap := map[string]Resource{}
   323  	paths := []string{}
   324  	for _, r := range resources {
   325  		absPath := r.path
   326  		if !filepath.IsAbs(absPath) {
   327  			absPath = "/" + absPath
   328  		}
   329  		switch r.kind {
   330  		case rfile:
   331  			f := &regularFile{
   332  				resource: resource{
   333  					paths: []string{absPath},
   334  					mode:  r.mode,
   335  					uid:   r.uid,
   336  					gid:   r.gid,
   337  				},
   338  				size:    int64(r.size),
   339  				digests: []digest.Digest{r.digest},
   340  			}
   341  			resourceMap[absPath] = f
   342  			paths = append(paths, absPath)
   343  		case rdirectory:
   344  			d := &directory{
   345  				resource: resource{
   346  					paths: []string{absPath},
   347  					mode:  r.mode,
   348  					uid:   r.uid,
   349  					gid:   r.gid,
   350  				},
   351  			}
   352  			resourceMap[absPath] = d
   353  			paths = append(paths, absPath)
   354  		case rhardlink:
   355  			targetPath := r.target
   356  			if !filepath.IsAbs(targetPath) {
   357  				targetPath = "/" + targetPath
   358  			}
   359  			target, ok := resourceMap[targetPath]
   360  			if !ok {
   361  				return nil, errors.New("must specify target before hardlink for test resources")
   362  			}
   363  			rf, ok := target.(*regularFile)
   364  			if !ok {
   365  				return nil, errors.New("hardlink target must be regular file")
   366  			}
   367  			// TODO(dmcgowan): full merge
   368  			rf.paths = append(rf.paths, absPath)
   369  			// TODO(dmcgowan): check if first path is now different, changes source order and should update
   370  			// resource map key, to avoid canonically ordered first should be regular file
   371  			sort.Stable(sort.StringSlice(rf.paths))
   372  		case rrelsymlink, rabssymlink:
   373  			targetPath := r.target
   374  			if r.kind == rabssymlink && !filepath.IsAbs(r.target) {
   375  				// for absolute links, we join with root.
   376  				targetPath = filepath.Join(root, targetPath)
   377  			}
   378  			s := &symLink{
   379  				resource: resource{
   380  					paths: []string{absPath},
   381  					mode:  r.mode,
   382  					uid:   r.uid,
   383  					gid:   r.gid,
   384  				},
   385  				target: targetPath,
   386  			}
   387  			resourceMap[absPath] = s
   388  			paths = append(paths, absPath)
   389  		case rchardev:
   390  			d := &device{
   391  				resource: resource{
   392  					paths: []string{absPath},
   393  					mode:  r.mode,
   394  					uid:   r.uid,
   395  					gid:   r.gid,
   396  				},
   397  				major: uint64(r.major),
   398  				minor: uint64(r.minor),
   399  			}
   400  			resourceMap[absPath] = d
   401  			paths = append(paths, absPath)
   402  		case rnamedpipe:
   403  			p := &namedPipe{
   404  				resource: resource{
   405  					paths: []string{absPath},
   406  					mode:  r.mode,
   407  					uid:   r.uid,
   408  					gid:   r.gid,
   409  				},
   410  			}
   411  			resourceMap[absPath] = p
   412  			paths = append(paths, absPath)
   413  		default:
   414  			return nil, fmt.Errorf("unknown resource type: %v", r.kind)
   415  		}
   416  	}
   417  
   418  	if len(resourceMap) < len(paths) {
   419  		return nil, errors.New("resource list has duplicated paths")
   420  	}
   421  
   422  	sort.Strings(paths)
   423  
   424  	manifestResources := make([]Resource, len(paths))
   425  	for i, p := range paths {
   426  		manifestResources[i] = resourceMap[p]
   427  	}
   428  
   429  	return manifestResources, nil
   430  }
   431
View as plain text