diff_test.go

Documentation: github.com/containerd/continuity/fs

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package fs
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"path/filepath"
    24  	"runtime"
    25  	"strings"
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/containerd/continuity/fs/fstest"
    30  )
    31  
    32  // TODO: Additional tests
    33  // - capability test (requires privilege)
    34  // - chown test (requires privilege)
    35  // - symlink test
    36  // - hardlink test
    37  
    38  func skipDiffTestOnWindows(t *testing.T) {
    39  	if runtime.GOOS == "windows" {
    40  		t.Skip("diff implementation is incomplete on windows")
    41  	}
    42  }
    43  
    44  func TestSimpleDiff(t *testing.T) {
    45  	skipDiffTestOnWindows(t)
    46  	l1 := fstest.Apply(
    47  		fstest.CreateDir("/etc", 0o755),
    48  		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0o644),
    49  		fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0o644),
    50  		fstest.CreateFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0o644),
    51  		fstest.CreateFile("/etc/unexpected", []byte("#!/bin/sh"), 0o644),
    52  	)
    53  	l2 := fstest.Apply(
    54  		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0o644),
    55  		fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0o666),
    56  		fstest.CreateDir("/root", 0o700),
    57  		fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0o644),
    58  		fstest.Remove("/etc/unexpected"),
    59  	)
    60  	diff := []TestChange{
    61  		Modify("/etc/hosts"),
    62  		Modify("/etc/profile"),
    63  		Delete("/etc/unexpected"),
    64  		Add("/root"),
    65  		Add("/root/.bashrc"),
    66  	}
    67  
    68  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
    69  		t.Fatalf("Failed diff with base: %+v", err)
    70  	}
    71  }
    72  
    73  func TestEmptyFileDiff(t *testing.T) {
    74  	skipDiffTestOnWindows(t)
    75  	tt := time.Now().Truncate(time.Second)
    76  	l1 := fstest.Apply(
    77  		fstest.CreateDir("/etc", 0o755),
    78  		fstest.CreateFile("/etc/empty", []byte(""), 0o644),
    79  		fstest.Chtimes("/etc/empty", tt, tt),
    80  	)
    81  	l2 := fstest.Apply()
    82  	diff := []TestChange{}
    83  
    84  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
    85  		t.Fatalf("Failed diff with base: %+v", err)
    86  	}
    87  }
    88  
    89  func TestNestedDeletion(t *testing.T) {
    90  	skipDiffTestOnWindows(t)
    91  	l1 := fstest.Apply(
    92  		fstest.CreateDir("/d0", 0o755),
    93  		fstest.CreateDir("/d1", 0o755),
    94  		fstest.CreateDir("/d1/d2", 0o755),
    95  		fstest.CreateFile("/d1/d2/f1", []byte("mydomain 10.0.0.1"), 0o644),
    96  	)
    97  	l2 := fstest.Apply(
    98  		fstest.RemoveAll("/d0"),
    99  		fstest.RemoveAll("/d1"),
   100  	)
   101  	diff := []TestChange{
   102  		Delete("/d0"),
   103  		Delete("/d1"),
   104  	}
   105  
   106  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   107  		t.Fatalf("Failed diff with base: %+v", err)
   108  	}
   109  }
   110  
   111  func TestDirectoryReplace(t *testing.T) {
   112  	skipDiffTestOnWindows(t)
   113  	l1 := fstest.Apply(
   114  		fstest.CreateDir("/dir1", 0o755),
   115  		fstest.CreateFile("/dir1/f1", []byte("#####"), 0o644),
   116  		fstest.CreateDir("/dir1/f2", 0o755),
   117  		fstest.CreateFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0o644),
   118  	)
   119  	l2 := fstest.Apply(
   120  		fstest.CreateFile("/dir1/f11", []byte("#New file here"), 0o644),
   121  		fstest.RemoveAll("/dir1/f2"),
   122  		fstest.CreateFile("/dir1/f2", []byte("Now file"), 0o666),
   123  	)
   124  	diff := []TestChange{
   125  		Add("/dir1/f11"),
   126  		Modify("/dir1/f2"),
   127  	}
   128  
   129  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   130  		t.Fatalf("Failed diff with base: %+v", err)
   131  	}
   132  }
   133  
   134  func TestRemoveDirectoryTree(t *testing.T) {
   135  	l1 := fstest.Apply(
   136  		fstest.CreateDir("/dir1/dir2/dir3", 0o755),
   137  		fstest.CreateFile("/dir1/f1", []byte("f1"), 0o644),
   138  		fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0o644),
   139  	)
   140  	l2 := fstest.Apply(
   141  		fstest.RemoveAll("/dir1"),
   142  	)
   143  	diff := []TestChange{
   144  		Delete("/dir1"),
   145  	}
   146  
   147  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   148  		t.Fatalf("Failed diff with base: %+v", err)
   149  	}
   150  }
   151  
   152  func TestRemoveDirectoryTreeWithDash(t *testing.T) {
   153  	if runtime.GOOS == "windows" {
   154  		t.Skip("windows fails this test with `-` files reported as modified")
   155  	}
   156  	l1 := fstest.Apply(
   157  		fstest.CreateDir("/dir1/dir2/dir3", 0o755),
   158  		fstest.CreateFile("/dir1/f1", []byte("f1"), 0o644),
   159  		fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0o644),
   160  		fstest.CreateDir("/dir1-before", 0o755),
   161  		fstest.CreateFile("/dir1-before/f2", []byte("f2"), 0o644),
   162  	)
   163  	l2 := fstest.Apply(
   164  		fstest.RemoveAll("/dir1"),
   165  	)
   166  	diff := []TestChange{
   167  		Delete("/dir1"),
   168  	}
   169  
   170  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   171  		t.Fatalf("Failed diff with base: %+v", err)
   172  	}
   173  }
   174  
   175  func TestFileReplace(t *testing.T) {
   176  	l1 := fstest.Apply(
   177  		fstest.CreateFile("/dir1", []byte("a file, not a directory"), 0o644),
   178  	)
   179  	l2 := fstest.Apply(
   180  		fstest.Remove("/dir1"),
   181  		fstest.CreateDir("/dir1/dir2", 0o755),
   182  		fstest.CreateFile("/dir1/dir2/f1", []byte("also a file"), 0o644),
   183  	)
   184  	diff := []TestChange{
   185  		Modify("/dir1"),
   186  		Add("/dir1/dir2"),
   187  		Add("/dir1/dir2/f1"),
   188  	}
   189  
   190  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   191  		t.Fatalf("Failed diff with base: %+v", err)
   192  	}
   193  }
   194  
   195  func TestParentDirectoryPermission(t *testing.T) {
   196  	skipDiffTestOnWindows(t)
   197  	l1 := fstest.Apply(
   198  		fstest.CreateDir("/dir1", 0o700),
   199  		fstest.CreateDir("/dir2", 0o751),
   200  		fstest.CreateDir("/dir3", 0o777),
   201  	)
   202  	l2 := fstest.Apply(
   203  		fstest.CreateDir("/dir1/d", 0o700),
   204  		fstest.CreateFile("/dir1/d/f", []byte("irrelevant"), 0o644),
   205  		fstest.CreateFile("/dir1/f", []byte("irrelevant"), 0o644),
   206  		fstest.CreateFile("/dir2/f", []byte("irrelevant"), 0o644),
   207  		fstest.CreateFile("/dir3/f", []byte("irrelevant"), 0o644),
   208  	)
   209  	diff := []TestChange{
   210  		Add("/dir1/d"),
   211  		Add("/dir1/d/f"),
   212  		Add("/dir1/f"),
   213  		Add("/dir2/f"),
   214  		Add("/dir3/f"),
   215  	}
   216  
   217  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   218  		t.Fatalf("Failed diff with base: %+v", err)
   219  	}
   220  }
   221  
   222  func TestUpdateWithSameTime(t *testing.T) {
   223  	skipDiffTestOnWindows(t)
   224  	tt := time.Now().Truncate(time.Second)
   225  	t1 := tt.Add(5 * time.Nanosecond)
   226  	t2 := tt.Add(6 * time.Nanosecond)
   227  	l1 := fstest.Apply(
   228  		fstest.CreateFile("/file-modified-time", []byte("1"), 0o644),
   229  		fstest.Chtimes("/file-modified-time", t1, t1),
   230  		fstest.CreateFile("/file-no-change", []byte("1"), 0o644),
   231  		fstest.Chtimes("/file-no-change", t1, t1),
   232  		fstest.CreateFile("/file-same-time", []byte("1"), 0o644),
   233  		fstest.Chtimes("/file-same-time", t1, t1),
   234  		fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0o644),
   235  		fstest.Chtimes("/file-truncated-time-1", tt, tt),
   236  		fstest.CreateFile("/file-truncated-time-2", []byte("1"), 0o644),
   237  		fstest.Chtimes("/file-truncated-time-2", tt, tt),
   238  		fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0o644),
   239  		fstest.Chtimes("/file-truncated-time-3", t1, t1),
   240  	)
   241  	l2 := fstest.Apply(
   242  		fstest.CreateFile("/file-modified-time", []byte("2"), 0o644),
   243  		fstest.Chtimes("/file-modified-time", t2, t2),
   244  		fstest.CreateFile("/file-no-change", []byte("1"), 0o644),
   245  		fstest.Chtimes("/file-no-change", t1, t1),
   246  		fstest.CreateFile("/file-same-time", []byte("2"), 0o644),
   247  		fstest.Chtimes("/file-same-time", t1, t1),
   248  		fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0o644),
   249  		fstest.Chtimes("/file-truncated-time-1", t1, t1),
   250  		fstest.CreateFile("/file-truncated-time-2", []byte("2"), 0o644),
   251  		fstest.Chtimes("/file-truncated-time-2", tt, tt),
   252  		fstest.CreateFile("/file-truncated-time-3", []byte("1"), 0o644),
   253  		fstest.Chtimes("/file-truncated-time-3", tt, tt),
   254  	)
   255  	diff := []TestChange{
   256  		Modify("/file-modified-time"),
   257  		// Include changes with truncated timestamps. Comparing newly
   258  		// extracted tars which have truncated timestamps will be
   259  		// expected to produce changes. The expectation is that diff
   260  		// archives are generated once and kept, newly generated diffs
   261  		// will not consider cases where only one side is truncated.
   262  		Modify("/file-truncated-time-1"),
   263  		Modify("/file-truncated-time-2"),
   264  		Modify("/file-truncated-time-3"),
   265  	}
   266  
   267  	if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   268  		t.Fatalf("Failed diff with base: %+v", err)
   269  	}
   270  }
   271  
   272  // buildkit#172
   273  func TestLchtimes(t *testing.T) {
   274  	skipDiffTestOnWindows(t)
   275  	mtimes := []time.Time{
   276  		time.Unix(0, 0),  // nsec is 0
   277  		time.Unix(0, 42), // nsec > 0
   278  	}
   279  	for _, mtime := range mtimes {
   280  		atime := time.Unix(424242, 42)
   281  		l1 := fstest.Apply(
   282  			fstest.CreateFile("/foo", []byte("foo"), 0o644),
   283  			fstest.Symlink("/foo", "/lnk0"),
   284  			fstest.Lchtimes("/lnk0", atime, mtime),
   285  		)
   286  		l2 := fstest.Apply() // empty
   287  		diff := []TestChange{}
   288  		if err := testDiffWithBase(t, l1, l2, diff); err != nil {
   289  			t.Fatalf("Failed diff with base: %+v", err)
   290  		}
   291  	}
   292  }
   293  
   294  func testDiffWithBase(t testing.TB, base, diff fstest.Applier, expected []TestChange) error {
   295  	t1 := t.TempDir()
   296  	t2 := t.TempDir()
   297  
   298  	if err := base.Apply(t1); err != nil {
   299  		return fmt.Errorf("failed to apply base filesystem: %w", err)
   300  	}
   301  
   302  	if err := CopyDir(t2, t1); err != nil {
   303  		return fmt.Errorf("failed to copy base directory: %w", err)
   304  	}
   305  
   306  	if err := diff.Apply(t2); err != nil {
   307  		return fmt.Errorf("failed to apply diff filesystem: %w", err)
   308  	}
   309  
   310  	changes, err := collectChanges(t1, t2)
   311  	if err != nil {
   312  		return fmt.Errorf("failed to collect changes: %w", err)
   313  	}
   314  
   315  	return checkChanges(t2, changes, expected)
   316  }
   317  
   318  func TestBaseDirectoryChanges(t *testing.T) {
   319  	apply := fstest.Apply(
   320  		fstest.CreateDir("/etc", 0o755),
   321  		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0o644),
   322  		fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0o644),
   323  		fstest.CreateDir("/root", 0o700),
   324  		fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0o644),
   325  	)
   326  	changes := []TestChange{
   327  		Add("/etc"),
   328  		Add("/etc/hosts"),
   329  		Add("/etc/profile"),
   330  		Add("/root"),
   331  		Add("/root/.bashrc"),
   332  	}
   333  
   334  	if err := testDiffWithoutBase(t, apply, changes); err != nil {
   335  		t.Fatalf("Failed diff without base: %+v", err)
   336  	}
   337  }
   338  
   339  func testDiffWithoutBase(t testing.TB, apply fstest.Applier, expected []TestChange) error {
   340  	tmp := t.TempDir()
   341  	if err := apply.Apply(tmp); err != nil {
   342  		return fmt.Errorf("failed to apply filesytem changes: %w", err)
   343  	}
   344  
   345  	changes, err := collectChanges("", tmp)
   346  	if err != nil {
   347  		return fmt.Errorf("failed to collect changes: %w", err)
   348  	}
   349  
   350  	return checkChanges(tmp, changes, expected)
   351  }
   352  
   353  func checkChanges(root string, changes, expected []TestChange) error {
   354  	if len(changes) != len(expected) {
   355  		return fmt.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected))
   356  	}
   357  	for i := range changes {
   358  		if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind {
   359  			return fmt.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected))
   360  		}
   361  		if changes[i].Kind != ChangeKindDelete {
   362  			filename := filepath.Join(root, changes[i].Path)
   363  			efi, err := os.Stat(filename)
   364  			if err != nil {
   365  				return fmt.Errorf("failed to stat %q: %w", filename, err)
   366  			}
   367  			afi := changes[i].FileInfo
   368  			if afi.Size() != efi.Size() {
   369  				return fmt.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size())
   370  			}
   371  			if afi.Mode() != efi.Mode() {
   372  				return fmt.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode())
   373  			}
   374  			if afi.ModTime() != efi.ModTime() {
   375  				return fmt.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime())
   376  			}
   377  			if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected {
   378  				return fmt.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected)
   379  			}
   380  		}
   381  	}
   382  
   383  	return nil
   384  }
   385  
   386  type TestChange struct {
   387  	Kind     ChangeKind
   388  	Path     string
   389  	FileInfo os.FileInfo
   390  	Source   string
   391  }
   392  
   393  func collectChanges(a, b string) ([]TestChange, error) {
   394  	changes := []TestChange{}
   395  	err := Changes(context.Background(), a, b, func(k ChangeKind, p string, f os.FileInfo, err error) error {
   396  		if err != nil {
   397  			return err
   398  		}
   399  		changes = append(changes, TestChange{
   400  			Kind:     k,
   401  			Path:     p,
   402  			FileInfo: f,
   403  			Source:   filepath.Join(b, p),
   404  		})
   405  		return nil
   406  	})
   407  	if err != nil {
   408  		return nil, fmt.Errorf("failed to compute changes: %w", err)
   409  	}
   410  
   411  	return changes, nil
   412  }
   413  
   414  func diffString(c1, c2 []TestChange) string {
   415  	return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2))
   416  }
   417  
   418  func changesString(c []TestChange) string {
   419  	strs := make([]string, len(c))
   420  	for i := range c {
   421  		strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path)
   422  	}
   423  	return strings.Join(strs, "\n")
   424  }
   425  
   426  func Add(p string) TestChange {
   427  	return TestChange{
   428  		Kind: ChangeKindAdd,
   429  		Path: filepath.FromSlash(p),
   430  	}
   431  }
   432  
   433  func Delete(p string) TestChange {
   434  	return TestChange{
   435  		Kind: ChangeKindDelete,
   436  		Path: filepath.FromSlash(p),
   437  	}
   438  }
   439  
   440  func Modify(p string) TestChange {
   441  	return TestChange{
   442  		Kind: ChangeKindModify,
   443  		Path: filepath.FromSlash(p),
   444  	}
   445  }
   446
View as plain text