pwalkdir_test.go

Documentation: github.com/opencontainers/selinux/pkg/pwalkdir

     1  //go:build go1.16
     2  // +build go1.16
     3  
     4  package pwalkdir
     5  
     6  import (
     7  	"errors"
     8  	"io/fs"
     9  	"math/rand"
    10  	"os"
    11  	"path/filepath"
    12  	"runtime"
    13  	"sync/atomic"
    14  	"testing"
    15  	"time"
    16  )
    17  
    18  func TestWalkDir(t *testing.T) {
    19  	var count uint32
    20  	concurrency := runtime.NumCPU() * 2
    21  
    22  	dir, total, err := prepareTestSet(3, 2, 1)
    23  	if err != nil {
    24  		t.Fatalf("dataset creation failed: %v", err)
    25  	}
    26  	defer os.RemoveAll(dir)
    27  
    28  	err = WalkN(dir,
    29  		func(_ string, _ fs.DirEntry, _ error) error {
    30  			atomic.AddUint32(&count, 1)
    31  			return nil
    32  		},
    33  		concurrency)
    34  
    35  	if err != nil {
    36  		t.Errorf("Walk failed: %v", err)
    37  	}
    38  	if count != uint32(total) {
    39  		t.Errorf("File count mismatch: found %d, expected %d", count, total)
    40  	}
    41  
    42  	t.Logf("concurrency: %d, files found: %d\n", concurrency, count)
    43  }
    44  
    45  func TestWalkDirManyErrors(t *testing.T) {
    46  	var count uint32
    47  
    48  	dir, total, err := prepareTestSet(3, 3, 2)
    49  	if err != nil {
    50  		t.Fatalf("dataset creation failed: %v", err)
    51  	}
    52  	defer os.RemoveAll(dir)
    53  
    54  	max := uint32(total / 2)
    55  	e42 := errors.New("42")
    56  	err = Walk(dir,
    57  		func(p string, e fs.DirEntry, _ error) error {
    58  			if atomic.AddUint32(&count, 1) > max {
    59  				return e42
    60  			}
    61  			return nil
    62  		})
    63  	t.Logf("found %d of %d files", count, total)
    64  
    65  	if err == nil {
    66  		t.Error("Walk succeeded, but error is expected")
    67  		if count != uint32(total) {
    68  			t.Errorf("File count mismatch: found %d, expected %d", count, total)
    69  		}
    70  	}
    71  }
    72  
    73  func makeManyDirs(prefix string, levels, dirs, files int) (count int, err error) {
    74  	for d := 0; d < dirs; d++ {
    75  		var dir string
    76  		dir, err = os.MkdirTemp(prefix, "d-")
    77  		if err != nil {
    78  			return
    79  		}
    80  		count++
    81  		for f := 0; f < files; f++ {
    82  			var fi *os.File
    83  			fi, err = os.CreateTemp(dir, "f-")
    84  			if err != nil {
    85  				return count, err
    86  			}
    87  			fi.Close()
    88  			count++
    89  		}
    90  		if levels == 0 {
    91  			continue
    92  		}
    93  		var c int
    94  		if c, err = makeManyDirs(dir, levels-1, dirs, files); err != nil {
    95  			return
    96  		}
    97  		count += c
    98  	}
    99  
   100  	return
   101  }
   102  
   103  // prepareTestSet() creates a directory tree of shallow files,
   104  // to be used for testing or benchmarking.
   105  //
   106  // Total dirs: dirs^levels + dirs^(levels-1) + ... + dirs^1
   107  // Total files: total_dirs * files
   108  func prepareTestSet(levels, dirs, files int) (dir string, total int, err error) {
   109  	dir, err = os.MkdirTemp(".", "pwalk-test-")
   110  	if err != nil {
   111  		return
   112  	}
   113  	total, err = makeManyDirs(dir, levels, dirs, files)
   114  	if err != nil && total > 0 {
   115  		_ = os.RemoveAll(dir)
   116  		dir = ""
   117  		total = 0
   118  		return
   119  	}
   120  	total++ // this dir
   121  
   122  	return
   123  }
   124  
   125  type walkerFunc func(root string, walkFn fs.WalkDirFunc) error
   126  
   127  func genWalkN(n int) walkerFunc {
   128  	return func(root string, walkFn fs.WalkDirFunc) error {
   129  		return WalkN(root, walkFn, n)
   130  	}
   131  }
   132  
   133  func BenchmarkWalk(b *testing.B) {
   134  	const (
   135  		levels = 5 // how deep
   136  		dirs   = 3 // dirs on each levels
   137  		files  = 8 // files on each levels
   138  	)
   139  
   140  	benchmarks := []struct {
   141  		walk fs.WalkDirFunc
   142  		name string
   143  	}{
   144  		{name: "Empty", walk: cbEmpty},
   145  		{name: "ReadFile", walk: cbReadFile},
   146  		{name: "ChownChmod", walk: cbChownChmod},
   147  		{name: "RandomSleep", walk: cbRandomSleep},
   148  	}
   149  
   150  	walkers := []struct {
   151  		walker walkerFunc
   152  		name   string
   153  	}{
   154  		{name: "filepath.WalkDir", walker: filepath.WalkDir},
   155  		{name: "pwalkdir.Walk", walker: Walk},
   156  		// test WalkN with various values of N
   157  		{name: "pwalkdir.Walk1", walker: genWalkN(1)},
   158  		{name: "pwalkdir.Walk2", walker: genWalkN(2)},
   159  		{name: "pwalkdir.Walk4", walker: genWalkN(4)},
   160  		{name: "pwalkdir.Walk8", walker: genWalkN(8)},
   161  		{name: "pwalkdir.Walk16", walker: genWalkN(16)},
   162  		{name: "pwalkdir.Walk32", walker: genWalkN(32)},
   163  		{name: "pwalkdir.Walk64", walker: genWalkN(64)},
   164  		{name: "pwalkdir.Walk128", walker: genWalkN(128)},
   165  		{name: "pwalkdir.Walk256", walker: genWalkN(256)},
   166  	}
   167  
   168  	dir, total, err := prepareTestSet(levels, dirs, files)
   169  	if err != nil {
   170  		b.Fatalf("dataset creation failed: %v", err)
   171  	}
   172  	defer os.RemoveAll(dir)
   173  	b.Logf("dataset: %d levels x %d dirs x %d files, total entries: %d", levels, dirs, files, total)
   174  
   175  	for _, bm := range benchmarks {
   176  		for _, w := range walkers {
   177  			walker := w.walker
   178  			walkFn := bm.walk
   179  			// preheat
   180  			if err := w.walker(dir, bm.walk); err != nil {
   181  				b.Errorf("walk failed: %v", err)
   182  			}
   183  			// benchmark
   184  			b.Run(bm.name+"/"+w.name, func(b *testing.B) {
   185  				for i := 0; i < b.N; i++ {
   186  					if err := walker(dir, walkFn); err != nil {
   187  						b.Errorf("walk failed: %v", err)
   188  					}
   189  				}
   190  			})
   191  		}
   192  	}
   193  }
   194  
   195  func cbEmpty(_ string, _ fs.DirEntry, _ error) error {
   196  	return nil
   197  }
   198  
   199  func cbChownChmod(path string, e fs.DirEntry, _ error) error {
   200  	_ = os.Chown(path, 0, 0)
   201  	mode := os.FileMode(0o644)
   202  	if e.IsDir() {
   203  		mode = os.FileMode(0o755)
   204  	}
   205  	_ = os.Chmod(path, mode)
   206  
   207  	return nil
   208  }
   209  
   210  func cbReadFile(path string, e fs.DirEntry, _ error) error {
   211  	var err error
   212  	if e.Type().IsRegular() {
   213  		_, err = os.ReadFile(path)
   214  	}
   215  	return err
   216  }
   217  
   218  func cbRandomSleep(_ string, _ fs.DirEntry, _ error) error {
   219  	time.Sleep(time.Duration(rand.Intn(500)) * time.Microsecond) //nolint:gosec // ignore G404: Use of weak random number generator
   220  	return nil
   221  }
   222
View as plain text