pkgio_reader.go

Documentation: sigs.k8s.io/kustomize/kyaml/kio

     1  // Copyright 2019 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package kio
     5  
     6  import (
     7  	"fmt"
     8  	"os"
     9  	"path/filepath"
    10  
    11  	"sigs.k8s.io/kustomize/kyaml/errors"
    12  	"sigs.k8s.io/kustomize/kyaml/filesys"
    13  	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
    14  	"sigs.k8s.io/kustomize/kyaml/sets"
    15  	"sigs.k8s.io/kustomize/kyaml/yaml"
    16  )
    17  
    18  // requiredResourcePackageAnnotations are annotations that are required to write resources back to
    19  // files.
    20  var requiredResourcePackageAnnotations = []string{kioutil.IndexAnnotation, kioutil.PathAnnotation}
    21  
    22  // PackageBuffer implements Reader and Writer, storing Resources in a local field.
    23  type PackageBuffer struct {
    24  	Nodes []*yaml.RNode
    25  }
    26  
    27  func (r *PackageBuffer) Read() ([]*yaml.RNode, error) {
    28  	return r.Nodes, nil
    29  }
    30  
    31  func (r *PackageBuffer) Write(nodes []*yaml.RNode) error {
    32  	r.Nodes = nodes
    33  	return nil
    34  }
    35  
    36  // LocalPackageReadWriter reads and writes Resources from / to a local directory.
    37  // When writing, LocalPackageReadWriter will delete files if all of the Resources from
    38  // that file have been removed from the output.
    39  type LocalPackageReadWriter struct {
    40  	Kind string `yaml:"kind,omitempty"`
    41  
    42  	KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
    43  
    44  	// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
    45  	PreserveSeqIndent bool
    46  
    47  	// PackagePath is the path to the package directory.
    48  	PackagePath string `yaml:"path,omitempty"`
    49  
    50  	// PackageFileName is the name of file containing package metadata.
    51  	// It will be used to identify package.
    52  	PackageFileName string `yaml:"packageFileName,omitempty"`
    53  
    54  	// MatchFilesGlob configures Read to only read Resources from files matching any of the
    55  	// provided patterns.
    56  	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
    57  	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
    58  
    59  	// IncludeSubpackages will configure Read to read Resources from subpackages.
    60  	// Subpackages are identified by presence of PackageFileName.
    61  	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
    62  
    63  	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
    64  	// apiVersion or kind is read.
    65  	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
    66  
    67  	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
    68  	// path and mode.
    69  	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
    70  
    71  	// SetAnnotations are annotations to set on the Resources as they are read.
    72  	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
    73  
    74  	// NoDeleteFiles if set to true, LocalPackageReadWriter won't delete any files
    75  	NoDeleteFiles bool `yaml:"noDeleteFiles,omitempty"`
    76  
    77  	files sets.String
    78  
    79  	// FileSkipFunc is a function which returns true if reader should ignore
    80  	// the file
    81  	FileSkipFunc LocalPackageSkipFileFunc
    82  
    83  	// FileSystem can be used to mock the disk file system.
    84  	FileSystem filesys.FileSystemOrOnDisk
    85  
    86  	// WrapBareSeqNode wraps the bare sequence node document with map node,
    87  	// kyaml uses reader annotations to track resources, it is not possible to
    88  	// add them to bare sequence nodes, this option enables wrapping such bare
    89  	// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
    90  	// note that this wrapping is different and not related to ResourceList wrapping
    91  	WrapBareSeqNode bool
    92  }
    93  
    94  func (r *LocalPackageReadWriter) Read() ([]*yaml.RNode, error) {
    95  	nodes, err := LocalPackageReader{
    96  		PackagePath:         r.PackagePath,
    97  		MatchFilesGlob:      r.MatchFilesGlob,
    98  		IncludeSubpackages:  r.IncludeSubpackages,
    99  		ErrorIfNonResources: r.ErrorIfNonResources,
   100  		SetAnnotations:      r.SetAnnotations,
   101  		PackageFileName:     r.PackageFileName,
   102  		FileSkipFunc:        r.FileSkipFunc,
   103  		PreserveSeqIndent:   r.PreserveSeqIndent,
   104  		FileSystem:          r.FileSystem,
   105  		WrapBareSeqNode:     r.WrapBareSeqNode,
   106  	}.Read()
   107  	if err != nil {
   108  		return nil, errors.Wrap(err)
   109  	}
   110  	// keep track of all the files
   111  	if !r.NoDeleteFiles {
   112  		r.files, err = r.getFiles(nodes)
   113  		if err != nil {
   114  			return nil, errors.Wrap(err)
   115  		}
   116  	}
   117  	return nodes, nil
   118  }
   119  
   120  func (r *LocalPackageReadWriter) Write(nodes []*yaml.RNode) error {
   121  	newFiles, err := r.getFiles(nodes)
   122  	if err != nil {
   123  		return errors.Wrap(err)
   124  	}
   125  	var clear []string
   126  	for k := range r.SetAnnotations {
   127  		clear = append(clear, k)
   128  	}
   129  	err = LocalPackageWriter{
   130  		PackagePath:           r.PackagePath,
   131  		ClearAnnotations:      clear,
   132  		KeepReaderAnnotations: r.KeepReaderAnnotations,
   133  		FileSystem:            r.FileSystem,
   134  	}.Write(nodes)
   135  	if err != nil {
   136  		return errors.Wrap(err)
   137  	}
   138  	deleteFiles := r.files.Difference(newFiles)
   139  	for f := range deleteFiles {
   140  		if err = r.FileSystem.RemoveAll(filepath.Join(r.PackagePath, f)); err != nil {
   141  			return errors.Wrap(err)
   142  		}
   143  	}
   144  	return nil
   145  }
   146  
   147  func (r *LocalPackageReadWriter) getFiles(nodes []*yaml.RNode) (sets.String, error) {
   148  	val := sets.String{}
   149  	for _, n := range nodes {
   150  		path, _, err := kioutil.GetFileAnnotations(n)
   151  		if err != nil {
   152  			return nil, errors.Wrap(err)
   153  		}
   154  		val.Insert(path)
   155  	}
   156  	return val, nil
   157  }
   158  
   159  // LocalPackageSkipFileFunc is a function which returns true if the file
   160  // in the package should be ignored by reader.
   161  // relPath is an OS specific relative path
   162  type LocalPackageSkipFileFunc func(relPath string) bool
   163  
   164  // LocalPackageReader reads ResourceNodes from a local package.
   165  type LocalPackageReader struct {
   166  	Kind string `yaml:"kind,omitempty"`
   167  
   168  	// PackagePath is the path to the package directory.
   169  	PackagePath string `yaml:"path,omitempty"`
   170  
   171  	// PackageFileName is the name of file containing package metadata.
   172  	// It will be used to identify package.
   173  	PackageFileName string `yaml:"packageFileName,omitempty"`
   174  
   175  	// MatchFilesGlob configures Read to only read Resources from files matching any of the
   176  	// provided patterns.
   177  	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
   178  	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
   179  
   180  	// IncludeSubpackages will configure Read to read Resources from subpackages.
   181  	// Subpackages are identified by presence of PackageFileName.
   182  	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
   183  
   184  	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
   185  	// apiVersion or kind is read.
   186  	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
   187  
   188  	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
   189  	// path and mode.
   190  	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
   191  
   192  	// SetAnnotations are annotations to set on the Resources as they are read.
   193  	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
   194  
   195  	// FileSkipFunc is a function which returns true if reader should ignore
   196  	// the file
   197  	FileSkipFunc LocalPackageSkipFileFunc
   198  
   199  	// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
   200  	PreserveSeqIndent bool
   201  
   202  	// FileSystem can be used to mock the disk file system.
   203  	FileSystem filesys.FileSystemOrOnDisk
   204  
   205  	// WrapBareSeqNode wraps the bare sequence node document with map node,
   206  	// kyaml uses reader annotations to track resources, it is not possible to
   207  	// add them to bare sequence nodes, this option enables wrapping such bare
   208  	// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
   209  	// note that this wrapping is different and not related to ResourceList wrapping
   210  	WrapBareSeqNode bool
   211  }
   212  
   213  var _ Reader = LocalPackageReader{}
   214  
   215  var DefaultMatch = []string{"*.yaml", "*.yml"}
   216  var JSONMatch = []string{"*.json"}
   217  var MatchAll = append(DefaultMatch, JSONMatch...)
   218  
   219  // Read reads the Resources.
   220  func (r LocalPackageReader) Read() ([]*yaml.RNode, error) {
   221  	if r.PackagePath == "" {
   222  		return nil, fmt.Errorf("must specify package path")
   223  	}
   224  
   225  	// use slash for path
   226  	r.PackagePath = filepath.ToSlash(r.PackagePath)
   227  	if len(r.MatchFilesGlob) == 0 {
   228  		r.MatchFilesGlob = DefaultMatch
   229  	}
   230  
   231  	var operand ResourceNodeSlice
   232  	var pathRelativeTo string
   233  	var err error
   234  	ignoreFilesMatcher := &ignoreFilesMatcher{
   235  		fs: r.FileSystem,
   236  	}
   237  	dir, file, err := r.FileSystem.CleanedAbs(r.PackagePath)
   238  	if err != nil {
   239  		return nil, errors.Wrap(err)
   240  	}
   241  	r.PackagePath = filepath.Join(string(dir), file)
   242  	err = r.FileSystem.Walk(r.PackagePath, func(
   243  		path string, info os.FileInfo, err error) error {
   244  		if err != nil {
   245  			return errors.Wrap(err)
   246  		}
   247  
   248  		// is this the user specified path?
   249  		if path == r.PackagePath {
   250  			if info.IsDir() {
   251  				// skip the root package directory, but check for a
   252  				// .krmignore file first.
   253  				pathRelativeTo = r.PackagePath
   254  				return ignoreFilesMatcher.readIgnoreFile(path)
   255  			}
   256  
   257  			// user specified path is a file rather than a directory.
   258  			// make its path relative to its parent so it can be written to another file.
   259  			pathRelativeTo = filepath.Dir(r.PackagePath)
   260  		}
   261  
   262  		// check if we should skip the directory or file
   263  		if info.IsDir() {
   264  			return r.shouldSkipDir(path, ignoreFilesMatcher)
   265  		}
   266  
   267  		// get the relative path to file within the package so we can write the files back out
   268  		// to another location.
   269  		relPath, err := filepath.Rel(pathRelativeTo, path)
   270  		if err != nil {
   271  			return errors.WrapPrefixf(err, pathRelativeTo)
   272  		}
   273  		if match, err := r.shouldSkipFile(path, relPath, ignoreFilesMatcher); err != nil {
   274  			return err
   275  		} else if match {
   276  			// skip this file
   277  			return nil
   278  		}
   279  
   280  		r.initReaderAnnotations(relPath, info)
   281  		nodes, err := r.readFile(path, info)
   282  		if err != nil {
   283  			return errors.WrapPrefixf(err, path)
   284  		}
   285  		operand = append(operand, nodes...)
   286  		return nil
   287  	})
   288  	return operand, err
   289  }
   290  
   291  // readFile reads the ResourceNodes from a file
   292  func (r *LocalPackageReader) readFile(path string, _ os.FileInfo) ([]*yaml.RNode, error) {
   293  	f, err := r.FileSystem.Open(path)
   294  	if err != nil {
   295  		return nil, err
   296  	}
   297  	defer f.Close()
   298  
   299  	rr := &ByteReader{
   300  		DisableUnwrapping:     true,
   301  		Reader:                f,
   302  		OmitReaderAnnotations: r.OmitReaderAnnotations,
   303  		SetAnnotations:        r.SetAnnotations,
   304  		PreserveSeqIndent:     r.PreserveSeqIndent,
   305  		WrapBareSeqNode:       r.WrapBareSeqNode,
   306  	}
   307  	return rr.Read()
   308  }
   309  
   310  // shouldSkipFile returns true if the file should be skipped
   311  func (r *LocalPackageReader) shouldSkipFile(path, relPath string, matcher *ignoreFilesMatcher) (bool, error) {
   312  	// check if the file is covered by a .krmignore file.
   313  	if matcher.matchFile(path) {
   314  		return true, nil
   315  	}
   316  
   317  	if r.FileSkipFunc != nil && r.FileSkipFunc(relPath) {
   318  		return true, nil
   319  	}
   320  
   321  	// check if the files are in scope
   322  	for _, g := range r.MatchFilesGlob {
   323  		if match, err := filepath.Match(g, filepath.Base(path)); err != nil {
   324  			return true, errors.Wrap(err)
   325  		} else if match {
   326  			return false, nil
   327  		}
   328  	}
   329  	return true, nil
   330  }
   331  
   332  // initReaderAnnotations adds the LocalPackageReader Annotations to r.SetAnnotations
   333  func (r *LocalPackageReader) initReaderAnnotations(path string, _ os.FileInfo) {
   334  	if r.SetAnnotations == nil {
   335  		r.SetAnnotations = map[string]string{}
   336  	}
   337  	if !r.OmitReaderAnnotations {
   338  		r.SetAnnotations[kioutil.PathAnnotation] = path
   339  		r.SetAnnotations[kioutil.LegacyPathAnnotation] = path
   340  	}
   341  }
   342  
   343  // shouldSkipDir returns a filepath.SkipDir if the directory should be skipped
   344  func (r *LocalPackageReader) shouldSkipDir(path string, matcher *ignoreFilesMatcher) error {
   345  	if matcher.matchDir(path) {
   346  		return filepath.SkipDir
   347  	}
   348  
   349  	if r.PackageFileName == "" {
   350  		return nil
   351  	}
   352  	// check if this is a subpackage
   353  	if !r.FileSystem.Exists(filepath.Join(path, r.PackageFileName)) {
   354  		return nil
   355  	}
   356  	if !r.IncludeSubpackages {
   357  		return filepath.SkipDir
   358  	}
   359  	return matcher.readIgnoreFile(path)
   360  }
   361
View as plain text