runfn.go

Documentation: sigs.k8s.io/kustomize/kyaml/runfn

     1  // Copyright 2019 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package runfn
     5  
     6  import (
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"os/user"
    11  	"path"
    12  	"path/filepath"
    13  	"sort"
    14  	"strconv"
    15  	"strings"
    16  	"sync/atomic"
    17  
    18  	"sigs.k8s.io/kustomize/kyaml/errors"
    19  	"sigs.k8s.io/kustomize/kyaml/fn/runtime/container"
    20  	"sigs.k8s.io/kustomize/kyaml/fn/runtime/exec"
    21  	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
    22  	"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
    23  	"sigs.k8s.io/kustomize/kyaml/kio"
    24  	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
    25  	"sigs.k8s.io/kustomize/kyaml/yaml"
    26  )
    27  
    28  // RunFns runs the set of configuration functions in a local directory against
    29  // the Resources in that directory
    30  type RunFns struct {
    31  	StorageMounts []runtimeutil.StorageMount
    32  
    33  	// Path is the path to the directory containing functions
    34  	Path string
    35  
    36  	// FunctionPaths Paths allows functions to be specified outside the configuration
    37  	// directory.
    38  	// Functions provided on FunctionPaths are globally scoped.
    39  	// If FunctionPaths length is > 0, then NoFunctionsFromInput defaults to true
    40  	FunctionPaths []string
    41  
    42  	// Functions is an explicit list of functions to run against the input.
    43  	// Functions provided on Functions are globally scoped.
    44  	// If Functions length is > 0, then NoFunctionsFromInput defaults to true
    45  	Functions []*yaml.RNode
    46  
    47  	// GlobalScope if true, functions read from input will be scoped globally rather
    48  	// than only to Resources under their subdirs.
    49  	GlobalScope bool
    50  
    51  	// Input can be set to read the Resources from Input rather than from a directory
    52  	Input io.Reader
    53  
    54  	// Network enables network access for functions that declare it
    55  	Network bool
    56  
    57  	// Output can be set to write the result to Output rather than back to the directory
    58  	Output io.Writer
    59  
    60  	// NoFunctionsFromInput if set to true will not read any functions from the input,
    61  	// and only use explicit sources
    62  	NoFunctionsFromInput *bool
    63  
    64  	// EnableStarlark will enable functions run as starlark scripts
    65  	EnableStarlark bool
    66  
    67  	// EnableExec will enable exec functions
    68  	EnableExec bool
    69  
    70  	// DisableContainers will disable functions run as containers
    71  	DisableContainers bool
    72  
    73  	// ResultsDir is where to write each functions results
    74  	ResultsDir string
    75  
    76  	// LogSteps enables logging the function that is running.
    77  	LogSteps bool
    78  
    79  	// LogWriter can be set to write the logs to LogWriter rather than stderr if LogSteps is enabled.
    80  	LogWriter io.Writer
    81  
    82  	// resultsCount is used to generate the results filename for each container
    83  	resultsCount uint32
    84  
    85  	// functionFilterProvider provides a filter to perform the function.
    86  	// this is a variable so it can be mocked in tests
    87  	functionFilterProvider func(
    88  		filter runtimeutil.FunctionSpec, api *yaml.RNode, currentUser currentUserFunc) (kio.Filter, error)
    89  
    90  	// AsCurrentUser is a boolean to indicate whether docker container should use
    91  	// the uid and gid that run the command
    92  	AsCurrentUser bool
    93  
    94  	// Env contains environment variables that will be exported to container
    95  	Env []string
    96  
    97  	// ContinueOnEmptyResult configures what happens when the underlying pipeline
    98  	// returns an empty result.
    99  	// If it is false (default), subsequent functions will be skipped and the
   100  	// result will be returned immediately.
   101  	// If it is true, the empty result will be provided as input to the next
   102  	// function in the list.
   103  	ContinueOnEmptyResult bool
   104  
   105  	// WorkingDir specifies which working directory an exec function should run in.
   106  	WorkingDir string
   107  }
   108  
   109  // Execute runs the command
   110  func (r RunFns) Execute() error {
   111  	// make the path absolute so it works on mac
   112  	var err error
   113  	r.Path, err = filepath.Abs(r.Path)
   114  	if err != nil {
   115  		return errors.Wrap(err)
   116  	}
   117  
   118  	// default the containerFilterProvider if it hasn't been override.  Split out for testing.
   119  	(&r).init()
   120  	nodes, fltrs, output, err := r.getNodesAndFilters()
   121  	if err != nil {
   122  		return err
   123  	}
   124  	return r.runFunctions(nodes, output, fltrs)
   125  }
   126  
   127  func (r RunFns) getNodesAndFilters() (
   128  	*kio.PackageBuffer, []kio.Filter, *kio.LocalPackageReadWriter, error) {
   129  	// Read Resources from Directory or Input
   130  	buff := &kio.PackageBuffer{}
   131  	p := kio.Pipeline{Outputs: []kio.Writer{buff}}
   132  	// save the output dir because we will need it to write back
   133  	// the same one for reading must be used for writing if deleting Resources
   134  	var outputPkg *kio.LocalPackageReadWriter
   135  	if r.Path != "" {
   136  		outputPkg = &kio.LocalPackageReadWriter{PackagePath: r.Path, MatchFilesGlob: kio.MatchAll}
   137  	}
   138  
   139  	if r.Input == nil {
   140  		p.Inputs = []kio.Reader{outputPkg}
   141  	} else {
   142  		p.Inputs = []kio.Reader{&kio.ByteReader{Reader: r.Input}}
   143  	}
   144  	if err := p.Execute(); err != nil {
   145  		return nil, nil, outputPkg, err
   146  	}
   147  
   148  	fltrs, err := r.getFilters(buff.Nodes)
   149  	if err != nil {
   150  		return nil, nil, outputPkg, err
   151  	}
   152  	return buff, fltrs, outputPkg, nil
   153  }
   154  
   155  func (r RunFns) getFilters(nodes []*yaml.RNode) ([]kio.Filter, error) {
   156  	var fltrs []kio.Filter
   157  
   158  	// fns from annotations on the input resources
   159  	f, err := r.getFunctionsFromInput(nodes)
   160  	if err != nil {
   161  		return nil, err
   162  	}
   163  	fltrs = append(fltrs, f...)
   164  
   165  	// fns from directories specified on the struct
   166  	f, err = r.getFunctionsFromFunctionPaths()
   167  	if err != nil {
   168  		return nil, err
   169  	}
   170  	fltrs = append(fltrs, f...)
   171  
   172  	// explicit fns specified on the struct
   173  	f, err = r.getFunctionsFromFunctions()
   174  	if err != nil {
   175  		return nil, err
   176  	}
   177  	fltrs = append(fltrs, f...)
   178  
   179  	return fltrs, nil
   180  }
   181  
   182  // runFunctions runs the fltrs against the input and writes to either r.Output or output
   183  func (r RunFns) runFunctions(
   184  	input kio.Reader, output kio.Writer, fltrs []kio.Filter) error {
   185  	// use the previously read Resources as input
   186  	var outputs []kio.Writer
   187  	if r.Output == nil {
   188  		// write back to the package
   189  		outputs = append(outputs, output)
   190  	} else {
   191  		// write to the output instead of the directory if r.Output is specified or
   192  		// the output is nil (reading from Input)
   193  		outputs = append(outputs, kio.ByteWriter{Writer: r.Output})
   194  	}
   195  
   196  	var err error
   197  	pipeline := kio.Pipeline{
   198  		Inputs:                []kio.Reader{input},
   199  		Filters:               fltrs,
   200  		Outputs:               outputs,
   201  		ContinueOnEmptyResult: r.ContinueOnEmptyResult,
   202  	}
   203  	if r.LogSteps {
   204  		err = pipeline.ExecuteWithCallback(func(op kio.Filter) {
   205  			var identifier string
   206  
   207  			switch filter := op.(type) {
   208  			case *container.Filter:
   209  				identifier = filter.Image
   210  			case *exec.Filter:
   211  				identifier = filter.Path
   212  			case *starlark.Filter:
   213  				identifier = filter.String()
   214  			default:
   215  				identifier = "unknown-type function"
   216  			}
   217  
   218  			_, _ = fmt.Fprintf(r.LogWriter, "Running %s\n", identifier)
   219  		})
   220  	} else {
   221  		err = pipeline.Execute()
   222  	}
   223  	if err != nil {
   224  		return err
   225  	}
   226  
   227  	// check for deferred function errors
   228  	var errs []string
   229  	for i := range fltrs {
   230  		cf, ok := fltrs[i].(runtimeutil.DeferFailureFunction)
   231  		if !ok {
   232  			continue
   233  		}
   234  		if cf.GetExit() != nil {
   235  			errs = append(errs, cf.GetExit().Error())
   236  		}
   237  	}
   238  	if len(errs) > 0 {
   239  		return fmt.Errorf(strings.Join(errs, "\n---\n"))
   240  	}
   241  	return nil
   242  }
   243  
   244  // getFunctionsFromInput scans the input for functions and runs them
   245  func (r RunFns) getFunctionsFromInput(nodes []*yaml.RNode) ([]kio.Filter, error) {
   246  	if *r.NoFunctionsFromInput {
   247  		return nil, nil
   248  	}
   249  
   250  	buff := &kio.PackageBuffer{}
   251  	err := kio.Pipeline{
   252  		Inputs:  []kio.Reader{&kio.PackageBuffer{Nodes: nodes}},
   253  		Filters: []kio.Filter{&runtimeutil.IsReconcilerFilter{}},
   254  		Outputs: []kio.Writer{buff},
   255  	}.Execute()
   256  	if err != nil {
   257  		return nil, err
   258  	}
   259  	err = sortFns(buff)
   260  	if err != nil {
   261  		return nil, err
   262  	}
   263  	return r.getFunctionFilters(false, buff.Nodes...)
   264  }
   265  
   266  // getFunctionsFromFunctionPaths returns the set of functions read from r.FunctionPaths
   267  // as a slice of Filters
   268  func (r RunFns) getFunctionsFromFunctionPaths() ([]kio.Filter, error) {
   269  	buff := &kio.PackageBuffer{}
   270  	for i := range r.FunctionPaths {
   271  		err := kio.Pipeline{
   272  			Inputs: []kio.Reader{
   273  				kio.LocalPackageReader{PackagePath: r.FunctionPaths[i]},
   274  			},
   275  			Outputs: []kio.Writer{buff},
   276  		}.Execute()
   277  		if err != nil {
   278  			return nil, err
   279  		}
   280  	}
   281  	return r.getFunctionFilters(true, buff.Nodes...)
   282  }
   283  
   284  // getFunctionsFromFunctions returns the set of explicitly provided functions as
   285  // Filters
   286  func (r RunFns) getFunctionsFromFunctions() ([]kio.Filter, error) {
   287  	return r.getFunctionFilters(true, r.Functions...)
   288  }
   289  
   290  // mergeContainerEnv will merge the envs specified by command line (imperative) and config
   291  // file (declarative). If they have same key, the imperative value will be respected.
   292  func (r RunFns) mergeContainerEnv(envs []string) []string {
   293  	imperative := runtimeutil.NewContainerEnvFromStringSlice(r.Env)
   294  	declarative := runtimeutil.NewContainerEnvFromStringSlice(envs)
   295  	for key, value := range imperative.EnvVars {
   296  		declarative.AddKeyValue(key, value)
   297  	}
   298  
   299  	for _, key := range imperative.VarsToExport {
   300  		declarative.AddKey(key)
   301  	}
   302  
   303  	return declarative.Raw()
   304  }
   305  
   306  func (r RunFns) getFunctionFilters(global bool, fns ...*yaml.RNode) (
   307  	[]kio.Filter, error) {
   308  	var fltrs []kio.Filter
   309  	for i := range fns {
   310  		api := fns[i]
   311  		spec, err := runtimeutil.GetFunctionSpec(api)
   312  		if err != nil {
   313  			return nil, fmt.Errorf("failed to get FunctionSpec: %w", err)
   314  		}
   315  		if spec == nil {
   316  			// resource doesn't have function spec
   317  			continue
   318  		}
   319  		if spec.Container.Network && !r.Network {
   320  			// TODO(eddiezane): Provide error info about which function needs the network
   321  			return fltrs, errors.Errorf("network required but not enabled with --network")
   322  		}
   323  		// merge envs from imperative and declarative
   324  		spec.Container.Env = r.mergeContainerEnv(spec.Container.Env)
   325  
   326  		c, err := r.functionFilterProvider(*spec, api, user.Current)
   327  		if err != nil {
   328  			return nil, err
   329  		}
   330  
   331  		if c == nil {
   332  			continue
   333  		}
   334  		cf, ok := c.(*container.Filter)
   335  		if ok {
   336  			if global {
   337  				cf.Exec.GlobalScope = true
   338  			}
   339  			cf.Exec.WorkingDir = r.WorkingDir
   340  		}
   341  		fltrs = append(fltrs, c)
   342  	}
   343  	return fltrs, nil
   344  }
   345  
   346  // sortFns sorts functions so that functions with the longest paths come first
   347  func sortFns(buff *kio.PackageBuffer) error {
   348  	var outerErr error
   349  	// sort the nodes so that we traverse them depth first
   350  	// functions deeper in the file system tree should be run first
   351  	sort.Slice(buff.Nodes, func(i, j int) bool {
   352  		if err := kioutil.CopyLegacyAnnotations(buff.Nodes[i]); err != nil {
   353  			return false
   354  		}
   355  		if err := kioutil.CopyLegacyAnnotations(buff.Nodes[j]); err != nil {
   356  			return false
   357  		}
   358  		mi, _ := buff.Nodes[i].GetMeta()
   359  		pi := filepath.ToSlash(mi.Annotations[kioutil.PathAnnotation])
   360  
   361  		mj, _ := buff.Nodes[j].GetMeta()
   362  		pj := filepath.ToSlash(mj.Annotations[kioutil.PathAnnotation])
   363  
   364  		// If the path is the same, we decide the ordering based on the
   365  		// index annotation.
   366  		if pi == pj {
   367  			iIndex, err := strconv.Atoi(mi.Annotations[kioutil.IndexAnnotation])
   368  			if err != nil {
   369  				outerErr = err
   370  				return false
   371  			}
   372  			jIndex, err := strconv.Atoi(mj.Annotations[kioutil.IndexAnnotation])
   373  			if err != nil {
   374  				outerErr = err
   375  				return false
   376  			}
   377  			return iIndex < jIndex
   378  		}
   379  
   380  		if filepath.Base(path.Dir(pi)) == "functions" {
   381  			// don't count the functions dir, the functions are scoped 1 level above
   382  			pi = filepath.Dir(path.Dir(pi))
   383  		} else {
   384  			pi = filepath.Dir(pi)
   385  		}
   386  
   387  		if filepath.Base(path.Dir(pj)) == "functions" {
   388  			// don't count the functions dir, the functions are scoped 1 level above
   389  			pj = filepath.Dir(path.Dir(pj))
   390  		} else {
   391  			pj = filepath.Dir(pj)
   392  		}
   393  
   394  		// i is "less" than j (comes earlier) if its depth is greater -- e.g. run
   395  		// i before j if it is deeper in the directory structure
   396  		li := len(strings.Split(pi, "/"))
   397  		if pi == "." {
   398  			// local dir should have 0 path elements instead of 1
   399  			li = 0
   400  		}
   401  		lj := len(strings.Split(pj, "/"))
   402  		if pj == "." {
   403  			// local dir should have 0 path elements instead of 1
   404  			lj = 0
   405  		}
   406  		if li != lj {
   407  			// use greater-than because we want to sort with the longest
   408  			// paths FIRST rather than last
   409  			return li > lj
   410  		}
   411  
   412  		// sort by path names if depths are equal
   413  		return pi < pj
   414  	})
   415  	return outerErr
   416  }
   417  
   418  // init initializes the RunFns with a containerFilterProvider.
   419  func (r *RunFns) init() {
   420  	if r.NoFunctionsFromInput == nil {
   421  		// default no functions from input if any function sources are explicitly provided
   422  		nfn := len(r.FunctionPaths) > 0 || len(r.Functions) > 0
   423  		r.NoFunctionsFromInput = &nfn
   424  	}
   425  
   426  	// if no path is specified, default reading from stdin and writing to stdout
   427  	if r.Path == "" {
   428  		if r.Output == nil {
   429  			r.Output = os.Stdout
   430  		}
   431  		if r.Input == nil {
   432  			r.Input = os.Stdin
   433  		}
   434  	}
   435  
   436  	// functionFilterProvider set the filter provider
   437  	if r.functionFilterProvider == nil {
   438  		r.functionFilterProvider = r.ffp
   439  	}
   440  
   441  	// if LogSteps is enabled and LogWriter is not specified, use stderr
   442  	if r.LogSteps && r.LogWriter == nil {
   443  		r.LogWriter = os.Stderr
   444  	}
   445  }
   446  
   447  type currentUserFunc func() (*user.User, error)
   448  
   449  // getUIDGID will return "nobody" if asCurrentUser is false. Otherwise
   450  // return "uid:gid" according to the return from currentUser function.
   451  func getUIDGID(asCurrentUser bool, currentUser currentUserFunc) (string, error) {
   452  	if !asCurrentUser {
   453  		return "nobody", nil
   454  	}
   455  
   456  	u, err := currentUser()
   457  	if err != nil {
   458  		return "", err
   459  	}
   460  	return fmt.Sprintf("%s:%s", u.Uid, u.Gid), nil
   461  }
   462  
   463  // ffp provides function filters
   464  func (r *RunFns) ffp(spec runtimeutil.FunctionSpec, api *yaml.RNode, currentUser currentUserFunc) (kio.Filter, error) {
   465  	var resultsFile string
   466  	if r.ResultsDir != "" {
   467  		resultsFile = filepath.Join(r.ResultsDir, fmt.Sprintf(
   468  			"results-%v.yaml", r.resultsCount))
   469  		atomic.AddUint32(&r.resultsCount, 1)
   470  	}
   471  	if !r.DisableContainers && spec.Container.Image != "" {
   472  		// TODO: Add a test for this behavior
   473  		uidgid, err := getUIDGID(r.AsCurrentUser, currentUser)
   474  		if err != nil {
   475  			return nil, err
   476  		}
   477  
   478  		// Storage mounts can either come from kustomize fn run --mounts,
   479  		// or from the declarative function mounts field.
   480  		storageMounts := spec.Container.StorageMounts
   481  		storageMounts = append(storageMounts, r.StorageMounts...)
   482  
   483  		c := container.NewContainer(
   484  			runtimeutil.ContainerSpec{
   485  				Image:         spec.Container.Image,
   486  				Network:       spec.Container.Network,
   487  				StorageMounts: storageMounts,
   488  				Env:           spec.Container.Env,
   489  			},
   490  			uidgid,
   491  		)
   492  		cf := &c
   493  		cf.Exec.FunctionConfig = api
   494  		cf.Exec.GlobalScope = r.GlobalScope
   495  		cf.Exec.ResultsFile = resultsFile
   496  		cf.Exec.DeferFailure = spec.DeferFailure
   497  		return cf, nil
   498  	}
   499  	if r.EnableStarlark && (spec.Starlark.Path != "" || spec.Starlark.URL != "") {
   500  		// the script path is relative to the function config file
   501  		m, err := api.GetMeta()
   502  		if err != nil {
   503  			return nil, errors.Wrap(err)
   504  		}
   505  
   506  		var p string
   507  		if spec.Starlark.Path != "" {
   508  			pathAnno := m.Annotations[kioutil.PathAnnotation]
   509  			if pathAnno == "" {
   510  				pathAnno = m.Annotations[kioutil.LegacyPathAnnotation]
   511  			}
   512  			p = filepath.ToSlash(path.Clean(pathAnno))
   513  
   514  			spec.Starlark.Path = filepath.ToSlash(path.Clean(spec.Starlark.Path))
   515  			if filepath.IsAbs(spec.Starlark.Path) || path.IsAbs(spec.Starlark.Path) {
   516  				return nil, errors.Errorf(
   517  					"absolute function path %s not allowed", spec.Starlark.Path)
   518  			}
   519  			if strings.HasPrefix(spec.Starlark.Path, "..") {
   520  				return nil, errors.Errorf(
   521  					"function path %s not allowed to start with ../", spec.Starlark.Path)
   522  			}
   523  			p = filepath.ToSlash(filepath.Join(r.Path, filepath.Dir(p), spec.Starlark.Path))
   524  		}
   525  
   526  		sf := &starlark.Filter{Name: spec.Starlark.Name, Path: p, URL: spec.Starlark.URL}
   527  
   528  		sf.FunctionConfig = api
   529  		sf.GlobalScope = r.GlobalScope
   530  		sf.ResultsFile = resultsFile
   531  		sf.DeferFailure = spec.DeferFailure
   532  		return sf, nil
   533  	}
   534  
   535  	if r.EnableExec && spec.Exec.Path != "" {
   536  		ef := &exec.Filter{
   537  			Path:       spec.Exec.Path,
   538  			WorkingDir: r.WorkingDir,
   539  		}
   540  
   541  		ef.FunctionConfig = api
   542  		ef.GlobalScope = r.GlobalScope
   543  		ef.ResultsFile = resultsFile
   544  		ef.DeferFailure = spec.DeferFailure
   545  		return ef, nil
   546  	}
   547  
   548  	return nil, nil
   549  }
   550
View as plain text