kusttarget.go

Documentation: sigs.k8s.io/kustomize/api/internal/target

     1  // Copyright 2019 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package target
     5  
     6  import (
     7  	"encoding/json"
     8  	"fmt"
     9  	"os"
    10  	"strings"
    11  
    12  	"sigs.k8s.io/kustomize/api/ifc"
    13  	"sigs.k8s.io/kustomize/api/internal/accumulator"
    14  	"sigs.k8s.io/kustomize/api/internal/builtins"
    15  	"sigs.k8s.io/kustomize/api/internal/kusterr"
    16  	load "sigs.k8s.io/kustomize/api/internal/loader"
    17  	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
    18  	"sigs.k8s.io/kustomize/api/internal/plugins/builtinhelpers"
    19  	"sigs.k8s.io/kustomize/api/internal/plugins/loader"
    20  	"sigs.k8s.io/kustomize/api/internal/utils"
    21  	"sigs.k8s.io/kustomize/api/konfig"
    22  	"sigs.k8s.io/kustomize/api/resmap"
    23  	"sigs.k8s.io/kustomize/api/resource"
    24  	"sigs.k8s.io/kustomize/api/types"
    25  	"sigs.k8s.io/kustomize/kyaml/errors"
    26  	"sigs.k8s.io/kustomize/kyaml/openapi"
    27  	"sigs.k8s.io/yaml"
    28  )
    29  
    30  // KustTarget encapsulates the entirety of a kustomization build.
    31  type KustTarget struct {
    32  	kustomization *types.Kustomization
    33  	kustFileName  string
    34  	ldr           ifc.Loader
    35  	validator     ifc.Validator
    36  	rFactory      *resmap.Factory
    37  	pLdr          *loader.Loader
    38  	origin        *resource.Origin
    39  }
    40  
    41  // NewKustTarget returns a new instance of KustTarget.
    42  func NewKustTarget(
    43  	ldr ifc.Loader,
    44  	validator ifc.Validator,
    45  	rFactory *resmap.Factory,
    46  	pLdr *loader.Loader) *KustTarget {
    47  	return &KustTarget{
    48  		ldr:       ldr,
    49  		validator: validator,
    50  		rFactory:  rFactory,
    51  		pLdr:      pLdr.LoaderWithWorkingDir(ldr.Root()),
    52  	}
    53  }
    54  
    55  // Load attempts to load the target's kustomization file.
    56  func (kt *KustTarget) Load() error {
    57  	content, kustFileName, err := LoadKustFile(kt.ldr)
    58  	if err != nil {
    59  		return err
    60  	}
    61  
    62  	var k types.Kustomization
    63  	if err := k.Unmarshal(content); err != nil {
    64  		return err
    65  	}
    66  
    67  	// show warning message when using deprecated fields.
    68  	if warningMessages := k.CheckDeprecatedFields(); warningMessages != nil {
    69  		for _, msg := range *warningMessages {
    70  			fmt.Fprintf(os.Stderr, "%v\n", msg)
    71  		}
    72  	}
    73  
    74  	k.FixKustomization()
    75  
    76  	// check that Kustomization is empty
    77  	if err := k.CheckEmpty(); err != nil {
    78  		return err
    79  	}
    80  
    81  	errs := k.EnforceFields()
    82  	if len(errs) > 0 {
    83  		return fmt.Errorf(
    84  			"Failed to read kustomization file under %s:\n"+
    85  				strings.Join(errs, "\n"), kt.ldr.Root())
    86  	}
    87  	kt.kustomization = &k
    88  	kt.kustFileName = kustFileName
    89  	return nil
    90  }
    91  
    92  // Kustomization returns a copy of the immutable, internal kustomization object.
    93  func (kt *KustTarget) Kustomization() types.Kustomization {
    94  	var result types.Kustomization
    95  	b, _ := json.Marshal(*kt.kustomization)
    96  	json.Unmarshal(b, &result)
    97  	return result
    98  }
    99  
   100  func LoadKustFile(ldr ifc.Loader) ([]byte, string, error) {
   101  	var content []byte
   102  	match := 0
   103  	var kustFileName string
   104  	for _, kf := range konfig.RecognizedKustomizationFileNames() {
   105  		c, err := ldr.Load(kf)
   106  		if err == nil {
   107  			match += 1
   108  			content = c
   109  			kustFileName = kf
   110  		}
   111  	}
   112  	switch match {
   113  	case 0:
   114  		return nil, "", NewErrMissingKustomization(ldr.Root())
   115  	case 1:
   116  		return content, kustFileName, nil
   117  	default:
   118  		return nil, "", fmt.Errorf(
   119  			"Found multiple kustomization files under: %s\n", ldr.Root())
   120  	}
   121  }
   122  
   123  // MakeCustomizedResMap creates a fully customized ResMap
   124  // per the instructions contained in its kustomization instance.
   125  func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) {
   126  	return kt.makeCustomizedResMap()
   127  }
   128  
   129  func (kt *KustTarget) makeCustomizedResMap() (resmap.ResMap, error) {
   130  	var origin *resource.Origin
   131  	if len(kt.kustomization.BuildMetadata) != 0 {
   132  		origin = &resource.Origin{}
   133  	}
   134  	kt.origin = origin
   135  	ra, err := kt.AccumulateTarget()
   136  	if err != nil {
   137  		return nil, err
   138  	}
   139  
   140  	// The following steps must be done last, not as part of
   141  	// the recursion implicit in AccumulateTarget.
   142  
   143  	err = kt.addHashesToNames(ra)
   144  	if err != nil {
   145  		return nil, err
   146  	}
   147  
   148  	// Given that names have changed (prefixs/suffixes added),
   149  	// fix all the back references to those names.
   150  	err = ra.FixBackReferences()
   151  	if err != nil {
   152  		return nil, err
   153  	}
   154  
   155  	// With all the back references fixed, it's OK to resolve Vars.
   156  	err = ra.ResolveVars()
   157  	if err != nil {
   158  		return nil, err
   159  	}
   160  
   161  	err = kt.IgnoreLocal(ra)
   162  	if err != nil {
   163  		return nil, err
   164  	}
   165  
   166  	return ra.ResMap(), nil
   167  }
   168  
   169  func (kt *KustTarget) addHashesToNames(
   170  	ra *accumulator.ResAccumulator) error {
   171  	p := builtins.NewHashTransformerPlugin()
   172  	err := kt.configureBuiltinPlugin(p, nil, builtinhelpers.HashTransformer)
   173  	if err != nil {
   174  		return err
   175  	}
   176  	return ra.Transform(p)
   177  }
   178  
   179  // AccumulateTarget returns a new ResAccumulator,
   180  // holding customized resources and the data/rules used
   181  // to do so.  The name back references and vars are
   182  // not yet fixed.
   183  // The origin parameter is used through the recursive calls
   184  // to annotate each resource with information about where
   185  // the resource came from, e.g. the file and/or the repository
   186  // it originated from.
   187  // As an entrypoint, one can pass an empty resource.Origin object to
   188  // AccumulateTarget. As AccumulateTarget moves recursively
   189  // through kustomization directories, it updates `origin.path`
   190  // accordingly. When a remote base is found, it updates `origin.repo`
   191  // and `origin.ref` accordingly.
   192  func (kt *KustTarget) AccumulateTarget() (
   193  	ra *accumulator.ResAccumulator, err error) {
   194  	return kt.accumulateTarget(accumulator.MakeEmptyAccumulator())
   195  }
   196  
   197  // ra should be empty when this KustTarget is a Kustomization, or the ra of the parent if this KustTarget is a Component
   198  // (or empty if the Component does not have a parent).
   199  func (kt *KustTarget) accumulateTarget(ra *accumulator.ResAccumulator) (
   200  	resRa *accumulator.ResAccumulator, err error) {
   201  	ra, err = kt.accumulateResources(ra, kt.kustomization.Resources)
   202  	if err != nil {
   203  		return nil, errors.WrapPrefixf(err, "accumulating resources")
   204  	}
   205  	tConfig, err := builtinconfig.MakeTransformerConfig(
   206  		kt.ldr, kt.kustomization.Configurations)
   207  	if err != nil {
   208  		return nil, err
   209  	}
   210  	err = ra.MergeConfig(tConfig)
   211  	if err != nil {
   212  		return nil, errors.WrapPrefixf(
   213  			err, "merging config %v", tConfig)
   214  	}
   215  	crdTc, err := accumulator.LoadConfigFromCRDs(kt.ldr, kt.kustomization.Crds)
   216  	if err != nil {
   217  		return nil, errors.WrapPrefixf(
   218  			err, "loading CRDs %v", kt.kustomization.Crds)
   219  	}
   220  	err = ra.MergeConfig(crdTc)
   221  	if err != nil {
   222  		return nil, errors.WrapPrefixf(
   223  			err, "merging CRDs %v", crdTc)
   224  	}
   225  	err = kt.runGenerators(ra)
   226  	if err != nil {
   227  		return nil, err
   228  	}
   229  
   230  	// components are expected to execute after reading resources and adding generators ,before applying transformers and validation.
   231  	// https://github.com/kubernetes-sigs/kustomize/pull/5170#discussion_r1212101287
   232  	ra, err = kt.accumulateComponents(ra, kt.kustomization.Components)
   233  	if err != nil {
   234  		return nil, errors.WrapPrefixf(err, "accumulating components")
   235  	}
   236  
   237  	err = kt.runTransformers(ra)
   238  	if err != nil {
   239  		return nil, err
   240  	}
   241  	err = kt.runValidators(ra)
   242  	if err != nil {
   243  		return nil, err
   244  	}
   245  	err = ra.MergeVars(kt.kustomization.Vars)
   246  	if err != nil {
   247  		return nil, errors.WrapPrefixf(
   248  			err, "merging vars %v", kt.kustomization.Vars)
   249  	}
   250  	return ra, nil
   251  }
   252  
   253  // IgnoreLocal drops the local resource by checking the annotation "config.kubernetes.io/local-config".
   254  func (kt *KustTarget) IgnoreLocal(ra *accumulator.ResAccumulator) error {
   255  	rf := kt.rFactory.RF()
   256  	if rf.IncludeLocalConfigs {
   257  		return nil
   258  	}
   259  	remainRes, err := rf.DropLocalNodes(ra.ResMap().ToRNodeSlice())
   260  	if err != nil {
   261  		return err
   262  	}
   263  	return ra.Intersection(kt.rFactory.FromResourceSlice(remainRes))
   264  }
   265  
   266  func (kt *KustTarget) runGenerators(
   267  	ra *accumulator.ResAccumulator) error {
   268  	var generators []*resmap.GeneratorWithProperties
   269  	gs, err := kt.configureBuiltinGenerators()
   270  	if err != nil {
   271  		return err
   272  	}
   273  	generators = append(generators, gs...)
   274  
   275  	gs, err = kt.configureExternalGenerators()
   276  	if err != nil {
   277  		return errors.WrapPrefixf(err, "loading generator plugins")
   278  	}
   279  	generators = append(generators, gs...)
   280  	for i, g := range generators {
   281  		resMap, err := g.Generate()
   282  		if err != nil {
   283  			return err
   284  		}
   285  		if resMap != nil {
   286  			err = resMap.AddOriginAnnotation(generators[i].Origin)
   287  			if err != nil {
   288  				return errors.WrapPrefixf(err, "adding origin annotations for generator %v", g)
   289  			}
   290  		}
   291  		err = ra.AbsorbAll(resMap)
   292  		if err != nil {
   293  			return errors.WrapPrefixf(err, "merging from generator %v", g)
   294  		}
   295  	}
   296  	return nil
   297  }
   298  
   299  func (kt *KustTarget) configureExternalGenerators() (
   300  	[]*resmap.GeneratorWithProperties, error) {
   301  	ra := accumulator.MakeEmptyAccumulator()
   302  	var generatorPaths []string
   303  	for _, p := range kt.kustomization.Generators {
   304  		// handle inline generators
   305  		rm, err := kt.rFactory.NewResMapFromBytes([]byte(p))
   306  		if err != nil {
   307  			// not an inline config
   308  			generatorPaths = append(generatorPaths, p)
   309  			continue
   310  		}
   311  		// inline config, track the origin
   312  		if kt.origin != nil {
   313  			resources := rm.Resources()
   314  			for _, r := range resources {
   315  				r.SetOrigin(kt.origin.Append(kt.kustFileName))
   316  				rm.Replace(r)
   317  			}
   318  		}
   319  		if err = ra.AppendAll(rm); err != nil {
   320  			return nil, errors.WrapPrefixf(err, "configuring external generator")
   321  		}
   322  	}
   323  	ra, err := kt.accumulateResources(ra, generatorPaths)
   324  	if err != nil {
   325  		return nil, err
   326  	}
   327  	return kt.pLdr.LoadGenerators(kt.ldr, kt.validator, ra.ResMap())
   328  }
   329  
   330  func (kt *KustTarget) runTransformers(ra *accumulator.ResAccumulator) error {
   331  	var r []*resmap.TransformerWithProperties
   332  	tConfig := ra.GetTransformerConfig()
   333  	lts, err := kt.configureBuiltinTransformers(tConfig)
   334  	if err != nil {
   335  		return err
   336  	}
   337  	r = append(r, lts...)
   338  	lts, err = kt.configureExternalTransformers(kt.kustomization.Transformers)
   339  	if err != nil {
   340  		return err
   341  	}
   342  	r = append(r, lts...)
   343  	return ra.Transform(newMultiTransformer(r))
   344  }
   345  
   346  func (kt *KustTarget) configureExternalTransformers(transformers []string) ([]*resmap.TransformerWithProperties, error) {
   347  	ra := accumulator.MakeEmptyAccumulator()
   348  	var transformerPaths []string
   349  	for _, p := range transformers {
   350  		// handle inline transformers
   351  		rm, err := kt.rFactory.NewResMapFromBytes([]byte(p))
   352  		if err != nil {
   353  			// not an inline config
   354  			transformerPaths = append(transformerPaths, p)
   355  			continue
   356  		}
   357  		// inline config, track the origin
   358  		if kt.origin != nil {
   359  			resources := rm.Resources()
   360  			for _, r := range resources {
   361  				r.SetOrigin(kt.origin.Append(kt.kustFileName))
   362  				rm.Replace(r)
   363  			}
   364  		}
   365  
   366  		if err = ra.AppendAll(rm); err != nil {
   367  			return nil, errors.WrapPrefixf(err, "configuring external transformer")
   368  		}
   369  	}
   370  	ra, err := kt.accumulateResources(ra, transformerPaths)
   371  	if err != nil {
   372  		return nil, err
   373  	}
   374  	return kt.pLdr.LoadTransformers(kt.ldr, kt.validator, ra.ResMap())
   375  }
   376  
   377  func (kt *KustTarget) runValidators(ra *accumulator.ResAccumulator) error {
   378  	validators, err := kt.configureExternalTransformers(kt.kustomization.Validators)
   379  	if err != nil {
   380  		return err
   381  	}
   382  	for _, v := range validators {
   383  		// Validators shouldn't modify the resource map
   384  		orignal := ra.ResMap().DeepCopy()
   385  		err = v.Transform(ra.ResMap())
   386  		if err != nil {
   387  			return err
   388  		}
   389  		newMap := ra.ResMap().DeepCopy()
   390  		if err = kt.removeValidatedByLabel(newMap); err != nil {
   391  			return err
   392  		}
   393  		if err = orignal.ErrorIfNotEqualSets(newMap); err != nil {
   394  			return fmt.Errorf("validator shouldn't modify the resource map: %v", err)
   395  		}
   396  	}
   397  	return nil
   398  }
   399  
   400  func (kt *KustTarget) removeValidatedByLabel(rm resmap.ResMap) error {
   401  	resources := rm.Resources()
   402  	for _, r := range resources {
   403  		labels := r.GetLabels()
   404  		if _, found := labels[konfig.ValidatedByLabelKey]; !found {
   405  			continue
   406  		}
   407  		delete(labels, konfig.ValidatedByLabelKey)
   408  		if err := r.SetLabels(labels); err != nil {
   409  			return err
   410  		}
   411  	}
   412  	return nil
   413  }
   414  
   415  // accumulateResources fills the given resourceAccumulator
   416  // with resources read from the given list of paths.
   417  func (kt *KustTarget) accumulateResources(
   418  	ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) {
   419  	for _, path := range paths {
   420  		// try loading resource as file then as base (directory or git repository)
   421  		if errF := kt.accumulateFile(ra, path); errF != nil {
   422  			// not much we can do if the error is an HTTP error so we bail out
   423  			if errors.Is(errF, load.ErrHTTP) {
   424  				return nil, errF
   425  			}
   426  			ldr, err := kt.ldr.New(path)
   427  			if err != nil {
   428  				// If accumulateFile found malformed YAML and there was a failure
   429  				// loading the resource as a base, then the resource is likely a
   430  				// file. The loader failure message is unnecessary, and could be
   431  				// confusing. Report only the file load error.
   432  				//
   433  				// However, a loader timeout implies there is a git repo at the
   434  				// path. In that case, both errors could be important.
   435  				if kusterr.IsMalformedYAMLError(errF) && !utils.IsErrTimeout(err) {
   436  					return nil, errF
   437  				}
   438  				return nil, errors.WrapPrefixf(
   439  					err, "accumulation err='%s'", errF.Error())
   440  			}
   441  			// store the origin, we'll need it later
   442  			origin := kt.origin.Copy()
   443  			if kt.origin != nil {
   444  				kt.origin = kt.origin.Append(path)
   445  				ra, err = kt.accumulateDirectory(ra, ldr, false)
   446  				// after we are done recursing through the directory, reset the origin
   447  				kt.origin = &origin
   448  			} else {
   449  				ra, err = kt.accumulateDirectory(ra, ldr, false)
   450  			}
   451  			if err != nil {
   452  				if kusterr.IsMalformedYAMLError(errF) { // Some error occurred while tyring to decode YAML file
   453  					return nil, errF
   454  				}
   455  				return nil, errors.WrapPrefixf(
   456  					err, "accumulation err='%s'", errF.Error())
   457  			}
   458  		}
   459  	}
   460  	return ra, nil
   461  }
   462  
   463  // accumulateResources fills the given resourceAccumulator
   464  // with resources read from the given list of paths.
   465  func (kt *KustTarget) accumulateComponents(
   466  	ra *accumulator.ResAccumulator, paths []string) (*accumulator.ResAccumulator, error) {
   467  	for _, path := range paths {
   468  		// Components always refer to directories
   469  		ldr, errL := kt.ldr.New(path)
   470  		if errL != nil {
   471  			return nil, fmt.Errorf("loader.New %q", errL)
   472  		}
   473  		var errD error
   474  		// store the origin, we'll need it later
   475  		origin := kt.origin.Copy()
   476  		if kt.origin != nil {
   477  			kt.origin = kt.origin.Append(path)
   478  			ra, errD = kt.accumulateDirectory(ra, ldr, true)
   479  			// after we are done recursing through the directory, reset the origin
   480  			kt.origin = &origin
   481  		} else {
   482  			ra, errD = kt.accumulateDirectory(ra, ldr, true)
   483  		}
   484  		if errD != nil {
   485  			return nil, fmt.Errorf("accumulateDirectory: %q", errD)
   486  		}
   487  	}
   488  	return ra, nil
   489  }
   490  
   491  func (kt *KustTarget) accumulateDirectory(
   492  	ra *accumulator.ResAccumulator, ldr ifc.Loader, isComponent bool) (*accumulator.ResAccumulator, error) {
   493  	defer ldr.Cleanup()
   494  	subKt := NewKustTarget(ldr, kt.validator, kt.rFactory, kt.pLdr)
   495  	err := subKt.Load()
   496  	if err != nil {
   497  		return nil, errors.WrapPrefixf(
   498  			err, "couldn't make target for path '%s'", ldr.Root())
   499  	}
   500  	subKt.kustomization.BuildMetadata = kt.kustomization.BuildMetadata
   501  	subKt.origin = kt.origin
   502  	var bytes []byte
   503  	if openApiPath, exists := subKt.Kustomization().OpenAPI["path"]; exists {
   504  		bytes, err = ldr.Load(openApiPath)
   505  		if err != nil {
   506  			return nil, err
   507  		}
   508  	}
   509  	err = openapi.SetSchema(subKt.Kustomization().OpenAPI, bytes, false)
   510  	if err != nil {
   511  		return nil, err
   512  	}
   513  	if isComponent && subKt.kustomization.Kind != types.ComponentKind {
   514  		return nil, fmt.Errorf(
   515  			"expected kind '%s' for path '%s' but got '%s'", types.ComponentKind, ldr.Root(), subKt.kustomization.Kind)
   516  	} else if !isComponent && subKt.kustomization.Kind == types.ComponentKind {
   517  		return nil, fmt.Errorf(
   518  			"expected kind != '%s' for path '%s'", types.ComponentKind, ldr.Root())
   519  	}
   520  
   521  	var subRa *accumulator.ResAccumulator
   522  	if isComponent {
   523  		// Components don't create a new accumulator: the kustomization directives are added to the current accumulator
   524  		subRa, err = subKt.accumulateTarget(ra)
   525  		ra = accumulator.MakeEmptyAccumulator()
   526  	} else {
   527  		// Child Kustomizations create a new accumulator which resolves their kustomization directives, which will later
   528  		// be merged into the current accumulator.
   529  		subRa, err = subKt.AccumulateTarget()
   530  	}
   531  	if err != nil {
   532  		return nil, errors.WrapPrefixf(
   533  			err, "recursed accumulation of path '%s'", ldr.Root())
   534  	}
   535  	err = ra.MergeAccumulator(subRa)
   536  	if err != nil {
   537  		return nil, errors.WrapPrefixf(
   538  			err, "recursed merging from path '%s'", ldr.Root())
   539  	}
   540  	return ra, nil
   541  }
   542  
   543  func (kt *KustTarget) accumulateFile(
   544  	ra *accumulator.ResAccumulator, path string) error {
   545  	resources, err := kt.rFactory.FromFile(kt.ldr, path)
   546  	if err != nil {
   547  		return errors.WrapPrefixf(err, "accumulating resources from '%s'", path)
   548  	}
   549  	if kt.origin != nil {
   550  		originAnno, err := kt.origin.Append(path).String()
   551  		if err != nil {
   552  			return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path)
   553  		}
   554  		err = resources.AnnotateAll(utils.OriginAnnotationKey, originAnno)
   555  		if err != nil || originAnno == "" {
   556  			return errors.WrapPrefixf(err, "cannot add path annotation for '%s'", path)
   557  		}
   558  	}
   559  	err = ra.AppendAll(resources)
   560  	if err != nil {
   561  		return errors.WrapPrefixf(err, "merging resources from '%s'", path)
   562  	}
   563  	return nil
   564  }
   565  
   566  func (kt *KustTarget) configureBuiltinPlugin(
   567  	p resmap.Configurable, c interface{}, bpt builtinhelpers.BuiltinPluginType) (err error) {
   568  	var y []byte
   569  	if c != nil {
   570  		y, err = yaml.Marshal(c)
   571  		if err != nil {
   572  			return errors.WrapPrefixf(
   573  				err, "builtin %s marshal", bpt)
   574  		}
   575  	}
   576  	err = p.Config(
   577  		resmap.NewPluginHelpers(
   578  			kt.ldr, kt.validator, kt.rFactory, kt.pLdr.Config()),
   579  		y)
   580  	if err != nil {
   581  		return errors.WrapPrefixf(
   582  			err, "trouble configuring builtin %s with config: `\n%s`", bpt, string(y))
   583  	}
   584  	return nil
   585  }
   586
View as plain text