kio.go

Documentation: sigs.k8s.io/kustomize/kyaml/kio

     1  // Copyright 2019 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  // Package kio contains low-level libraries for reading, modifying and writing
     5  // Resource Configuration and packages.
     6  package kio
     7  
     8  import (
     9  	"fmt"
    10  	"strconv"
    11  
    12  	"sigs.k8s.io/kustomize/kyaml/errors"
    13  	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
    14  	"sigs.k8s.io/kustomize/kyaml/yaml"
    15  )
    16  
    17  // Reader reads ResourceNodes. Analogous to io.Reader.
    18  type Reader interface {
    19  	Read() ([]*yaml.RNode, error)
    20  }
    21  
    22  // ResourceNodeSlice is a collection of ResourceNodes.
    23  // While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
    24  // Readers, Filters or Writers may have constraints.
    25  type ResourceNodeSlice []*yaml.RNode
    26  
    27  var _ Reader = ResourceNodeSlice{}
    28  
    29  func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
    30  	return o, nil
    31  }
    32  
    33  // Writer writes ResourceNodes. Analogous to io.Writer.
    34  type Writer interface {
    35  	Write([]*yaml.RNode) error
    36  }
    37  
    38  // WriterFunc implements a Writer as a function.
    39  type WriterFunc func([]*yaml.RNode) error
    40  
    41  func (fn WriterFunc) Write(o []*yaml.RNode) error {
    42  	return fn(o)
    43  }
    44  
    45  // ReaderWriter implements both Reader and Writer interfaces
    46  type ReaderWriter interface {
    47  	Reader
    48  	Writer
    49  }
    50  
    51  // Filter modifies a collection of Resource Configuration by returning the modified slice.
    52  // When possible, Filters should be serializable to yaml so that they can be described
    53  // as either data or code.
    54  //
    55  // Analogous to http://www.linfo.org/filters.html
    56  type Filter interface {
    57  	Filter([]*yaml.RNode) ([]*yaml.RNode, error)
    58  }
    59  
    60  // TrackableFilter is an extension of Filter which is also capable of tracking
    61  // which fields were mutated by the filter.
    62  type TrackableFilter interface {
    63  	Filter
    64  	WithMutationTracker(func(key, value, tag string, node *yaml.RNode))
    65  }
    66  
    67  // FilterFunc implements a Filter as a function.
    68  type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
    69  
    70  func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
    71  	return fn(o)
    72  }
    73  
    74  // Pipeline reads Resource Configuration from a set of Inputs, applies some
    75  // transformation filters, and writes the results to a set of Outputs.
    76  //
    77  // Analogous to http://www.linfo.org/pipes.html
    78  type Pipeline struct {
    79  	// Inputs provide sources for Resource Configuration to be read.
    80  	Inputs []Reader `yaml:"inputs,omitempty"`
    81  
    82  	// Filters are transformations applied to the Resource Configuration.
    83  	// They are applied in the order they are specified.
    84  	// Analogous to http://www.linfo.org/filters.html
    85  	Filters []Filter `yaml:"filters,omitempty"`
    86  
    87  	// Outputs are where the transformed Resource Configuration is written.
    88  	Outputs []Writer `yaml:"outputs,omitempty"`
    89  
    90  	// ContinueOnEmptyResult configures what happens when a filter in the pipeline
    91  	// returns an empty result.
    92  	// If it is false (default), subsequent filters will be skipped and the result
    93  	// will be returned immediately. This is useful as an optimization when you
    94  	// know that subsequent filters will not alter the empty result.
    95  	// If it is true, the empty result will be provided as input to the next
    96  	// filter in the list. This is useful when subsequent functions in the
    97  	// pipeline may generate new resources.
    98  	ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
    99  }
   100  
   101  // Execute executes each step in the sequence, returning immediately after encountering
   102  // any error as part of the Pipeline.
   103  func (p Pipeline) Execute() error {
   104  	return p.ExecuteWithCallback(nil)
   105  }
   106  
   107  // PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
   108  type PipelineExecuteCallbackFunc = func(op Filter)
   109  
   110  // ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
   111  // any error as part of the Pipeline. The callback will be called each time a step succeeds.
   112  func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
   113  	var result []*yaml.RNode
   114  
   115  	// read from the inputs
   116  	for _, i := range p.Inputs {
   117  		nodes, err := i.Read()
   118  		if err != nil {
   119  			return errors.Wrap(err)
   120  		}
   121  		result = append(result, nodes...)
   122  	}
   123  
   124  	// apply operations
   125  	for i := range p.Filters {
   126  		// Not all RNodes passed through kio.Pipeline have metadata nor should
   127  		// they all be required to.
   128  		nodeAnnos, err := PreprocessResourcesForInternalAnnotationMigration(result)
   129  		if err != nil {
   130  			return err
   131  		}
   132  
   133  		op := p.Filters[i]
   134  		if callback != nil {
   135  			callback(op)
   136  		}
   137  		result, err = op.Filter(result)
   138  		// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
   139  		// handled by outputs. However currently some writer like LocalPackageReadWriter
   140  		// will clear the output directory and which will cause unpredictable results
   141  		if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
   142  			return errors.Wrap(err)
   143  		}
   144  
   145  		// If either the internal annotations for path, index, and id OR the legacy
   146  		// annotations for path, index, and id are changed, we have to update the other.
   147  		err = ReconcileInternalAnnotations(result, nodeAnnos)
   148  		if err != nil {
   149  			return err
   150  		}
   151  	}
   152  
   153  	// write to the outputs
   154  	for _, o := range p.Outputs {
   155  		if err := o.Write(result); err != nil {
   156  			return errors.Wrap(err)
   157  		}
   158  	}
   159  	return nil
   160  }
   161  
   162  // FilterAll runs the yaml.Filter against all inputs
   163  func FilterAll(filter yaml.Filter) Filter {
   164  	return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
   165  		for i := range nodes {
   166  			_, err := filter.Filter(nodes[i])
   167  			if err != nil {
   168  				return nil, errors.Wrap(err)
   169  			}
   170  		}
   171  		return nodes, nil
   172  	})
   173  }
   174  
   175  // PreprocessResourcesForInternalAnnotationMigration returns a mapping from id to all
   176  // internal annotations, so that we can use it to reconcile the annotations
   177  // later. This is necessary because currently both internal-prefixed annotations
   178  // and legacy annotations are currently supported, and a change to one must be
   179  // reflected in the other if needed.
   180  func PreprocessResourcesForInternalAnnotationMigration(result []*yaml.RNode) (map[string]map[string]string, error) {
   181  	idToAnnosMap := make(map[string]map[string]string)
   182  	for i := range result {
   183  		idStr := strconv.Itoa(i)
   184  		err := result[i].PipeE(yaml.SetAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation, idStr))
   185  		if err != nil {
   186  			return nil, err
   187  		}
   188  		idToAnnosMap[idStr] = kioutil.GetInternalAnnotations(result[i])
   189  		if err = kioutil.CopyLegacyAnnotations(result[i]); err != nil {
   190  			return nil, err
   191  		}
   192  		meta, _ := result[i].GetMeta()
   193  		if err = checkMismatchedAnnos(meta.Annotations); err != nil {
   194  			return nil, err
   195  		}
   196  	}
   197  	return idToAnnosMap, nil
   198  }
   199  
   200  func checkMismatchedAnnos(annotations map[string]string) error {
   201  	path := annotations[kioutil.PathAnnotation]
   202  	index := annotations[kioutil.IndexAnnotation]
   203  	id := annotations[kioutil.IdAnnotation]
   204  
   205  	legacyPath := annotations[kioutil.LegacyPathAnnotation]
   206  	legacyIndex := annotations[kioutil.LegacyIndexAnnotation]
   207  	legacyId := annotations[kioutil.LegacyIdAnnotation]
   208  
   209  	// if prior to running the functions, the legacy and internal annotations differ,
   210  	// throw an error as we cannot infer the user's intent.
   211  	if path != "" && legacyPath != "" && path != legacyPath {
   212  		return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
   213  	}
   214  	if index != "" && legacyIndex != "" && index != legacyIndex {
   215  		return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
   216  	}
   217  	if id != "" && legacyId != "" && id != legacyId {
   218  		return fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
   219  	}
   220  	return nil
   221  }
   222  
   223  type nodeAnnotations struct {
   224  	path  string
   225  	index string
   226  	id    string
   227  }
   228  
   229  // ReconcileInternalAnnotations reconciles the annotation format for path, index and id annotations.
   230  // It will ensure the output annotation format matches the format in the input. e.g. if the input
   231  // format uses the legacy format and the output will be converted to the legacy format if it's not.
   232  func ReconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
   233  	useInternal, useLegacy, err := determineAnnotationsFormat(nodeAnnosMap)
   234  	if err != nil {
   235  		return err
   236  	}
   237  
   238  	for i := range result {
   239  		// if only one annotation is set, set the other.
   240  		err = missingInternalOrLegacyAnnotations(result[i])
   241  		if err != nil {
   242  			return err
   243  		}
   244  		// we must check to see if the function changed either the new internal annotations
   245  		// or the old legacy annotations. If one is changed, the change must be reflected
   246  		// in the other.
   247  		err = checkAnnotationsAltered(result[i], nodeAnnosMap)
   248  		if err != nil {
   249  			return err
   250  		}
   251  		// We invoke determineAnnotationsFormat to find out if the original annotations
   252  		// use the internal or (and) the legacy format. We format the resources to
   253  		// make them consistent with original format.
   254  		err = formatInternalAnnotations(result[i], useInternal, useLegacy)
   255  		if err != nil {
   256  			return err
   257  		}
   258  		// if the annotations are still somehow out of sync, throw an error
   259  		meta, _ := result[i].GetMeta()
   260  		err = checkMismatchedAnnos(meta.Annotations)
   261  		if err != nil {
   262  			return err
   263  		}
   264  
   265  		if _, err = result[i].Pipe(yaml.ClearAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation)); err != nil {
   266  			return err
   267  		}
   268  	}
   269  	return nil
   270  }
   271  
   272  // determineAnnotationsFormat determines if the resources are using one of the internal and legacy annotation format or both of them.
   273  func determineAnnotationsFormat(nodeAnnosMap map[string]map[string]string) (bool, bool, error) {
   274  	var useInternal, useLegacy bool
   275  	var err error
   276  
   277  	if len(nodeAnnosMap) == 0 {
   278  		return true, true, nil
   279  	}
   280  
   281  	var internal, legacy *bool
   282  	for _, annos := range nodeAnnosMap {
   283  		_, foundPath := annos[kioutil.PathAnnotation]
   284  		_, foundIndex := annos[kioutil.IndexAnnotation]
   285  		_, foundId := annos[kioutil.IdAnnotation]
   286  		_, foundLegacyPath := annos[kioutil.LegacyPathAnnotation]
   287  		_, foundLegacyIndex := annos[kioutil.LegacyIndexAnnotation]
   288  		_, foundLegacyId := annos[kioutil.LegacyIdAnnotation]
   289  
   290  		if !(foundPath || foundIndex || foundId || foundLegacyPath || foundLegacyIndex || foundLegacyId) {
   291  			continue
   292  		}
   293  
   294  		foundOneOf := foundPath || foundIndex || foundId
   295  		if internal == nil {
   296  			f := foundOneOf
   297  			internal = &f
   298  		}
   299  		if (foundOneOf && !*internal) || (!foundOneOf && *internal) {
   300  			err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
   301  			return useInternal, useLegacy, err
   302  		}
   303  
   304  		foundOneOf = foundLegacyPath || foundLegacyIndex || foundLegacyId
   305  		if legacy == nil {
   306  			f := foundOneOf
   307  			legacy = &f
   308  		}
   309  		if (foundOneOf && !*legacy) || (!foundOneOf && *legacy) {
   310  			err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
   311  			return useInternal, useLegacy, err
   312  		}
   313  	}
   314  	if internal != nil {
   315  		useInternal = *internal
   316  	}
   317  	if legacy != nil {
   318  		useLegacy = *legacy
   319  	}
   320  	return useInternal, useLegacy, err
   321  }
   322  
   323  func missingInternalOrLegacyAnnotations(rn *yaml.RNode) error {
   324  	if err := missingInternalOrLegacyAnnotation(rn, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
   325  		return err
   326  	}
   327  	if err := missingInternalOrLegacyAnnotation(rn, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
   328  		return err
   329  	}
   330  	if err := missingInternalOrLegacyAnnotation(rn, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
   331  		return err
   332  	}
   333  	return nil
   334  }
   335  
   336  func missingInternalOrLegacyAnnotation(rn *yaml.RNode, newKey string, legacyKey string) error {
   337  	meta, _ := rn.GetMeta()
   338  	annotations := meta.Annotations
   339  	value := annotations[newKey]
   340  	legacyValue := annotations[legacyKey]
   341  
   342  	if value == "" && legacyValue == "" {
   343  		// do nothing
   344  		return nil
   345  	}
   346  
   347  	if value == "" {
   348  		// new key is not set, copy from legacy key
   349  		if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
   350  			return err
   351  		}
   352  	} else if legacyValue == "" {
   353  		// legacy key is not set, copy from new key
   354  		if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
   355  			return err
   356  		}
   357  	}
   358  	return nil
   359  }
   360  
   361  func checkAnnotationsAltered(rn *yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
   362  	meta, _ := rn.GetMeta()
   363  	annotations := meta.Annotations
   364  	// get the resource's current path, index, and ids from the new annotations
   365  	internal := nodeAnnotations{
   366  		path:  annotations[kioutil.PathAnnotation],
   367  		index: annotations[kioutil.IndexAnnotation],
   368  		id:    annotations[kioutil.IdAnnotation],
   369  	}
   370  
   371  	// get the resource's current path, index, and ids from the legacy annotations
   372  	legacy := nodeAnnotations{
   373  		path:  annotations[kioutil.LegacyPathAnnotation],
   374  		index: annotations[kioutil.LegacyIndexAnnotation],
   375  		id:    annotations[kioutil.LegacyIdAnnotation],
   376  	}
   377  
   378  	rid := annotations[kioutil.InternalAnnotationsMigrationResourceIDAnnotation]
   379  	originalAnnotations, found := nodeAnnosMap[rid]
   380  	if !found {
   381  		return nil
   382  	}
   383  	originalPath, found := originalAnnotations[kioutil.PathAnnotation]
   384  	if !found {
   385  		originalPath = originalAnnotations[kioutil.LegacyPathAnnotation]
   386  	}
   387  	if originalPath != "" {
   388  		switch {
   389  		case originalPath != internal.path && originalPath != legacy.path && internal.path != legacy.path:
   390  			return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
   391  		case originalPath != internal.path:
   392  			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyPathAnnotation, internal.path)); err != nil {
   393  				return err
   394  			}
   395  		case originalPath != legacy.path:
   396  			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, legacy.path)); err != nil {
   397  				return err
   398  			}
   399  		}
   400  	}
   401  
   402  	originalIndex, found := originalAnnotations[kioutil.IndexAnnotation]
   403  	if !found {
   404  		originalIndex = originalAnnotations[kioutil.LegacyIndexAnnotation]
   405  	}
   406  	if originalIndex != "" {
   407  		switch {
   408  		case originalIndex != internal.index && originalIndex != legacy.index && internal.index != legacy.index:
   409  			return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
   410  		case originalIndex != internal.index:
   411  			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyIndexAnnotation, internal.index)); err != nil {
   412  				return err
   413  			}
   414  		case originalIndex != legacy.index:
   415  			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.IndexAnnotation, legacy.index)); err != nil {
   416  				return err
   417  			}
   418  		}
   419  	}
   420  	return nil
   421  }
   422  
   423  func formatInternalAnnotations(rn *yaml.RNode, useInternal, useLegacy bool) error {
   424  	if !useInternal {
   425  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IdAnnotation)); err != nil {
   426  			return err
   427  		}
   428  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.PathAnnotation)); err != nil {
   429  			return err
   430  		}
   431  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IndexAnnotation)); err != nil {
   432  			return err
   433  		}
   434  	}
   435  	if !useLegacy {
   436  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIdAnnotation)); err != nil {
   437  			return err
   438  		}
   439  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyPathAnnotation)); err != nil {
   440  			return err
   441  		}
   442  		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIndexAnnotation)); err != nil {
   443  			return err
   444  		}
   445  	}
   446  	return nil
   447  }
   448
View as plain text