processors.go

Documentation: sigs.k8s.io/kustomize/kyaml/fn/framework

     1  // Copyright 2021 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package framework
     5  
     6  import (
     7  	"strings"
     8  
     9  	validationErrors "k8s.io/kube-openapi/pkg/validation/errors"
    10  	"k8s.io/kube-openapi/pkg/validation/spec"
    11  	"k8s.io/kube-openapi/pkg/validation/strfmt"
    12  	"k8s.io/kube-openapi/pkg/validation/validate"
    13  	"sigs.k8s.io/kustomize/kyaml/errors"
    14  	"sigs.k8s.io/kustomize/kyaml/kio"
    15  	"sigs.k8s.io/kustomize/kyaml/kio/filters"
    16  	"sigs.k8s.io/kustomize/kyaml/openapi"
    17  	"sigs.k8s.io/kustomize/kyaml/yaml"
    18  	k8syaml "sigs.k8s.io/yaml"
    19  )
    20  
    21  // SimpleProcessor processes a ResourceList by loading the FunctionConfig into
    22  // the given Config type and then running the provided Filter on the ResourceList.
    23  // The provided Config MAY implement Defaulter and Validator to have Default and Validate
    24  // respectively called between unmarshalling and filter execution.
    25  //
    26  // Typical uses include functions that do not actually require config, and simple functions built
    27  // with a filter that closes over the Config instance to access ResourceList.functionConfig values.
    28  type SimpleProcessor struct {
    29  	// Filter is the kio.Filter that will be used to process the ResourceList's items.
    30  	// Note that kio.FilterFunc is available to transform a compatible func into a kio.Filter.
    31  	Filter kio.Filter
    32  	// Config must be a struct capable of receiving the data from ResourceList.functionConfig.
    33  	// Filter functions may close over this struct to access its data.
    34  	Config interface{}
    35  }
    36  
    37  // Process makes SimpleProcessor implement the ResourceListProcessor interface.
    38  // It loads the ResourceList.functionConfig into the provided Config type, applying
    39  // defaulting and validation if supported by Config. It then executes the processor's filter.
    40  func (p SimpleProcessor) Process(rl *ResourceList) error {
    41  	if err := LoadFunctionConfig(rl.FunctionConfig, p.Config); err != nil {
    42  		return errors.WrapPrefixf(err, "loading function config")
    43  	}
    44  	return errors.WrapPrefixf(rl.Filter(p.Filter), "processing filter")
    45  }
    46  
    47  // GVKFilterMap is a FilterProvider that resolves Filters through a simple lookup in a map.
    48  // It is intended for use in VersionedAPIProcessor.
    49  type GVKFilterMap map[string]map[string]kio.Filter
    50  
    51  // ProviderFor makes GVKFilterMap implement the FilterProvider interface.
    52  // It uses the given apiVersion and kind to do a simple lookup in the map and
    53  // returns an error if no exact match is found.
    54  func (m GVKFilterMap) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
    55  	if kind == "" {
    56  		return nil, errors.Errorf("kind is required")
    57  	}
    58  	if apiVersion == "" {
    59  		return nil, errors.Errorf("apiVersion is required")
    60  	}
    61  
    62  	var ok bool
    63  	var versionMap map[string]kio.Filter
    64  	if versionMap, ok = m[kind]; !ok {
    65  		return nil, errors.Errorf("kind %q is not supported", kind)
    66  	}
    67  
    68  	var p kio.Filter
    69  	if p, ok = versionMap[apiVersion]; !ok {
    70  		return nil, errors.Errorf("apiVersion %q is not supported for kind %q", apiVersion, kind)
    71  	}
    72  	return p, nil
    73  }
    74  
    75  // FilterProvider is implemented by types that provide a way to look up which Filter
    76  // should be used to process a ResourceList based on the ApiVersion and Kind of the
    77  // ResourceList.functionConfig in the input. FilterProviders are intended to be used
    78  // as part of VersionedAPIProcessor.
    79  type FilterProvider interface {
    80  	// ProviderFor returns the appropriate filter for the given APIVersion and Kind.
    81  	ProviderFor(apiVersion, kind string) (kio.Filter, error)
    82  }
    83  
    84  // FilterProviderFunc converts a compatible function to a FilterProvider.
    85  type FilterProviderFunc func(apiVersion, kind string) (kio.Filter, error)
    86  
    87  // ProviderFor makes FilterProviderFunc implement FilterProvider.
    88  func (f FilterProviderFunc) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
    89  	return f(apiVersion, kind)
    90  }
    91  
    92  // VersionedAPIProcessor selects the appropriate kio.Filter based on the ApiVersion
    93  // and Kind of the ResourceList.functionConfig in the input.
    94  // It can be used to implement configuration function APIs that evolve over time,
    95  // or create processors that support multiple configuration APIs with a single entrypoint.
    96  // All provided Filters MUST be structs capable of receiving ResourceList.functionConfig data.
    97  // Provided Filters MAY implement Defaulter and Validator to have Default and Validate
    98  // respectively called between unmarshalling and filter execution.
    99  type VersionedAPIProcessor struct {
   100  	// FilterProvider resolves a kio.Filter for each supported API, based on its APIVersion and Kind.
   101  	// GVKFilterMap is a simple FilterProvider implementation for use here.
   102  	FilterProvider FilterProvider
   103  }
   104  
   105  // Process makes VersionedAPIProcessor implement the ResourceListProcessor interface.
   106  // It looks up the configuration object to use based on the ApiVersion and Kind of the
   107  // input ResourceList.functionConfig, loads ResourceList.functionConfig into that object,
   108  // invokes Validate and Default if supported, and finally invokes Filter.
   109  func (p *VersionedAPIProcessor) Process(rl *ResourceList) error {
   110  	api, err := p.FilterProvider.ProviderFor(extractGVK(rl.FunctionConfig))
   111  	if err != nil {
   112  		return errors.WrapPrefixf(err, "unable to identify provider for resource")
   113  	}
   114  	if err := LoadFunctionConfig(rl.FunctionConfig, api); err != nil {
   115  		return errors.Wrap(err)
   116  	}
   117  	return errors.Wrap(rl.Filter(api))
   118  }
   119  
   120  // extractGVK returns the apiVersion and kind fields from the given RNodes if it contains
   121  // valid TypeMeta. It returns an empty string if a value is not found.
   122  func extractGVK(src *yaml.RNode) (apiVersion, kind string) {
   123  	if src == nil {
   124  		return "", ""
   125  	}
   126  	if versionNode := src.Field("apiVersion"); versionNode != nil {
   127  		if a, err := versionNode.Value.String(); err == nil {
   128  			apiVersion = strings.TrimSpace(a)
   129  		}
   130  	}
   131  	if kindNode := src.Field("kind"); kindNode != nil {
   132  		if k, err := kindNode.Value.String(); err == nil {
   133  			kind = strings.TrimSpace(k)
   134  		}
   135  	}
   136  	return apiVersion, kind
   137  }
   138  
   139  // LoadFunctionConfig reads a configuration resource from YAML into the provided data structure
   140  // and then prepares it for use by running defaulting and validation on it, if supported.
   141  // ResourceListProcessors should use this function to load ResourceList.functionConfig.
   142  func LoadFunctionConfig(src *yaml.RNode, api interface{}) error {
   143  	if api == nil {
   144  		return nil
   145  	}
   146  	// Run this before unmarshalling to avoid nasty unmarshal failure error messages
   147  	var schemaValidationError error
   148  	if s, ok := api.(ValidationSchemaProvider); ok {
   149  		schema, err := s.Schema()
   150  		if err != nil {
   151  			return errors.WrapPrefixf(err, "loading provided schema")
   152  		}
   153  		schemaValidationError = errors.Wrap(validate.AgainstSchema(schema, src, strfmt.Default))
   154  		// don't return it yet--try to make it to custom validation stage to combine errors
   155  	}
   156  
   157  	// using sigs.k8s.io/yaml here lets the custom types embed core types
   158  	// that only have json tags, notably types from k8s.io/apimachinery/pkg/apis/meta/v1
   159  	if err := k8syaml.Unmarshal([]byte(src.MustString()), api); err != nil {
   160  		if schemaValidationError != nil {
   161  			// if we got a validation error, report it instead as it is likely a nicer version of the same message
   162  			return schemaValidationError
   163  		}
   164  		return errors.Wrap(err)
   165  	}
   166  
   167  	if d, ok := api.(Defaulter); ok {
   168  		if err := d.Default(); err != nil {
   169  			return errors.Wrap(err)
   170  		}
   171  	}
   172  
   173  	if v, ok := api.(Validator); ok {
   174  		return combineErrors(schemaValidationError, v.Validate())
   175  	}
   176  	return schemaValidationError
   177  }
   178  
   179  func combineErrors(schemaErr, customErr error) error {
   180  	combined := validationErrors.CompositeValidationError()
   181  	if compositeSchemaErr, ok := schemaErr.(*validationErrors.CompositeError); ok {
   182  		combined.Errors = append(combined.Errors, compositeSchemaErr.Errors...)
   183  	} else if schemaErr != nil {
   184  		combined.Errors = append(combined.Errors, schemaErr)
   185  	}
   186  	if compositeCustomErr, ok := customErr.(*validationErrors.CompositeError); ok {
   187  		combined.Errors = append(combined.Errors, compositeCustomErr.Errors...)
   188  	} else if customErr != nil {
   189  		combined.Errors = append(combined.Errors, customErr)
   190  	}
   191  	if len(combined.Errors) > 0 {
   192  		return combined
   193  	}
   194  	return nil
   195  }
   196  
   197  // TemplateProcessor is a ResourceListProcessor based on rendering templates with the data in
   198  // ResourceList.functionConfig. It works as follows:
   199  // - loads ResourceList.functionConfig into TemplateData
   200  // - runs PreProcessFilters
   201  // - renders ResourceTemplates and adds them to ResourceList.items
   202  // - renders PatchTemplates and applies them to ResourceList.items
   203  // - executes a merge on ResourceList.items if configured to
   204  // - runs PostProcessFilters
   205  // The TemplateData struct MAY implement Defaulter and Validator to have Default and Validate
   206  // respectively called between unmarshalling and filter execution.
   207  //
   208  // TemplateProcessor also implements kio.Filter directly and can be used in the construction of
   209  // higher-level processors. For example, you might use TemplateProcessors as the filters for each
   210  // API supported by a VersionedAPIProcessor (see VersionedAPIProcessor examples).
   211  type TemplateProcessor struct {
   212  	// TemplateData will will be exposed to all the templates in the processor (unless explicitly
   213  	// overridden for a template).
   214  	// If TemplateProcessor is used directly as a ResourceListProcessor, TemplateData will contain the
   215  	// value of ResourceList.functionConfig.
   216  	TemplateData interface{}
   217  
   218  	// ResourceTemplates returns a list of templates to render into resources.
   219  	// If MergeResources is set, any matching resources in ResourceList.items will be used as patches
   220  	// modifying the rendered templates. Otherwise, the rendered resources will be appended to
   221  	// the input resources as-is.
   222  	ResourceTemplates []ResourceTemplate
   223  
   224  	// PatchTemplates is a list of templates to render into patches that apply to ResourceList.items.
   225  	// ResourcePatchTemplate can be used here to patch entire resources.
   226  	// ContainerPatchTemplate can be used here to patch specific containers within resources.
   227  	PatchTemplates []PatchTemplate
   228  
   229  	// MergeResources, if set to true, will cause the resources in ResourceList.items to be
   230  	// will be applied as patches on any matching resources generated by ResourceTemplates.
   231  	MergeResources bool
   232  
   233  	// PreProcessFilters provides a hook to manipulate the ResourceList's items or config after
   234  	// TemplateData has been populated but before template-based filters are applied.
   235  	PreProcessFilters []kio.Filter
   236  
   237  	// PostProcessFilters provides a hook to manipulate the ResourceList's items after template
   238  	// filters are applied.
   239  	PostProcessFilters []kio.Filter
   240  
   241  	// AdditionalSchemas is a function that returns a list of schema definitions to add to openapi.
   242  	// This enables correct merging of custom resource fields.
   243  	AdditionalSchemas SchemaParser
   244  }
   245  
   246  type SchemaParser interface {
   247  	Parse() ([]*spec.Definitions, error)
   248  }
   249  
   250  type SchemaParserFunc func() ([]*spec.Definitions, error)
   251  
   252  func (s SchemaParserFunc) Parse() ([]*spec.Definitions, error) {
   253  	return s()
   254  }
   255  
   256  // Filter implements the kio.Filter interface, enabling you to use TemplateProcessor
   257  // as part of a higher-level ResourceListProcessor like VersionedAPIProcessor.
   258  // It sets up all the features of TemplateProcessors as a pipeline of filters and executes them.
   259  func (tp TemplateProcessor) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
   260  	if tp.AdditionalSchemas != nil {
   261  		defs, err := tp.AdditionalSchemas.Parse()
   262  		if err != nil {
   263  			return nil, errors.WrapPrefixf(err, "parsing AdditionalSchemas")
   264  		}
   265  		defer openapi.ResetOpenAPI()
   266  		for i := range defs {
   267  			openapi.AddDefinitions(*defs[i])
   268  		}
   269  	}
   270  
   271  	buf := &kio.PackageBuffer{Nodes: items}
   272  	pipeline := kio.Pipeline{
   273  		Inputs: []kio.Reader{buf},
   274  		Filters: []kio.Filter{
   275  			kio.FilterFunc(tp.doPreProcess),
   276  			kio.FilterFunc(tp.doResourceTemplates),
   277  			kio.FilterFunc(tp.doPatchTemplates),
   278  			kio.FilterFunc(tp.doMerge),
   279  			kio.FilterFunc(tp.doPostProcess),
   280  		},
   281  		Outputs:               []kio.Writer{buf},
   282  		ContinueOnEmptyResult: true,
   283  	}
   284  	if err := pipeline.Execute(); err != nil {
   285  		return nil, err
   286  	}
   287  
   288  	return buf.Nodes, nil
   289  }
   290  
   291  // Process implements the ResourceListProcessor interface, enabling you to use TemplateProcessor
   292  // directly as a processor. As a Processor, it loads the ResourceList.functionConfig into the
   293  // TemplateData field, exposing it to all templates by default.
   294  func (tp TemplateProcessor) Process(rl *ResourceList) error {
   295  	if err := LoadFunctionConfig(rl.FunctionConfig, tp.TemplateData); err != nil {
   296  		return errors.Wrap(err)
   297  	}
   298  	return errors.Wrap(rl.Filter(tp))
   299  }
   300  
   301  // PatchTemplate is implemented by kio.Filters that work by rendering patches and applying them to
   302  // the given resource nodes.
   303  type PatchTemplate interface {
   304  	// Filter is a kio.Filter-compliant function that applies PatchTemplate's templates as patches
   305  	// on the given resource nodes.
   306  	Filter(items []*yaml.RNode) ([]*yaml.RNode, error)
   307  	// DefaultTemplateData accepts default data to be used in template rendering when no template
   308  	// data was explicitly provided to the PatchTemplate.
   309  	DefaultTemplateData(interface{})
   310  }
   311  
   312  func (tp *TemplateProcessor) doPreProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
   313  	if tp.PreProcessFilters == nil {
   314  		return items, nil
   315  	}
   316  	for i := range tp.PreProcessFilters {
   317  		filter := tp.PreProcessFilters[i]
   318  		var err error
   319  		items, err = filter.Filter(items)
   320  		if err != nil {
   321  			return nil, err
   322  		}
   323  	}
   324  	return items, nil
   325  }
   326  
   327  func (tp *TemplateProcessor) doMerge(items []*yaml.RNode) ([]*yaml.RNode, error) {
   328  	var err error
   329  	if tp.MergeResources {
   330  		items, err = filters.MergeFilter{}.Filter(items)
   331  	}
   332  	return items, err
   333  }
   334  
   335  func (tp *TemplateProcessor) doPostProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
   336  	if tp.PostProcessFilters == nil {
   337  		return items, nil
   338  	}
   339  	for i := range tp.PostProcessFilters {
   340  		filter := tp.PostProcessFilters[i]
   341  		var err error
   342  		items, err = filter.Filter(items)
   343  		if err != nil {
   344  			return nil, err
   345  		}
   346  	}
   347  	return items, nil
   348  }
   349  
   350  func (tp *TemplateProcessor) doResourceTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
   351  	if tp.ResourceTemplates == nil {
   352  		return items, nil
   353  	}
   354  
   355  	for i := range tp.ResourceTemplates {
   356  		tp.ResourceTemplates[i].DefaultTemplateData(tp.TemplateData)
   357  		newItems, err := tp.ResourceTemplates[i].Render()
   358  		if err != nil {
   359  			return nil, err
   360  		}
   361  		if tp.MergeResources {
   362  			// apply inputs as patches -- add the new items to the front of the list
   363  			items = append(newItems, items...)
   364  		} else {
   365  			// assume these are new unique resources--append to the list
   366  			items = append(items, newItems...)
   367  		}
   368  	}
   369  	return items, nil
   370  }
   371  
   372  func (tp *TemplateProcessor) doPatchTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
   373  	if tp.PatchTemplates == nil {
   374  		return items, nil
   375  	}
   376  
   377  	for i := range tp.PatchTemplates {
   378  		// Default the template data for the patch to the processor's data
   379  		tp.PatchTemplates[i].DefaultTemplateData(tp.TemplateData)
   380  		var err error
   381  		if items, err = tp.PatchTemplates[i].Filter(items); err != nil {
   382  			return nil, err
   383  		}
   384  	}
   385  	return items, nil
   386  }
   387
View as plain text