packer.go

Documentation: edge-infra.dev/pkg/f8n/warehouse/lift/pack

     1  // Package pack contains functionality for building Edge Pallet packages from
     2  // files on disk and Pallet package definitions (pallet.yaml).
     3  // TODO: move docstring
     4  package pack
     5  
     6  import (
     7  	"fmt"
     8  	"path/filepath"
     9  	"regexp"
    10  	"sort"
    11  
    12  	"slices"
    13  
    14  	"sigs.k8s.io/kustomize/api/filters/annotations"
    15  	"sigs.k8s.io/kustomize/api/hasher"
    16  	"sigs.k8s.io/kustomize/api/resmap"
    17  	"sigs.k8s.io/kustomize/api/resource"
    18  	"sigs.k8s.io/kustomize/kyaml/kio"
    19  	"sigs.k8s.io/kustomize/kyaml/yaml"
    20  
    21  	"edge-infra.dev/pkg/f8n/warehouse/capability"
    22  	"edge-infra.dev/pkg/f8n/warehouse/lift"
    23  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/filters/transformers/ambiguous"
    24  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/filters/transformers/labels"
    25  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/filters/transformers/palletmetadata"
    26  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/internal"
    27  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/types"
    28  	"edge-infra.dev/pkg/f8n/warehouse/oci/layer"
    29  	"edge-infra.dev/pkg/f8n/warehouse/pallet"
    30  	"edge-infra.dev/pkg/k8s/eyaml/fieldspecs"
    31  )
    32  
    33  // Packer produces Warehouse OCI artifacts from source manifests.
    34  type Packer struct {
    35  	Context                    // execution context used for package lookups and accessing package sources
    36  	buildInfo pallet.BuildInfo // build information added to all built packages
    37  
    38  	capabilities map[capability.Capability]internal.Capability
    39  	cache        map[string]pallet.Pallet // cache of build results
    40  	pkgCache     map[string]*Package      // cache of loaded package definition files
    41  	cfgCache     map[string]lift.Config   // cache of package-specific warehouse configs
    42  }
    43  
    44  func New(c Context, build pallet.BuildInfo) (*Packer, error) {
    45  	c, err := c.Default()
    46  	if err != nil {
    47  		return nil, fmt.Errorf("failed to instantiate packing context: %w", err)
    48  	}
    49  
    50  	b := &Packer{
    51  		Context:      c,
    52  		buildInfo:    build,
    53  		capabilities: make(map[capability.Capability]internal.Capability, len(c.Capabilities())),
    54  		cache:        make(map[string]pallet.Pallet),
    55  		pkgCache:     make(map[string]*Package),
    56  		cfgCache:     make(map[string]lift.Config),
    57  	}
    58  
    59  	for _, c := range b.Context.Capabilities() {
    60  		var name capability.Capability
    61  		switch {
    62  		// infrastructure is only capability not named by package name
    63  		case c.Package == b.Infrastructure.Package:
    64  			name = capability.CapabilityInfra
    65  		default:
    66  			pkg, err := b.loadPkg(c.Package)
    67  			if err != nil {
    68  				return nil, err
    69  			}
    70  			name = capability.Capability(pkg.Config.Name)
    71  		}
    72  
    73  		var err error
    74  		b.capabilities[name], err = internal.NewCapability(name, c)
    75  		if err != nil {
    76  			return nil, err
    77  		}
    78  	}
    79  
    80  	return b, nil
    81  }
    82  
    83  // Pack compiles the sources at the input dir as well as any dependencies,
    84  // returning the packaged Pallet and its dependencies, if any, as an ordered
    85  // map. The provided build metadata is applied to all Pallets that are packed
    86  // from source.
    87  // TODO: honor build information from pallet.yaml if it exists
    88  // TODO: check each path as we parse Pallets and CapabilityProviders dependencies/capabilities so that we can fail early with a clear error message if its invalid
    89  func (b *Packer) Pack(dir string) (pallet.Pallet, error) {
    90  	if filepath.IsAbs(dir) {
    91  		// if path is absolute, it is invalid
    92  		return nil, fmt.Errorf(
    93  			"%s is not a valid warehouse package path, it should be relative to "+
    94  				"WAREHOUSE_PATH", dir,
    95  		)
    96  	}
    97  	return b.packPkg(dir)
    98  }
    99  
   100  func (b *Packer) packPkg(path string) (pallet.Pallet, error) {
   101  	if plt, ok := b.cache[path]; ok {
   102  		return plt, nil
   103  	}
   104  
   105  	if !b.FS.Exists(path) {
   106  		return nil, fmt.Errorf("failed to find package %s", path)
   107  	}
   108  
   109  	pkg, err := b.loadPkg(path)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  
   114  	if err := b.Compile(pkg); err != nil {
   115  		return nil, err
   116  	}
   117  
   118  	// a single pallet with no dependencies means we have reached leaf node
   119  	if len(pkg.Artifacts) == 1 && pkg.Config.Dependencies.IsEmpty() {
   120  		// we can safely cast because all artifacts we are building are Pallets
   121  		plt := pkg.Artifacts[0].(pallet.Pallet)
   122  		// add result to our pkg cache
   123  		b.cache[path] = plt
   124  		b.pkgCache[path] = pkg
   125  		return plt, nil
   126  	}
   127  
   128  	// now we know we are dealing with an image index
   129  
   130  	// if providers is still empty, this is a composite/umbrella pallet.
   131  	// if it doesn't explicitly set providers, assume it supports all providers
   132  	// that are configured globally
   133  	if len(pkg.Providers) == 0 && len(pkg.Config.Providers) == 0 {
   134  		pkg.Providers = b.ClusterProviders
   135  	}
   136  
   137  	// build dependencies
   138  	for _, d := range pkg.Config.Dependencies.Paths {
   139  		dep, err := b.packPkg(d)
   140  		if err != nil {
   141  			return nil, err
   142  		}
   143  		pkg.Artifacts = append(pkg.Artifacts, dep)
   144  		// aggregate dependency capabilities for final result
   145  		for _, c := range dep.Capabilities() {
   146  			if !slices.Contains(pkg.Capabilities, c) {
   147  				pkg.Capabilities = append(pkg.Capabilities, c)
   148  			}
   149  		}
   150  		// aggregate dependency parameters
   151  		for _, depParam := range dep.Parameters() {
   152  			if !slices.Contains(pkg.Parameters, depParam) {
   153  				pkg.Parameters = append(pkg.Parameters, depParam)
   154  			}
   155  		}
   156  	}
   157  
   158  	// finally assemble final result now
   159  	opts := pallet.Options{
   160  		Metadata:         pkg.Metadata,
   161  		ClusterProviders: pkg.Providers,
   162  		Capabilities:     pkg.Capabilities,
   163  		DisableRendering: pkg.Config.DisableRendering,
   164  		Parameters:       pkg.Parameters,
   165  	}
   166  
   167  	plt, err := pallet.ImageIndex(opts, pkg.Artifacts...)
   168  	if err != nil {
   169  		return nil, err
   170  	}
   171  	// add result to our cache
   172  	b.cache[path] = plt
   173  
   174  	return plt, nil
   175  }
   176  
   177  // Compile renders the kustomize targets defined by the package, sorting manifests
   178  // into the appropriate layers and then packing the layers into OCI Images
   179  func (b *Packer) Compile(p *Package) error {
   180  	if p.Config.ManifestTarget != nil {
   181  		for _, m := range p.Config.ManifestTarget {
   182  			layers, err := b.packRawLayers(p, m.Path)
   183  			if err != nil {
   184  				return err
   185  			}
   186  			p.Providers = append(p.Providers, m.Providers...)
   187  			opts := pallet.Options{
   188  				ClusterProviders: m.Providers,
   189  				Capabilities:     p.Capabilities,
   190  				Metadata:         p.Metadata,
   191  				DisableRendering: p.Config.DisableRendering,
   192  				Parameters:       p.Parameters,
   193  			}
   194  			img, err := pallet.Image(opts, layers...)
   195  			if err != nil {
   196  				return err
   197  			}
   198  			p.Artifacts = append(p.Artifacts, img)
   199  		}
   200  	}
   201  	for _, t := range p.Config.Kustomize {
   202  		layers, err := b.packKustomizeLayers(p, filepath.Join(p.Dir, t.Target))
   203  		if err != nil {
   204  			return err
   205  		}
   206  
   207  		// track provider in overall result
   208  		p.Providers = append(p.Providers, t.Providers...)
   209  
   210  		opts := pallet.Options{
   211  			ClusterProviders: t.Providers,
   212  			Capabilities:     p.Capabilities,
   213  			Metadata:         p.Metadata,
   214  			DisableRendering: p.Config.DisableRendering,
   215  			Parameters:       p.Parameters,
   216  		}
   217  		img, err := pallet.Image(opts, layers...)
   218  		if err != nil {
   219  			return err
   220  		}
   221  		p.Artifacts = append(p.Artifacts, img)
   222  	}
   223  
   224  	return nil
   225  }
   226  
   227  // packLayers executes kustomize against the target and packs the appropriate
   228  // layers based on the package definition and rendered contents. It returns
   229  // two arrays, the layers built and the total list of capabilities (by name)
   230  // that were found in the manifests.
   231  func (b *Packer) packLayers(p *Package, kres resmap.ResMap) ([]layer.Layer, error) {
   232  	// TODO: better error context (package name, etc) for all errors throughout
   233  	// build up list of filters we are going to apply against the result
   234  	filters := append(b.Filters,
   235  		&palletmetadata.Filter{
   236  			Annotations: annotations.Filter{
   237  				Annotations: p.Metadata.K8sAnnotations(),
   238  				FsSlice:     fieldspecs.Annotations,
   239  			},
   240  		},
   241  		// TODO: allow configuring the fieldspecs here
   242  		&ambiguous.Filter{},
   243  	)
   244  	// the labels filter adds variables to manifests, so can only be ran if
   245  	// rendering is not disabled
   246  	if !p.Config.DisableRendering {
   247  		filters = append(filters, &labels.Filter{})
   248  	}
   249  	for _, f := range filters {
   250  		if err := kres.ApplyFilter(f); err != nil {
   251  			return nil, fmt.Errorf("failed to execute filter %s: %w", f.Name(), err)
   252  		}
   253  	}
   254  
   255  	// collect capabilities defined for this package to filter rendered objects
   256  	// into the appropriate layers
   257  	// +1 for built-in Infrastructure capabilities that any package can integrate
   258  	// with
   259  	caps := make([]internal.Capability, len(p.Config.Capabilities)+1)
   260  	// add infra provider
   261  	// TODO: support per-package infra provider config
   262  	caps[0] = b.capabilities[capability.CapabilityInfra]
   263  	for i, name := range p.Config.Capabilities {
   264  		if c, ok := b.capabilities[name]; ok {
   265  			// offset for global infra provider
   266  			caps[i+1] = c
   267  			continue
   268  		}
   269  		return nil, fmt.Errorf(
   270  			"%s references unknown capability %s", p.Config.Name, name,
   271  		)
   272  	}
   273  
   274  	// sort resources into individual arrays based on which capability matches
   275  	// resources that match none end up in the generic runtime layer
   276  	//
   277  	// NOTE: this code leverages the fact that pallet.LayerInfra and pallet.CapabilityInfra
   278  	// 			 are the same string constant. probably not worth addressing until
   279  	//			 multiple infra providers are supported
   280  	var (
   281  		resources = map[string][]*yaml.RNode{}
   282  		yy        = kres.ToRNodeSlice()
   283  	)
   284  
   285  	for _, y := range yy {
   286  		matched := false // whether or not current resource has been matched
   287  		var match string // capability that matched, used for error messages
   288  
   289  		// loop over capabilities, see if any match on this object
   290  		for _, c := range caps {
   291  			switch {
   292  			// each resource can only match once
   293  			case c.Matches(y) && matched:
   294  				return nil, fmt.Errorf(
   295  					"%s matched for both %s and %s. manifests can't be duplicated "+
   296  						"across layers. check your capability configuration to ensure that "+
   297  						"they do not cover overlapping resources.",
   298  					fmtRNodeMetadata(y), c.Name, match,
   299  				)
   300  			case c.Matches(y):
   301  				matched = true
   302  				match = string(c.Name)
   303  
   304  				if resources[match] == nil {
   305  					resources[match] = make([]*yaml.RNode, 0)
   306  				}
   307  
   308  				resources[match] = append(resources[match], y)
   309  			}
   310  		}
   311  		// if not matched, drop into the base runtime layer
   312  		if !matched {
   313  			match = string(layer.Runtime)
   314  			if resources[match] == nil {
   315  				resources[match] = make([]*yaml.RNode, 0)
   316  			}
   317  			resources[match] = append(resources[match], y)
   318  		}
   319  		// reset match for next iteration
   320  		matched = false
   321  		match = ""
   322  	}
   323  	if resources[layer.Infra.String()] != nil {
   324  		filt := labels.Filter{}
   325  		filtered, err := filt.Filter(resources[layer.Infra.String()])
   326  		if err != nil {
   327  			return nil, err
   328  		}
   329  		resources[layer.Infra.String()] = filtered
   330  	}
   331  
   332  	// order resources prior to layer packing to maintain order of capabilities
   333  	resourceKeys := make([]string, 0, len(resources))
   334  	for k := range resources {
   335  		resourceKeys = append(resourceKeys, k)
   336  	}
   337  	sort.Strings(resourceKeys)
   338  
   339  	// finally pack layers with the appropriate objects
   340  	var layers []layer.Layer
   341  
   342  	for _, key := range resourceKeys {
   343  		objs := resources[key]
   344  		switch {
   345  		// if there are no resources and it is a capability that was explicitly
   346  		// set by the package definition, issa error
   347  		case len(objs) == 0 && !layer.IsType(key):
   348  			return nil, fmt.Errorf(
   349  				"capability %s was specified but package produced no matching manifests",
   350  				key,
   351  			)
   352  		case len(objs) == 0:
   353  			continue
   354  		}
   355  
   356  		// convert this layers documents into single string representation so we can
   357  		// pack it
   358  		ydata, err := kio.StringAll(objs)
   359  		if err != nil {
   360  			return nil, err
   361  		}
   362  
   363  		if !p.Config.DisableRendering {
   364  			err := b.handleParams(p, ydata)
   365  			if err != nil {
   366  				return nil, err
   367  			}
   368  		}
   369  
   370  		// pack the data based on the kind of layer it is
   371  		switch key {
   372  		// if its the base runtime layer or the infra layer, create it as normal
   373  		case layer.Infra.String():
   374  			l, err := layer.New(layer.Infra, []byte(ydata))
   375  			if err != nil {
   376  				return nil, err
   377  			}
   378  
   379  			layers = append(layers, l)
   380  			// Reflect integration with capability in built package
   381  			p.Capabilities = addCapability(p.Capabilities, capability.CapabilityInfra)
   382  		case layer.Runtime.String():
   383  			l, err := layer.New(layer.Runtime, []byte(ydata))
   384  			if err != nil {
   385  				return nil, err
   386  			}
   387  
   388  			layers = append(layers, l)
   389  
   390  		// if its not base runtime or infra layer, it must be a runtime capability
   391  		// so we add an additional annotation for the runtime capability name
   392  		default:
   393  			l, err := layer.New(
   394  				layer.Runtime,
   395  				[]byte(ydata),
   396  				layer.ForCapability(capability.Capability(key)),
   397  			)
   398  			if err != nil {
   399  				return nil, err
   400  			}
   401  
   402  			layers = append(layers, l)
   403  			// Reflect integration with capability in built package
   404  			p.Capabilities = addCapability(p.Capabilities, capability.Capability(key))
   405  		}
   406  	}
   407  
   408  	// sort layers, ordering of layers in images needs to be deterministic or else
   409  	// digest of package with same contents will churn
   410  	layer.Sort(layers)
   411  	return layers, nil
   412  }
   413  
   414  func (b *Packer) packRawLayers(p *Package, t string) ([]layer.Layer, error) {
   415  	file, err := b.FS.ReadFile(t)
   416  	if err != nil {
   417  		return nil, err
   418  	}
   419  	resourceFac := resource.NewFactory(&hasher.Hasher{})
   420  	resmapFac := resmap.NewFactory(resourceFac)
   421  	kres, err := resmapFac.NewResMapFromBytes(file)
   422  	if err != nil {
   423  		return nil, err
   424  	}
   425  	return b.packLayers(p, kres)
   426  }
   427  
   428  func (b *Packer) packKustomizeLayers(p *Package, t string) ([]layer.Layer, error) {
   429  	kres, err := b.Kustomizer.Run(b.FS, t)
   430  	if err != nil {
   431  		return nil, fmt.Errorf("failed to kustomize %s: %w", t, err)
   432  	}
   433  	return b.packLayers(p, kres)
   434  }
   435  
   436  // Artifacts returns all of the built Pallets from its internal cache
   437  func (b *Packer) Artifacts() []pallet.Pallet {
   438  	r := make([]pallet.Pallet, 0, len(b.cache))
   439  	for _, v := range b.cache {
   440  		r = append(r, v)
   441  	}
   442  	return r
   443  }
   444  
   445  // loadPkg loads a package definition file from the filesystem or the cache of
   446  // loaded package defs if present
   447  func (b *Packer) loadPkg(path string) (*Package, error) {
   448  	if pkg, ok := b.pkgCache[path]; ok {
   449  		return pkg, nil
   450  	}
   451  	pkgCfg, err := b.LoadPkg(path)
   452  	if err != nil {
   453  		return nil, err
   454  	}
   455  
   456  	pkg := &Package{
   457  		Config:    pkgCfg,
   458  		Warehouse: *b.Config,
   459  		Dir:       path,
   460  		Metadata:  b.pkgMeta(pkgCfg),
   461  	}
   462  
   463  	pkg.Warehouse, err = b.Config.WithParameters(pkg.Config.Parameters)
   464  	if err != nil {
   465  		return nil, err
   466  	}
   467  
   468  	for _, cfgpath := range pkg.Config.Configurations {
   469  		cfg, err := b.loadCfg(cfgpath)
   470  		if err != nil {
   471  			return nil, err
   472  		}
   473  		pkg.Warehouse, err = mergeCfg(pkg.Warehouse, cfg)
   474  		if err != nil {
   475  			return nil, err
   476  		}
   477  	}
   478  
   479  	return pkg, nil
   480  }
   481  
   482  // loadCfg loads a warehouse config file from the file system or the cache
   483  // of loaded configs if present
   484  func (b *Packer) loadCfg(path string) (lift.Config, error) {
   485  	path = b.ResolveConfigPath(path)
   486  	if cfg, ok := b.cfgCache[path]; ok {
   487  		return cfg, nil
   488  	}
   489  	return b.LoadConfig(path)
   490  }
   491  
   492  func (b *Packer) pkgMeta(pkg types.Pallet) pallet.Metadata {
   493  	version := b.buildInfo.Version
   494  
   495  	// If version is set in pallet.yaml, honor it if it passes validation
   496  	if err := pkg.PalletSpec.Version.Validate(); err == nil {
   497  		version = pkg.PalletSpec.Version.String()
   498  	}
   499  
   500  	return pallet.Metadata{
   501  		Name:        pkg.Name,
   502  		Description: pkg.Description,
   503  		Vendor:      pkg.Vendor,
   504  		Team:        pkg.Team,
   505  		BuildInfo: pallet.BuildInfo{
   506  			Source:   b.buildInfo.Source,
   507  			Version:  version,
   508  			Revision: b.buildInfo.Revision,
   509  			Created:  b.buildInfo.Created,
   510  		},
   511  		// TODO: update build info based on package specs
   512  		// TODO: implement readme support
   513  	}
   514  }
   515  
   516  func fmtRNodeMetadata(y *yaml.RNode) string {
   517  	return fmt.Sprintf("%s/%s/%s", y.GetKind(), y.GetNamespace(), y.GetName())
   518  }
   519  
   520  func getParameters(data string) []string {
   521  	var parameters []string
   522  
   523  	re := regexp.MustCompile(`[^\$]\${1}{[[:word:]]*?}`)
   524  	results := re.FindAllString(data, -1)
   525  
   526  	// Exit early if no parameters are found
   527  	if len(results) == 0 {
   528  		return nil
   529  	}
   530  
   531  	trimmer := regexp.MustCompile(`[^[:word:]]+`)
   532  
   533  	for _, result := range results {
   534  		// trim result down such that only the rendering parameter name remains
   535  		result = trimmer.ReplaceAllString(result, "")
   536  		if !slices.Contains(parameters, result) {
   537  			parameters = append(parameters, result)
   538  		}
   539  	}
   540  	return parameters
   541  }
   542  
   543  func addCapability(cc capability.Capabilities, c capability.Capability) capability.Capabilities {
   544  	if !slices.Contains(cc, c) {
   545  		cc = append(cc, c)
   546  	}
   547  	return cc
   548  }
   549  
   550  func (b *Packer) handleParams(p *Package, ydata string) error {
   551  	keys := make([]string, len(b.Config.Parameters))
   552  	for i, p := range b.Config.Parameters {
   553  		keys[i] = p.Key
   554  	}
   555  	parameters := getParameters(ydata)
   556  	for _, manifestP := range parameters {
   557  		valid := false
   558  		for _, p := range keys {
   559  			if p == manifestP {
   560  				// found valid parameter, exit early
   561  				valid = true
   562  				continue
   563  			}
   564  		}
   565  		if !valid {
   566  			return fmt.Errorf("rendering parameter %s is invalid: expected one of %v",
   567  				manifestP, keys)
   568  		}
   569  		if !slices.Contains(p.Parameters, manifestP) {
   570  			p.Parameters = append(p.Parameters, manifestP)
   571  		}
   572  		if manifestP == lift.ClusterHashRenderingParameter &&
   573  			!slices.Contains(p.Parameters, lift.ClusterUUIDRenderingParameter) {
   574  			p.Parameters = append(p.Parameters, lift.ClusterUUIDRenderingParameter)
   575  		}
   576  	}
   577  
   578  	return nil
   579  }
   580
View as plain text