packer_test.go

Documentation: edge-infra.dev/pkg/f8n/warehouse/lift/pack

     1  package pack
     2  
     3  import (
     4  	"embed"
     5  	"io/fs"
     6  	"strings"
     7  	"testing"
     8  
     9  	"github.com/stretchr/testify/assert"
    10  	"gopkg.in/yaml.v3"
    11  
    12  	"edge-infra.dev/pkg/f8n/warehouse"
    13  	"edge-infra.dev/pkg/f8n/warehouse/lift"
    14  	"edge-infra.dev/pkg/f8n/warehouse/lift/pack/types"
    15  	"edge-infra.dev/pkg/f8n/warehouse/oci"
    16  	"edge-infra.dev/pkg/f8n/warehouse/oci/validate"
    17  	"edge-infra.dev/pkg/f8n/warehouse/pallet"
    18  	"edge-infra.dev/pkg/k8s/kustomize"
    19  	"edge-infra.dev/pkg/lib/build"
    20  	"edge-infra.dev/test/fixtures"
    21  )
    22  
    23  var (
    24  	b = pallet.BuildInfo{
    25  		Created:  "2023-06-21T15:00:00Z",
    26  		Source:   "https://github.com/ncrvoyix-swt-retail/edge-infra",
    27  		Revision: "abcde1234",
    28  		Version:  "0.0.1",
    29  	}
    30  	//go:embed testdata/fs/*
    31  	data embed.FS
    32  )
    33  
    34  func TestParameterValidation(t *testing.T) {
    35  	validParameters := []string{"cluster_uuid", "cluster_hash", "gcp_project_id", "foreman_gcp_project_id"}
    36  	tcs := map[string]struct {
    37  		data       string
    38  		parameters []string
    39  	}{
    40  		"without parameters": {
    41  			data: `apiVersion: v1
    42  			kind: ServiceAccount
    43  			metadata:
    44  			  name: shoot`,
    45  			parameters: nil,
    46  		},
    47  		"with invalid parameters": {
    48  			data: `apiVersion: v1
    49  			kind: ServiceAccount
    50  			metadata:
    51  			  name: ${var:=invalid%)!@}`,
    52  			parameters: nil,
    53  		},
    54  		"with valid parameters": {
    55  			data: `apiVersion: iam.cnrm.cloud.google.com/v1beta1
    56  			kind: IAMPolicyMember
    57  			metadata:
    58  			  name: shoot-publisher
    59  			spec:
    60  			  member: serviceAccount:shoot-${cluster_hash}@${gcp_project_id}.iam.gserviceaccount.com
    61  			  resourceRef:
    62  				apiVersion: pubsub.cnrm.cloud.google.com/v1beta1
    63  				kind: PubSubTopic
    64  				external: projects/${foreman_gcp_project_id}/topics/data-sync-e2c
    65  			  role: roles/pubsub.publisher`,
    66  			parameters: []string{
    67  				"cluster_hash",
    68  				"gcp_project_id",
    69  				"foreman_gcp_project_id",
    70  			},
    71  		},
    72  		"with valid and invalid parameters": {
    73  			data: `apiVersion: iam.cnrm.cloud.google.com/v1beta1
    74  			kind: IAMPolicyMember
    75  			metadata:
    76  			  name: ${var:=default}
    77  			spec:
    78  			  member: serviceAccount:shoot-${cluster_hash}@${gcp_project_id}.iam.gserviceaccount.com
    79  			  resourceRef:
    80  				apiVersion: pubsub.cnrm.cloud.google.com/v1beta1
    81  				kind: PubSubTopic
    82  				external: projects/${foreman_gcp_project_id}/topics/data-sync-e2c
    83  			  role: roles/pubsub.publisher`,
    84  			parameters: []string{
    85  				"cluster_hash",
    86  				"gcp_project_id",
    87  				"foreman_gcp_project_id",
    88  			},
    89  		},
    90  		"with multiple $ prefix": {
    91  			data: `apiVersion: iam.cnrm.cloud.google.com/v1beta1
    92  			kind: IAMPolicyMember
    93  			metadata:
    94  			  name: $${NOPE}
    95  			spec:
    96  			  member: serviceAccount:shoot-$$${cluster_hash}@${gcp_project_id}.iam.gserviceaccount.com
    97  			  resourceRef:
    98  				apiVersion: pubsub.cnrm.cloud.google.com/v1beta1
    99  				kind: PubSubTopic
   100  				external: projects/${foreman_gcp_project_id}/topics/data-sync-e2c
   101  			  role: roles/pubsub.publisher`,
   102  			parameters: []string{
   103  				"gcp_project_id",
   104  				"foreman_gcp_project_id",
   105  			},
   106  		},
   107  	}
   108  
   109  	for test, tc := range tcs {
   110  		t.Run(test, func(t *testing.T) {
   111  			parameters := getParameters(tc.data)
   112  			assert.Equal(t, tc.parameters, parameters, "Expected parameters do not match actual parameters: %s", test)
   113  		})
   114  		t.Run(test, func(t *testing.T) {
   115  			parameters := getParameters(tc.data)
   116  			for _, p := range parameters {
   117  				valid := false
   118  				for _, vp := range validParameters {
   119  					if p == vp {
   120  						// found valid parameter, exit early
   121  						valid = true
   122  						continue
   123  					}
   124  				}
   125  				assert.True(t, valid, "rendering parameter %s is invalid: expected one of %v",
   126  					p, validParameters)
   127  			}
   128  		})
   129  	}
   130  }
   131  
   132  // TestPacker tests various functions in packer.go and pallets resulting from packer.Pack()
   133  func TestPacker(t *testing.T) {
   134  	packer := fixturesFS(t)
   135  
   136  	t.Run("Version", func(t *testing.T) {
   137  		// Create pkgMeta using inferred version from pallet.BuildInfo
   138  		testPallet := types.Pallet{}
   139  		inferredVersion := packer.pkgMeta(testPallet)
   140  		assert.Contains(t, inferredVersion.Version, b.Version)
   141  
   142  		testPallet.Version = build.Version{
   143  			Commit:           "abcd1234",
   144  			ReleaseCandidate: false,
   145  			SemVer:           "9.9.9",
   146  			Timestamp:        1686062747,
   147  		}
   148  
   149  		// Create pkgMeta using declared version in the testPallet
   150  		declaredVersion := packer.pkgMeta(testPallet)
   151  		assert.Contains(t, declaredVersion.Version, testPallet.Version.SemVer)
   152  	})
   153  
   154  	// TestParameterAnnotation confirms that the pallet's parameters are annotated on the packed artifact.
   155  	// This is particularly important as lumperctl relies on the accuracy of these annotations during unpacking.
   156  	t.Run("ParameterAnnotation", func(t *testing.T) {
   157  		// Pack the pallet
   158  		plt, err := packer.Pack("pallets/real/stores/store")
   159  		assert.NoError(t, err)
   160  		packedParameters := plt.Parameters()
   161  		assert.NotEmpty(t, packedParameters)
   162  
   163  		// Unwrap underlying artifact
   164  		a := plt.Unwrap()
   165  		annos, err := oci.Annotations(a)
   166  		assert.NoError(t, err)
   167  
   168  		// Compare the pallet parameters to those annotated on the underlying artifact
   169  		// These must match as lumperctl relies on these parameters to render & apply the pallet properly
   170  		assert.Equal(t, strings.Join(packedParameters, ","), annos[warehouse.AnnotationParameters])
   171  	})
   172  
   173  	// Artifacts confirms that the Artifacts() exposure func contains pallets recently packed
   174  	t.Run("Artifacts", func(t *testing.T) {
   175  		packedPallet, err := packer.Pack("pallets/test-pallets/cert-manager-test")
   176  		assert.NoError(t, err)
   177  		packedParameters := packedPallet.Parameters()
   178  		assert.NotEmpty(t, packedParameters)
   179  
   180  		fetchedPallets := packer.Artifacts()
   181  		assert.NotEmpty(t, fetchedPallets)
   182  		assert.Contains(t, fetchedPallets, packedPallet)
   183  	})
   184  
   185  	// ValidArtifacts ensures packed pallets are valid OCI artifacts
   186  	t.Run("ValidArtifact", func(t *testing.T) {
   187  		plt, err := packer.Pack("pallets/real/stores/store")
   188  		assert.NoError(t, err)
   189  
   190  		err = validate.Pallet(plt)
   191  		assert.NoError(t, err)
   192  
   193  		// Test underlying artifact is valid for good measure
   194  		a := plt.Unwrap()
   195  		err = validate.Warehouse(a)
   196  		assert.NoError(t, err)
   197  	})
   198  }
   199  
   200  // ReproducibleBuilds packs the same pallet multiple times to ensure build reproducibility
   201  // The cache is cleared prior to each packing session to ensure the final pallet is not pulled from the packer's cache.
   202  // Each packing session occurs in series to ensure the cache is indeed empty prior to packing and not affected by the
   203  // other packing sessions.
   204  //
   205  // Investigating the cache during this repeated packing process will show dependencies pulled from cache only after
   206  // they have been packed. The cache will always be empty at the start of each packing session but will still be used
   207  // after packing dependencies.
   208  // e.g. when packing the external-secrets pallet with a clean cache, external-secrets-operator, a dependency of every
   209  // other pallet inside the external-secrets pallet, will be packed first and then pulled from cache when the remaining
   210  // dependencies of the external-secrets pallet are being packed from scratch.
   211  func TestReproducibleBuilds(t *testing.T) {
   212  	packer := fixturesFS(t)
   213  
   214  	// Pack external-secrets test pallet and inspect raw manifest
   215  	ctrlPallet, err := packer.Pack("pallets/real/external-secrets")
   216  	assert.NoError(t, err)
   217  
   218  	ctrlManifest, err := ctrlPallet.RawManifest()
   219  	assert.NoError(t, err)
   220  
   221  	// Pack the same pallet multiple times and compare to the control
   222  	for i := 0; i < 10; i++ {
   223  		t.Run("RepeatBuild", func(t *testing.T) {
   224  			// Clear cache to assert packing
   225  			packer.cache = make(map[string]pallet.Pallet)
   226  
   227  			// Pack same pallet again
   228  			testPallet, err := packer.Pack("pallets/real/external-secrets")
   229  			assert.NoError(t, err)
   230  
   231  			// Pull raw manifest and compare
   232  			testManifest, err := testPallet.RawManifest()
   233  			assert.NoError(t, err)
   234  			assert.Equal(t, string(ctrlManifest), string(testManifest))
   235  		})
   236  	}
   237  }
   238  
   239  // Create packer with test fixtures FS
   240  func fixturesFS(t *testing.T) *Packer {
   241  	t.Helper()
   242  	t.Setenv("WAREHOUSE_PATH", "/")
   243  	t.Setenv("WAREHOUSE_CACHE", "/cache")
   244  
   245  	// Set up & re-root fs for packer
   246  	testFS, err := fs.Sub(fixtures.PalletFixtures, "warehouse/src")
   247  	if err != nil {
   248  		t.Fatalf("failed to reroot test fs: %s", err)
   249  	}
   250  	packerFS := &kustomize.FS{FS: testFS}
   251  
   252  	return packerFromFS(t, packerFS)
   253  }
   254  
   255  func testdataFS(t *testing.T) *Packer {
   256  	t.Helper()
   257  	t.Setenv("WAREHOUSE_PATH", "/")
   258  	t.Setenv("WAREHOUSE_CACHE", "/cache")
   259  	s, err := fs.Sub(data, "testdata/fs")
   260  	if err != nil {
   261  		t.Fatalf("failed to reroot test fs: %s", err)
   262  	}
   263  	packerFS := &kustomize.FS{FS: s}
   264  
   265  	return packerFromFS(t, packerFS)
   266  }
   267  
   268  func packerFromFS(t *testing.T, packerFS *kustomize.FS) *Packer {
   269  	// Read and convert test .warehouse.yaml to lift.Config
   270  	testCfgBytes, err := packerFS.ReadFile(".warehouse.yaml")
   271  	if err != nil {
   272  		t.Fatalf("failed to read .warehouse.yaml in test fs: %s", err)
   273  	}
   274  	testCfg := lift.Config{}
   275  	err = yaml.Unmarshal(testCfgBytes, &testCfg)
   276  	if err != nil {
   277  		t.Fatalf("failed to unmarshal test cfg: %s", err)
   278  	}
   279  
   280  	// Instantiate basic lift.Config & merge with test config
   281  	cfg, err := lift.NewConfig()
   282  	if err != nil {
   283  		t.Fatal(err)
   284  	}
   285  	cfg, err = cfg.FromConfig(testCfg)
   286  	if err != nil {
   287  		t.Fatalf("failed to merge test cfg: %s", err)
   288  	}
   289  
   290  	// Instantiate test packer
   291  	c := Context{
   292  		Config: &cfg,
   293  		FS:     packerFS,
   294  	}
   295  
   296  	packer, err := New(c, b)
   297  	if err != nil {
   298  		t.Fatal(err)
   299  	}
   300  
   301  	return packer
   302  }
   303  
   304  func TestRawLayers(t *testing.T) {
   305  	packer := testdataFS(t)
   306  	packedRawPallet, err := packer.Pack("cert-manager-test")
   307  	assert.NoError(t, err)
   308  	rawMeta := packedRawPallet.Metadata()
   309  	packedKustomizePallet, err := packer.Pack("cert-manager-kustomize")
   310  	assert.NoError(t, err)
   311  	kustomizeMeta := packedKustomizePallet.Metadata()
   312  	assert.Equal(t, rawMeta, kustomizeMeta)
   313  }
   314
View as plain text