compatibility.go

Documentation: kubevirt.io/api/apitesting/roundtrip

     1  /*
     2  Copyright 2024 The KubeVirt Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package roundtrip
    18  
    19  import (
    20  	"bytes"
    21  	gojson "encoding/json"
    22  	"fmt"
    23  	"os"
    24  	"path/filepath"
    25  	"reflect"
    26  	"sort"
    27  	"strings"
    28  	"testing"
    29  
    30  	"github.com/google/go-cmp/cmp"
    31  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    32  	"k8s.io/apimachinery/pkg/runtime"
    33  	"k8s.io/apimachinery/pkg/runtime/schema"
    34  	"k8s.io/apimachinery/pkg/runtime/serializer/json"
    35  	"k8s.io/apimachinery/pkg/util/sets"
    36  	v1 "kubevirt.io/api/core/v1"
    37  )
    38  
    39  // This tests are largely take from k8s apitesting, in the future when it is possible it could be imported directly:
    40  // https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/roundtrip.go
    41  
    42  // CompatibilityTestOptions holds configuration for running a compatibility test using in-memory objects
    43  // and serialized files on disk representing the current code and serialized data from previous versions.
    44  //
    45  // Example use: `NewCompatibilityTestOptions(scheme).Complete(t).Run(t)`
    46  type CompatibilityTestOptions struct {
    47  	// Scheme is used to create new objects for filling, decoding, and for constructing serializers.
    48  	// Required.
    49  	Scheme *runtime.Scheme
    50  
    51  	// TestDataDir points to a directory containing compatibility test data.
    52  	// Complete() populates this with "testdata" if unset.
    53  	TestDataDir string
    54  
    55  	// TestDataDirCurrentVersion points to a directory containing compatibility test data for the current version.
    56  	// Complete() populates this with "<TestDataDir>/HEAD" if unset.
    57  	// Within this directory, `<group>.<version>.<kind>.[json|yaml|pb]` files are required to exist, and are:
    58  	// * verified to match serialized FilledObjects[GVK]
    59  	// * verified to decode without error
    60  	// * verified to round-trip byte-for-byte when re-encoded
    61  	// * verified to be semantically equal when decoded into memory
    62  	TestDataDirCurrentVersion string
    63  
    64  	// TestDataDirsPreviousVersions is a list of directories containing compatibility test data for previous versions.
    65  	// Complete() populates this with "<TestDataDir>/v*" directories if nil.
    66  	// Within these directories, `<group>.<version>.<kind>.[json|yaml|pb]` files are optional. If present, they are:
    67  	// * verified to decode without error
    68  	// * verified to round-trip byte-for-byte when re-encoded (or to match a `<group>.<version>.<kind>.[json|yaml|pb].after_roundtrip.[json|yaml|pb]` file if it exists)
    69  	// * verified to be semantically equal when decoded into memory
    70  	TestDataDirsPreviousVersions []string
    71  
    72  	// Kinds is a list of fully qualified kinds to test.
    73  	// Complete() populates this with Scheme.AllKnownTypes() if unset.
    74  	Kinds []schema.GroupVersionKind
    75  
    76  	// FilledObjects is an optional set of pre-filled objects to use for verifying HEAD fixtures.
    77  	// Complete() populates this with the result of CompatibilityTestObject(Kinds[*], Scheme, FillFuncs) for any missing kinds.
    78  	// Objects must deterministically populate every field and be identical on every invocation.
    79  	FilledObjects map[schema.GroupVersionKind]runtime.Object
    80  
    81  	// FillFuncs is an optional map of custom functions to use to fill instances of particular types.
    82  	FillFuncs map[reflect.Type]FillFunc
    83  
    84  	JSON runtime.Serializer
    85  	YAML runtime.Serializer
    86  }
    87  
    88  // FillFunc is a function that populates all serializable fields in obj.
    89  // s and i are string and integer values relevant to the object being populated
    90  // (for example, the json key containing the object)
    91  // that can be used when filling the object to make the object content identifiable
    92  type FillFunc func(s string, i int, obj interface{})
    93  
    94  type Data struct {
    95  	Base64Data string `json:"data"`
    96  }
    97  
    98  func NewCompatibilityTestOptions(scheme *runtime.Scheme) *CompatibilityTestOptions {
    99  	return &CompatibilityTestOptions{Scheme: scheme}
   100  }
   101  
   102  // This includes kinds that typically only need to be tested in a single API group.
   103  // Ignoring these kinds as they are covered in k8s.
   104  var ignoreCoreKinds = sets.NewString(
   105  	"CreateOptions", "UpdateOptions", "PatchOptions", "DeleteOptions",
   106  	"GetOptions", "ListOptions", "ExportOptions",
   107  	"WatchEvent", "APIGroup", "APIVersions", "Status",
   108  )
   109  
   110  func (c *CompatibilityTestOptions) Complete(t *testing.T) *CompatibilityTestOptions {
   111  	t.Helper()
   112  
   113  	// Verify scheme
   114  	if c.Scheme == nil {
   115  		t.Fatal("scheme is required")
   116  	}
   117  
   118  	// Populate testdata dirs
   119  	if c.TestDataDir == "" {
   120  		c.TestDataDir = "testdata"
   121  	}
   122  	if c.TestDataDirCurrentVersion == "" {
   123  		c.TestDataDirCurrentVersion = filepath.Join(c.TestDataDir, "HEAD")
   124  	}
   125  	if c.TestDataDirsPreviousVersions == nil {
   126  		dirs, err := filepath.Glob(filepath.Join(c.TestDataDir, "release-*"))
   127  		if err != nil {
   128  			t.Fatal(err)
   129  		}
   130  		sort.Strings(dirs)
   131  		c.TestDataDirsPreviousVersions = dirs
   132  	}
   133  
   134  	// Populate kinds
   135  	if len(c.Kinds) == 0 {
   136  		gvks := []schema.GroupVersionKind{}
   137  		for gvk := range c.Scheme.AllKnownTypes() {
   138  			if gvk.Version == "" || gvk.Version == runtime.APIVersionInternal {
   139  				// only test external types
   140  				continue
   141  			}
   142  			if strings.HasSuffix(gvk.Kind, "List") {
   143  				// omit list types
   144  				continue
   145  			}
   146  			if ignoreCoreKinds.Has(gvk.Kind) {
   147  				// only test options types in the core API group
   148  				continue
   149  			}
   150  			if gvk == v1.VirtualMachineInstanceMigrationGroupVersionKind || gvk == v1.VirtualMachineInstancePresetGroupVersionKind || gvk == v1.VirtualMachineInstanceReplicaSetGroupVersionKind {
   151  				continue
   152  			}
   153  			gvks = append(gvks, gvk)
   154  		}
   155  		c.Kinds = gvks
   156  	}
   157  
   158  	// Sort kinds to get deterministic test order
   159  	sort.Slice(c.Kinds, func(i, j int) bool {
   160  		if c.Kinds[i].Group != c.Kinds[j].Group {
   161  			return c.Kinds[i].Group < c.Kinds[j].Group
   162  		}
   163  		if c.Kinds[i].Version != c.Kinds[j].Version {
   164  			return c.Kinds[i].Version < c.Kinds[j].Version
   165  		}
   166  		if c.Kinds[i].Kind != c.Kinds[j].Kind {
   167  			return c.Kinds[i].Kind < c.Kinds[j].Kind
   168  		}
   169  		return false
   170  	})
   171  
   172  	// Fill any missing objects
   173  	if c.FilledObjects == nil {
   174  		c.FilledObjects = map[schema.GroupVersionKind]runtime.Object{}
   175  	}
   176  	fillFuncs := defaultFillFuncs()
   177  	for k, v := range c.FillFuncs {
   178  		fillFuncs[k] = v
   179  	}
   180  	for _, gvk := range c.Kinds {
   181  		if _, ok := c.FilledObjects[gvk]; ok {
   182  			continue
   183  		}
   184  
   185  		obj, err := CompatibilityTestObject(c.Scheme, gvk, fillFuncs)
   186  		if err != nil {
   187  			t.Fatal(err)
   188  		}
   189  		c.FilledObjects[gvk] = obj
   190  	}
   191  
   192  	if c.JSON == nil {
   193  		c.JSON = json.NewSerializer(json.DefaultMetaFactory, c.Scheme, c.Scheme, true)
   194  	}
   195  	if c.YAML == nil {
   196  		c.YAML = json.NewYAMLSerializer(json.DefaultMetaFactory, c.Scheme, c.Scheme)
   197  	}
   198  
   199  	return c
   200  }
   201  
   202  func (c *CompatibilityTestOptions) Run(t *testing.T) {
   203  	usedHEADFixtures := sets.NewString()
   204  
   205  	var ranCurrentTests bool
   206  	for _, gvk := range c.Kinds {
   207  		t.Run(makeName(gvk), func(t *testing.T) {
   208  
   209  			t.Run("HEAD", func(t *testing.T) {
   210  				c.runCurrentVersionTest(t, gvk, usedHEADFixtures)
   211  				ranCurrentTests = true
   212  			})
   213  
   214  			for _, previousVersionDir := range c.TestDataDirsPreviousVersions {
   215  				t.Run(filepath.Base(previousVersionDir), func(t *testing.T) {
   216  					c.runPreviousVersionTest(t, gvk, previousVersionDir, nil)
   217  				})
   218  			}
   219  
   220  		})
   221  	}
   222  
   223  	// Check for unused HEAD fixtures
   224  	t.Run("unused_fixtures", func(t *testing.T) {
   225  		if !ranCurrentTests {
   226  			return
   227  		}
   228  		files, err := os.ReadDir(c.TestDataDirCurrentVersion)
   229  		if err != nil {
   230  			t.Fatal(err)
   231  		}
   232  		allFixtures := sets.NewString()
   233  		for _, file := range files {
   234  			allFixtures.Insert(file.Name())
   235  		}
   236  
   237  		if unused := allFixtures.Difference(usedHEADFixtures); len(unused) > 0 {
   238  			t.Fatalf("remove unused fixtures from %s:\n%s", c.TestDataDirCurrentVersion, strings.Join(unused.List(), "\n"))
   239  		}
   240  	})
   241  }
   242  
   243  func (c *CompatibilityTestOptions) runCurrentVersionTest(t *testing.T, gvk schema.GroupVersionKind, usedFiles sets.String) {
   244  	expectedObject := c.FilledObjects[gvk]
   245  	expectedJSON, expectedYAML := c.encode(t, expectedObject)
   246  
   247  	actualJSON, actualYAML, err := read(c.TestDataDirCurrentVersion, gvk, "", usedFiles)
   248  	if err != nil && !os.IsNotExist(err) {
   249  		t.Fatal(err)
   250  	}
   251  
   252  	needsUpdate := false
   253  	if os.IsNotExist(err) {
   254  		t.Errorf("current version compatibility files did not exist: %v", err)
   255  		needsUpdate = true
   256  	} else {
   257  		if !bytes.Equal(expectedJSON, actualJSON) {
   258  			t.Errorf("json differs")
   259  			t.Log(cmp.Diff(string(actualJSON), string(expectedJSON)))
   260  			needsUpdate = true
   261  		}
   262  
   263  		if !bytes.Equal(expectedYAML, actualYAML) {
   264  			t.Errorf("yaml differs")
   265  			t.Log(cmp.Diff(string(actualYAML), string(expectedYAML)))
   266  			needsUpdate = true
   267  		}
   268  	}
   269  
   270  	if needsUpdate {
   271  		const updateEnvVar = "UPDATE_COMPATIBILITY_FIXTURE_DATA"
   272  		if os.Getenv(updateEnvVar) == "true" {
   273  			writeFile(t, c.TestDataDirCurrentVersion, gvk, "", "json", expectedJSON)
   274  			writeFile(t, c.TestDataDirCurrentVersion, gvk, "", "yaml", expectedYAML)
   275  			t.Logf("wrote expected compatibility data... verify, commit, and rerun tests")
   276  		} else {
   277  			t.Logf("if the diff is expected because of a new type or a new field, re-run with %s=true to update the compatibility data", updateEnvVar)
   278  		}
   279  		return
   280  	}
   281  
   282  	emptyObj, err := c.Scheme.New(gvk)
   283  	if err != nil {
   284  		t.Fatal(err)
   285  	}
   286  	{
   287  		// compact before decoding since embedded RawExtension fields retain indenting
   288  		compacted := &bytes.Buffer{}
   289  		if err := gojson.Compact(compacted, actualJSON); err != nil {
   290  			t.Error(err)
   291  		}
   292  
   293  		jsonDecoded := emptyObj.DeepCopyObject()
   294  		jsonDecoded, _, err = c.JSON.Decode(compacted.Bytes(), &gvk, jsonDecoded)
   295  		if err != nil {
   296  			t.Error(err)
   297  		} else if !apiequality.Semantic.DeepEqual(expectedObject, jsonDecoded) {
   298  			t.Errorf("expected and decoded json objects differed:\n%s", cmp.Diff(expectedObject, jsonDecoded))
   299  		}
   300  	}
   301  	{
   302  		yamlDecoded := emptyObj.DeepCopyObject()
   303  		yamlDecoded, _, err = c.YAML.Decode(actualYAML, &gvk, yamlDecoded)
   304  		if err != nil {
   305  			t.Error(err)
   306  		} else if !apiequality.Semantic.DeepEqual(expectedObject, yamlDecoded) {
   307  			t.Errorf("expected and decoded yaml objects differed:\n%s", cmp.Diff(expectedObject, yamlDecoded))
   308  		}
   309  	}
   310  }
   311  
   312  func (c *CompatibilityTestOptions) encode(t *testing.T, obj runtime.Object) (json, yaml []byte) {
   313  	jsonBytes := bytes.NewBuffer(nil)
   314  	if err := c.JSON.Encode(obj, jsonBytes); err != nil {
   315  		t.Fatalf("error encoding json: %v", err)
   316  	}
   317  	yamlBytes := bytes.NewBuffer(nil)
   318  	if err := c.YAML.Encode(obj, yamlBytes); err != nil {
   319  		t.Fatalf("error encoding yaml: %v", err)
   320  	}
   321  
   322  	return jsonBytes.Bytes(), yamlBytes.Bytes()
   323  }
   324  
   325  func read(dir string, gvk schema.GroupVersionKind, suffix string, usedFiles sets.String) (json, yaml []byte, err error) {
   326  	jsonFilename := makeName(gvk) + suffix + ".json"
   327  	actualJSON, jsonErr := os.ReadFile(filepath.Join(dir, jsonFilename))
   328  	yamlFilename := makeName(gvk) + suffix + ".yaml"
   329  	actualYAML, yamlErr := os.ReadFile(filepath.Join(dir, yamlFilename))
   330  
   331  	if usedFiles != nil {
   332  		usedFiles.Insert(jsonFilename)
   333  		usedFiles.Insert(yamlFilename)
   334  	}
   335  	if jsonErr != nil {
   336  		return actualJSON, actualYAML, jsonErr
   337  	}
   338  	if yamlErr != nil {
   339  		return actualJSON, actualYAML, yamlErr
   340  	}
   341  
   342  	return actualJSON, actualYAML, nil
   343  }
   344  
   345  func writeFile(t *testing.T, dir string, gvk schema.GroupVersionKind, suffix, extension string, data []byte) {
   346  	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
   347  		t.Fatal("error making directory", err)
   348  	}
   349  	if err := os.WriteFile(filepath.Join(dir, makeName(gvk)+suffix+"."+extension), data, os.FileMode(0644)); err != nil {
   350  		t.Fatalf("error writing %s: %v", extension, err)
   351  	}
   352  }
   353  
   354  func (c *CompatibilityTestOptions) runPreviousVersionTest(t *testing.T, gvk schema.GroupVersionKind, previousVersionDir string, usedFiles sets.String) {
   355  	jsonBeforeRoundTrip, yamlBeforeRoundTrip, err := read(previousVersionDir, gvk, "", usedFiles)
   356  	if os.IsNotExist(err) || (len(jsonBeforeRoundTrip) == 0 && len(yamlBeforeRoundTrip) == 0) {
   357  		fmt.Printf("error reading the %s, skipping", previousVersionDir)
   358  		t.SkipNow()
   359  		return
   360  	}
   361  	if err != nil {
   362  		t.Fatal(err)
   363  	}
   364  
   365  	emptyObj, err := c.Scheme.New(gvk)
   366  	if err != nil {
   367  		t.Fatal(err)
   368  	}
   369  
   370  	// compact before decoding since embedded RawExtension fields retain indenting
   371  	compacted := &bytes.Buffer{}
   372  	if err := gojson.Compact(compacted, jsonBeforeRoundTrip); err != nil {
   373  		t.Fatal(err)
   374  	}
   375  
   376  	jsonDecoded := emptyObj.DeepCopyObject()
   377  	jsonDecoded, _, err = c.JSON.Decode(compacted.Bytes(), &gvk, jsonDecoded)
   378  	if err != nil {
   379  		t.Fatal(err)
   380  	}
   381  	jsonBytes := bytes.NewBuffer(nil)
   382  	if err := c.JSON.Encode(jsonDecoded, jsonBytes); err != nil {
   383  		t.Fatalf("error encoding json: %v", err)
   384  	}
   385  	jsonAfterRoundTrip := jsonBytes.Bytes()
   386  
   387  	yamlDecoded := emptyObj.DeepCopyObject()
   388  	yamlDecoded, _, err = c.YAML.Decode(yamlBeforeRoundTrip, &gvk, yamlDecoded)
   389  	if err != nil {
   390  		t.Fatal(err)
   391  	} else if !apiequality.Semantic.DeepEqual(jsonDecoded, yamlDecoded) {
   392  		t.Errorf("decoded json and yaml objects differ:\n%s", cmp.Diff(jsonDecoded, yamlDecoded))
   393  	}
   394  	yamlBytes := bytes.NewBuffer(nil)
   395  	if err := c.YAML.Encode(yamlDecoded, yamlBytes); err != nil {
   396  		t.Fatalf("error encoding yaml: %v", err)
   397  	}
   398  	yamlAfterRoundTrip := yamlBytes.Bytes()
   399  
   400  	expectedJSONAfterRoundTrip, expectedYAMLAfterRoundTrip, _ := read(previousVersionDir, gvk, ".after_roundtrip", usedFiles)
   401  	if len(expectedJSONAfterRoundTrip) == 0 {
   402  		expectedJSONAfterRoundTrip = jsonBeforeRoundTrip
   403  	}
   404  	if len(expectedYAMLAfterRoundTrip) == 0 {
   405  		expectedYAMLAfterRoundTrip = yamlBeforeRoundTrip
   406  	}
   407  
   408  	jsonNeedsUpdate := false
   409  	yamlNeedsUpdate := false
   410  
   411  	if !bytes.Equal(expectedJSONAfterRoundTrip, jsonAfterRoundTrip) {
   412  		t.Errorf("json differs")
   413  		t.Log(cmp.Diff(string(expectedJSONAfterRoundTrip), string(jsonAfterRoundTrip)))
   414  		jsonNeedsUpdate = true
   415  	}
   416  
   417  	if !bytes.Equal(expectedYAMLAfterRoundTrip, yamlAfterRoundTrip) {
   418  		t.Errorf("yaml differs")
   419  		t.Log(cmp.Diff(string(expectedYAMLAfterRoundTrip), string(yamlAfterRoundTrip)))
   420  		yamlNeedsUpdate = true
   421  	}
   422  
   423  	if jsonNeedsUpdate || yamlNeedsUpdate {
   424  		const updateEnvVar = "UPDATE_COMPATIBILITY_FIXTURE_DATA"
   425  		if os.Getenv(updateEnvVar) == "true" {
   426  			if jsonNeedsUpdate {
   427  				writeFile(t, previousVersionDir, gvk, ".after_roundtrip", "json", jsonAfterRoundTrip)
   428  			}
   429  			if yamlNeedsUpdate {
   430  				writeFile(t, previousVersionDir, gvk, ".after_roundtrip", "yaml", yamlAfterRoundTrip)
   431  			}
   432  			t.Logf("wrote expected compatibility data... verify, commit, and rerun tests")
   433  		} else {
   434  			t.Logf("if the diff is expected because of a new type or a new field, re-run with %s=true to update the compatibility data", updateEnvVar)
   435  		}
   436  		return
   437  	}
   438  }
   439  
   440  func makeName(gvk schema.GroupVersionKind) string {
   441  	g := gvk.Group
   442  	if g == "" {
   443  		g = "core"
   444  	}
   445  	return g + "." + gvk.Version + "." + gvk.Kind
   446  }
   447
View as plain text