conversion_test.go

Documentation: k8s.io/apiextensions-apiserver/test/integration/conversion

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package conversion
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"net/http"
    24  	"reflect"
    25  	"strings"
    26  	"sync"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/google/go-cmp/cmp"
    31  
    32  	"k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    35  	"k8s.io/apimachinery/pkg/runtime"
    36  	"k8s.io/apimachinery/pkg/runtime/schema"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/util/sets"
    39  	"k8s.io/apimachinery/pkg/util/uuid"
    40  	"k8s.io/apimachinery/pkg/util/wait"
    41  	etcd3watcher "k8s.io/apiserver/pkg/storage/etcd3"
    42  	"k8s.io/client-go/dynamic"
    43  	_ "k8s.io/component-base/logs/testinit" // enable logging flags
    44  
    45  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    46  	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
    47  	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    48  	serveroptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
    49  	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
    50  	"k8s.io/apiextensions-apiserver/test/integration/storage"
    51  )
    52  
    53  type Checker func(t *testing.T, ctc *conversionTestContext)
    54  
    55  func checks(checkers ...Checker) []Checker {
    56  	return checkers
    57  }
    58  
    59  func TestWebhookConverterWithWatchCache(t *testing.T) {
    60  	testWebhookConverter(t, true)
    61  }
    62  func TestWebhookConverterWithoutWatchCache(t *testing.T) {
    63  	testWebhookConverter(t, false)
    64  }
    65  
    66  func testWebhookConverter(t *testing.T, watchCache bool) {
    67  	tests := []struct {
    68  		group          string
    69  		handler        http.Handler
    70  		reviewVersions []string
    71  		checks         []Checker
    72  	}{
    73  		{
    74  			group:          "noop-converter-v1",
    75  			handler:        NewObjectConverterWebhookHandler(t, noopConverter),
    76  			reviewVersions: []string{"v1", "v1beta1"},
    77  			checks:         checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1")), // no v1beta2 as the schema differs
    78  		},
    79  		{
    80  			group:          "noop-converter-v1beta1",
    81  			handler:        NewObjectConverterWebhookHandler(t, noopConverter),
    82  			reviewVersions: []string{"v1beta1", "v1"},
    83  			checks:         checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1")), // no v1beta2 as the schema differs
    84  		},
    85  		{
    86  			group:          "nontrivial-converter-v1",
    87  			handler:        NewObjectConverterWebhookHandler(t, nontrivialConverter),
    88  			reviewVersions: []string{"v1", "v1beta1"},
    89  			checks:         checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1", "v1beta2"), validateNonTrivialConverted, validateNonTrivialConvertedList, validateStoragePruning, validateDefaulting),
    90  		},
    91  		{
    92  			group:          "nontrivial-converter-v1beta1",
    93  			handler:        NewObjectConverterWebhookHandler(t, nontrivialConverter),
    94  			reviewVersions: []string{"v1beta1", "v1"},
    95  			checks:         checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1", "v1beta2"), validateNonTrivialConverted, validateNonTrivialConvertedList, validateStoragePruning, validateDefaulting),
    96  		},
    97  		{
    98  			group:          "metadata-mutating-v1",
    99  			handler:        NewObjectConverterWebhookHandler(t, metadataMutatingConverter),
   100  			reviewVersions: []string{"v1", "v1beta1"},
   101  			checks:         checks(validateObjectMetaMutation),
   102  		},
   103  		{
   104  			group:          "metadata-mutating-v1beta1",
   105  			handler:        NewObjectConverterWebhookHandler(t, metadataMutatingConverter),
   106  			reviewVersions: []string{"v1beta1", "v1"},
   107  			checks:         checks(validateObjectMetaMutation),
   108  		},
   109  		{
   110  			group:          "metadata-uid-mutating-v1",
   111  			handler:        NewObjectConverterWebhookHandler(t, uidMutatingConverter),
   112  			reviewVersions: []string{"v1", "v1beta1"},
   113  			checks:         checks(validateUIDMutation),
   114  		},
   115  		{
   116  			group:          "metadata-uid-mutating-v1beta1",
   117  			handler:        NewObjectConverterWebhookHandler(t, uidMutatingConverter),
   118  			reviewVersions: []string{"v1beta1", "v1"},
   119  			checks:         checks(validateUIDMutation),
   120  		},
   121  		{
   122  			group:          "empty-response-v1",
   123  			handler:        NewReviewWebhookHandler(t, nil, emptyV1ResponseConverter),
   124  			reviewVersions: []string{"v1", "v1beta1"},
   125  			checks:         checks(expectConversionFailureMessage("empty-response", "returned 0 objects, expected 1")),
   126  		},
   127  		{
   128  			group:          "empty-response-v1beta1",
   129  			handler:        NewReviewWebhookHandler(t, emptyV1Beta1ResponseConverter, nil),
   130  			reviewVersions: []string{"v1beta1", "v1"},
   131  			checks:         checks(expectConversionFailureMessage("empty-response", "returned 0 objects, expected 1")),
   132  		},
   133  		{
   134  			group:          "failure-message-v1",
   135  			handler:        NewReviewWebhookHandler(t, nil, failureV1ResponseConverter("custom webhook conversion error")),
   136  			reviewVersions: []string{"v1", "v1beta1"},
   137  			checks:         checks(expectConversionFailureMessage("failure-message", "custom webhook conversion error")),
   138  		},
   139  		{
   140  			group:          "failure-message-v1beta1",
   141  			handler:        NewReviewWebhookHandler(t, failureV1Beta1ResponseConverter("custom webhook conversion error"), nil),
   142  			reviewVersions: []string{"v1beta1", "v1"},
   143  			checks:         checks(expectConversionFailureMessage("failure-message", "custom webhook conversion error")),
   144  		},
   145  		{
   146  			group:          "unhandled-v1",
   147  			handler:        NewReviewWebhookHandler(t, nil, nil),
   148  			reviewVersions: []string{"v1", "v1beta1"},
   149  			checks:         checks(expectConversionFailureMessage("server-error", "the server rejected our request")),
   150  		},
   151  		{
   152  			group:          "unhandled-v1beta1",
   153  			handler:        NewReviewWebhookHandler(t, nil, nil),
   154  			reviewVersions: []string{"v1beta1", "v1"},
   155  			checks:         checks(expectConversionFailureMessage("server-error", "the server rejected our request")),
   156  		},
   157  	}
   158  
   159  	// TODO: Added for integration testing of conversion webhooks, where decode errors due to conversion webhook failures need to be tested.
   160  	// Maybe we should identify conversion webhook related errors in decoding to avoid triggering this? Or maybe having this special casing
   161  	// of test cases in production code should be removed?
   162  	etcd3watcher.TestOnlySetFatalOnDecodeError(false)
   163  	defer etcd3watcher.TestOnlySetFatalOnDecodeError(true)
   164  
   165  	tearDown, config, options, err := fixtures.StartDefaultServer(t, fmt.Sprintf("--watch-cache=%v", watchCache))
   166  	if err != nil {
   167  		t.Fatal(err)
   168  	}
   169  
   170  	apiExtensionsClient, err := clientset.NewForConfig(config)
   171  	if err != nil {
   172  		tearDown()
   173  		t.Fatal(err)
   174  	}
   175  
   176  	dynamicClient, err := dynamic.NewForConfig(config)
   177  	if err != nil {
   178  		tearDown()
   179  		t.Fatal(err)
   180  	}
   181  	defer tearDown()
   182  
   183  	crd := multiVersionFixture.DeepCopy()
   184  
   185  	RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd, nil, nil)
   186  	restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural})
   187  	if err != nil {
   188  		t.Fatal(err)
   189  	}
   190  	etcdClient, _, err := storage.GetEtcdClients(restOptions.StorageConfig.Transport)
   191  	if err != nil {
   192  		t.Fatal(err)
   193  	}
   194  	defer etcdClient.Close()
   195  
   196  	etcdObjectReader := storage.NewEtcdObjectReader(etcdClient, &restOptions, crd)
   197  	ctcTearDown, ctc := newConversionTestContext(t, apiExtensionsClient, dynamicClient, etcdObjectReader, crd)
   198  	defer ctcTearDown()
   199  
   200  	// read only object to read at a different version than stored when we need to force conversion
   201  	marker, err := ctc.versionedClient("marker", "v1beta1").Create(context.TODO(), newConversionMultiVersionFixture("marker", "marker", "v1beta1"), metav1.CreateOptions{})
   202  	if err != nil {
   203  		t.Fatal(err)
   204  	}
   205  
   206  	for _, test := range tests {
   207  		t.Run(test.group, func(t *testing.T) {
   208  			upCh, handler := closeOnCall(test.handler)
   209  			tearDown, webhookClientConfig, err := StartConversionWebhookServer(handler)
   210  			if err != nil {
   211  				t.Fatal(err)
   212  			}
   213  			defer tearDown()
   214  
   215  			ctc.setConversionWebhook(t, webhookClientConfig, test.reviewVersions)
   216  			defer ctc.removeConversionWebhook(t)
   217  
   218  			// wait until new webhook is called the first time
   219  			if err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
   220  				_, err := ctc.versionedClient(marker.GetNamespace(), "v1alpha1").Get(context.TODO(), marker.GetName(), metav1.GetOptions{})
   221  				select {
   222  				case <-upCh:
   223  					return true, nil
   224  				default:
   225  					t.Logf("Waiting for webhook to become effective, getting marker object: %v", err)
   226  					return false, nil
   227  				}
   228  			}); err != nil {
   229  				t.Fatal(err)
   230  			}
   231  
   232  			for i, checkFn := range test.checks {
   233  				name := fmt.Sprintf("check-%d", i)
   234  				t.Run(name, func(t *testing.T) {
   235  					defer ctc.setAndWaitStorageVersion(t, "v1beta1")
   236  					ctc.namespace = fmt.Sprintf("webhook-conversion-%s-%s", test.group, name)
   237  					checkFn(t, ctc)
   238  				})
   239  			}
   240  		})
   241  	}
   242  }
   243  
   244  func validateStorageVersion(t *testing.T, ctc *conversionTestContext) {
   245  	ns := ctc.namespace
   246  
   247  	for _, version := range ctc.crd.Spec.Versions {
   248  		t.Run(version.Name, func(t *testing.T) {
   249  			name := "storageversion-" + version.Name
   250  			client := ctc.versionedClient(ns, version.Name)
   251  			obj, err := client.Create(context.TODO(), newConversionMultiVersionFixture(ns, name, version.Name), metav1.CreateOptions{})
   252  			if err != nil {
   253  				t.Fatal(err)
   254  			}
   255  			ctc.setAndWaitStorageVersion(t, "v1beta2")
   256  
   257  			if _, err = client.Get(context.TODO(), obj.GetName(), metav1.GetOptions{}); err != nil {
   258  				t.Fatal(err)
   259  			}
   260  
   261  			ctc.setAndWaitStorageVersion(t, "v1beta1")
   262  		})
   263  	}
   264  }
   265  
   266  // validateMixedStorageVersions ensures that identical custom resources written at different storage versions
   267  // are readable and remain the same.
   268  func validateMixedStorageVersions(versions ...string) func(t *testing.T, ctc *conversionTestContext) {
   269  	return func(t *testing.T, ctc *conversionTestContext) {
   270  		ns := ctc.namespace
   271  		clients := ctc.versionedClients(ns)
   272  
   273  		// Create CRs at all storage versions
   274  		objNames := []string{}
   275  		for _, version := range versions {
   276  			ctc.setAndWaitStorageVersion(t, version)
   277  
   278  			name := "mixedstorage-stored-as-" + version
   279  			obj, err := clients[version].Create(context.TODO(), newConversionMultiVersionFixture(ns, name, version), metav1.CreateOptions{})
   280  			if err != nil {
   281  				t.Fatal(err)
   282  			}
   283  			objNames = append(objNames, obj.GetName())
   284  		}
   285  
   286  		// Ensure copies of an object have the same fields and values at each custom resource definition version regardless of storage version
   287  		for clientVersion, client := range clients {
   288  			t.Run(clientVersion, func(t *testing.T) {
   289  				o1, err := client.Get(context.TODO(), objNames[0], metav1.GetOptions{})
   290  				if err != nil {
   291  					t.Fatal(err)
   292  				}
   293  				for _, objName := range objNames[1:] {
   294  					o2, err := client.Get(context.TODO(), objName, metav1.GetOptions{})
   295  					if err != nil {
   296  						t.Fatal(err)
   297  					}
   298  
   299  					// ignore metadata for comparison purposes
   300  					delete(o1.Object, "metadata")
   301  					delete(o2.Object, "metadata")
   302  					if !reflect.DeepEqual(o1.Object, o2.Object) {
   303  						t.Errorf("Expected custom resource to be same regardless of which storage version is used to create, but got: %s", cmp.Diff(o1, o2))
   304  					}
   305  				}
   306  			})
   307  		}
   308  	}
   309  }
   310  
   311  func validateServed(t *testing.T, ctc *conversionTestContext) {
   312  	ns := ctc.namespace
   313  
   314  	for _, version := range ctc.crd.Spec.Versions {
   315  		t.Run(version.Name, func(t *testing.T) {
   316  			name := "served-" + version.Name
   317  			client := ctc.versionedClient(ns, version.Name)
   318  			obj, err := client.Create(context.TODO(), newConversionMultiVersionFixture(ns, name, version.Name), metav1.CreateOptions{})
   319  			if err != nil {
   320  				t.Fatal(err)
   321  			}
   322  			ctc.setServed(t, version.Name, false)
   323  			ctc.waitForServed(t, version.Name, false, client, obj)
   324  			ctc.setServed(t, version.Name, true)
   325  			ctc.waitForServed(t, version.Name, true, client, obj)
   326  		})
   327  	}
   328  }
   329  
   330  func validateNonTrivialConverted(t *testing.T, ctc *conversionTestContext) {
   331  	ns := ctc.namespace
   332  
   333  	for _, createVersion := range ctc.crd.Spec.Versions {
   334  		t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) {
   335  			name := "converted-" + createVersion.Name
   336  			client := ctc.versionedClient(ns, createVersion.Name)
   337  
   338  			fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name)
   339  			if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil {
   340  				t.Fatal(err)
   341  			}
   342  			if _, err := client.Create(context.TODO(), fixture, metav1.CreateOptions{}); err != nil {
   343  				t.Fatal(err)
   344  			}
   345  
   346  			// verify that the right, pruned version is in storage
   347  			obj, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name)
   348  			if err != nil {
   349  				t.Fatal(err)
   350  			}
   351  			verifyMultiVersionObject(t, "v1beta1", obj)
   352  
   353  			for _, getVersion := range ctc.crd.Spec.Versions {
   354  				client := ctc.versionedClient(ns, getVersion.Name)
   355  				obj, err := client.Get(context.TODO(), name, metav1.GetOptions{})
   356  				if err != nil {
   357  					t.Fatal(err)
   358  				}
   359  				verifyMultiVersionObject(t, getVersion.Name, obj)
   360  			}
   361  
   362  			// send a non-trivial patch to the main resource to verify the oldObject is in the right version
   363  			if _, err := client.Patch(context.TODO(), name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"main":"true"}}}`), metav1.PatchOptions{}); err != nil {
   364  				t.Fatal(err)
   365  			}
   366  			// verify that the right, pruned version is in storage
   367  			obj, err = ctc.etcdObjectReader.GetStoredCustomResource(ns, name)
   368  			if err != nil {
   369  				t.Fatal(err)
   370  			}
   371  			verifyMultiVersionObject(t, "v1beta1", obj)
   372  
   373  			// send a non-trivial patch to the status subresource to verify the oldObject is in the right version
   374  			if _, err := client.Patch(context.TODO(), name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"status":"true"}}}`), metav1.PatchOptions{}, "status"); err != nil {
   375  				t.Fatal(err)
   376  			}
   377  			// verify that the right, pruned version is in storage
   378  			obj, err = ctc.etcdObjectReader.GetStoredCustomResource(ns, name)
   379  			if err != nil {
   380  				t.Fatal(err)
   381  			}
   382  			verifyMultiVersionObject(t, "v1beta1", obj)
   383  		})
   384  	}
   385  }
   386  
   387  func validateNonTrivialConvertedList(t *testing.T, ctc *conversionTestContext) {
   388  	ns := ctc.namespace + "-list"
   389  
   390  	names := sets.String{}
   391  	for _, createVersion := range ctc.crd.Spec.Versions {
   392  		name := "converted-" + createVersion.Name
   393  		client := ctc.versionedClient(ns, createVersion.Name)
   394  		fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name)
   395  		if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil {
   396  			t.Fatal(err)
   397  		}
   398  		_, err := client.Create(context.TODO(), fixture, metav1.CreateOptions{})
   399  		if err != nil {
   400  			t.Fatal(err)
   401  		}
   402  		names.Insert(name)
   403  	}
   404  
   405  	for _, listVersion := range ctc.crd.Spec.Versions {
   406  		t.Run(fmt.Sprintf("listing objects as %s", listVersion.Name), func(t *testing.T) {
   407  			client := ctc.versionedClient(ns, listVersion.Name)
   408  			obj, err := client.List(context.TODO(), metav1.ListOptions{})
   409  			if err != nil {
   410  				t.Fatal(err)
   411  			}
   412  			if len(obj.Items) != len(ctc.crd.Spec.Versions) {
   413  				t.Fatal("unexpected number of items")
   414  			}
   415  			foundNames := sets.String{}
   416  			for _, u := range obj.Items {
   417  				foundNames.Insert(u.GetName())
   418  				verifyMultiVersionObject(t, listVersion.Name, &u)
   419  			}
   420  			if !foundNames.Equal(names) {
   421  				t.Errorf("unexpected set of returned items: %s", foundNames.Difference(names))
   422  			}
   423  		})
   424  	}
   425  }
   426  
   427  func validateStoragePruning(t *testing.T, ctc *conversionTestContext) {
   428  	ns := ctc.namespace
   429  
   430  	for _, createVersion := range ctc.crd.Spec.Versions {
   431  		t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) {
   432  			name := "storagepruning-" + createVersion.Name
   433  			client := ctc.versionedClient(ns, createVersion.Name)
   434  
   435  			fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name)
   436  			if err := unstructured.SetNestedField(fixture.Object, "foo", "garbage"); err != nil {
   437  				t.Fatal(err)
   438  			}
   439  			_, err := client.Create(context.TODO(), fixture, metav1.CreateOptions{})
   440  			if err != nil {
   441  				t.Fatal(err)
   442  			}
   443  
   444  			// verify that the right, pruned version is in storage
   445  			obj, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name)
   446  			if err != nil {
   447  				t.Fatal(err)
   448  			}
   449  			verifyMultiVersionObject(t, "v1beta1", obj)
   450  
   451  			// add garbage and set a label
   452  			if err := unstructured.SetNestedField(obj.Object, "foo", "garbage"); err != nil {
   453  				t.Fatal(err)
   454  			}
   455  			labels := obj.GetLabels()
   456  			if labels == nil {
   457  				labels = map[string]string{}
   458  			}
   459  			labels["mutated"] = "true"
   460  			obj.SetLabels(labels)
   461  			if err := ctc.etcdObjectReader.SetStoredCustomResource(ns, name, obj); err != nil {
   462  				t.Fatal(err)
   463  			}
   464  
   465  			for _, getVersion := range ctc.crd.Spec.Versions {
   466  				client := ctc.versionedClient(ns, getVersion.Name)
   467  				obj, err := client.Get(context.TODO(), name, metav1.GetOptions{})
   468  				if err != nil {
   469  					t.Fatal(err)
   470  				}
   471  
   472  				// check that the direct mutation in etcd worked
   473  				labels := obj.GetLabels()
   474  				if labels["mutated"] != "true" {
   475  					t.Errorf("expected object %s in version %s to have label 'mutated=true'", name, getVersion.Name)
   476  				}
   477  
   478  				verifyMultiVersionObject(t, getVersion.Name, obj)
   479  			}
   480  		})
   481  	}
   482  }
   483  
   484  func validateObjectMetaMutation(t *testing.T, ctc *conversionTestContext) {
   485  	ns := ctc.namespace
   486  
   487  	t.Logf("Creating object in storage version v1beta1")
   488  	storageVersion := "v1beta1"
   489  	ctc.setAndWaitStorageVersion(t, storageVersion)
   490  	name := "objectmeta-mutation-" + storageVersion
   491  	client := ctc.versionedClient(ns, storageVersion)
   492  	obj, err := client.Create(context.TODO(), newConversionMultiVersionFixture(ns, name, storageVersion), metav1.CreateOptions{})
   493  	if err != nil {
   494  		t.Fatal(err)
   495  	}
   496  	validateObjectMetaMutationObject(t, false, false, obj)
   497  
   498  	t.Logf("Getting object in other version v1beta2")
   499  	client = ctc.versionedClient(ns, "v1beta2")
   500  	obj, err = client.Get(context.TODO(), name, metav1.GetOptions{})
   501  	if err != nil {
   502  		t.Fatal(err)
   503  	}
   504  	validateObjectMetaMutationObject(t, true, true, obj)
   505  
   506  	t.Logf("Creating object in non-storage version")
   507  	name = "objectmeta-mutation-v1beta2"
   508  	client = ctc.versionedClient(ns, "v1beta2")
   509  	obj, err = client.Create(context.TODO(), newConversionMultiVersionFixture(ns, name, "v1beta2"), metav1.CreateOptions{})
   510  	if err != nil {
   511  		t.Fatal(err)
   512  	}
   513  	validateObjectMetaMutationObject(t, true, true, obj)
   514  
   515  	t.Logf("Listing objects in non-storage version")
   516  	client = ctc.versionedClient(ns, "v1beta2")
   517  	list, err := client.List(context.TODO(), metav1.ListOptions{})
   518  	if err != nil {
   519  		t.Fatal(err)
   520  	}
   521  	for _, obj := range list.Items {
   522  		validateObjectMetaMutationObject(t, true, true, &obj)
   523  	}
   524  }
   525  
   526  func validateObjectMetaMutationObject(t *testing.T, expectAnnotations, expectLabels bool, obj *unstructured.Unstructured) {
   527  	if expectAnnotations {
   528  		if _, found := obj.GetAnnotations()["from"]; !found {
   529  			t.Errorf("expected 'from=stable.example.com/v1beta1' annotation")
   530  		}
   531  		if _, found := obj.GetAnnotations()["to"]; !found {
   532  			t.Errorf("expected 'to=stable.example.com/v1beta2' annotation")
   533  		}
   534  	} else {
   535  		if v, found := obj.GetAnnotations()["from"]; found {
   536  			t.Errorf("unexpected 'from' annotation: %s", v)
   537  		}
   538  		if v, found := obj.GetAnnotations()["to"]; found {
   539  			t.Errorf("unexpected 'to' annotation: %s", v)
   540  		}
   541  	}
   542  	if expectLabels {
   543  		if _, found := obj.GetLabels()["from"]; !found {
   544  			t.Errorf("expected 'from=stable.example.com.v1beta1' label")
   545  		}
   546  		if _, found := obj.GetLabels()["to"]; !found {
   547  			t.Errorf("expected 'to=stable.example.com.v1beta2' label")
   548  		}
   549  	} else {
   550  		if v, found := obj.GetLabels()["from"]; found {
   551  			t.Errorf("unexpected 'from' label: %s", v)
   552  		}
   553  		if v, found := obj.GetLabels()["to"]; found {
   554  			t.Errorf("unexpected 'to' label: %s", v)
   555  		}
   556  	}
   557  	if sets.NewString(obj.GetFinalizers()...).Has("foo") {
   558  		t.Errorf("unexpected 'foo' finalizer")
   559  	}
   560  	if obj.GetGeneration() == 42 {
   561  		t.Errorf("unexpected generation 42")
   562  	}
   563  	if v, found, err := unstructured.NestedString(obj.Object, "metadata", "garbage"); err != nil {
   564  		t.Errorf("unexpected error accessing 'metadata.garbage': %v", err)
   565  	} else if found {
   566  		t.Errorf("unexpected 'metadata.garbage': %s", v)
   567  	}
   568  }
   569  
   570  func validateUIDMutation(t *testing.T, ctc *conversionTestContext) {
   571  	ns := ctc.namespace
   572  
   573  	t.Logf("Creating object in non-storage version v1beta1")
   574  	storageVersion := "v1beta1"
   575  	ctc.setAndWaitStorageVersion(t, storageVersion)
   576  	name := "uid-mutation-" + storageVersion
   577  	client := ctc.versionedClient(ns, "v1beta2")
   578  	obj, err := client.Create(context.TODO(), newConversionMultiVersionFixture(ns, name, "v1beta2"), metav1.CreateOptions{})
   579  	if err == nil {
   580  		t.Fatalf("expected creation error, but got: %v", obj)
   581  	} else if !strings.Contains(err.Error(), "must have the same UID") {
   582  		t.Errorf("expected 'must have the same UID' error message, but got: %v", err)
   583  	}
   584  }
   585  
   586  func validateDefaulting(t *testing.T, ctc *conversionTestContext) {
   587  	if _, defaulting := ctc.crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["defaults"]; !defaulting {
   588  		return
   589  	}
   590  
   591  	ns := ctc.namespace
   592  	storageVersion := "v1beta1"
   593  
   594  	for _, createVersion := range ctc.crd.Spec.Versions {
   595  		t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) {
   596  			name := "defaulting-" + createVersion.Name
   597  			client := ctc.versionedClient(ns, createVersion.Name)
   598  
   599  			fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name)
   600  			if err := unstructured.SetNestedField(fixture.Object, map[string]interface{}{}, "defaults"); err != nil {
   601  				t.Fatal(err)
   602  			}
   603  			created, err := client.Create(context.TODO(), fixture, metav1.CreateOptions{})
   604  			if err != nil {
   605  				t.Fatal(err)
   606  			}
   607  
   608  			// check that defaulting happens
   609  			// - in the request version when doing no-op conversion when deserializing
   610  			// - when reading back from storage in the storage version
   611  			// only the first is persisted.
   612  			defaults, found, err := unstructured.NestedMap(created.Object, "defaults")
   613  			if err != nil {
   614  				t.Fatal(err)
   615  			} else if !found {
   616  				t.Fatalf("expected .defaults to exist")
   617  			}
   618  			expectedLen := 1
   619  			if !createVersion.Storage {
   620  				expectedLen++
   621  			}
   622  			if len(defaults) != expectedLen {
   623  				t.Fatalf("after %s create expected .defaults to have %d values, but got: %v", createVersion.Name, expectedLen, defaults)
   624  			}
   625  			if _, found := defaults[createVersion.Name].(bool); !found {
   626  				t.Errorf("after %s create expected .defaults[%s] to be true, but .defaults is: %v", createVersion.Name, createVersion.Name, defaults)
   627  			}
   628  			if _, found := defaults[storageVersion].(bool); !found {
   629  				t.Errorf("after %s create expected .defaults[%s] to be true because it is the storage version, but .defaults is: %v", createVersion.Name, storageVersion, defaults)
   630  			}
   631  
   632  			// verify that only the request version default is persisted
   633  			persisted, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name)
   634  			if err != nil {
   635  				t.Fatal(err)
   636  			}
   637  			if _, found, err := unstructured.NestedBool(persisted.Object, "defaults", storageVersion); err != nil {
   638  				t.Fatal(err)
   639  			} else if createVersion.Name != storageVersion && found {
   640  				t.Errorf("after %s create .defaults[storage version %s] not to be persisted, but got in etcd: %v", createVersion.Name, storageVersion, defaults)
   641  			}
   642  
   643  			// check that when reading any other version, we do not default that version, but only the (non-persisted) storage version default
   644  			for _, v := range ctc.crd.Spec.Versions {
   645  				if v.Name == createVersion.Name {
   646  					// create version is persisted anyway, nothing to verify
   647  					continue
   648  				}
   649  
   650  				got, err := ctc.versionedClient(ns, v.Name).Get(context.TODO(), created.GetName(), metav1.GetOptions{})
   651  				if err != nil {
   652  					t.Fatal(err)
   653  				}
   654  
   655  				if _, found, err := unstructured.NestedBool(got.Object, "defaults", v.Name); err != nil {
   656  					t.Fatal(err)
   657  				} else if v.Name != storageVersion && found {
   658  					t.Errorf("after %s GET expected .defaults[%s] not to be true because only storage version %s is defaulted on read, but .defaults is: %v", v.Name, v.Name, storageVersion, defaults)
   659  				}
   660  
   661  				if _, found, err := unstructured.NestedBool(got.Object, "defaults", storageVersion); err != nil {
   662  					t.Fatal(err)
   663  				} else if !found {
   664  					t.Errorf("after non-create, non-storage %s GET expected .defaults[storage version %s] to be true, but .defaults is: %v", v.Name, storageVersion, defaults)
   665  				}
   666  			}
   667  		})
   668  	}
   669  }
   670  
   671  func expectConversionFailureMessage(id, message string) func(t *testing.T, ctc *conversionTestContext) {
   672  	return func(t *testing.T, ctc *conversionTestContext) {
   673  		ns := ctc.namespace
   674  		clients := ctc.versionedClients(ns)
   675  		var err error
   676  		// storage version is v1beta1, so this skips conversion
   677  		obj, err := clients["v1beta1"].Create(context.TODO(), newConversionMultiVersionFixture(ns, id, "v1beta1"), metav1.CreateOptions{})
   678  		if err != nil {
   679  			t.Fatal(err)
   680  		}
   681  
   682  		// manually convert
   683  		objv1beta2 := newConversionMultiVersionFixture(ns, id, "v1beta2")
   684  		meta, _, _ := unstructured.NestedFieldCopy(obj.Object, "metadata")
   685  		unstructured.SetNestedField(objv1beta2.Object, meta, "metadata")
   686  
   687  		for _, verb := range []string{"get", "list", "create", "update", "patch", "delete", "deletecollection"} {
   688  			t.Run(verb, func(t *testing.T) {
   689  				switch verb {
   690  				case "get":
   691  					_, err = clients["v1beta2"].Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
   692  				case "list":
   693  					_, err = clients["v1beta2"].List(context.TODO(), metav1.ListOptions{})
   694  				case "create":
   695  					_, err = clients["v1beta2"].Create(context.TODO(), newConversionMultiVersionFixture(ns, id, "v1beta2"), metav1.CreateOptions{})
   696  				case "update":
   697  					_, err = clients["v1beta2"].Update(context.TODO(), objv1beta2, metav1.UpdateOptions{})
   698  				case "patch":
   699  					_, err = clients["v1beta2"].Patch(context.TODO(), obj.GetName(), types.MergePatchType, []byte(`{"metadata":{"annotations":{"patch":"true"}}}`), metav1.PatchOptions{})
   700  				case "delete":
   701  					err = clients["v1beta2"].Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{})
   702  				case "deletecollection":
   703  					err = clients["v1beta2"].DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
   704  				default:
   705  					t.Errorf("unknown verb %q", verb)
   706  				}
   707  
   708  				if err == nil {
   709  					t.Errorf("expected error with message %s, but got no error", message)
   710  				} else if !strings.Contains(err.Error(), message) {
   711  					t.Errorf("expected error with message %s, but got %v", message, err)
   712  				}
   713  			})
   714  		}
   715  		for _, subresource := range []string{"status", "scale"} {
   716  			for _, verb := range []string{"get", "update", "patch"} {
   717  				t.Run(fmt.Sprintf("%s-%s", subresource, verb), func(t *testing.T) {
   718  					switch verb {
   719  					case "get":
   720  						_, err = clients["v1beta2"].Get(context.TODO(), obj.GetName(), metav1.GetOptions{}, subresource)
   721  					case "update":
   722  						o := objv1beta2
   723  						if subresource == "scale" {
   724  							o = &unstructured.Unstructured{
   725  								Object: map[string]interface{}{
   726  									"apiVersion": "autoscaling/v1",
   727  									"kind":       "Scale",
   728  									"metadata": map[string]interface{}{
   729  										"name": obj.GetName(),
   730  									},
   731  									"spec": map[string]interface{}{
   732  										"replicas": 42,
   733  									},
   734  								},
   735  							}
   736  						}
   737  						_, err = clients["v1beta2"].Update(context.TODO(), o, metav1.UpdateOptions{}, subresource)
   738  					case "patch":
   739  						_, err = clients["v1beta2"].Patch(context.TODO(), obj.GetName(), types.MergePatchType, []byte(`{"metadata":{"annotations":{"patch":"true"}}}`), metav1.PatchOptions{}, subresource)
   740  					default:
   741  						t.Errorf("unknown subresource verb %q", verb)
   742  					}
   743  
   744  					if err == nil {
   745  						t.Errorf("expected error with message %s, but got no error", message)
   746  					} else if !strings.Contains(err.Error(), message) {
   747  						t.Errorf("expected error with message %s, but got %v", message, err)
   748  					}
   749  				})
   750  			}
   751  		}
   752  	}
   753  }
   754  
   755  func noopConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) {
   756  	u := &unstructured.Unstructured{Object: map[string]interface{}{}}
   757  	if err := json.Unmarshal(obj.Raw, u); err != nil {
   758  		return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err)
   759  	}
   760  	u.Object["apiVersion"] = desiredAPIVersion
   761  	raw, err := json.Marshal(u)
   762  	if err != nil {
   763  		return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err)
   764  	}
   765  	return runtime.RawExtension{Raw: raw}, nil
   766  }
   767  
   768  func emptyV1ResponseConverter(review *apiextensionsv1.ConversionReview) (*apiextensionsv1.ConversionReview, error) {
   769  	review.Response = &apiextensionsv1.ConversionResponse{
   770  		UID:              review.Request.UID,
   771  		ConvertedObjects: []runtime.RawExtension{},
   772  		Result:           metav1.Status{Status: "Success"},
   773  	}
   774  	return review, nil
   775  }
   776  func emptyV1Beta1ResponseConverter(review *apiextensionsv1beta1.ConversionReview) (*apiextensionsv1beta1.ConversionReview, error) {
   777  	review.Response = &apiextensionsv1beta1.ConversionResponse{
   778  		UID:              review.Request.UID,
   779  		ConvertedObjects: []runtime.RawExtension{},
   780  		Result:           metav1.Status{Status: "Success"},
   781  	}
   782  	return review, nil
   783  }
   784  
   785  func failureV1ResponseConverter(message string) func(review *apiextensionsv1.ConversionReview) (*apiextensionsv1.ConversionReview, error) {
   786  	return func(review *apiextensionsv1.ConversionReview) (*apiextensionsv1.ConversionReview, error) {
   787  		review.Response = &apiextensionsv1.ConversionResponse{
   788  			UID:              review.Request.UID,
   789  			ConvertedObjects: []runtime.RawExtension{},
   790  			Result:           metav1.Status{Message: message, Status: "Failure"},
   791  		}
   792  		return review, nil
   793  	}
   794  }
   795  
   796  func failureV1Beta1ResponseConverter(message string) func(review *apiextensionsv1beta1.ConversionReview) (*apiextensionsv1beta1.ConversionReview, error) {
   797  	return func(review *apiextensionsv1beta1.ConversionReview) (*apiextensionsv1beta1.ConversionReview, error) {
   798  		review.Response = &apiextensionsv1beta1.ConversionResponse{
   799  			UID:              review.Request.UID,
   800  			ConvertedObjects: []runtime.RawExtension{},
   801  			Result:           metav1.Status{Message: message, Status: "Failure"},
   802  		}
   803  		return review, nil
   804  	}
   805  }
   806  
   807  func nontrivialConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) {
   808  	u := &unstructured.Unstructured{Object: map[string]interface{}{}}
   809  	if err := json.Unmarshal(obj.Raw, u); err != nil {
   810  		return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err)
   811  	}
   812  
   813  	currentAPIVersion := u.GetAPIVersion()
   814  
   815  	if currentAPIVersion == "stable.example.com/v1beta2" && (desiredAPIVersion == "stable.example.com/v1alpha1" || desiredAPIVersion == "stable.example.com/v1beta1") {
   816  		u.Object["num"] = u.Object["numv2"]
   817  		u.Object["content"] = u.Object["contentv2"]
   818  		delete(u.Object, "numv2")
   819  		delete(u.Object, "contentv2")
   820  	} else if (currentAPIVersion == "stable.example.com/v1alpha1" || currentAPIVersion == "stable.example.com/v1beta1") && desiredAPIVersion == "stable.example.com/v1beta2" {
   821  		u.Object["numv2"] = u.Object["num"]
   822  		u.Object["contentv2"] = u.Object["content"]
   823  		delete(u.Object, "num")
   824  		delete(u.Object, "content")
   825  	} else if currentAPIVersion == "stable.example.com/v1alpha1" && desiredAPIVersion == "stable.example.com/v1beta1" {
   826  		// same schema
   827  	} else if currentAPIVersion == "stable.example.com/v1beta1" && desiredAPIVersion == "stable.example.com/v1alpha1" {
   828  		// same schema
   829  	} else if currentAPIVersion != desiredAPIVersion {
   830  		return runtime.RawExtension{}, fmt.Errorf("cannot convert from %s to %s", currentAPIVersion, desiredAPIVersion)
   831  	}
   832  	u.Object["apiVersion"] = desiredAPIVersion
   833  	raw, err := json.Marshal(u)
   834  	if err != nil {
   835  		return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err)
   836  	}
   837  	return runtime.RawExtension{Raw: raw}, nil
   838  }
   839  
   840  func metadataMutatingConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) {
   841  	obj, err := nontrivialConverter(desiredAPIVersion, obj)
   842  	if err != nil {
   843  		return runtime.RawExtension{}, err
   844  	}
   845  
   846  	u := &unstructured.Unstructured{Object: map[string]interface{}{}}
   847  	if err := json.Unmarshal(obj.Raw, u); err != nil {
   848  		return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err)
   849  	}
   850  
   851  	// do not mutate the marker or the probe objects
   852  	if !strings.Contains(u.GetName(), "mutation") {
   853  		return obj, nil
   854  	}
   855  
   856  	currentAPIVersion := u.GetAPIVersion()
   857  
   858  	// mutate annotations. This should be persisted.
   859  	annotations := u.GetAnnotations()
   860  	if annotations == nil {
   861  		annotations = map[string]string{}
   862  	}
   863  	annotations["from"] = currentAPIVersion
   864  	annotations["to"] = desiredAPIVersion
   865  	u.SetAnnotations(annotations)
   866  
   867  	// mutate labels. This should be persisted.
   868  	labels := u.GetLabels()
   869  	if labels == nil {
   870  		labels = map[string]string{}
   871  	}
   872  	labels["from"] = strings.Replace(currentAPIVersion, "/", ".", 1) // replace / with . because label values do not allow /
   873  	labels["to"] = strings.Replace(desiredAPIVersion, "/", ".", 1)
   874  	u.SetLabels(labels)
   875  
   876  	// mutate other fields. This should be ignored.
   877  	u.SetGeneration(42)
   878  	u.SetOwnerReferences([]metav1.OwnerReference{{
   879  		APIVersion:         "v1",
   880  		Kind:               "Namespace",
   881  		Name:               "default",
   882  		UID:                "1234",
   883  		Controller:         nil,
   884  		BlockOwnerDeletion: nil,
   885  	}})
   886  	u.SetResourceVersion("42")
   887  	u.SetFinalizers([]string{"foo"})
   888  	if err := unstructured.SetNestedField(u.Object, "foo", "metadata", "garbage"); err != nil {
   889  		return runtime.RawExtension{}, err
   890  	}
   891  
   892  	raw, err := json.Marshal(u)
   893  	if err != nil {
   894  		return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err)
   895  	}
   896  	return runtime.RawExtension{Raw: raw}, nil
   897  }
   898  
   899  func uidMutatingConverter(desiredAPIVersion string, obj runtime.RawExtension) (runtime.RawExtension, error) {
   900  	u := &unstructured.Unstructured{Object: map[string]interface{}{}}
   901  	if err := json.Unmarshal(obj.Raw, u); err != nil {
   902  		return runtime.RawExtension{}, fmt.Errorf("failed to deserialize object: %s with error: %v", string(obj.Raw), err)
   903  	}
   904  
   905  	// do not mutate the marker or the probe objects
   906  	if strings.Contains(u.GetName(), "mutation") {
   907  		// mutate other fields. This should be ignored.
   908  		if err := unstructured.SetNestedField(u.Object, "42", "metadata", "uid"); err != nil {
   909  			return runtime.RawExtension{}, err
   910  		}
   911  	}
   912  
   913  	u.Object["apiVersion"] = desiredAPIVersion
   914  	raw, err := json.Marshal(u)
   915  	if err != nil {
   916  		return runtime.RawExtension{}, fmt.Errorf("failed to serialize object: %v with error: %v", u, err)
   917  	}
   918  	return runtime.RawExtension{Raw: raw}, nil
   919  }
   920  
   921  func newConversionTestContext(t *testing.T, apiExtensionsClient clientset.Interface, dynamicClient dynamic.Interface, etcdObjectReader *storage.EtcdObjectReader, v1CRD *apiextensionsv1.CustomResourceDefinition) (func(), *conversionTestContext) {
   922  	v1CRD, err := fixtures.CreateNewV1CustomResourceDefinition(v1CRD, apiExtensionsClient, dynamicClient)
   923  	if err != nil {
   924  		t.Fatal(err)
   925  	}
   926  	crd, err := apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), v1CRD.Name, metav1.GetOptions{})
   927  	if err != nil {
   928  		t.Fatal(err)
   929  	}
   930  
   931  	tearDown := func() {
   932  		if err := fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionsClient); err != nil {
   933  			t.Fatal(err)
   934  		}
   935  	}
   936  
   937  	return tearDown, &conversionTestContext{apiExtensionsClient: apiExtensionsClient, dynamicClient: dynamicClient, crd: crd, etcdObjectReader: etcdObjectReader}
   938  }
   939  
   940  type conversionTestContext struct {
   941  	namespace           string
   942  	apiExtensionsClient clientset.Interface
   943  	dynamicClient       dynamic.Interface
   944  	crd                 *apiextensionsv1.CustomResourceDefinition
   945  	etcdObjectReader    *storage.EtcdObjectReader
   946  }
   947  
   948  func (c *conversionTestContext) versionedClient(ns string, version string) dynamic.ResourceInterface {
   949  	gvr := schema.GroupVersionResource{Group: c.crd.Spec.Group, Version: version, Resource: c.crd.Spec.Names.Plural}
   950  	if c.crd.Spec.Scope != apiextensionsv1.ClusterScoped {
   951  		return c.dynamicClient.Resource(gvr).Namespace(ns)
   952  	}
   953  	return c.dynamicClient.Resource(gvr)
   954  }
   955  
   956  func (c *conversionTestContext) versionedClients(ns string) map[string]dynamic.ResourceInterface {
   957  	ret := map[string]dynamic.ResourceInterface{}
   958  	for _, v := range c.crd.Spec.Versions {
   959  		ret[v.Name] = c.versionedClient(ns, v.Name)
   960  	}
   961  	return ret
   962  }
   963  
   964  func (c *conversionTestContext) setConversionWebhook(t *testing.T, webhookClientConfig *apiextensionsv1.WebhookClientConfig, reviewVersions []string) {
   965  	crd, err := c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), c.crd.Name, metav1.GetOptions{})
   966  	if err != nil {
   967  		t.Fatal(err)
   968  	}
   969  	crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
   970  		Strategy: apiextensionsv1.WebhookConverter,
   971  		Webhook: &apiextensionsv1.WebhookConversion{
   972  			ClientConfig:             webhookClientConfig,
   973  			ConversionReviewVersions: reviewVersions,
   974  		},
   975  	}
   976  	crd, err = c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
   977  	if err != nil {
   978  		t.Fatal(err)
   979  	}
   980  	c.crd = crd
   981  
   982  }
   983  
   984  func (c *conversionTestContext) removeConversionWebhook(t *testing.T) {
   985  	crd, err := c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), c.crd.Name, metav1.GetOptions{})
   986  	if err != nil {
   987  		t.Fatal(err)
   988  	}
   989  	crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
   990  		Strategy: apiextensionsv1.NoneConverter,
   991  	}
   992  
   993  	crd, err = c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
   994  	if err != nil {
   995  		t.Fatal(err)
   996  	}
   997  	c.crd = crd
   998  }
   999  
  1000  func (c *conversionTestContext) setAndWaitStorageVersion(t *testing.T, version string) {
  1001  	c.setStorageVersion(t, version)
  1002  
  1003  	// create probe object. Version should be the default one to avoid webhook calls during test setup.
  1004  	client := c.versionedClient("probe", "v1beta1")
  1005  	name := fmt.Sprintf("probe-%v", uuid.NewUUID())
  1006  	storageProbe, err := client.Create(context.TODO(), newConversionMultiVersionFixture("probe", name, "v1beta1"), metav1.CreateOptions{})
  1007  	if err != nil {
  1008  		t.Fatal(err)
  1009  	}
  1010  
  1011  	// update object continuously and wait for etcd to have the target storage version.
  1012  	c.waitForStorageVersion(t, version, c.versionedClient(storageProbe.GetNamespace(), "v1beta1"), storageProbe)
  1013  
  1014  	err = client.Delete(context.TODO(), name, metav1.DeleteOptions{})
  1015  	if err != nil {
  1016  		t.Fatal(err)
  1017  	}
  1018  }
  1019  
  1020  func (c *conversionTestContext) setStorageVersion(t *testing.T, version string) {
  1021  	crd, err := c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), c.crd.Name, metav1.GetOptions{})
  1022  	if err != nil {
  1023  		t.Fatal(err)
  1024  	}
  1025  	for i, v := range crd.Spec.Versions {
  1026  		crd.Spec.Versions[i].Storage = v.Name == version
  1027  	}
  1028  	crd, err = c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
  1029  	if err != nil {
  1030  		t.Fatal(err)
  1031  	}
  1032  	c.crd = crd
  1033  }
  1034  
  1035  func (c *conversionTestContext) waitForStorageVersion(t *testing.T, version string, versionedClient dynamic.ResourceInterface, obj *unstructured.Unstructured) *unstructured.Unstructured {
  1036  	if err := c.etcdObjectReader.WaitForStorageVersion(version, obj.GetNamespace(), obj.GetName(), 30*time.Second, func() {
  1037  		if _, err := versionedClient.Patch(context.TODO(), obj.GetName(), types.MergePatchType, []byte(`{}`), metav1.PatchOptions{}); err != nil {
  1038  			t.Fatalf("failed to update object: %v", err)
  1039  		}
  1040  	}); err != nil {
  1041  		t.Fatalf("failed waiting for storage version %s: %v", version, err)
  1042  	}
  1043  
  1044  	t.Logf("Effective storage version: %s", version)
  1045  
  1046  	return obj
  1047  }
  1048  
  1049  func (c *conversionTestContext) setServed(t *testing.T, version string, served bool) {
  1050  	crd, err := c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), c.crd.Name, metav1.GetOptions{})
  1051  	if err != nil {
  1052  		t.Fatal(err)
  1053  	}
  1054  	for i, v := range crd.Spec.Versions {
  1055  		if v.Name == version {
  1056  			crd.Spec.Versions[i].Served = served
  1057  		}
  1058  	}
  1059  	crd, err = c.apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
  1060  	if err != nil {
  1061  		t.Fatal(err)
  1062  	}
  1063  	c.crd = crd
  1064  }
  1065  
  1066  func (c *conversionTestContext) waitForServed(t *testing.T, version string, served bool, versionedClient dynamic.ResourceInterface, obj *unstructured.Unstructured) {
  1067  	timeout := 30 * time.Second
  1068  	waitCh := time.After(timeout)
  1069  	for {
  1070  		obj, err := versionedClient.Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
  1071  		if (err == nil && served) || (errors.IsNotFound(err) && served == false) {
  1072  			return
  1073  		}
  1074  		select {
  1075  		case <-waitCh:
  1076  			t.Fatalf("Timed out after %v waiting for CRD served=%t for version %s for %v. Last error: %v", timeout, served, version, obj, err)
  1077  		case <-time.After(10 * time.Millisecond):
  1078  		}
  1079  	}
  1080  }
  1081  
  1082  var multiVersionFixture = &apiextensionsv1.CustomResourceDefinition{
  1083  	ObjectMeta: metav1.ObjectMeta{Name: "multiversion.stable.example.com"},
  1084  	Spec: apiextensionsv1.CustomResourceDefinitionSpec{
  1085  		Group: "stable.example.com",
  1086  		Names: apiextensionsv1.CustomResourceDefinitionNames{
  1087  			Plural:     "multiversion",
  1088  			Singular:   "multiversion",
  1089  			Kind:       "MultiVersion",
  1090  			ShortNames: []string{"mv"},
  1091  			ListKind:   "MultiVersionList",
  1092  			Categories: []string{"all"},
  1093  		},
  1094  		Scope:                 apiextensionsv1.NamespaceScoped,
  1095  		PreserveUnknownFields: false,
  1096  		Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
  1097  			{
  1098  				// storage version, same schema as v1alpha1
  1099  				Name:    "v1beta1",
  1100  				Served:  true,
  1101  				Storage: true,
  1102  				Subresources: &apiextensionsv1.CustomResourceSubresources{
  1103  					Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
  1104  					Scale: &apiextensionsv1.CustomResourceSubresourceScale{
  1105  						SpecReplicasPath:   ".spec.num.num1",
  1106  						StatusReplicasPath: ".status.num.num2",
  1107  					},
  1108  				},
  1109  				Schema: &apiextensionsv1.CustomResourceValidation{
  1110  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  1111  						Type: "object",
  1112  						Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1113  							"content": {
  1114  								Type: "object",
  1115  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1116  									"key": {Type: "string"},
  1117  								},
  1118  							},
  1119  							"num": {
  1120  								Type: "object",
  1121  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1122  									"num1": {Type: "integer"},
  1123  									"num2": {Type: "integer"},
  1124  								},
  1125  							},
  1126  							"defaults": {
  1127  								Type: "object",
  1128  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1129  									"v1alpha1": {Type: "boolean"},
  1130  									"v1beta1":  {Type: "boolean", Default: jsonPtr(true)},
  1131  									"v1beta2":  {Type: "boolean"},
  1132  								},
  1133  							},
  1134  						},
  1135  					},
  1136  				},
  1137  			},
  1138  			{
  1139  				// same schema as v1beta1
  1140  				Name:    "v1alpha1",
  1141  				Served:  true,
  1142  				Storage: false,
  1143  				Subresources: &apiextensionsv1.CustomResourceSubresources{
  1144  					Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
  1145  					Scale: &apiextensionsv1.CustomResourceSubresourceScale{
  1146  						SpecReplicasPath:   ".spec.num.num1",
  1147  						StatusReplicasPath: ".status.num.num2",
  1148  					},
  1149  				},
  1150  				Schema: &apiextensionsv1.CustomResourceValidation{
  1151  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  1152  						Type: "object",
  1153  						Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1154  							"content": {
  1155  								Type: "object",
  1156  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1157  									"key": {Type: "string"},
  1158  								},
  1159  							},
  1160  							"num": {
  1161  								Type: "object",
  1162  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1163  									"num1": {Type: "integer"},
  1164  									"num2": {Type: "integer"},
  1165  								},
  1166  							},
  1167  							"defaults": {
  1168  								Type: "object",
  1169  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1170  									"v1alpha1": {Type: "boolean", Default: jsonPtr(true)},
  1171  									"v1beta1":  {Type: "boolean"},
  1172  									"v1beta2":  {Type: "boolean"},
  1173  								},
  1174  							},
  1175  						},
  1176  					},
  1177  				},
  1178  			},
  1179  			{
  1180  				// different schema than v1beta1 and v1alpha1
  1181  				Name:    "v1beta2",
  1182  				Served:  true,
  1183  				Storage: false,
  1184  				Subresources: &apiextensionsv1.CustomResourceSubresources{
  1185  					Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
  1186  					Scale: &apiextensionsv1.CustomResourceSubresourceScale{
  1187  						SpecReplicasPath:   ".spec.num.num1",
  1188  						StatusReplicasPath: ".status.num.num2",
  1189  					},
  1190  				},
  1191  				Schema: &apiextensionsv1.CustomResourceValidation{
  1192  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  1193  						Type: "object",
  1194  						Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1195  							"contentv2": {
  1196  								Type: "object",
  1197  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1198  									"key": {Type: "string"},
  1199  								},
  1200  							},
  1201  							"numv2": {
  1202  								Type: "object",
  1203  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1204  									"num1": {Type: "integer"},
  1205  									"num2": {Type: "integer"},
  1206  								},
  1207  							},
  1208  							"defaults": {
  1209  								Type: "object",
  1210  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1211  									"v1alpha1": {Type: "boolean"},
  1212  									"v1beta1":  {Type: "boolean"},
  1213  									"v1beta2":  {Type: "boolean", Default: jsonPtr(true)},
  1214  								},
  1215  							},
  1216  						},
  1217  					},
  1218  				},
  1219  			},
  1220  		},
  1221  	},
  1222  }
  1223  
  1224  func newConversionMultiVersionFixture(namespace, name, version string) *unstructured.Unstructured {
  1225  	u := &unstructured.Unstructured{
  1226  		Object: map[string]interface{}{
  1227  			"apiVersion": "stable.example.com/" + version,
  1228  			"kind":       "MultiVersion",
  1229  			"metadata": map[string]interface{}{
  1230  				"namespace": namespace,
  1231  				"name":      name,
  1232  			},
  1233  		},
  1234  	}
  1235  
  1236  	switch version {
  1237  	case "v1alpha1":
  1238  		u.Object["content"] = map[string]interface{}{
  1239  			"key": "value",
  1240  		}
  1241  		u.Object["num"] = map[string]interface{}{
  1242  			"num1": int64(1),
  1243  			"num2": int64(1000000),
  1244  		}
  1245  	case "v1beta1":
  1246  		u.Object["content"] = map[string]interface{}{
  1247  			"key": "value",
  1248  		}
  1249  		u.Object["num"] = map[string]interface{}{
  1250  			"num1": int64(1),
  1251  			"num2": int64(1000000),
  1252  		}
  1253  	case "v1beta2":
  1254  		u.Object["contentv2"] = map[string]interface{}{
  1255  			"key": "value",
  1256  		}
  1257  		u.Object["numv2"] = map[string]interface{}{
  1258  			"num1": int64(1),
  1259  			"num2": int64(1000000),
  1260  		}
  1261  	default:
  1262  		panic(fmt.Sprintf("unknown version %s", version))
  1263  	}
  1264  
  1265  	return u
  1266  }
  1267  
  1268  func verifyMultiVersionObject(t *testing.T, v string, obj *unstructured.Unstructured) {
  1269  	j := runtime.DeepCopyJSON(obj.Object)
  1270  
  1271  	if expected := "stable.example.com/" + v; obj.GetAPIVersion() != expected {
  1272  		t.Errorf("unexpected apiVersion %q, expected %q", obj.GetAPIVersion(), expected)
  1273  		return
  1274  	}
  1275  
  1276  	delete(j, "metadata")
  1277  
  1278  	var expected = map[string]map[string]interface{}{
  1279  		"v1alpha1": {
  1280  			"apiVersion": "stable.example.com/v1alpha1",
  1281  			"kind":       "MultiVersion",
  1282  			"content": map[string]interface{}{
  1283  				"key": "value",
  1284  			},
  1285  			"num": map[string]interface{}{
  1286  				"num1": int64(1),
  1287  				"num2": int64(1000000),
  1288  			},
  1289  		},
  1290  		"v1beta1": {
  1291  			"apiVersion": "stable.example.com/v1beta1",
  1292  			"kind":       "MultiVersion",
  1293  			"content": map[string]interface{}{
  1294  				"key": "value",
  1295  			},
  1296  			"num": map[string]interface{}{
  1297  				"num1": int64(1),
  1298  				"num2": int64(1000000),
  1299  			},
  1300  		},
  1301  		"v1beta2": {
  1302  			"apiVersion": "stable.example.com/v1beta2",
  1303  			"kind":       "MultiVersion",
  1304  			"contentv2": map[string]interface{}{
  1305  				"key": "value",
  1306  			},
  1307  			"numv2": map[string]interface{}{
  1308  				"num1": int64(1),
  1309  				"num2": int64(1000000),
  1310  			},
  1311  		},
  1312  	}
  1313  	if !reflect.DeepEqual(expected[v], j) {
  1314  		t.Errorf("unexpected %s object: %s", v, cmp.Diff(expected[v], j))
  1315  	}
  1316  }
  1317  
  1318  func closeOnCall(h http.Handler) (chan struct{}, http.Handler) {
  1319  	ch := make(chan struct{})
  1320  	once := sync.Once{}
  1321  	return ch, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1322  		once.Do(func() {
  1323  			close(ch)
  1324  		})
  1325  		h.ServeHTTP(w, r)
  1326  	})
  1327  }
  1328  
  1329  func jsonPtr(x interface{}) *apiextensionsv1.JSON {
  1330  	bs, err := json.Marshal(x)
  1331  	if err != nil {
  1332  		panic(err)
  1333  	}
  1334  	ret := apiextensionsv1.JSON{Raw: bs}
  1335  	return &ret
  1336  }
  1337
View as plain text