crd_conversion_webhook.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/onsi/ginkgo/v2"
    25  	"github.com/onsi/gomega"
    26  
    27  	appsv1 "k8s.io/api/apps/v1"
    28  	v1 "k8s.io/api/core/v1"
    29  	rbacv1 "k8s.io/api/rbac/v1"
    30  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    33  	"k8s.io/apimachinery/pkg/util/intstr"
    34  	"k8s.io/apimachinery/pkg/util/wait"
    35  	"k8s.io/client-go/dynamic"
    36  	clientset "k8s.io/client-go/kubernetes"
    37  	"k8s.io/kubernetes/test/e2e/framework"
    38  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    39  	"k8s.io/kubernetes/test/utils/crd"
    40  	"k8s.io/kubernetes/test/utils/format"
    41  	imageutils "k8s.io/kubernetes/test/utils/image"
    42  	admissionapi "k8s.io/pod-security-admission/api"
    43  	"k8s.io/utils/pointer"
    44  
    45  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    46  	"k8s.io/apiextensions-apiserver/test/integration"
    47  
    48  	// ensure libs have a chance to initialize
    49  	_ "github.com/stretchr/testify/assert"
    50  )
    51  
    52  const (
    53  	secretCRDName      = "sample-custom-resource-conversion-webhook-secret"
    54  	deploymentCRDName  = "sample-crd-conversion-webhook-deployment"
    55  	serviceCRDName     = "e2e-test-crd-conversion-webhook"
    56  	roleBindingCRDName = "crd-conversion-webhook-auth-reader"
    57  )
    58  
    59  var apiVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
    60  	{
    61  		Name:    "v1",
    62  		Served:  true,
    63  		Storage: true,
    64  		Schema: &apiextensionsv1.CustomResourceValidation{
    65  			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
    66  				Type: "object",
    67  				Properties: map[string]apiextensionsv1.JSONSchemaProps{
    68  					"hostPort": {Type: "string"},
    69  				},
    70  			},
    71  		},
    72  	},
    73  	{
    74  		Name:    "v2",
    75  		Served:  true,
    76  		Storage: false,
    77  		Schema: &apiextensionsv1.CustomResourceValidation{
    78  			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
    79  				Type: "object",
    80  				Properties: map[string]apiextensionsv1.JSONSchemaProps{
    81  					"host": {Type: "string"},
    82  					"port": {Type: "string"},
    83  				},
    84  			},
    85  		},
    86  	},
    87  }
    88  
    89  var alternativeAPIVersions = []apiextensionsv1.CustomResourceDefinitionVersion{
    90  	{
    91  		Name:    "v1",
    92  		Served:  true,
    93  		Storage: false,
    94  		Schema: &apiextensionsv1.CustomResourceValidation{
    95  			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
    96  				Type: "object",
    97  				Properties: map[string]apiextensionsv1.JSONSchemaProps{
    98  					"hostPort": {Type: "string"},
    99  				},
   100  			},
   101  		},
   102  	},
   103  	{
   104  		Name:    "v2",
   105  		Served:  true,
   106  		Storage: true,
   107  		Schema: &apiextensionsv1.CustomResourceValidation{
   108  			OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
   109  				Type: "object",
   110  				Properties: map[string]apiextensionsv1.JSONSchemaProps{
   111  					"host": {Type: "string"},
   112  					"port": {Type: "string"},
   113  				},
   114  			},
   115  		},
   116  	},
   117  }
   118  
   119  var _ = SIGDescribe("CustomResourceConversionWebhook [Privileged:ClusterAdmin]", func() {
   120  	var certCtx *certContext
   121  	f := framework.NewDefaultFramework("crd-webhook")
   122  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
   123  	servicePort := int32(9443)
   124  	containerPort := int32(9444)
   125  
   126  	ginkgo.BeforeEach(func(ctx context.Context) {
   127  		ginkgo.DeferCleanup(cleanCRDWebhookTest, f.ClientSet, f.Namespace.Name)
   128  
   129  		ginkgo.By("Setting up server cert")
   130  		certCtx = setupServerCert(f.Namespace.Name, serviceCRDName)
   131  		createAuthReaderRoleBindingForCRDConversion(ctx, f, f.Namespace.Name)
   132  
   133  		deployCustomResourceWebhookAndService(ctx, f, imageutils.GetE2EImage(imageutils.Agnhost), certCtx, servicePort, containerPort)
   134  	})
   135  
   136  	/*
   137  		Release: v1.16
   138  		Testname: Custom Resource Definition Conversion Webhook, conversion custom resource
   139  		Description: Register a conversion webhook and a custom resource definition. Create a v1 custom
   140  		resource. Attempts to read it at v2 MUST succeed.
   141  	*/
   142  	framework.ConformanceIt("should be able to convert from CR v1 to CR v2", func(ctx context.Context) {
   143  		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
   144  			crd.Spec.Versions = apiVersions
   145  			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
   146  				Strategy: apiextensionsv1.WebhookConverter,
   147  				Webhook: &apiextensionsv1.WebhookConversion{
   148  					ClientConfig: &apiextensionsv1.WebhookClientConfig{
   149  						CABundle: certCtx.signingCert,
   150  						Service: &apiextensionsv1.ServiceReference{
   151  							Namespace: f.Namespace.Name,
   152  							Name:      serviceCRDName,
   153  							Path:      pointer.String("/crdconvert"),
   154  							Port:      pointer.Int32(servicePort),
   155  						},
   156  					},
   157  					ConversionReviewVersions: []string{"v1", "v1beta1"},
   158  				},
   159  			}
   160  			crd.Spec.PreserveUnknownFields = false
   161  		})
   162  		if err != nil {
   163  			return
   164  		}
   165  		ginkgo.DeferCleanup(testcrd.CleanUp)
   166  		waitWebhookConversionReady(ctx, f, testcrd.Crd, testcrd.DynamicClients, "v2")
   167  		testCustomResourceConversionWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients)
   168  	})
   169  
   170  	/*
   171  		Release: v1.16
   172  		Testname: Custom Resource Definition Conversion Webhook, convert mixed version list
   173  		Description: Register a conversion webhook and a custom resource definition. Create a custom resource stored at
   174  		v1. Change the custom resource definition storage to v2. Create a custom resource stored at v2. Attempt to list
   175  		the custom resources at v2; the list result MUST contain both custom resources at v2.
   176  	*/
   177  	framework.ConformanceIt("should be able to convert a non homogeneous list of CRs", func(ctx context.Context) {
   178  		testcrd, err := crd.CreateMultiVersionTestCRD(f, "stable.example.com", func(crd *apiextensionsv1.CustomResourceDefinition) {
   179  			crd.Spec.Versions = apiVersions
   180  			crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
   181  				Strategy: apiextensionsv1.WebhookConverter,
   182  				Webhook: &apiextensionsv1.WebhookConversion{
   183  					ClientConfig: &apiextensionsv1.WebhookClientConfig{
   184  						CABundle: certCtx.signingCert,
   185  						Service: &apiextensionsv1.ServiceReference{
   186  							Namespace: f.Namespace.Name,
   187  							Name:      serviceCRDName,
   188  							Path:      pointer.String("/crdconvert"),
   189  							Port:      pointer.Int32(servicePort),
   190  						},
   191  					},
   192  					ConversionReviewVersions: []string{"v1", "v1beta1"},
   193  				},
   194  			}
   195  			crd.Spec.PreserveUnknownFields = false
   196  		})
   197  		if err != nil {
   198  			return
   199  		}
   200  		ginkgo.DeferCleanup(testcrd.CleanUp)
   201  		waitWebhookConversionReady(ctx, f, testcrd.Crd, testcrd.DynamicClients, "v2")
   202  		testCRListConversion(ctx, f, testcrd)
   203  	})
   204  })
   205  
   206  func cleanCRDWebhookTest(ctx context.Context, client clientset.Interface, namespaceName string) {
   207  	_ = client.CoreV1().Services(namespaceName).Delete(ctx, serviceCRDName, metav1.DeleteOptions{})
   208  	_ = client.AppsV1().Deployments(namespaceName).Delete(ctx, deploymentCRDName, metav1.DeleteOptions{})
   209  	_ = client.CoreV1().Secrets(namespaceName).Delete(ctx, secretCRDName, metav1.DeleteOptions{})
   210  	_ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, roleBindingCRDName, metav1.DeleteOptions{})
   211  }
   212  
   213  func createAuthReaderRoleBindingForCRDConversion(ctx context.Context, f *framework.Framework, namespace string) {
   214  	ginkgo.By("Create role binding to let cr conversion webhook read extension-apiserver-authentication")
   215  	client := f.ClientSet
   216  	// Create the role binding to allow the webhook read the extension-apiserver-authentication configmap
   217  	_, err := client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{
   218  		ObjectMeta: metav1.ObjectMeta{
   219  			Name: roleBindingCRDName,
   220  		},
   221  		RoleRef: rbacv1.RoleRef{
   222  			APIGroup: "",
   223  			Kind:     "Role",
   224  			Name:     "extension-apiserver-authentication-reader",
   225  		},
   226  
   227  		Subjects: []rbacv1.Subject{
   228  			{
   229  				Kind:      "ServiceAccount",
   230  				Name:      "default",
   231  				Namespace: namespace,
   232  			},
   233  		},
   234  	}, metav1.CreateOptions{})
   235  	if err != nil && apierrors.IsAlreadyExists(err) {
   236  		framework.Logf("role binding %s already exists", roleBindingCRDName)
   237  	} else {
   238  		framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace)
   239  	}
   240  }
   241  
   242  func deployCustomResourceWebhookAndService(ctx context.Context, f *framework.Framework, image string, certCtx *certContext, servicePort int32, containerPort int32) {
   243  	ginkgo.By("Deploying the custom resource conversion webhook pod")
   244  	client := f.ClientSet
   245  
   246  	// Creating the secret that contains the webhook's cert.
   247  	secret := &v1.Secret{
   248  		ObjectMeta: metav1.ObjectMeta{
   249  			Name: secretCRDName,
   250  		},
   251  		Type: v1.SecretTypeOpaque,
   252  		Data: map[string][]byte{
   253  			"tls.crt": certCtx.cert,
   254  			"tls.key": certCtx.key,
   255  		},
   256  	}
   257  	namespace := f.Namespace.Name
   258  	_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
   259  	framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace)
   260  
   261  	// Create the deployment of the webhook
   262  	podLabels := map[string]string{"app": "sample-crd-conversion-webhook", "crd-webhook": "true"}
   263  	replicas := int32(1)
   264  	mounts := []v1.VolumeMount{
   265  		{
   266  			Name:      "crd-conversion-webhook-certs",
   267  			ReadOnly:  true,
   268  			MountPath: "/webhook.local.config/certificates",
   269  		},
   270  	}
   271  	volumes := []v1.Volume{
   272  		{
   273  			Name: "crd-conversion-webhook-certs",
   274  			VolumeSource: v1.VolumeSource{
   275  				Secret: &v1.SecretVolumeSource{SecretName: secretCRDName},
   276  			},
   277  		},
   278  	}
   279  	containers := []v1.Container{
   280  		{
   281  			Name:         "sample-crd-conversion-webhook",
   282  			VolumeMounts: mounts,
   283  			Args: []string{
   284  				"crd-conversion-webhook",
   285  				"--tls-cert-file=/webhook.local.config/certificates/tls.crt",
   286  				"--tls-private-key-file=/webhook.local.config/certificates/tls.key",
   287  				"-v=4",
   288  				// Use a non-default port for containers.
   289  				fmt.Sprintf("--port=%d", containerPort),
   290  			},
   291  			ReadinessProbe: &v1.Probe{
   292  				ProbeHandler: v1.ProbeHandler{
   293  					HTTPGet: &v1.HTTPGetAction{
   294  						Scheme: v1.URISchemeHTTPS,
   295  						Port:   intstr.FromInt32(containerPort),
   296  						Path:   "/readyz",
   297  					},
   298  				},
   299  				PeriodSeconds:    1,
   300  				SuccessThreshold: 1,
   301  				FailureThreshold: 30,
   302  			},
   303  			Image: image,
   304  			Ports: []v1.ContainerPort{{ContainerPort: containerPort}},
   305  		},
   306  	}
   307  	d := e2edeployment.NewDeployment(deploymentCRDName, replicas, podLabels, "", "", appsv1.RollingUpdateDeploymentStrategyType)
   308  	d.Spec.Template.Spec.Containers = containers
   309  	d.Spec.Template.Spec.Volumes = volumes
   310  
   311  	deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, d, metav1.CreateOptions{})
   312  	framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentCRDName, namespace)
   313  
   314  	ginkgo.By("Wait for the deployment to be ready")
   315  
   316  	err = e2edeployment.WaitForDeploymentRevisionAndImage(client, namespace, deploymentCRDName, "1", image)
   317  	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentCRDName, namespace)
   318  
   319  	err = e2edeployment.WaitForDeploymentComplete(client, deployment)
   320  	framework.ExpectNoError(err, "waiting for %s deployment status valid", deploymentCRDName)
   321  
   322  	ginkgo.By("Deploying the webhook service")
   323  
   324  	serviceLabels := map[string]string{"crd-webhook": "true"}
   325  	service := &v1.Service{
   326  		ObjectMeta: metav1.ObjectMeta{
   327  			Namespace: namespace,
   328  			Name:      serviceCRDName,
   329  			Labels:    map[string]string{"test": "crd-webhook"},
   330  		},
   331  		Spec: v1.ServiceSpec{
   332  			Selector: serviceLabels,
   333  			Ports: []v1.ServicePort{
   334  				{
   335  					Protocol:   v1.ProtocolTCP,
   336  					Port:       servicePort,
   337  					TargetPort: intstr.FromInt32(containerPort),
   338  				},
   339  			},
   340  		},
   341  	}
   342  	_, err = client.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{})
   343  	framework.ExpectNoError(err, "creating service %s in namespace %s", serviceCRDName, namespace)
   344  
   345  	ginkgo.By("Verifying the service has paired with the endpoint")
   346  	err = framework.WaitForServiceEndpointsNum(ctx, client, namespace, serviceCRDName, 1, 1*time.Second, 30*time.Second)
   347  	framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1)
   348  }
   349  
   350  func verifyV1Object(crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) {
   351  	gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v1"))
   352  	hostPort, exists := obj.Object["hostPort"]
   353  	if !exists {
   354  		framework.Failf("HostPort not found.")
   355  	}
   356  
   357  	gomega.Expect(hostPort).To(gomega.BeEquivalentTo("localhost:8080"))
   358  	_, hostExists := obj.Object["host"]
   359  	if hostExists {
   360  		framework.Failf("Host should not have been declared.")
   361  	}
   362  	_, portExists := obj.Object["port"]
   363  	if portExists {
   364  		framework.Failf("Port should not have been declared.")
   365  	}
   366  }
   367  
   368  func verifyV2Object(crd *apiextensionsv1.CustomResourceDefinition, obj *unstructured.Unstructured) {
   369  	gomega.Expect(obj.GetAPIVersion()).To(gomega.BeEquivalentTo(crd.Spec.Group + "/v2"))
   370  	_, hostPortExists := obj.Object["hostPort"]
   371  	if hostPortExists {
   372  		framework.Failf("HostPort should not have been declared.")
   373  	}
   374  	host, hostExists := obj.Object["host"]
   375  	if !hostExists {
   376  		framework.Failf("Host declaration not found.")
   377  	}
   378  	gomega.Expect(host).To(gomega.BeEquivalentTo("localhost"))
   379  	port, portExists := obj.Object["port"]
   380  	if !portExists {
   381  		framework.Failf("Port declaration not found.")
   382  	}
   383  	gomega.Expect(port).To(gomega.BeEquivalentTo("8080"))
   384  }
   385  
   386  func testCustomResourceConversionWebhook(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) {
   387  	name := "cr-instance-1"
   388  	ginkgo.By("Creating a v1 custom resource")
   389  	crInstance := &unstructured.Unstructured{
   390  		Object: map[string]interface{}{
   391  			"kind":       crd.Spec.Names.Kind,
   392  			"apiVersion": crd.Spec.Group + "/v1",
   393  			"metadata": map[string]interface{}{
   394  				"name":      name,
   395  				"namespace": f.Namespace.Name,
   396  			},
   397  			"hostPort": "localhost:8080",
   398  		},
   399  	}
   400  	_, err := customResourceClients["v1"].Create(ctx, crInstance, metav1.CreateOptions{})
   401  	framework.ExpectNoError(err)
   402  	ginkgo.By("v2 custom resource should be converted")
   403  	v2crd, err := customResourceClients["v2"].Get(ctx, name, metav1.GetOptions{})
   404  	framework.ExpectNoError(err, "Getting v2 of custom resource %s", name)
   405  	verifyV2Object(crd, v2crd)
   406  }
   407  
   408  func testCRListConversion(ctx context.Context, f *framework.Framework, testCrd *crd.TestCrd) {
   409  	crd := testCrd.Crd
   410  	customResourceClients := testCrd.DynamicClients
   411  	name1 := "cr-instance-1"
   412  	name2 := "cr-instance-2"
   413  	ginkgo.By("Creating a v1 custom resource")
   414  	crInstance := &unstructured.Unstructured{
   415  		Object: map[string]interface{}{
   416  			"kind":       crd.Spec.Names.Kind,
   417  			"apiVersion": crd.Spec.Group + "/v1",
   418  			"metadata": map[string]interface{}{
   419  				"name":      name1,
   420  				"namespace": f.Namespace.Name,
   421  			},
   422  			"hostPort": "localhost:8080",
   423  		},
   424  	}
   425  	_, err := customResourceClients["v1"].Create(ctx, crInstance, metav1.CreateOptions{})
   426  	framework.ExpectNoError(err)
   427  
   428  	// Now cr-instance-1 is stored as v1. lets change storage version
   429  	crd, err = integration.UpdateV1CustomResourceDefinitionWithRetry(testCrd.APIExtensionClient, crd.Name, func(c *apiextensionsv1.CustomResourceDefinition) {
   430  		c.Spec.Versions = alternativeAPIVersions
   431  	})
   432  	framework.ExpectNoError(err)
   433  	ginkgo.By("Create a v2 custom resource")
   434  	crInstance = &unstructured.Unstructured{
   435  		Object: map[string]interface{}{
   436  			"kind":       crd.Spec.Names.Kind,
   437  			"apiVersion": crd.Spec.Group + "/v1",
   438  			"metadata": map[string]interface{}{
   439  				"name":      name2,
   440  				"namespace": f.Namespace.Name,
   441  			},
   442  			"hostPort": "localhost:8080",
   443  		},
   444  	}
   445  
   446  	// After changing a CRD, the resources for versions will be re-created that can be result in
   447  	// cancelled connection (e.g. "grpc connection closed" or "context canceled").
   448  	// Just retrying fixes that.
   449  	//
   450  	// TODO: we have to wait for the storage version to become effective. Storage version changes are not instant.
   451  	for i := 0; i < 5; i++ {
   452  		_, err = customResourceClients["v1"].Create(ctx, crInstance, metav1.CreateOptions{})
   453  		if err == nil {
   454  			break
   455  		}
   456  	}
   457  	framework.ExpectNoError(err)
   458  
   459  	// Now that we have a v1 and v2 object, both list operation in v1 and v2 should work as expected.
   460  
   461  	ginkgo.By("List CRs in v1")
   462  	list, err := customResourceClients["v1"].List(ctx, metav1.ListOptions{})
   463  	framework.ExpectNoError(err)
   464  	gomega.Expect(list.Items).To(gomega.HaveLen(2))
   465  	if !((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
   466  		(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)) {
   467  		framework.Failf("failed to find v1 objects with names %s and %s in the list: \n%s", name1, name2, format.Object(list.Items, 1))
   468  	}
   469  	verifyV1Object(crd, &list.Items[0])
   470  	verifyV1Object(crd, &list.Items[1])
   471  
   472  	ginkgo.By("List CRs in v2")
   473  	list, err = customResourceClients["v2"].List(ctx, metav1.ListOptions{})
   474  	framework.ExpectNoError(err)
   475  	gomega.Expect(list.Items).To(gomega.HaveLen(2))
   476  	if !((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
   477  		(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)) {
   478  		framework.Failf("failed to find v2 objects with names %s and %s in the list: \n%s", name1, name2, format.Object(list.Items, 1))
   479  	}
   480  	verifyV2Object(crd, &list.Items[0])
   481  	verifyV2Object(crd, &list.Items[1])
   482  }
   483  
   484  // waitWebhookConversionReady sends stub custom resource creation requests requiring conversion until one succeeds.
   485  func waitWebhookConversionReady(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface, version string) {
   486  	framework.ExpectNoError(wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (bool, error) {
   487  		crInstance := &unstructured.Unstructured{
   488  			Object: map[string]interface{}{
   489  				"kind":       crd.Spec.Names.Kind,
   490  				"apiVersion": crd.Spec.Group + "/" + version,
   491  				"metadata": map[string]interface{}{
   492  					"name":      f.UniqueName,
   493  					"namespace": f.Namespace.Name,
   494  				},
   495  			},
   496  		}
   497  		_, err := customResourceClients[version].Create(ctx, crInstance, metav1.CreateOptions{})
   498  		if err != nil {
   499  			// tolerate clusters that do not set --enable-aggregator-routing and have to wait for kube-proxy
   500  			// to program the service network, during which conversion requests return errors
   501  			framework.Logf("error waiting for conversion to succeed during setup: %v", err)
   502  			return false, nil
   503  		}
   504  
   505  		framework.ExpectNoError(customResourceClients[version].Delete(ctx, crInstance.GetName(), metav1.DeleteOptions{}), "cleaning up stub object")
   506  		return true, nil
   507  	}))
   508  }
   509
View as plain text