webhook.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"reflect"
    24  	"strings"
    25  	"time"
    26  
    27  	"k8s.io/utils/pointer"
    28  
    29  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    30  	appsv1 "k8s.io/api/apps/v1"
    31  	v1 "k8s.io/api/core/v1"
    32  	rbacv1 "k8s.io/api/rbac/v1"
    33  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    34  	crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    35  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    38  	"k8s.io/apimachinery/pkg/types"
    39  	"k8s.io/apimachinery/pkg/util/intstr"
    40  	"k8s.io/apimachinery/pkg/util/uuid"
    41  	"k8s.io/apimachinery/pkg/util/wait"
    42  	"k8s.io/client-go/dynamic"
    43  	clientset "k8s.io/client-go/kubernetes"
    44  	"k8s.io/client-go/util/retry"
    45  	"k8s.io/kubernetes/test/e2e/framework"
    46  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    47  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    48  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    49  	"k8s.io/kubernetes/test/utils/crd"
    50  	imageutils "k8s.io/kubernetes/test/utils/image"
    51  	admissionapi "k8s.io/pod-security-admission/api"
    52  
    53  	"github.com/onsi/ginkgo/v2"
    54  	"github.com/onsi/gomega"
    55  
    56  	// ensure libs have a chance to initialize
    57  	_ "github.com/stretchr/testify/assert"
    58  )
    59  
    60  const (
    61  	secretName      = "sample-webhook-secret"
    62  	deploymentName  = "sample-webhook-deployment"
    63  	serviceName     = "e2e-test-webhook"
    64  	roleBindingName = "webhook-auth-reader"
    65  
    66  	skipNamespaceLabelKey   = "skip-webhook-admission"
    67  	skipNamespaceLabelValue = "yes"
    68  	skipNamespaceBaseName   = "exempted-namespace"
    69  	disallowedPodName       = "disallowed-pod"
    70  	toBeAttachedPodName     = "to-be-attached-pod"
    71  	hangingPodName          = "hanging-pod"
    72  	disallowedConfigMapName = "disallowed-configmap"
    73  	allowedConfigMapName    = "allowed-configmap"
    74  	failNamespaceLabelKey   = "fail-closed-webhook"
    75  	failNamespaceLabelValue = "yes"
    76  	failNamespaceBaseName   = "fail-closed-namespace"
    77  	addedLabelKey           = "added-label"
    78  	addedLabelValue         = "yes"
    79  )
    80  
    81  var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() {
    82  	var certCtx *certContext
    83  	f := framework.NewDefaultFramework("webhook")
    84  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    85  	servicePort := int32(8443)
    86  	containerPort := int32(8444)
    87  
    88  	var client clientset.Interface
    89  	var namespaceName string
    90  	var markersNamespaceName string
    91  
    92  	ginkgo.BeforeEach(func(ctx context.Context) {
    93  		client = f.ClientSet
    94  		namespaceName = f.Namespace.Name
    95  
    96  		// Make sure the namespace created for the test is labeled to be selected by the webhooks
    97  		labelNamespace(ctx, f, f.Namespace.Name)
    98  		markersNamespaceName = createWebhookConfigurationReadyNamespace(ctx, f)
    99  
   100  		ginkgo.By("Setting up server cert")
   101  		certCtx = setupServerCert(namespaceName, serviceName)
   102  		createAuthReaderRoleBinding(ctx, f, namespaceName)
   103  
   104  		deployWebhookAndService(ctx, f, imageutils.GetE2EImage(imageutils.Agnhost), certCtx, servicePort, containerPort)
   105  	})
   106  
   107  	ginkgo.AfterEach(func(ctx context.Context) {
   108  		cleanWebhookTest(ctx, client, namespaceName)
   109  	})
   110  
   111  	/*
   112  		Release: v1.16
   113  		Testname: Admission webhook, discovery document
   114  		Description: The admissionregistration.k8s.io API group MUST exists in the /apis discovery document.
   115  		The admissionregistration.k8s.io/v1 API group/version MUST exists in the /apis discovery document.
   116  		The mutatingwebhookconfigurations and validatingwebhookconfigurations resources MUST exist in the
   117  		/apis/admissionregistration.k8s.io/v1 discovery document.
   118  	*/
   119  	framework.ConformanceIt("should include webhook resources in discovery documents", func(ctx context.Context) {
   120  		{
   121  			ginkgo.By("fetching the /apis discovery document")
   122  			apiGroupList := &metav1.APIGroupList{}
   123  			err := client.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Into(apiGroupList)
   124  			framework.ExpectNoError(err, "fetching /apis")
   125  
   126  			ginkgo.By("finding the admissionregistration.k8s.io API group in the /apis discovery document")
   127  			var group *metav1.APIGroup
   128  			for _, g := range apiGroupList.Groups {
   129  				if g.Name == admissionregistrationv1.GroupName {
   130  					group = &g
   131  					break
   132  				}
   133  			}
   134  			gomega.Expect(group).ToNot(gomega.BeNil(), "admissionregistration.k8s.io API group not found in /apis discovery document")
   135  
   136  			ginkgo.By("finding the admissionregistration.k8s.io/v1 API group/version in the /apis discovery document")
   137  			var version *metav1.GroupVersionForDiscovery
   138  			for _, v := range group.Versions {
   139  				if v.Version == admissionregistrationv1.SchemeGroupVersion.Version {
   140  					version = &v
   141  					break
   142  				}
   143  			}
   144  			gomega.Expect(version).ToNot(gomega.BeNil(), "admissionregistration.k8s.io/v1 API group version not found in /apis discovery document")
   145  		}
   146  
   147  		{
   148  			ginkgo.By("fetching the /apis/admissionregistration.k8s.io discovery document")
   149  			group := &metav1.APIGroup{}
   150  			err := client.Discovery().RESTClient().Get().AbsPath("/apis/admissionregistration.k8s.io").Do(ctx).Into(group)
   151  			framework.ExpectNoError(err, "fetching /apis/admissionregistration.k8s.io")
   152  			gomega.Expect(group.Name).To(gomega.Equal(admissionregistrationv1.GroupName), "verifying API group name in /apis/admissionregistration.k8s.io discovery document")
   153  
   154  			ginkgo.By("finding the admissionregistration.k8s.io/v1 API group/version in the /apis/admissionregistration.k8s.io discovery document")
   155  			var version *metav1.GroupVersionForDiscovery
   156  			for _, v := range group.Versions {
   157  				if v.Version == admissionregistrationv1.SchemeGroupVersion.Version {
   158  					version = &v
   159  					break
   160  				}
   161  			}
   162  			gomega.Expect(version).ToNot(gomega.BeNil(), "admissionregistration.k8s.io/v1 API group version not found in /apis/admissionregistration.k8s.io discovery document")
   163  		}
   164  
   165  		{
   166  			ginkgo.By("fetching the /apis/admissionregistration.k8s.io/v1 discovery document")
   167  			apiResourceList := &metav1.APIResourceList{}
   168  			err := client.Discovery().RESTClient().Get().AbsPath("/apis/admissionregistration.k8s.io/v1").Do(ctx).Into(apiResourceList)
   169  			framework.ExpectNoError(err, "fetching /apis/admissionregistration.k8s.io/v1")
   170  			gomega.Expect(apiResourceList.GroupVersion).To(gomega.Equal(admissionregistrationv1.SchemeGroupVersion.String()), "verifying API group/version in /apis/admissionregistration.k8s.io/v1 discovery document")
   171  
   172  			ginkgo.By("finding mutatingwebhookconfigurations and validatingwebhookconfigurations resources in the /apis/admissionregistration.k8s.io/v1 discovery document")
   173  			var (
   174  				mutatingWebhookResource   *metav1.APIResource
   175  				validatingWebhookResource *metav1.APIResource
   176  			)
   177  			for i := range apiResourceList.APIResources {
   178  				if apiResourceList.APIResources[i].Name == "mutatingwebhookconfigurations" {
   179  					mutatingWebhookResource = &apiResourceList.APIResources[i]
   180  				}
   181  				if apiResourceList.APIResources[i].Name == "validatingwebhookconfigurations" {
   182  					validatingWebhookResource = &apiResourceList.APIResources[i]
   183  				}
   184  			}
   185  			gomega.Expect(mutatingWebhookResource).ToNot(gomega.BeNil(), "mutatingwebhookconfigurations resource not found in /apis/admissionregistration.k8s.io/v1 discovery document")
   186  			gomega.Expect(validatingWebhookResource).ToNot(gomega.BeNil(), "validatingwebhookconfigurations resource not found in /apis/admissionregistration.k8s.io/v1 discovery document")
   187  		}
   188  	})
   189  
   190  	/*
   191  		Release: v1.16
   192  		Testname: Admission webhook, deny create
   193  		Description: Register an admission webhook configuration that admits pod and configmap. Attempts to create
   194  		non-compliant pods and configmaps, or update/patch compliant pods and configmaps to be non-compliant MUST
   195  		be denied. An attempt to create a pod that causes a webhook to hang MUST result in a webhook timeout error,
   196  		and the pod creation MUST be denied. An attempt to create a non-compliant configmap in a whitelisted
   197  		namespace based on the webhook namespace selector MUST be allowed.
   198  	*/
   199  	framework.ConformanceIt("should be able to deny pod and configmap creation", func(ctx context.Context) {
   200  		registerWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   201  		testWebhook(ctx, f)
   202  	})
   203  
   204  	/*
   205  		Release: v1.16
   206  		Testname: Admission webhook, deny attach
   207  		Description: Register an admission webhook configuration that denies connecting to a pod's attach sub-resource.
   208  		Attempts to attach MUST be denied.
   209  	*/
   210  	framework.ConformanceIt("should be able to deny attaching pod", func(ctx context.Context) {
   211  		registerWebhookForAttachingPod(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   212  		testAttachingPodWebhook(ctx, f)
   213  	})
   214  
   215  	/*
   216  		Release: v1.16
   217  		Testname: Admission webhook, deny custom resource create and delete
   218  		Description: Register an admission webhook configuration that denies creation, update and deletion of
   219  		custom resources. Attempts to create, update and delete custom resources MUST be denied.
   220  	*/
   221  	framework.ConformanceIt("should be able to deny custom resource creation, update and deletion", func(ctx context.Context) {
   222  		testcrd, err := crd.CreateTestCRD(f)
   223  		if err != nil {
   224  			return
   225  		}
   226  		ginkgo.DeferCleanup(testcrd.CleanUp)
   227  		registerWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort)
   228  		testCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"])
   229  		testBlockingCustomResourceUpdateDeletion(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"])
   230  	})
   231  
   232  	/*
   233  		Release: v1.16
   234  		Testname: Admission webhook, fail closed
   235  		Description: Register a webhook with a fail closed policy and without CA bundle so that it cannot be called.
   236  		Attempt operations that require the admission webhook; all MUST be denied.
   237  	*/
   238  	framework.ConformanceIt("should unconditionally reject operations on fail closed webhook", func(ctx context.Context) {
   239  		registerFailClosedWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   240  		testFailClosedWebhook(ctx, f)
   241  	})
   242  
   243  	/*
   244  		Release: v1.16
   245  		Testname: Admission webhook, ordered mutation
   246  		Description: Register a mutating webhook configuration with two webhooks that admit configmaps, one that
   247  		adds a data key if the configmap already has a specific key, and another that adds a key if the key added by
   248  		the first webhook is present. Attempt to create a config map; both keys MUST be added to the config map.
   249  	*/
   250  	framework.ConformanceIt("should mutate configmap", func(ctx context.Context) {
   251  		registerMutatingWebhookForConfigMap(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   252  		testMutatingConfigMapWebhook(ctx, f)
   253  	})
   254  
   255  	/*
   256  		Release: v1.16
   257  		Testname: Admission webhook, mutation with defaulting
   258  		Description: Register a mutating webhook that adds an InitContainer to pods. Attempt to create a pod;
   259  		the InitContainer MUST be added the TerminationMessagePolicy MUST be defaulted.
   260  	*/
   261  	framework.ConformanceIt("should mutate pod and apply defaults after mutation", func(ctx context.Context) {
   262  		registerMutatingWebhookForPod(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   263  		testMutatingPodWebhook(ctx, f)
   264  	})
   265  
   266  	/*
   267  		Release: v1.16
   268  		Testname: Admission webhook, admission control not allowed on webhook configuration objects
   269  		Description: Register webhooks that mutate and deny deletion of webhook configuration objects. Attempt to create
   270  		and delete a webhook configuration object; both operations MUST be allowed and the webhook configuration object
   271  		MUST NOT be mutated the webhooks.
   272  	*/
   273  	framework.ConformanceIt("should not be able to mutate or prevent deletion of webhook configuration objects", func(ctx context.Context) {
   274  		registerValidatingWebhookForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName+"blocking", certCtx, servicePort)
   275  		registerMutatingWebhookForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName+"blocking", certCtx, servicePort)
   276  		testWebhooksForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   277  	})
   278  
   279  	/*
   280  		Release: v1.16
   281  		Testname: Admission webhook, mutate custom resource
   282  		Description: Register a webhook that mutates a custom resource. Attempt to create custom resource object;
   283  		the custom resource MUST be mutated.
   284  	*/
   285  	framework.ConformanceIt("should mutate custom resource", func(ctx context.Context) {
   286  		testcrd, err := crd.CreateTestCRD(f)
   287  		if err != nil {
   288  			return
   289  		}
   290  		ginkgo.DeferCleanup(testcrd.CleanUp)
   291  		registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort)
   292  		testMutatingCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"], false)
   293  	})
   294  
   295  	/*
   296  		Release: v1.16
   297  		Testname: Admission webhook, deny custom resource definition
   298  		Description: Register a webhook that denies custom resource definition create. Attempt to create a
   299  		custom resource definition; the create request MUST be denied.
   300  	*/
   301  	framework.ConformanceIt("should deny crd creation", func(ctx context.Context) {
   302  		registerValidatingWebhookForCRD(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort)
   303  
   304  		testCRDDenyWebhook(ctx, f)
   305  	})
   306  
   307  	/*
   308  		Release: v1.16
   309  		Testname: Admission webhook, mutate custom resource with different stored version
   310  		Description: Register a webhook that mutates custom resources on create and update. Register a custom resource
   311  		definition using v1 as stored version. Create a custom resource. Patch the custom resource definition to use v2 as
   312  		the stored version. Attempt to patch the custom resource with a new field and value; the patch MUST be applied
   313  		successfully.
   314  	*/
   315  	framework.ConformanceIt("should mutate custom resource with different stored version", func(ctx context.Context) {
   316  		testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f)
   317  		if err != nil {
   318  			return
   319  		}
   320  		ginkgo.DeferCleanup(testcrd.CleanUp)
   321  		registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort)
   322  		testMultiVersionCustomResourceWebhook(ctx, f, testcrd)
   323  	})
   324  
   325  	/*
   326  		Release: v1.16
   327  		Testname: Admission webhook, mutate custom resource with pruning
   328  		Description: Register mutating webhooks that adds fields to custom objects. Register a custom resource definition
   329  		with a schema that includes only one of the data keys added by the webhooks. Attempt to a custom resource;
   330  		the fields included in the schema MUST be present and field not included in the schema MUST NOT be present.
   331  	*/
   332  	framework.ConformanceIt("should mutate custom resource with pruning", func(ctx context.Context) {
   333  		const prune = true
   334  		testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1.CustomResourceDefinition) {
   335  			crd.Spec.PreserveUnknownFields = false
   336  			for i := range crd.Spec.Versions {
   337  				crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{
   338  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
   339  						Type: "object",
   340  						Properties: map[string]apiextensionsv1.JSONSchemaProps{
   341  							"data": {
   342  								Type: "object",
   343  								Properties: map[string]apiextensionsv1.JSONSchemaProps{
   344  									"mutation-start":   {Type: "string"},
   345  									"mutation-stage-1": {Type: "string"},
   346  									// mutation-stage-2 is intentionally missing such that it is pruned
   347  								},
   348  							},
   349  						},
   350  					},
   351  				}
   352  			}
   353  		})
   354  		if err != nil {
   355  			return
   356  		}
   357  		ginkgo.DeferCleanup(testcrd.CleanUp)
   358  		registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort)
   359  		testMutatingCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"], prune)
   360  	})
   361  
   362  	/*
   363  		Release: v1.16
   364  		Testname: Admission webhook, honor timeout
   365  		Description: Using a webhook that waits 5 seconds before admitting objects, configure the webhook with combinations
   366  		of timeouts and failure policy values. Attempt to create a config map with each combination. Requests MUST
   367  		timeout if the configured webhook timeout is less than 5 seconds and failure policy is fail. Requests must not timeout if
   368  		the failure policy is ignore. Requests MUST NOT timeout if configured webhook timeout is 10 seconds (much longer
   369  		than the webhook wait duration).
   370  	*/
   371  	framework.ConformanceIt("should honor timeout", func(ctx context.Context) {
   372  		policyFail := admissionregistrationv1.Fail
   373  		policyIgnore := admissionregistrationv1.Ignore
   374  
   375  		ginkgo.By("Setting timeout (1s) shorter than webhook latency (5s)")
   376  		slowWebhookCleanup := registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, pointer.Int32(1), servicePort)
   377  		testSlowWebhookTimeoutFailEarly(ctx, f)
   378  		slowWebhookCleanup(ctx)
   379  
   380  		ginkgo.By("Having no error when timeout is shorter than webhook latency and failure policy is ignore")
   381  		slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyIgnore, pointer.Int32(1), servicePort)
   382  		testSlowWebhookTimeoutNoError(ctx, f)
   383  		slowWebhookCleanup(ctx)
   384  
   385  		ginkgo.By("Having no error when timeout is longer than webhook latency")
   386  		slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, pointer.Int32(10), servicePort)
   387  		testSlowWebhookTimeoutNoError(ctx, f)
   388  		slowWebhookCleanup(ctx)
   389  
   390  		ginkgo.By("Having no error when timeout is empty (defaulted to 10s in v1)")
   391  		slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, nil, servicePort)
   392  		testSlowWebhookTimeoutNoError(ctx, f)
   393  		slowWebhookCleanup(ctx)
   394  	})
   395  
   396  	/*
   397  		Release: v1.16
   398  		Testname: Admission webhook, update validating webhook
   399  		Description: Register a validating admission webhook configuration. Update the webhook to not apply to the create
   400  		operation and attempt to create an object; the webhook MUST NOT deny the create. Patch the webhook to apply to the
   401  		create operation again and attempt to create an object; the webhook MUST deny the create.
   402  	*/
   403  	framework.ConformanceIt("patching/updating a validating webhook should work", func(ctx context.Context) {
   404  		client := f.ClientSet
   405  		admissionClient := client.AdmissionregistrationV1()
   406  
   407  		ginkgo.By("Creating a validating webhook configuration")
   408  		hook, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
   409  			ObjectMeta: metav1.ObjectMeta{
   410  				Name: f.UniqueName,
   411  			},
   412  			Webhooks: []admissionregistrationv1.ValidatingWebhook{
   413  				newDenyConfigMapWebhookFixture(f, certCtx, servicePort),
   414  				newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
   415  			},
   416  		})
   417  		framework.ExpectNoError(err, "Creating validating webhook configuration")
   418  		defer func() {
   419  			err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, hook.Name, metav1.DeleteOptions{})
   420  			framework.ExpectNoError(err, "Deleting validating webhook configuration")
   421  		}()
   422  
   423  		// ensure backend is ready before proceeding
   424  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   425  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   426  
   427  		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
   428  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   429  			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
   430  			_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   431  			if err == nil {
   432  				err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   433  				framework.ExpectNoError(err, "Deleting successfully created configMap")
   434  				return false, nil
   435  			}
   436  			if !strings.Contains(err.Error(), "denied") {
   437  				return false, err
   438  			}
   439  			return true, nil
   440  		})
   441  
   442  		ginkgo.By("Updating a validating webhook configuration's rules to not include the create operation")
   443  		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   444  			h, err := admissionClient.ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   445  			framework.ExpectNoError(err, "Getting validating webhook configuration")
   446  			h.Webhooks[0].Rules[0].Operations = []admissionregistrationv1.OperationType{admissionregistrationv1.Update}
   447  			_, err = admissionClient.ValidatingWebhookConfigurations().Update(ctx, h, metav1.UpdateOptions{})
   448  			return err
   449  		})
   450  		framework.ExpectNoError(err, "Updating validating webhook configuration")
   451  
   452  		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
   453  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   454  			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
   455  			_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   456  			if err != nil {
   457  				if !strings.Contains(err.Error(), "denied") {
   458  					return false, err
   459  				}
   460  				return false, nil
   461  			}
   462  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   463  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   464  			return true, nil
   465  		})
   466  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be allowed creation since webhook was updated to not validate create", f.Namespace.Name)
   467  
   468  		ginkgo.By("Patching a validating webhook configuration's rules to include the create operation")
   469  		hook, err = admissionClient.ValidatingWebhookConfigurations().Patch(ctx, f.UniqueName,
   470  			types.JSONPatchType,
   471  			[]byte(`[{"op": "replace", "path": "/webhooks/0/rules/0/operations", "value": ["CREATE"]}]`), metav1.PatchOptions{})
   472  		framework.ExpectNoError(err, "Patching validating webhook configuration")
   473  
   474  		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
   475  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   476  			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
   477  			_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   478  			if err == nil {
   479  				err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   480  				framework.ExpectNoError(err, "Deleting successfully created configMap")
   481  				return false, nil
   482  			}
   483  			if !strings.Contains(err.Error(), "denied") {
   484  				return false, err
   485  			}
   486  			return true, nil
   487  		})
   488  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be denied creation by validating webhook", f.Namespace.Name)
   489  	})
   490  
   491  	/*
   492  		Release: v1.16
   493  		Testname: Admission webhook, update mutating webhook
   494  		Description: Register a mutating admission webhook configuration. Update the webhook to not apply to the create
   495  		operation and attempt to create an object; the webhook MUST NOT mutate the object. Patch the webhook to apply to the
   496  		create operation again and attempt to create an object; the webhook MUST mutate the object.
   497  	*/
   498  	framework.ConformanceIt("patching/updating a mutating webhook should work", func(ctx context.Context) {
   499  		client := f.ClientSet
   500  		admissionClient := client.AdmissionregistrationV1()
   501  
   502  		ginkgo.By("Creating a mutating webhook configuration")
   503  		hook, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
   504  			ObjectMeta: metav1.ObjectMeta{
   505  				Name: f.UniqueName,
   506  			},
   507  			Webhooks: []admissionregistrationv1.MutatingWebhook{
   508  				newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort),
   509  				newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
   510  			},
   511  		})
   512  		framework.ExpectNoError(err, "Creating mutating webhook configuration")
   513  		defer func() {
   514  			err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, hook.Name, metav1.DeleteOptions{})
   515  			framework.ExpectNoError(err, "Deleting mutating webhook configuration")
   516  		}()
   517  
   518  		// ensure backend is ready before proceeding
   519  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   520  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   521  
   522  		hook, err = admissionClient.MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   523  		framework.ExpectNoError(err, "Getting mutating webhook configuration")
   524  		ginkgo.By("Updating a mutating webhook configuration's rules to not include the create operation")
   525  		hook.Webhooks[0].Rules[0].Operations = []admissionregistrationv1.OperationType{admissionregistrationv1.Update}
   526  		hook, err = admissionClient.MutatingWebhookConfigurations().Update(ctx, hook, metav1.UpdateOptions{})
   527  		framework.ExpectNoError(err, "Updating mutating webhook configuration")
   528  
   529  		ginkgo.By("Creating a configMap that should not be mutated")
   530  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   531  			cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f)
   532  			created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   533  			if err != nil {
   534  				return false, err
   535  			}
   536  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   537  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   538  			_, ok := created.Data["mutation-stage-1"]
   539  			return !ok, nil
   540  		})
   541  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s this is not mutated", f.Namespace.Name)
   542  
   543  		ginkgo.By("Patching a mutating webhook configuration's rules to include the create operation")
   544  		hook, err = admissionClient.MutatingWebhookConfigurations().Patch(ctx, f.UniqueName,
   545  			types.JSONPatchType,
   546  			[]byte(`[{"op": "replace", "path": "/webhooks/0/rules/0/operations", "value": ["CREATE"]}]`), metav1.PatchOptions{})
   547  		framework.ExpectNoError(err, "Patching mutating webhook configuration")
   548  
   549  		ginkgo.By("Creating a configMap that should be mutated")
   550  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   551  			cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f)
   552  			created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   553  			if err != nil {
   554  				return false, err
   555  			}
   556  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   557  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   558  			_, ok := created.Data["mutation-stage-1"]
   559  			return ok, nil
   560  		})
   561  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be mutated", f.Namespace.Name)
   562  	})
   563  
   564  	/*
   565  		Release: v1.16
   566  		Testname: Admission webhook, list validating webhooks
   567  		Description: Create 10 validating webhook configurations, all with a label. Attempt to list the webhook
   568  		configurations matching the label; all the created webhook configurations MUST be present. Attempt to create an
   569  		object; the create MUST be denied. Attempt to remove the webhook configurations matching the label with deletecollection;
   570  		all webhook configurations MUST be deleted. Attempt to create an object; the create MUST NOT be denied.
   571  	*/
   572  	framework.ConformanceIt("listing validating webhooks should work", func(ctx context.Context) {
   573  		testListSize := 10
   574  		testUUID := string(uuid.NewUUID())
   575  
   576  		for i := 0; i < testListSize; i++ {
   577  			name := fmt.Sprintf("%s-%d", f.UniqueName, i)
   578  			_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
   579  				ObjectMeta: metav1.ObjectMeta{
   580  					Name:   name,
   581  					Labels: map[string]string{"e2e-list-test-uuid": testUUID},
   582  				},
   583  				Webhooks: []admissionregistrationv1.ValidatingWebhook{
   584  					newDenyConfigMapWebhookFixture(f, certCtx, servicePort),
   585  					newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
   586  				},
   587  			})
   588  			framework.ExpectNoError(err, "Creating validating webhook configuration")
   589  		}
   590  		selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID}
   591  
   592  		ginkgo.By("Listing all of the created validation webhooks")
   593  		list, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, selectorListOpts)
   594  		framework.ExpectNoError(err, "Listing validating webhook configurations")
   595  		gomega.Expect(list.Items).To(gomega.HaveLen(testListSize))
   596  
   597  		// ensure backend is ready before proceeding
   598  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   599  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   600  
   601  		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
   602  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   603  			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
   604  			_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   605  			if err == nil {
   606  				err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   607  				framework.ExpectNoError(err, "Deleting successfully created configMap")
   608  				return false, nil
   609  			}
   610  			if !strings.Contains(err.Error(), "denied") {
   611  				return false, err
   612  			}
   613  			return true, nil
   614  		})
   615  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be denied creation by validating webhook", f.Namespace.Name)
   616  
   617  		ginkgo.By("Deleting the collection of validation webhooks")
   618  		err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().DeleteCollection(ctx, metav1.DeleteOptions{}, selectorListOpts)
   619  		framework.ExpectNoError(err, "Deleting collection of validating webhook configurations")
   620  
   621  		ginkgo.By("Creating a configMap that does not comply to the validation webhook rules")
   622  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   623  			cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f)
   624  			_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   625  			if err != nil {
   626  				if !strings.Contains(err.Error(), "denied") {
   627  					return false, err
   628  				}
   629  				return false, nil
   630  			}
   631  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   632  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   633  			return true, nil
   634  		})
   635  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be allowed creation since there are no webhooks", f.Namespace.Name)
   636  	})
   637  
   638  	/*
   639  		Release: v1.16
   640  		Testname: Admission webhook, list mutating webhooks
   641  		Description: Create 10 mutating webhook configurations, all with a label. Attempt to list the webhook
   642  		configurations matching the label; all the created webhook configurations MUST be present. Attempt to create an
   643  		object; the object MUST be mutated. Attempt to remove the webhook configurations matching the label with deletecollection;
   644  		all webhook configurations MUST be deleted. Attempt to create an object; the object MUST NOT be mutated.
   645  	*/
   646  	framework.ConformanceIt("listing mutating webhooks should work", func(ctx context.Context) {
   647  		testListSize := 10
   648  		testUUID := string(uuid.NewUUID())
   649  
   650  		for i := 0; i < testListSize; i++ {
   651  			name := fmt.Sprintf("%s-%d", f.UniqueName, i)
   652  			_, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
   653  				ObjectMeta: metav1.ObjectMeta{
   654  					Name:   name,
   655  					Labels: map[string]string{"e2e-list-test-uuid": testUUID},
   656  				},
   657  				Webhooks: []admissionregistrationv1.MutatingWebhook{
   658  					newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort),
   659  					newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
   660  				},
   661  			})
   662  			framework.ExpectNoError(err, "Creating mutating webhook configuration")
   663  		}
   664  		selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID}
   665  
   666  		ginkgo.By("Listing all of the created validation webhooks")
   667  		list, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, selectorListOpts)
   668  		framework.ExpectNoError(err, "Listing mutating webhook configurations")
   669  		gomega.Expect(list.Items).To(gomega.HaveLen(testListSize))
   670  
   671  		// ensure backend is ready before proceeding
   672  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   673  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   674  
   675  		ginkgo.By("Creating a configMap that should be mutated")
   676  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   677  			cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f)
   678  			created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   679  			if err != nil {
   680  				return false, err
   681  			}
   682  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   683  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   684  			_, ok := created.Data["mutation-stage-1"]
   685  			return ok, nil
   686  		})
   687  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be mutated", f.Namespace.Name)
   688  
   689  		ginkgo.By("Deleting the collection of validation webhooks")
   690  		err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().DeleteCollection(ctx, metav1.DeleteOptions{}, selectorListOpts)
   691  		framework.ExpectNoError(err, "Deleting collection of mutating webhook configurations")
   692  
   693  		ginkgo.By("Creating a configMap that should not be mutated")
   694  		err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
   695  			cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f)
   696  			created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   697  			if err != nil {
   698  				return false, err
   699  			}
   700  			err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{})
   701  			framework.ExpectNoError(err, "Deleting successfully created configMap")
   702  			_, ok := created.Data["mutation-stage-1"]
   703  			return !ok, nil
   704  		})
   705  		framework.ExpectNoError(err, "Waiting for configMap in namespace %s this is not mutated", f.Namespace.Name)
   706  	})
   707  
   708  	/*
   709  		Release: v1.28
   710  		Testname: Validating Admission webhook, create and update validating webhook configuration with matchConditions
   711  		Description: Register a validating webhook configuration. Verify that the match conditions field are
   712  		properly stored in the api-server.  Update the validating webhook configuration and retrieve it; the
   713  		retrieved object must contain the newly update matchConditions fields.
   714  	*/
   715  	framework.ConformanceIt("should be able to create and update validating webhook configurations with match conditions", func(ctx context.Context) {
   716  		initalMatchConditions := []admissionregistrationv1.MatchCondition{
   717  			{
   718  				Name:       "expression-1",
   719  				Expression: "object.metadata.namespace == 'production'",
   720  			},
   721  		}
   722  
   723  		ginkgo.By("creating a validating webhook with match conditions")
   724  		validatingWebhookConfiguration := newValidatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions)
   725  
   726  		_, err := createValidatingWebhookConfiguration(ctx, f, validatingWebhookConfiguration)
   727  		framework.ExpectNoError(err)
   728  
   729  		ginkgo.By("verifying the validating webhook match conditions")
   730  		validatingWebhookConfiguration, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   731  		framework.ExpectNoError(err)
   732  		gomega.Expect(validatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(initalMatchConditions), "verifying that match conditions are created")
   733  		defer func() {
   734  			err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, validatingWebhookConfiguration.Name, metav1.DeleteOptions{})
   735  			framework.ExpectNoError(err, "deleting mutating webhook configuration")
   736  		}()
   737  
   738  		ginkgo.By("updating the validating webhook match conditions")
   739  		updatedMatchConditions := []admissionregistrationv1.MatchCondition{
   740  			{
   741  				Name:       "expression-1",
   742  				Expression: "object.metadata.namespace == 'production'",
   743  			},
   744  			{
   745  				Name:       "expression-2",
   746  				Expression: "object.metadata.namespace == 'staging'",
   747  			},
   748  		}
   749  		validatingWebhookConfiguration.Webhooks[0].MatchConditions = updatedMatchConditions
   750  		_, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, validatingWebhookConfiguration, metav1.UpdateOptions{})
   751  		framework.ExpectNoError(err)
   752  
   753  		ginkgo.By("verifying the validating webhook match conditions")
   754  		validatingWebhookConfiguration, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   755  		framework.ExpectNoError(err)
   756  		gomega.Expect(validatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(updatedMatchConditions), "verifying that match conditions are updated")
   757  	})
   758  
   759  	/*
   760  		Release: v1.28
   761  		Testname: Mutating Admission webhook, create and update mutating webhook configuration with matchConditions
   762  		Description: Register a mutating webhook configuration. Verify that the match conditions field are
   763  		properly stored in the api-server.  Update the mutating webhook configuration and retrieve it; the
   764  		retrieved object must contain the newly update matchConditions fields.
   765  	*/
   766  	framework.ConformanceIt("should be able to create and update mutating webhook configurations with match conditions", func(ctx context.Context) {
   767  		initalMatchConditions := []admissionregistrationv1.MatchCondition{
   768  			{
   769  				Name:       "expression-1",
   770  				Expression: "object.metadata.namespace == 'production'",
   771  			},
   772  		}
   773  
   774  		ginkgo.By("creating a mutating webhook with match conditions")
   775  		mutatingWebhookConfiguration := newMutatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions)
   776  
   777  		_, err := createMutatingWebhookConfiguration(ctx, f, mutatingWebhookConfiguration)
   778  		framework.ExpectNoError(err)
   779  
   780  		ginkgo.By("verifying the mutating webhook match conditions")
   781  		mutatingWebhookConfiguration, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   782  		framework.ExpectNoError(err)
   783  		gomega.Expect(mutatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(initalMatchConditions), "verifying that match conditions are created")
   784  		defer func() {
   785  			err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, mutatingWebhookConfiguration.Name, metav1.DeleteOptions{})
   786  			framework.ExpectNoError(err, "deleting mutating webhook configuration")
   787  		}()
   788  
   789  		ginkgo.By("updating the mutating webhook match conditions")
   790  		updatedMatchConditions := []admissionregistrationv1.MatchCondition{
   791  			{
   792  				Name:       "expression-1",
   793  				Expression: "object.metadata.namespace == 'production'",
   794  			},
   795  			{
   796  				Name:       "expression-2",
   797  				Expression: "object.metadata.namespace == 'staging'",
   798  			},
   799  		}
   800  		mutatingWebhookConfiguration.Webhooks[0].MatchConditions = updatedMatchConditions
   801  		_, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, mutatingWebhookConfiguration, metav1.UpdateOptions{})
   802  		framework.ExpectNoError(err)
   803  
   804  		ginkgo.By("verifying the mutating webhook match conditions")
   805  		mutatingWebhookConfiguration, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{})
   806  		framework.ExpectNoError(err)
   807  		gomega.Expect(mutatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(updatedMatchConditions), "verifying that match conditions are updated")
   808  	})
   809  
   810  	/*
   811  		Release: v1.28
   812  		Testname: Validing Admission webhook, reject validating webhook configurations with invalid matchConditions
   813  		Description: Creates a validating webhook configuration with an invalid CEL expression in it's
   814  		matchConditions field. The api-server server should reject the create request with a "compilation
   815  		failed" error message.
   816  	*/
   817  	framework.ConformanceIt("should reject validating webhook configurations with invalid match conditions", func(ctx context.Context) {
   818  		initalMatchConditions := []admissionregistrationv1.MatchCondition{
   819  			{
   820  				Name:       "invalid-expression-1",
   821  				Expression: "... [] bad expression",
   822  			},
   823  		}
   824  
   825  		ginkgo.By("creating a validating webhook with match conditions")
   826  		validatingWebhookConfiguration := newValidatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions)
   827  
   828  		_, err := createValidatingWebhookConfiguration(ctx, f, validatingWebhookConfiguration)
   829  		gomega.Expect(err).To(gomega.HaveOccurred(), "create validatingwebhookconfiguration should have been denied by the api-server")
   830  		expectedErrMsg := "compilation failed"
   831  		gomega.Expect(strings.Contains(err.Error(), expectedErrMsg)).To(gomega.BeTrue())
   832  	})
   833  
   834  	/*
   835  		Release: v1.28
   836  		Testname: Mutating Admission webhook, reject mutating webhook configurations with invalid matchConditions
   837  		Description: Creates a mutating webhook configuration with an invalid CEL expression in it's
   838  		matchConditions field. The api-server server should reject the create request with a "compilation
   839  		failed" error message.
   840  	*/
   841  	framework.ConformanceIt("should reject mutating webhook configurations with invalid match conditions", func(ctx context.Context) {
   842  		initalMatchConditions := []admissionregistrationv1.MatchCondition{
   843  			{
   844  				Name:       "invalid-expression-1",
   845  				Expression: "... [] bad expression",
   846  			},
   847  		}
   848  
   849  		ginkgo.By("creating a mutating webhook with match conditions")
   850  		mutatingWebhookConfiguration := newMutatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions)
   851  
   852  		_, err := createMutatingWebhookConfiguration(ctx, f, mutatingWebhookConfiguration)
   853  		gomega.Expect(err).To(gomega.HaveOccurred(), "create mutatingwebhookconfiguration should have been denied by the api-server")
   854  		expectedErrMsg := "compilation failed"
   855  		gomega.Expect(strings.Contains(err.Error(), expectedErrMsg)).To(gomega.BeTrue())
   856  	})
   857  
   858  	/*
   859  		Release: v1.28
   860  		Testname: Mutating Admission webhook, mutating webhook excluding object with specific name
   861  		Description: Create a mutating webhook configuration with matchConditions field that
   862  		will reject all resources except ones with a specific name 'skip-me'. Create
   863  		a configMap with the name 'skip-me' and verify that it's mutated. Create a
   864  		configMap with a different name than 'skip-me' and verify that it's mustated.
   865  	*/
   866  	framework.ConformanceIt("should mutate everything except 'skip-me' configmaps", func(ctx context.Context) {
   867  		skipMeMatchConditions := []admissionregistrationv1.MatchCondition{
   868  			{
   869  				Name:       "skip-me",
   870  				Expression: "object.metadata.name != 'skip-me'",
   871  			},
   872  		}
   873  
   874  		ginkgo.By("creating a mutating webhook with match conditions")
   875  		namespace := f.Namespace.Name
   876  
   877  		mutatingWebhook1 := newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort)
   878  		mutatingWebhook1.MatchConditions = skipMeMatchConditions
   879  		created, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
   880  			ObjectMeta: metav1.ObjectMeta{
   881  				Name: f.UniqueName,
   882  			},
   883  			Webhooks: []admissionregistrationv1.MutatingWebhook{
   884  				mutatingWebhook1,
   885  				// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
   886  				newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
   887  			},
   888  		})
   889  		framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", f.UniqueName, namespace)
   890  		defer func() {
   891  			err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, created.Name, metav1.DeleteOptions{})
   892  			framework.ExpectNoError(err, "deleting mutating webhook configuration")
   893  		}()
   894  
   895  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   896  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   897  		ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), f.UniqueName, metav1.DeleteOptions{})
   898  
   899  		// ensure backend is ready before proceeding
   900  		err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
   901  		framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
   902  
   903  		ginkgo.By("create the configmap with a random name")
   904  
   905  		cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f)
   906  		mutatedCM, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   907  		framework.ExpectNoError(err, "creating configMap object")
   908  
   909  		ginkgo.By("verify the configmap is mutated")
   910  		expectedConfigMapData := map[string]string{
   911  			"mutation-start":   "yes",
   912  			"mutation-stage-1": "yes",
   913  		}
   914  		gomega.Expect(reflect.DeepEqual(expectedConfigMapData, mutatedCM.Data)).To(gomega.BeTrue())
   915  
   916  		ginkgo.By("create the configmap with 'skip-me' name")
   917  
   918  		cm = namedToBeMutatedConfigMap("skip-me", f)
   919  		skippedCM, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{})
   920  		framework.ExpectNoError(err, "creating configMap object")
   921  		expectedConfigMapData = map[string]string{
   922  			"mutation-start": "yes",
   923  		}
   924  		gomega.Expect(reflect.DeepEqual(expectedConfigMapData, skippedCM.Data)).To(gomega.BeTrue())
   925  	})
   926  })
   927  
   928  func newValidatingWebhookWithMatchConditions(
   929  	f *framework.Framework,
   930  	servicePort int32,
   931  	certCtx *certContext,
   932  	matchConditions []admissionregistrationv1.MatchCondition,
   933  ) *admissionregistrationv1.ValidatingWebhookConfiguration {
   934  	sideEffects := admissionregistrationv1.SideEffectClassNone
   935  	equivalent := admissionregistrationv1.Equivalent
   936  	return &admissionregistrationv1.ValidatingWebhookConfiguration{
   937  		ObjectMeta: metav1.ObjectMeta{
   938  			Name: f.UniqueName,
   939  		},
   940  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
   941  			{
   942  				Name: "validation-webhook-with-match-conditions.k8s.io",
   943  				Rules: []admissionregistrationv1.RuleWithOperations{{
   944  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
   945  					Rule: admissionregistrationv1.Rule{
   946  						APIGroups:   []string{""},
   947  						APIVersions: []string{"v1"},
   948  						Resources:   []string{"configmaps"},
   949  					},
   950  				}},
   951  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
   952  					Service: &admissionregistrationv1.ServiceReference{
   953  						Namespace: f.Namespace.Name,
   954  						Name:      serviceName,
   955  						Path:      strPtr("/always-deny"),
   956  						Port:      pointer.Int32(servicePort),
   957  					},
   958  					CABundle: certCtx.signingCert,
   959  				},
   960  				SideEffects:             &sideEffects,
   961  				MatchPolicy:             &equivalent,
   962  				AdmissionReviewVersions: []string{"v1"},
   963  				// Scope the webhook to just the markers namespace
   964  				NamespaceSelector: &metav1.LabelSelector{
   965  					MatchLabels: map[string]string{f.UniqueName: "true"},
   966  				},
   967  				MatchConditions: matchConditions,
   968  			},
   969  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
   970  		},
   971  	}
   972  }
   973  
   974  func newMutatingWebhookWithMatchConditions(
   975  	f *framework.Framework,
   976  	servicePort int32,
   977  	certCtx *certContext,
   978  	matchConditions []admissionregistrationv1.MatchCondition,
   979  ) *admissionregistrationv1.MutatingWebhookConfiguration {
   980  	sideEffects := admissionregistrationv1.SideEffectClassNone
   981  	return &admissionregistrationv1.MutatingWebhookConfiguration{
   982  		ObjectMeta: metav1.ObjectMeta{
   983  			Name: f.UniqueName,
   984  		},
   985  		Webhooks: []admissionregistrationv1.MutatingWebhook{
   986  			{
   987  				Name: "adding-configmap-data.k8s.io",
   988  				Rules: []admissionregistrationv1.RuleWithOperations{{
   989  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
   990  					Rule: admissionregistrationv1.Rule{
   991  						APIGroups:   []string{""},
   992  						APIVersions: []string{"v1"},
   993  						Resources:   []string{"configmaps"},
   994  					},
   995  				}},
   996  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
   997  					Service: &admissionregistrationv1.ServiceReference{
   998  						Namespace: f.Namespace.Name,
   999  						Name:      serviceName,
  1000  						Path:      strPtr("/mutating-configmaps"),
  1001  						Port:      pointer.Int32(servicePort),
  1002  					},
  1003  					CABundle: certCtx.signingCert,
  1004  				},
  1005  				SideEffects:             &sideEffects,
  1006  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1007  				// Scope the webhook to just this namespace
  1008  				NamespaceSelector: &metav1.LabelSelector{
  1009  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1010  				},
  1011  				MatchConditions: matchConditions,
  1012  			},
  1013  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1014  		},
  1015  	}
  1016  }
  1017  
  1018  func createAuthReaderRoleBinding(ctx context.Context, f *framework.Framework, namespace string) {
  1019  	ginkgo.By("Create role binding to let webhook read extension-apiserver-authentication")
  1020  	client := f.ClientSet
  1021  	// Create the role binding to allow the webhook read the extension-apiserver-authentication configmap
  1022  	_, err := client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{
  1023  		ObjectMeta: metav1.ObjectMeta{
  1024  			Name: roleBindingName,
  1025  			Annotations: map[string]string{
  1026  				rbacv1.AutoUpdateAnnotationKey: "true",
  1027  			},
  1028  		},
  1029  		RoleRef: rbacv1.RoleRef{
  1030  			APIGroup: "",
  1031  			Kind:     "Role",
  1032  			Name:     "extension-apiserver-authentication-reader",
  1033  		},
  1034  
  1035  		Subjects: []rbacv1.Subject{
  1036  			{
  1037  				Kind:      "ServiceAccount",
  1038  				Name:      "default",
  1039  				Namespace: namespace,
  1040  			},
  1041  		},
  1042  	}, metav1.CreateOptions{})
  1043  	if err != nil && apierrors.IsAlreadyExists(err) {
  1044  		framework.Logf("role binding %s already exists", roleBindingName)
  1045  	} else {
  1046  		framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace)
  1047  	}
  1048  }
  1049  
  1050  func deployWebhookAndService(ctx context.Context, f *framework.Framework, image string, certCtx *certContext, servicePort int32, containerPort int32) {
  1051  	ginkgo.By("Deploying the webhook pod")
  1052  	client := f.ClientSet
  1053  
  1054  	// Creating the secret that contains the webhook's cert.
  1055  	secret := &v1.Secret{
  1056  		ObjectMeta: metav1.ObjectMeta{
  1057  			Name: secretName,
  1058  		},
  1059  		Type: v1.SecretTypeOpaque,
  1060  		Data: map[string][]byte{
  1061  			"tls.crt": certCtx.cert,
  1062  			"tls.key": certCtx.key,
  1063  		},
  1064  	}
  1065  	namespace := f.Namespace.Name
  1066  	_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
  1067  	framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace)
  1068  
  1069  	// Create the deployment of the webhook
  1070  	podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"}
  1071  	replicas := int32(1)
  1072  	mounts := []v1.VolumeMount{
  1073  		{
  1074  			Name:      "webhook-certs",
  1075  			ReadOnly:  true,
  1076  			MountPath: "/webhook.local.config/certificates",
  1077  		},
  1078  	}
  1079  	volumes := []v1.Volume{
  1080  		{
  1081  			Name: "webhook-certs",
  1082  			VolumeSource: v1.VolumeSource{
  1083  				Secret: &v1.SecretVolumeSource{SecretName: secretName},
  1084  			},
  1085  		},
  1086  	}
  1087  	containers := []v1.Container{
  1088  		{
  1089  			Name:         "sample-webhook",
  1090  			VolumeMounts: mounts,
  1091  			Args: []string{
  1092  				"webhook",
  1093  				"--tls-cert-file=/webhook.local.config/certificates/tls.crt",
  1094  				"--tls-private-key-file=/webhook.local.config/certificates/tls.key",
  1095  				"-v=4",
  1096  				// Use a non-default port for containers.
  1097  				fmt.Sprintf("--port=%d", containerPort),
  1098  			},
  1099  			ReadinessProbe: &v1.Probe{
  1100  				ProbeHandler: v1.ProbeHandler{
  1101  					HTTPGet: &v1.HTTPGetAction{
  1102  						Scheme: v1.URISchemeHTTPS,
  1103  						Port:   intstr.FromInt32(containerPort),
  1104  						Path:   "/readyz",
  1105  					},
  1106  				},
  1107  				PeriodSeconds:    1,
  1108  				SuccessThreshold: 1,
  1109  				FailureThreshold: 30,
  1110  			},
  1111  			Image: image,
  1112  			Ports: []v1.ContainerPort{{ContainerPort: containerPort}},
  1113  		},
  1114  	}
  1115  	d := e2edeployment.NewDeployment(deploymentName, replicas, podLabels, "", "", appsv1.RollingUpdateDeploymentStrategyType)
  1116  	d.Spec.Template.Spec.Containers = containers
  1117  	d.Spec.Template.Spec.Volumes = volumes
  1118  
  1119  	deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, d, metav1.CreateOptions{})
  1120  	framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace)
  1121  	ginkgo.By("Wait for the deployment to be ready")
  1122  	err = e2edeployment.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image)
  1123  	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace)
  1124  	err = e2edeployment.WaitForDeploymentComplete(client, deployment)
  1125  	framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace)
  1126  
  1127  	ginkgo.By("Deploying the webhook service")
  1128  
  1129  	serviceLabels := map[string]string{"webhook": "true"}
  1130  	service := &v1.Service{
  1131  		ObjectMeta: metav1.ObjectMeta{
  1132  			Namespace: namespace,
  1133  			Name:      serviceName,
  1134  			Labels:    map[string]string{"test": "webhook"},
  1135  		},
  1136  		Spec: v1.ServiceSpec{
  1137  			Selector: serviceLabels,
  1138  			Ports: []v1.ServicePort{
  1139  				{
  1140  					Protocol:   v1.ProtocolTCP,
  1141  					Port:       servicePort,
  1142  					TargetPort: intstr.FromInt32(containerPort),
  1143  				},
  1144  			},
  1145  		},
  1146  	}
  1147  	_, err = client.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{})
  1148  	framework.ExpectNoError(err, "creating service %s in namespace %s", serviceName, namespace)
  1149  
  1150  	ginkgo.By("Verifying the service has paired with the endpoint")
  1151  	err = framework.WaitForServiceEndpointsNum(ctx, client, namespace, serviceName, 1, 1*time.Second, 30*time.Second)
  1152  	framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceName, 1)
  1153  }
  1154  
  1155  func strPtr(s string) *string { return &s }
  1156  
  1157  func registerWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1158  	client := f.ClientSet
  1159  	ginkgo.By("Registering the webhook via the AdmissionRegistration API")
  1160  
  1161  	namespace := f.Namespace.Name
  1162  	// A webhook that cannot talk to server, with fail-open policy
  1163  	failOpenHook := failingWebhook(namespace, "fail-open.k8s.io", servicePort)
  1164  	policyIgnore := admissionregistrationv1.Ignore
  1165  	failOpenHook.FailurePolicy = &policyIgnore
  1166  	failOpenHook.NamespaceSelector = &metav1.LabelSelector{
  1167  		MatchLabels: map[string]string{f.UniqueName: "true"},
  1168  	}
  1169  
  1170  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1171  		ObjectMeta: metav1.ObjectMeta{
  1172  			Name: configName,
  1173  		},
  1174  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1175  			newDenyPodWebhookFixture(f, certCtx, servicePort),
  1176  			newDenyConfigMapWebhookFixture(f, certCtx, servicePort),
  1177  			// Server cannot talk to this webhook, so it always fails.
  1178  			// Because this webhook is configured fail-open, request should be admitted after the call fails.
  1179  			failOpenHook,
  1180  
  1181  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1182  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1183  		},
  1184  	})
  1185  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1186  
  1187  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1188  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1189  
  1190  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1191  }
  1192  
  1193  func registerWebhookForAttachingPod(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1194  	client := f.ClientSet
  1195  	ginkgo.By("Registering the webhook via the AdmissionRegistration API")
  1196  
  1197  	namespace := f.Namespace.Name
  1198  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1199  
  1200  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1201  		ObjectMeta: metav1.ObjectMeta{
  1202  			Name: configName,
  1203  		},
  1204  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1205  			{
  1206  				Name: "deny-attaching-pod.k8s.io",
  1207  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1208  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Connect},
  1209  					Rule: admissionregistrationv1.Rule{
  1210  						APIGroups:   []string{""},
  1211  						APIVersions: []string{"v1"},
  1212  						Resources:   []string{"pods/attach"},
  1213  					},
  1214  				}},
  1215  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1216  					Service: &admissionregistrationv1.ServiceReference{
  1217  						Namespace: namespace,
  1218  						Name:      serviceName,
  1219  						Path:      strPtr("/pods/attach"),
  1220  						Port:      pointer.Int32(servicePort),
  1221  					},
  1222  					CABundle: certCtx.signingCert,
  1223  				},
  1224  				SideEffects:             &sideEffectsNone,
  1225  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1226  				// Scope the webhook to just this namespace
  1227  				NamespaceSelector: &metav1.LabelSelector{
  1228  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1229  				},
  1230  			},
  1231  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1232  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1233  		},
  1234  	})
  1235  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1236  
  1237  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1238  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1239  
  1240  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1241  }
  1242  
  1243  func registerMutatingWebhookForConfigMap(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1244  	client := f.ClientSet
  1245  	ginkgo.By("Registering the mutating configmap webhook via the AdmissionRegistration API")
  1246  
  1247  	namespace := f.Namespace.Name
  1248  
  1249  	_, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
  1250  		ObjectMeta: metav1.ObjectMeta{
  1251  			Name: configName,
  1252  		},
  1253  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  1254  			newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort),
  1255  			newMutateConfigMapWebhookFixture(f, certCtx, 2, servicePort),
  1256  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1257  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1258  		},
  1259  	})
  1260  	framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace)
  1261  
  1262  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1263  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1264  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1265  }
  1266  
  1267  func testMutatingConfigMapWebhook(ctx context.Context, f *framework.Framework) {
  1268  	ginkgo.By("create a configmap that should be updated by the webhook")
  1269  	client := f.ClientSet
  1270  	configMap := toBeMutatedConfigMap(f)
  1271  	mutatedConfigMap, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configMap, metav1.CreateOptions{})
  1272  	framework.ExpectNoError(err)
  1273  	expectedConfigMapData := map[string]string{
  1274  		"mutation-start":   "yes",
  1275  		"mutation-stage-1": "yes",
  1276  		"mutation-stage-2": "yes",
  1277  	}
  1278  	if !reflect.DeepEqual(expectedConfigMapData, mutatedConfigMap.Data) {
  1279  		framework.Failf("\nexpected %#v\n, got %#v\n", expectedConfigMapData, mutatedConfigMap.Data)
  1280  	}
  1281  }
  1282  
  1283  func registerMutatingWebhookForPod(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1284  	client := f.ClientSet
  1285  	ginkgo.By("Registering the mutating pod webhook via the AdmissionRegistration API")
  1286  
  1287  	namespace := f.Namespace.Name
  1288  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1289  
  1290  	_, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
  1291  		ObjectMeta: metav1.ObjectMeta{
  1292  			Name: configName,
  1293  		},
  1294  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  1295  			{
  1296  				Name: "adding-init-container.k8s.io",
  1297  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1298  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  1299  					Rule: admissionregistrationv1.Rule{
  1300  						APIGroups:   []string{""},
  1301  						APIVersions: []string{"v1"},
  1302  						Resources:   []string{"pods"},
  1303  					},
  1304  				}},
  1305  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1306  					Service: &admissionregistrationv1.ServiceReference{
  1307  						Namespace: namespace,
  1308  						Name:      serviceName,
  1309  						Path:      strPtr("/mutating-pods"),
  1310  						Port:      pointer.Int32(servicePort),
  1311  					},
  1312  					CABundle: certCtx.signingCert,
  1313  				},
  1314  				SideEffects:             &sideEffectsNone,
  1315  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1316  				// Scope the webhook to just this namespace
  1317  				NamespaceSelector: &metav1.LabelSelector{
  1318  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1319  				},
  1320  			},
  1321  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1322  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1323  		},
  1324  	})
  1325  	framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace)
  1326  
  1327  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1328  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1329  
  1330  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1331  }
  1332  
  1333  func testMutatingPodWebhook(ctx context.Context, f *framework.Framework) {
  1334  	ginkgo.By("create a pod that should be updated by the webhook")
  1335  	client := f.ClientSet
  1336  	pod := toBeMutatedPod(f)
  1337  	mutatedPod, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
  1338  	framework.ExpectNoError(err)
  1339  	if len(mutatedPod.Spec.InitContainers) != 1 {
  1340  		framework.Failf("expect pod to have 1 init container, got %#v", mutatedPod.Spec.InitContainers)
  1341  	}
  1342  	if got, expected := mutatedPod.Spec.InitContainers[0].Name, "webhook-added-init-container"; got != expected {
  1343  		framework.Failf("expect the init container name to be %q, got %q", expected, got)
  1344  	}
  1345  	if got, expected := mutatedPod.Spec.InitContainers[0].TerminationMessagePolicy, v1.TerminationMessageReadFile; got != expected {
  1346  		framework.Failf("expect the init terminationMessagePolicy to be default to %q, got %q", expected, got)
  1347  	}
  1348  }
  1349  
  1350  func toBeMutatedPod(f *framework.Framework) *v1.Pod {
  1351  	return &v1.Pod{
  1352  		ObjectMeta: metav1.ObjectMeta{
  1353  			Name: "webhook-to-be-mutated",
  1354  		},
  1355  		Spec: v1.PodSpec{
  1356  			Containers: []v1.Container{
  1357  				{
  1358  					Name:  "example",
  1359  					Image: imageutils.GetPauseImageName(),
  1360  				},
  1361  			},
  1362  		},
  1363  	}
  1364  }
  1365  
  1366  func testWebhook(ctx context.Context, f *framework.Framework) {
  1367  	ginkgo.By("create a pod that should be denied by the webhook")
  1368  	client := f.ClientSet
  1369  	// Creating the pod, the request should be rejected
  1370  	pod := nonCompliantPod(f)
  1371  	_, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
  1372  	gomega.Expect(err).To(gomega.HaveOccurred(), "create pod %s in namespace %s should have been denied by webhook", pod.Name, f.Namespace.Name)
  1373  	expectedErrMsg1 := "the pod contains unwanted container name"
  1374  	if !strings.Contains(err.Error(), expectedErrMsg1) {
  1375  		framework.Failf("expect error contains %q, got %q", expectedErrMsg1, err.Error())
  1376  	}
  1377  	expectedErrMsg2 := "the pod contains unwanted label"
  1378  	if !strings.Contains(err.Error(), expectedErrMsg2) {
  1379  		framework.Failf("expect error contains %q, got %q", expectedErrMsg2, err.Error())
  1380  	}
  1381  
  1382  	ginkgo.By("create a pod that causes the webhook to hang")
  1383  	client = f.ClientSet
  1384  	// Creating the pod, the request should be rejected
  1385  	pod = hangingPod(f)
  1386  	_, err = client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
  1387  	gomega.Expect(err).To(gomega.HaveOccurred(), "create pod %s in namespace %s should have caused webhook to hang", pod.Name, f.Namespace.Name)
  1388  	// ensure the error is webhook-related, not client-side
  1389  	if !strings.Contains(err.Error(), "webhook") {
  1390  		framework.Failf("expect error %q, got %q", "webhook", err.Error())
  1391  	}
  1392  	// ensure the error is a timeout
  1393  	if !strings.Contains(err.Error(), "deadline") {
  1394  		framework.Failf("expect error %q, got %q", "deadline", err.Error())
  1395  	}
  1396  	// ensure the pod was not actually created
  1397  	if _, err := client.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) {
  1398  		framework.Failf("expect notfound error looking for rejected pod, got %v", err)
  1399  	}
  1400  
  1401  	ginkgo.By("create a configmap that should be denied by the webhook")
  1402  	// Creating the configmap, the request should be rejected
  1403  	configmap := nonCompliantConfigMap(f)
  1404  	_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configmap, metav1.CreateOptions{})
  1405  	gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap %s in namespace %s should have been denied by the webhook", configmap.Name, f.Namespace.Name)
  1406  	expectedErrMsg := "the configmap contains unwanted key and value"
  1407  	if !strings.Contains(err.Error(), expectedErrMsg) {
  1408  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  1409  	}
  1410  
  1411  	ginkgo.By("create a configmap that should be admitted by the webhook")
  1412  	// Creating the configmap, the request should be admitted
  1413  	configmap = &v1.ConfigMap{
  1414  		ObjectMeta: metav1.ObjectMeta{
  1415  			Name: allowedConfigMapName,
  1416  		},
  1417  		Data: map[string]string{
  1418  			"admit": "this",
  1419  		},
  1420  	}
  1421  	_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configmap, metav1.CreateOptions{})
  1422  	framework.ExpectNoError(err, "failed to create configmap %s in namespace: %s", configmap.Name, f.Namespace.Name)
  1423  
  1424  	ginkgo.By("update (PUT) the admitted configmap to a non-compliant one should be rejected by the webhook")
  1425  	toNonCompliantFn := func(cm *v1.ConfigMap) {
  1426  		if cm.Data == nil {
  1427  			cm.Data = map[string]string{}
  1428  		}
  1429  		cm.Data["webhook-e2e-test"] = "webhook-disallow"
  1430  	}
  1431  	_, err = updateConfigMap(ctx, client, f.Namespace.Name, allowedConfigMapName, toNonCompliantFn)
  1432  	gomega.Expect(err).To(gomega.HaveOccurred(), "update (PUT) admitted configmap %s in namespace %s to a non-compliant one should be rejected by webhook", allowedConfigMapName, f.Namespace.Name)
  1433  	if !strings.Contains(err.Error(), expectedErrMsg) {
  1434  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  1435  	}
  1436  
  1437  	ginkgo.By("update (PATCH) the admitted configmap to a non-compliant one should be rejected by the webhook")
  1438  	patch := nonCompliantConfigMapPatch()
  1439  	_, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Patch(ctx, allowedConfigMapName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
  1440  	gomega.Expect(err).To(gomega.HaveOccurred(), "update admitted configmap %s in namespace %s by strategic merge patch to a non-compliant one should be rejected by webhook. Patch: %+v", allowedConfigMapName, f.Namespace.Name, patch)
  1441  	if !strings.Contains(err.Error(), expectedErrMsg) {
  1442  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  1443  	}
  1444  
  1445  	ginkgo.By("create a namespace that bypass the webhook")
  1446  	// skipNamespace will be deleted by framework at the end of the test
  1447  	skipNamespace, err := f.CreateNamespace(ctx, skipNamespaceBaseName, map[string]string{
  1448  		skipNamespaceLabelKey: skipNamespaceLabelValue,
  1449  		f.UniqueName:          "true",
  1450  	})
  1451  	framework.ExpectNoError(err, "creating namespace %q", skipNamespaceBaseName)
  1452  	skipNamespaceName := skipNamespace.Name
  1453  
  1454  	ginkgo.By("create a configmap that violates the webhook policy but is in a whitelisted namespace")
  1455  	configmap = nonCompliantConfigMap(f)
  1456  	_, err = client.CoreV1().ConfigMaps(skipNamespaceName).Create(ctx, configmap, metav1.CreateOptions{})
  1457  	framework.ExpectNoError(err, "failed to create configmap %s in namespace: %s", configmap.Name, skipNamespaceName)
  1458  }
  1459  
  1460  func testAttachingPodWebhook(ctx context.Context, f *framework.Framework) {
  1461  	ginkgo.By("create a pod")
  1462  	client := f.ClientSet
  1463  	pod := toBeAttachedPod(f)
  1464  	_, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
  1465  	framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", pod.Name, f.Namespace.Name)
  1466  	err = e2epod.WaitForPodNameRunningInNamespace(ctx, client, pod.Name, f.Namespace.Name)
  1467  	framework.ExpectNoError(err, "error while waiting for pod %s to go to Running phase in namespace: %s", pod.Name, f.Namespace.Name)
  1468  
  1469  	ginkgo.By("'kubectl attach' the pod, should be denied by the webhook")
  1470  	timer := time.NewTimer(30 * time.Second)
  1471  	defer timer.Stop()
  1472  	_, err = e2ekubectl.NewKubectlCommand(f.Namespace.Name, "attach", fmt.Sprintf("--namespace=%v", f.Namespace.Name), pod.Name, "-i", "-c=container1").WithTimeout(timer.C).Exec()
  1473  	gomega.Expect(err).To(gomega.HaveOccurred(), "'kubectl attach' the pod, should be denied by the webhook")
  1474  	if e, a := "attaching to pod 'to-be-attached-pod' is not allowed", err.Error(); !strings.Contains(a, e) {
  1475  		framework.Failf("unexpected 'kubectl attach' error message. expected to contain %q, got %q", e, a)
  1476  	}
  1477  }
  1478  
  1479  // failingWebhook returns a webhook with rule of create configmaps,
  1480  // but with an invalid client config so that server cannot communicate with it
  1481  func failingWebhook(namespace, name string, servicePort int32) admissionregistrationv1.ValidatingWebhook {
  1482  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1483  
  1484  	return admissionregistrationv1.ValidatingWebhook{
  1485  		Name: name,
  1486  		Rules: []admissionregistrationv1.RuleWithOperations{{
  1487  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  1488  			Rule: admissionregistrationv1.Rule{
  1489  				APIGroups:   []string{""},
  1490  				APIVersions: []string{"v1"},
  1491  				Resources:   []string{"configmaps"},
  1492  			},
  1493  		}},
  1494  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1495  			Service: &admissionregistrationv1.ServiceReference{
  1496  				Namespace: namespace,
  1497  				Name:      serviceName,
  1498  				Path:      strPtr("/configmaps"),
  1499  				Port:      pointer.Int32(servicePort),
  1500  			},
  1501  			// Without CA bundle, the call to webhook always fails
  1502  			CABundle: nil,
  1503  		},
  1504  		SideEffects:             &sideEffectsNone,
  1505  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1506  	}
  1507  }
  1508  
  1509  func registerFailClosedWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1510  	ginkgo.By("Registering a webhook that server cannot talk to, with fail closed policy, via the AdmissionRegistration API")
  1511  
  1512  	namespace := f.Namespace.Name
  1513  	// A webhook that cannot talk to server, with fail-closed policy
  1514  	policyFail := admissionregistrationv1.Fail
  1515  	hook := failingWebhook(namespace, "fail-closed.k8s.io", servicePort)
  1516  	hook.FailurePolicy = &policyFail
  1517  	hook.NamespaceSelector = &metav1.LabelSelector{
  1518  		MatchLabels: map[string]string{f.UniqueName: "true"},
  1519  		MatchExpressions: []metav1.LabelSelectorRequirement{
  1520  			{
  1521  				Key:      failNamespaceLabelKey,
  1522  				Operator: metav1.LabelSelectorOpIn,
  1523  				Values:   []string{failNamespaceLabelValue},
  1524  			},
  1525  		},
  1526  	}
  1527  
  1528  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1529  		ObjectMeta: metav1.ObjectMeta{
  1530  			Name: configName,
  1531  		},
  1532  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1533  			// Server cannot talk to this webhook, so it always fails.
  1534  			// Because this webhook is configured fail-closed, request should be rejected after the call fails.
  1535  			hook,
  1536  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1537  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1538  		},
  1539  	})
  1540  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1541  
  1542  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1543  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1544  	ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1545  }
  1546  
  1547  func testFailClosedWebhook(ctx context.Context, f *framework.Framework) {
  1548  	client := f.ClientSet
  1549  	ginkgo.By("create a namespace for the webhook")
  1550  	// failNamespace will be deleted by framework at the end of the test
  1551  	failNamespace, err := f.CreateNamespace(ctx, failNamespaceBaseName, map[string]string{
  1552  		failNamespaceLabelKey: failNamespaceLabelValue,
  1553  		f.UniqueName:          "true",
  1554  	})
  1555  	framework.ExpectNoError(err, "creating namespace %q", failNamespaceBaseName)
  1556  	failNamespaceName := failNamespace.Name
  1557  
  1558  	ginkgo.By("create a configmap should be unconditionally rejected by the webhook")
  1559  	configmap := &v1.ConfigMap{
  1560  		ObjectMeta: metav1.ObjectMeta{
  1561  			Name: "foo",
  1562  		},
  1563  	}
  1564  	_, err = client.CoreV1().ConfigMaps(failNamespaceName).Create(ctx, configmap, metav1.CreateOptions{})
  1565  	gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap in namespace: %s should be unconditionally rejected by the webhook", failNamespaceName)
  1566  	if !apierrors.IsInternalError(err) {
  1567  		framework.Failf("expect an internal error, got %#v", err)
  1568  	}
  1569  }
  1570  
  1571  func registerValidatingWebhookForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1572  	var err error
  1573  	client := f.ClientSet
  1574  	ginkgo.By("Registering a validating webhook on ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects, via the AdmissionRegistration API")
  1575  
  1576  	namespace := f.Namespace.Name
  1577  	failurePolicy := admissionregistrationv1.Fail
  1578  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1579  
  1580  	// This webhook denies all requests to Delete validating webhook configuration and
  1581  	// mutating webhook configuration objects. It should never be called, however, because
  1582  	// dynamic admission webhooks should not be called on requests involving webhook configuration objects.
  1583  	_, err = createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1584  		ObjectMeta: metav1.ObjectMeta{
  1585  			Name: configName,
  1586  		},
  1587  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1588  			{
  1589  				Name: "deny-webhook-configuration-deletions.k8s.io",
  1590  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1591  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Delete},
  1592  					Rule: admissionregistrationv1.Rule{
  1593  						APIGroups:   []string{"admissionregistration.k8s.io"},
  1594  						APIVersions: []string{"*"},
  1595  						Resources: []string{
  1596  							"validatingwebhookconfigurations",
  1597  							"mutatingwebhookconfigurations",
  1598  						},
  1599  					},
  1600  				}},
  1601  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1602  					Service: &admissionregistrationv1.ServiceReference{
  1603  						Namespace: namespace,
  1604  						Name:      serviceName,
  1605  						Path:      strPtr("/always-deny"),
  1606  						Port:      pointer.Int32(servicePort),
  1607  					},
  1608  					CABundle: certCtx.signingCert,
  1609  				},
  1610  				SideEffects:             &sideEffectsNone,
  1611  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1612  				FailurePolicy:           &failurePolicy,
  1613  				// Scope the webhook to just this namespace
  1614  				NamespaceSelector: &metav1.LabelSelector{
  1615  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1616  				},
  1617  			},
  1618  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1619  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1620  		},
  1621  	})
  1622  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1623  
  1624  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1625  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1626  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1627  }
  1628  
  1629  func registerMutatingWebhookForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1630  	var err error
  1631  	client := f.ClientSet
  1632  	ginkgo.By("Registering a mutating webhook on ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects, via the AdmissionRegistration API")
  1633  
  1634  	namespace := f.Namespace.Name
  1635  	failurePolicy := admissionregistrationv1.Fail
  1636  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1637  
  1638  	// This webhook adds a label to all requests create to validating webhook configuration and
  1639  	// mutating webhook configuration objects. It should never be called, however, because
  1640  	// dynamic admission webhooks should not be called on requests involving webhook configuration objects.
  1641  	_, err = createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
  1642  		ObjectMeta: metav1.ObjectMeta{
  1643  			Name: configName,
  1644  		},
  1645  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  1646  			{
  1647  				Name: "add-label-to-webhook-configurations.k8s.io",
  1648  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1649  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  1650  					Rule: admissionregistrationv1.Rule{
  1651  						APIGroups:   []string{"admissionregistration.k8s.io"},
  1652  						APIVersions: []string{"*"},
  1653  						Resources: []string{
  1654  							"validatingwebhookconfigurations",
  1655  							"mutatingwebhookconfigurations",
  1656  						},
  1657  					},
  1658  				}},
  1659  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1660  					Service: &admissionregistrationv1.ServiceReference{
  1661  						Namespace: namespace,
  1662  						Name:      serviceName,
  1663  						Path:      strPtr("/add-label"),
  1664  						Port:      pointer.Int32(servicePort),
  1665  					},
  1666  					CABundle: certCtx.signingCert,
  1667  				},
  1668  				SideEffects:             &sideEffectsNone,
  1669  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1670  				FailurePolicy:           &failurePolicy,
  1671  				// Scope the webhook to just this namespace
  1672  				NamespaceSelector: &metav1.LabelSelector{
  1673  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1674  				},
  1675  			},
  1676  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1677  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1678  		},
  1679  	})
  1680  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1681  
  1682  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1683  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1684  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1685  }
  1686  
  1687  // This test assumes that the deletion-rejecting webhook defined in
  1688  // registerValidatingWebhookForWebhookConfigurations and the webhook-config-mutating
  1689  // webhook defined in registerMutatingWebhookForWebhookConfigurations already exist.
  1690  func testWebhooksForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  1691  	var err error
  1692  	client := f.ClientSet
  1693  	ginkgo.By("Creating a dummy validating-webhook-configuration object")
  1694  
  1695  	namespace := f.Namespace.Name
  1696  	failurePolicy := admissionregistrationv1.Ignore
  1697  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1698  
  1699  	mutatedValidatingWebhookConfiguration, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1700  		ObjectMeta: metav1.ObjectMeta{
  1701  			Name: configName,
  1702  		},
  1703  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1704  			{
  1705  				Name: "dummy-validating-webhook.k8s.io",
  1706  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1707  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  1708  					// This will not match any real resources so this webhook should never be called.
  1709  					Rule: admissionregistrationv1.Rule{
  1710  						APIGroups:   []string{""},
  1711  						APIVersions: []string{"v1"},
  1712  						Resources:   []string{"invalid"},
  1713  					},
  1714  				}},
  1715  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1716  					Service: &admissionregistrationv1.ServiceReference{
  1717  						Namespace: namespace,
  1718  						Name:      serviceName,
  1719  						// This path not recognized by the webhook service,
  1720  						// so the call to this webhook will always fail,
  1721  						// but because the failure policy is ignore, it will
  1722  						// have no effect on admission requests.
  1723  						Path: strPtr(""),
  1724  						Port: pointer.Int32(servicePort),
  1725  					},
  1726  					CABundle: nil,
  1727  				},
  1728  				SideEffects:             &sideEffectsNone,
  1729  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1730  				FailurePolicy:           &failurePolicy,
  1731  				// Scope the webhook to just this namespace
  1732  				NamespaceSelector: &metav1.LabelSelector{
  1733  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1734  				},
  1735  			},
  1736  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1737  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1738  		},
  1739  	})
  1740  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1741  	if mutatedValidatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedValidatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue {
  1742  		framework.Failf("expected %s not to be mutated by mutating webhooks but it was", configName)
  1743  	}
  1744  
  1745  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1746  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1747  
  1748  	ginkgo.By("Deleting the validating-webhook-configuration, which should be possible to remove")
  1749  
  1750  	err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{})
  1751  	framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace)
  1752  
  1753  	ginkgo.By("Creating a dummy mutating-webhook-configuration object")
  1754  
  1755  	mutatedMutatingWebhookConfiguration, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
  1756  		ObjectMeta: metav1.ObjectMeta{
  1757  			Name: configName,
  1758  		},
  1759  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  1760  			{
  1761  				Name: "dummy-mutating-webhook.k8s.io",
  1762  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1763  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  1764  					// This will not match any real resources so this webhook should never be called.
  1765  					Rule: admissionregistrationv1.Rule{
  1766  						APIGroups:   []string{""},
  1767  						APIVersions: []string{"v1"},
  1768  						Resources:   []string{"invalid"},
  1769  					},
  1770  				}},
  1771  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1772  					Service: &admissionregistrationv1.ServiceReference{
  1773  						Namespace: namespace,
  1774  						Name:      serviceName,
  1775  						// This path not recognized by the webhook service,
  1776  						// so the call to this webhook will always fail,
  1777  						// but because the failure policy is ignore, it will
  1778  						// have no effect on admission requests.
  1779  						Path: strPtr(""),
  1780  						Port: pointer.Int32(servicePort),
  1781  					},
  1782  					CABundle: nil,
  1783  				},
  1784  				SideEffects:             &sideEffectsNone,
  1785  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1786  				FailurePolicy:           &failurePolicy,
  1787  				// Scope the webhook to just this namespace
  1788  				NamespaceSelector: &metav1.LabelSelector{
  1789  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1790  				},
  1791  			},
  1792  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1793  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1794  		},
  1795  	})
  1796  	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
  1797  	if mutatedMutatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedMutatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue {
  1798  		framework.Failf("expected %s not to be mutated by mutating webhooks but it was", configName)
  1799  	}
  1800  
  1801  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1802  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1803  
  1804  	ginkgo.By("Deleting the mutating-webhook-configuration, which should be possible to remove")
  1805  
  1806  	err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{})
  1807  	framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace)
  1808  }
  1809  
  1810  func nonCompliantPod(f *framework.Framework) *v1.Pod {
  1811  	return &v1.Pod{
  1812  		ObjectMeta: metav1.ObjectMeta{
  1813  			Name: disallowedPodName,
  1814  			Labels: map[string]string{
  1815  				"webhook-e2e-test": "webhook-disallow",
  1816  			},
  1817  		},
  1818  		Spec: v1.PodSpec{
  1819  			Containers: []v1.Container{
  1820  				{
  1821  					Name:  "webhook-disallow",
  1822  					Image: imageutils.GetPauseImageName(),
  1823  				},
  1824  			},
  1825  		},
  1826  	}
  1827  }
  1828  
  1829  func hangingPod(f *framework.Framework) *v1.Pod {
  1830  	return &v1.Pod{
  1831  		ObjectMeta: metav1.ObjectMeta{
  1832  			Name: hangingPodName,
  1833  			Labels: map[string]string{
  1834  				"webhook-e2e-test": "wait-forever",
  1835  			},
  1836  		},
  1837  		Spec: v1.PodSpec{
  1838  			Containers: []v1.Container{
  1839  				{
  1840  					Name:  "wait-forever",
  1841  					Image: imageutils.GetPauseImageName(),
  1842  				},
  1843  			},
  1844  		},
  1845  	}
  1846  }
  1847  
  1848  func toBeAttachedPod(f *framework.Framework) *v1.Pod {
  1849  	return &v1.Pod{
  1850  		ObjectMeta: metav1.ObjectMeta{
  1851  			Name: toBeAttachedPodName,
  1852  		},
  1853  		Spec: v1.PodSpec{
  1854  			Containers: []v1.Container{
  1855  				{
  1856  					Name:  "container1",
  1857  					Image: imageutils.GetPauseImageName(),
  1858  				},
  1859  			},
  1860  		},
  1861  	}
  1862  }
  1863  
  1864  func nonCompliantConfigMap(f *framework.Framework) *v1.ConfigMap {
  1865  	return namedNonCompliantConfigMap(disallowedConfigMapName, f)
  1866  }
  1867  
  1868  func namedNonCompliantConfigMap(name string, f *framework.Framework) *v1.ConfigMap {
  1869  	return &v1.ConfigMap{
  1870  		ObjectMeta: metav1.ObjectMeta{
  1871  			Name: name,
  1872  		},
  1873  		Data: map[string]string{
  1874  			"webhook-e2e-test": "webhook-disallow",
  1875  		},
  1876  	}
  1877  }
  1878  
  1879  func toBeMutatedConfigMap(f *framework.Framework) *v1.ConfigMap {
  1880  	return namedToBeMutatedConfigMap("to-be-mutated", f)
  1881  }
  1882  
  1883  func namedToBeMutatedConfigMap(name string, f *framework.Framework) *v1.ConfigMap {
  1884  	return &v1.ConfigMap{
  1885  		ObjectMeta: metav1.ObjectMeta{
  1886  			Name: name,
  1887  		},
  1888  		Data: map[string]string{
  1889  			"mutation-start": "yes",
  1890  		},
  1891  	}
  1892  }
  1893  
  1894  func nonCompliantConfigMapPatch() string {
  1895  	return fmt.Sprint(`{"data":{"webhook-e2e-test":"webhook-disallow"}}`)
  1896  }
  1897  
  1898  type updateConfigMapFn func(cm *v1.ConfigMap)
  1899  
  1900  func updateConfigMap(ctx context.Context, c clientset.Interface, ns, name string, update updateConfigMapFn) (*v1.ConfigMap, error) {
  1901  	var cm *v1.ConfigMap
  1902  	pollErr := wait.PollImmediate(2*time.Second, 1*time.Minute, func() (bool, error) {
  1903  		var err error
  1904  		if cm, err = c.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}); err != nil {
  1905  			return false, err
  1906  		}
  1907  		update(cm)
  1908  		if cm, err = c.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{}); err == nil {
  1909  			return true, nil
  1910  		}
  1911  		// Only retry update on conflict
  1912  		if !apierrors.IsConflict(err) {
  1913  			return false, err
  1914  		}
  1915  		return false, nil
  1916  	})
  1917  	return cm, pollErr
  1918  }
  1919  
  1920  type updateCustomResourceFn func(cm *unstructured.Unstructured)
  1921  
  1922  func updateCustomResource(ctx context.Context, c dynamic.ResourceInterface, ns, name string, update updateCustomResourceFn) (*unstructured.Unstructured, error) {
  1923  	var cr *unstructured.Unstructured
  1924  	pollErr := wait.PollImmediate(2*time.Second, 1*time.Minute, func() (bool, error) {
  1925  		var err error
  1926  		if cr, err = c.Get(ctx, name, metav1.GetOptions{}); err != nil {
  1927  			return false, err
  1928  		}
  1929  		update(cr)
  1930  		if cr, err = c.Update(ctx, cr, metav1.UpdateOptions{}); err == nil {
  1931  			return true, nil
  1932  		}
  1933  		// Only retry update on conflict
  1934  		if !apierrors.IsConflict(err) {
  1935  			return false, err
  1936  		}
  1937  		return false, nil
  1938  	})
  1939  	return cr, pollErr
  1940  }
  1941  
  1942  func cleanWebhookTest(ctx context.Context, client clientset.Interface, namespaceName string) {
  1943  	_ = client.CoreV1().Services(namespaceName).Delete(ctx, serviceName, metav1.DeleteOptions{})
  1944  	_ = client.AppsV1().Deployments(namespaceName).Delete(ctx, deploymentName, metav1.DeleteOptions{})
  1945  	_ = client.CoreV1().Secrets(namespaceName).Delete(ctx, secretName, metav1.DeleteOptions{})
  1946  	_ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, roleBindingName, metav1.DeleteOptions{})
  1947  }
  1948  
  1949  func registerWebhookForCustomResource(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, testcrd *crd.TestCrd, servicePort int32) {
  1950  	client := f.ClientSet
  1951  	ginkgo.By("Registering the custom resource webhook via the AdmissionRegistration API")
  1952  
  1953  	namespace := f.Namespace.Name
  1954  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  1955  
  1956  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  1957  		ObjectMeta: metav1.ObjectMeta{
  1958  			Name: configName,
  1959  		},
  1960  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1961  			{
  1962  				Name: "deny-unwanted-custom-resource-data.k8s.io",
  1963  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1964  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete},
  1965  					Rule: admissionregistrationv1.Rule{
  1966  						APIGroups:   []string{testcrd.Crd.Spec.Group},
  1967  						APIVersions: servedAPIVersions(testcrd.Crd),
  1968  						Resources:   []string{testcrd.Crd.Spec.Names.Plural},
  1969  					},
  1970  				}},
  1971  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1972  					Service: &admissionregistrationv1.ServiceReference{
  1973  						Namespace: namespace,
  1974  						Name:      serviceName,
  1975  						Path:      strPtr("/custom-resource"),
  1976  						Port:      pointer.Int32(servicePort),
  1977  					},
  1978  					CABundle: certCtx.signingCert,
  1979  				},
  1980  				SideEffects:             &sideEffectsNone,
  1981  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1982  				// Scope the webhook to just this namespace
  1983  				NamespaceSelector: &metav1.LabelSelector{
  1984  					MatchLabels: map[string]string{f.UniqueName: "true"},
  1985  				},
  1986  			},
  1987  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  1988  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  1989  		},
  1990  	})
  1991  	framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace)
  1992  
  1993  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  1994  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  1995  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  1996  }
  1997  
  1998  func registerMutatingWebhookForCustomResource(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, testcrd *crd.TestCrd, servicePort int32) {
  1999  	client := f.ClientSet
  2000  	ginkgo.By(fmt.Sprintf("Registering the mutating webhook for custom resource %s via the AdmissionRegistration API", testcrd.Crd.Name))
  2001  
  2002  	namespace := f.Namespace.Name
  2003  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2004  
  2005  	_, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{
  2006  		ObjectMeta: metav1.ObjectMeta{
  2007  			Name: configName,
  2008  		},
  2009  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  2010  			{
  2011  				Name: "mutate-custom-resource-data-stage-1.k8s.io",
  2012  				Rules: []admissionregistrationv1.RuleWithOperations{{
  2013  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},
  2014  					Rule: admissionregistrationv1.Rule{
  2015  						APIGroups:   []string{testcrd.Crd.Spec.Group},
  2016  						APIVersions: servedAPIVersions(testcrd.Crd),
  2017  						Resources:   []string{testcrd.Crd.Spec.Names.Plural},
  2018  					},
  2019  				}},
  2020  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2021  					Service: &admissionregistrationv1.ServiceReference{
  2022  						Namespace: namespace,
  2023  						Name:      serviceName,
  2024  						Path:      strPtr("/mutating-custom-resource"),
  2025  						Port:      pointer.Int32(servicePort),
  2026  					},
  2027  					CABundle: certCtx.signingCert,
  2028  				},
  2029  				SideEffects:             &sideEffectsNone,
  2030  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2031  				// Scope the webhook to just this namespace
  2032  				NamespaceSelector: &metav1.LabelSelector{
  2033  					MatchLabels: map[string]string{f.UniqueName: "true"},
  2034  				},
  2035  			},
  2036  			{
  2037  				Name: "mutate-custom-resource-data-stage-2.k8s.io",
  2038  				Rules: []admissionregistrationv1.RuleWithOperations{{
  2039  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2040  					Rule: admissionregistrationv1.Rule{
  2041  						APIGroups:   []string{testcrd.Crd.Spec.Group},
  2042  						APIVersions: servedAPIVersions(testcrd.Crd),
  2043  						Resources:   []string{testcrd.Crd.Spec.Names.Plural},
  2044  					},
  2045  				}},
  2046  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2047  					Service: &admissionregistrationv1.ServiceReference{
  2048  						Namespace: namespace,
  2049  						Name:      serviceName,
  2050  						Path:      strPtr("/mutating-custom-resource"),
  2051  						Port:      pointer.Int32(servicePort),
  2052  					},
  2053  					CABundle: certCtx.signingCert,
  2054  				},
  2055  				SideEffects:             &sideEffectsNone,
  2056  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2057  				// Scope the webhook to just this namespace
  2058  				NamespaceSelector: &metav1.LabelSelector{
  2059  					MatchLabels: map[string]string{f.UniqueName: "true"},
  2060  				},
  2061  			},
  2062  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  2063  			newMutatingIsReadyWebhookFixture(f, certCtx, servicePort),
  2064  		},
  2065  	})
  2066  	framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace)
  2067  
  2068  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  2069  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  2070  
  2071  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  2072  }
  2073  
  2074  func testCustomResourceWebhook(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
  2075  	ginkgo.By("Creating a custom resource that should be denied by the webhook")
  2076  	crInstanceName := "cr-instance-1"
  2077  	crInstance := &unstructured.Unstructured{
  2078  		Object: map[string]interface{}{
  2079  			"kind":       crd.Spec.Names.Kind,
  2080  			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
  2081  			"metadata": map[string]interface{}{
  2082  				"name":      crInstanceName,
  2083  				"namespace": f.Namespace.Name,
  2084  			},
  2085  			"data": map[string]interface{}{
  2086  				"webhook-e2e-test": "webhook-disallow",
  2087  			},
  2088  		},
  2089  	}
  2090  	_, err := customResourceClient.Create(ctx, crInstance, metav1.CreateOptions{})
  2091  	gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource %s in namespace %s should be denied by webhook", crInstanceName, f.Namespace.Name)
  2092  	expectedErrMsg := "the custom resource contains unwanted data"
  2093  	if !strings.Contains(err.Error(), expectedErrMsg) {
  2094  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  2095  	}
  2096  }
  2097  
  2098  func testBlockingCustomResourceUpdateDeletion(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) {
  2099  	ginkgo.By("Creating a custom resource whose deletion would be denied by the webhook")
  2100  	crInstanceName := "cr-instance-2"
  2101  	crInstance := &unstructured.Unstructured{
  2102  		Object: map[string]interface{}{
  2103  			"kind":       crd.Spec.Names.Kind,
  2104  			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
  2105  			"metadata": map[string]interface{}{
  2106  				"name":      crInstanceName,
  2107  				"namespace": f.Namespace.Name,
  2108  			},
  2109  			"data": map[string]interface{}{
  2110  				"webhook-e2e-test": "webhook-nondeletable",
  2111  			},
  2112  		},
  2113  	}
  2114  	_, err := customResourceClient.Create(ctx, crInstance, metav1.CreateOptions{})
  2115  	framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name)
  2116  
  2117  	ginkgo.By("Updating the custom resource with disallowed data should be denied")
  2118  	toNonCompliantFn := func(cr *unstructured.Unstructured) {
  2119  		if _, ok := cr.Object["data"]; !ok {
  2120  			cr.Object["data"] = map[string]interface{}{}
  2121  		}
  2122  		data := cr.Object["data"].(map[string]interface{})
  2123  		data["webhook-e2e-test"] = "webhook-disallow"
  2124  	}
  2125  	_, err = updateCustomResource(ctx, customResourceClient, f.Namespace.Name, crInstanceName, toNonCompliantFn)
  2126  	gomega.Expect(err).To(gomega.HaveOccurred(), "updating custom resource %s in namespace: %s should be denied", crInstanceName, f.Namespace.Name)
  2127  
  2128  	expectedErrMsg := "the custom resource contains unwanted data"
  2129  	if !strings.Contains(err.Error(), expectedErrMsg) {
  2130  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  2131  	}
  2132  
  2133  	ginkgo.By("Deleting the custom resource should be denied")
  2134  	err = customResourceClient.Delete(ctx, crInstanceName, metav1.DeleteOptions{})
  2135  	gomega.Expect(err).To(gomega.HaveOccurred(), "deleting custom resource %s in namespace: %s should be denied", crInstanceName, f.Namespace.Name)
  2136  	expectedErrMsg1 := "the custom resource cannot be deleted because it contains unwanted key and value"
  2137  	if !strings.Contains(err.Error(), expectedErrMsg1) {
  2138  		framework.Failf("expect error contains %q, got %q", expectedErrMsg1, err.Error())
  2139  	}
  2140  
  2141  	ginkgo.By("Remove the offending key and value from the custom resource data")
  2142  	toCompliantFn := func(cr *unstructured.Unstructured) {
  2143  		if _, ok := cr.Object["data"]; !ok {
  2144  			cr.Object["data"] = map[string]interface{}{}
  2145  		}
  2146  		data := cr.Object["data"].(map[string]interface{})
  2147  		data["webhook-e2e-test"] = "webhook-allow"
  2148  	}
  2149  	_, err = updateCustomResource(ctx, customResourceClient, f.Namespace.Name, crInstanceName, toCompliantFn)
  2150  	framework.ExpectNoError(err, "failed to update custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name)
  2151  
  2152  	ginkgo.By("Deleting the updated custom resource should be successful")
  2153  	err = customResourceClient.Delete(ctx, crInstanceName, metav1.DeleteOptions{})
  2154  	framework.ExpectNoError(err, "failed to delete custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name)
  2155  
  2156  }
  2157  
  2158  func testMutatingCustomResourceWebhook(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) {
  2159  	ginkgo.By("Creating a custom resource that should be mutated by the webhook")
  2160  	crName := "cr-instance-1"
  2161  	cr := &unstructured.Unstructured{
  2162  		Object: map[string]interface{}{
  2163  			"kind":       crd.Spec.Names.Kind,
  2164  			"apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name,
  2165  			"metadata": map[string]interface{}{
  2166  				"name":      crName,
  2167  				"namespace": f.Namespace.Name,
  2168  			},
  2169  			"data": map[string]interface{}{
  2170  				"mutation-start": "yes",
  2171  			},
  2172  		},
  2173  	}
  2174  	mutatedCR, err := customResourceClient.Create(ctx, cr, metav1.CreateOptions{})
  2175  	framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name)
  2176  	expectedCRData := map[string]interface{}{
  2177  		"mutation-start":   "yes",
  2178  		"mutation-stage-1": "yes",
  2179  	}
  2180  	if !prune {
  2181  		expectedCRData["mutation-stage-2"] = "yes"
  2182  	}
  2183  	if !reflect.DeepEqual(expectedCRData, mutatedCR.Object["data"]) {
  2184  		framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"])
  2185  	}
  2186  }
  2187  
  2188  func testMultiVersionCustomResourceWebhook(ctx context.Context, f *framework.Framework, testcrd *crd.TestCrd) {
  2189  	customResourceClient := testcrd.DynamicClients["v1"]
  2190  	ginkgo.By("Creating a custom resource while v1 is storage version")
  2191  	crName := "cr-instance-1"
  2192  	cr := &unstructured.Unstructured{
  2193  		Object: map[string]interface{}{
  2194  			"kind":       testcrd.Crd.Spec.Names.Kind,
  2195  			"apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name,
  2196  			"metadata": map[string]interface{}{
  2197  				"name":      crName,
  2198  				"namespace": f.Namespace.Name,
  2199  			},
  2200  			"data": map[string]interface{}{
  2201  				"mutation-start": "yes",
  2202  			},
  2203  		},
  2204  	}
  2205  	_, err := customResourceClient.Create(ctx, cr, metav1.CreateOptions{})
  2206  	framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name)
  2207  
  2208  	ginkgo.By("Patching Custom Resource Definition to set v2 as storage")
  2209  	apiVersionWithV2StoragePatch := `{
  2210  		"spec": {
  2211  		  "versions": [
  2212  		    {
  2213  			  "name": "v1",
  2214  			  "storage": false,
  2215  			  "served": true,
  2216  			  "schema": {
  2217  			    "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"}
  2218  			  }
  2219              },
  2220  		    {
  2221  			  "name": "v2",
  2222  			  "storage": true,
  2223  			  "served": true,
  2224  			  "schema": {
  2225  			    "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"}
  2226  			  }
  2227              }
  2228            ]
  2229         }
  2230      }`
  2231  	_, err = testcrd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch), metav1.PatchOptions{})
  2232  	framework.ExpectNoError(err, "failed to patch custom resource definition %s in namespace: %s", testcrd.Crd.Name, f.Namespace.Name)
  2233  
  2234  	ginkgo.By("Patching the custom resource while v2 is storage version")
  2235  	crDummyPatch := fmt.Sprint(`[{ "op": "add", "path": "/dummy", "value": "test" }]`)
  2236  	mutatedCR, err := testcrd.DynamicClients["v2"].Patch(ctx, crName, types.JSONPatchType, []byte(crDummyPatch), metav1.PatchOptions{})
  2237  	framework.ExpectNoError(err, "failed to patch custom resource %s in namespace: %s", crName, f.Namespace.Name)
  2238  	expectedCRData := map[string]interface{}{
  2239  		"mutation-start":   "yes",
  2240  		"mutation-stage-1": "yes",
  2241  		"mutation-stage-2": "yes",
  2242  	}
  2243  	if !reflect.DeepEqual(expectedCRData, mutatedCR.Object["data"]) {
  2244  		framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"])
  2245  	}
  2246  	if !reflect.DeepEqual("test", mutatedCR.Object["dummy"]) {
  2247  		framework.Failf("\nexpected %#v\n, got %#v\n", "test", mutatedCR.Object["dummy"])
  2248  	}
  2249  }
  2250  
  2251  func registerValidatingWebhookForCRD(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) {
  2252  	client := f.ClientSet
  2253  	ginkgo.By("Registering the crd webhook via the AdmissionRegistration API")
  2254  
  2255  	namespace := f.Namespace.Name
  2256  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2257  
  2258  	// This webhook will deny the creation of CustomResourceDefinitions which have the
  2259  	// label "webhook-e2e-test":"webhook-disallow"
  2260  	// NOTE: Because tests are run in parallel and in an unpredictable order, it is critical
  2261  	// that no other test attempts to create CRD with that label.
  2262  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  2263  		ObjectMeta: metav1.ObjectMeta{
  2264  			Name: configName,
  2265  		},
  2266  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  2267  			{
  2268  				Name: "deny-crd-with-unwanted-label.k8s.io",
  2269  				Rules: []admissionregistrationv1.RuleWithOperations{{
  2270  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2271  					Rule: admissionregistrationv1.Rule{
  2272  						APIGroups:   []string{"apiextensions.k8s.io"},
  2273  						APIVersions: []string{"*"},
  2274  						Resources:   []string{"customresourcedefinitions"},
  2275  					},
  2276  				}},
  2277  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2278  					Service: &admissionregistrationv1.ServiceReference{
  2279  						Namespace: namespace,
  2280  						Name:      serviceName,
  2281  						Path:      strPtr("/crd"),
  2282  						Port:      pointer.Int32(servicePort),
  2283  					},
  2284  					CABundle: certCtx.signingCert,
  2285  				},
  2286  				SideEffects:             &sideEffectsNone,
  2287  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2288  				// Scope the webhook to just this test
  2289  				ObjectSelector: &metav1.LabelSelector{
  2290  					MatchLabels: map[string]string{f.UniqueName: "true"},
  2291  				},
  2292  			},
  2293  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  2294  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  2295  		},
  2296  	})
  2297  	framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace)
  2298  
  2299  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  2300  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  2301  	ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{})
  2302  }
  2303  
  2304  func testCRDDenyWebhook(ctx context.Context, f *framework.Framework) {
  2305  	ginkgo.By("Creating a custom resource definition that should be denied by the webhook")
  2306  	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
  2307  	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
  2308  	group := fmt.Sprintf("%s.example.com", f.BaseName)
  2309  	apiVersions := []apiextensionsv1.CustomResourceDefinitionVersion{
  2310  		{
  2311  			Name:    "v1",
  2312  			Served:  true,
  2313  			Storage: true,
  2314  			Schema: &apiextensionsv1.CustomResourceValidation{
  2315  				OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  2316  					XPreserveUnknownFields: pointer.BoolPtr(true),
  2317  					Type:                   "object",
  2318  				},
  2319  			},
  2320  		},
  2321  	}
  2322  
  2323  	// Creating a custom resource definition for use by assorted tests.
  2324  	config, err := framework.LoadConfig()
  2325  	if err != nil {
  2326  		framework.Failf("failed to load config: %v", err)
  2327  		return
  2328  	}
  2329  	apiExtensionClient, err := crdclientset.NewForConfig(config)
  2330  	if err != nil {
  2331  		framework.Failf("failed to initialize apiExtensionClient: %v", err)
  2332  		return
  2333  	}
  2334  	crd := &apiextensionsv1.CustomResourceDefinition{
  2335  		ObjectMeta: metav1.ObjectMeta{
  2336  			Name: name + "s." + group,
  2337  			Labels: map[string]string{
  2338  				// this label ensures our object is routed to this test's webhook
  2339  				f.UniqueName: "true",
  2340  				// this is the label the webhook disallows
  2341  				"webhook-e2e-test": "webhook-disallow",
  2342  			},
  2343  		},
  2344  		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
  2345  			Group:    group,
  2346  			Versions: apiVersions,
  2347  			Names: apiextensionsv1.CustomResourceDefinitionNames{
  2348  				Singular: name,
  2349  				Kind:     kind,
  2350  				ListKind: kind + "List",
  2351  				Plural:   name + "s",
  2352  			},
  2353  			Scope: apiextensionsv1.NamespaceScoped,
  2354  		},
  2355  	}
  2356  
  2357  	// create CRD
  2358  	_, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{})
  2359  	gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource definition %s should be denied by webhook", crd.Name)
  2360  	expectedErrMsg := "the crd contains unwanted label"
  2361  	if !strings.Contains(err.Error(), expectedErrMsg) {
  2362  		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
  2363  	}
  2364  }
  2365  
  2366  func labelNamespace(ctx context.Context, f *framework.Framework, namespace string) {
  2367  	client := f.ClientSet
  2368  
  2369  	// Add a unique label to the namespace
  2370  	nsPatch, err := json.Marshal(map[string]interface{}{
  2371  		"metadata": map[string]interface{}{
  2372  			"labels": map[string]string{f.UniqueName: "true"},
  2373  		},
  2374  	})
  2375  	framework.ExpectNoError(err, "error marshaling namespace %s", namespace)
  2376  	_, err = client.CoreV1().Namespaces().Patch(ctx, namespace, types.StrategicMergePatchType, nsPatch, metav1.PatchOptions{})
  2377  	framework.ExpectNoError(err, "error labeling namespace %s", namespace)
  2378  }
  2379  
  2380  func registerSlowWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, policy *admissionregistrationv1.FailurePolicyType, timeout *int32, servicePort int32) func(ctx context.Context) {
  2381  	client := f.ClientSet
  2382  	ginkgo.By("Registering slow webhook via the AdmissionRegistration API")
  2383  
  2384  	namespace := f.Namespace.Name
  2385  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2386  
  2387  	_, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{
  2388  		ObjectMeta: metav1.ObjectMeta{
  2389  			Name: configName,
  2390  		},
  2391  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  2392  			{
  2393  				Name: "allow-configmap-with-delay-webhook.k8s.io",
  2394  				Rules: []admissionregistrationv1.RuleWithOperations{{
  2395  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2396  					Rule: admissionregistrationv1.Rule{
  2397  						APIGroups:   []string{""},
  2398  						APIVersions: []string{"v1"},
  2399  						Resources:   []string{"configmaps"},
  2400  					},
  2401  				}},
  2402  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2403  					Service: &admissionregistrationv1.ServiceReference{
  2404  						Namespace: namespace,
  2405  						Name:      serviceName,
  2406  						Path:      strPtr("/always-allow-delay-5s"),
  2407  						Port:      pointer.Int32(servicePort),
  2408  					},
  2409  					CABundle: certCtx.signingCert,
  2410  				},
  2411  				// Scope the webhook to just this namespace
  2412  				NamespaceSelector: &metav1.LabelSelector{
  2413  					MatchLabels: map[string]string{f.UniqueName: "true"},
  2414  				},
  2415  				FailurePolicy:           policy,
  2416  				TimeoutSeconds:          timeout,
  2417  				SideEffects:             &sideEffectsNone,
  2418  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2419  			},
  2420  			// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
  2421  			newValidatingIsReadyWebhookFixture(f, certCtx, servicePort),
  2422  		},
  2423  	})
  2424  	framework.ExpectNoError(err, "registering slow webhook config %s with namespace %s", configName, namespace)
  2425  
  2426  	err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName)
  2427  	framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
  2428  
  2429  	cleanup := func(ctx context.Context) {
  2430  		err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{})
  2431  		if !apierrors.IsNotFound(err) {
  2432  			framework.ExpectNoError(err)
  2433  		}
  2434  	}
  2435  
  2436  	// We clean up ourselves if the caller doesn't get to it, but we also
  2437  	// give the caller a chance to do it in the middle of the test.
  2438  	ginkgo.DeferCleanup(cleanup)
  2439  	return cleanup
  2440  }
  2441  
  2442  func testSlowWebhookTimeoutFailEarly(ctx context.Context, f *framework.Framework) {
  2443  	ginkgo.By("Request fails when timeout (1s) is shorter than slow webhook latency (5s)")
  2444  	client := f.ClientSet
  2445  	name := "e2e-test-slow-webhook-configmap"
  2446  	_, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{})
  2447  	gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap in namespace %s should have timed-out reaching slow webhook", f.Namespace.Name)
  2448  	// http timeout message: context deadline exceeded
  2449  	// dial timeout message: dial tcp {address}: i/o timeout
  2450  	isTimeoutError := strings.Contains(err.Error(), `context deadline exceeded`) || strings.Contains(err.Error(), `timeout`)
  2451  	isErrorQueryingWebhook := strings.Contains(err.Error(), `/always-allow-delay-5s?timeout=1s`)
  2452  	if !isTimeoutError || !isErrorQueryingWebhook {
  2453  		framework.Failf("expect an HTTP/dial timeout error querying the slow webhook, got: %q", err.Error())
  2454  	}
  2455  }
  2456  
  2457  func testSlowWebhookTimeoutNoError(ctx context.Context, f *framework.Framework) {
  2458  	client := f.ClientSet
  2459  	name := "e2e-test-slow-webhook-configmap"
  2460  	_, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{})
  2461  	framework.ExpectNoError(err)
  2462  	err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, name, metav1.DeleteOptions{})
  2463  	framework.ExpectNoError(err)
  2464  }
  2465  
  2466  // createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically
  2467  // for the admission webhook calling test.
  2468  func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework, opts ...crd.Option) (*crd.TestCrd, error) {
  2469  	group := fmt.Sprintf("%s.example.com", f.BaseName)
  2470  	return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1.CustomResourceDefinition) {
  2471  		crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{
  2472  			{
  2473  				Name:    "v1",
  2474  				Served:  true,
  2475  				Storage: true,
  2476  				Schema: &apiextensionsv1.CustomResourceValidation{
  2477  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  2478  						XPreserveUnknownFields: pointer.BoolPtr(true),
  2479  						Type:                   "object",
  2480  					},
  2481  				},
  2482  			},
  2483  			{
  2484  				Name:    "v2",
  2485  				Served:  true,
  2486  				Storage: false,
  2487  				Schema: &apiextensionsv1.CustomResourceValidation{
  2488  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  2489  						XPreserveUnknownFields: pointer.BoolPtr(true),
  2490  						Type:                   "object",
  2491  					},
  2492  				},
  2493  			},
  2494  		}
  2495  	}}, opts...)...)
  2496  }
  2497  
  2498  // servedAPIVersions returns the API versions served by the CRD.
  2499  func servedAPIVersions(crd *apiextensionsv1.CustomResourceDefinition) []string {
  2500  	ret := []string{}
  2501  	for _, v := range crd.Spec.Versions {
  2502  		if v.Served {
  2503  			ret = append(ret, v.Name)
  2504  		}
  2505  	}
  2506  	return ret
  2507  }
  2508  
  2509  // createValidatingWebhookConfiguration ensures the webhook config scopes object or namespace selection
  2510  // to avoid interfering with other tests, then creates the config.
  2511  func createValidatingWebhookConfiguration(ctx context.Context, f *framework.Framework, config *admissionregistrationv1.ValidatingWebhookConfiguration) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
  2512  	for _, webhook := range config.Webhooks {
  2513  		if webhook.NamespaceSelector != nil && webhook.NamespaceSelector.MatchLabels[f.UniqueName] == "true" {
  2514  			continue
  2515  		}
  2516  		if webhook.ObjectSelector != nil && webhook.ObjectSelector.MatchLabels[f.UniqueName] == "true" {
  2517  			continue
  2518  		}
  2519  		framework.Failf(`webhook %s in config %s has no namespace or object selector with %s="true", and can interfere with other tests`, webhook.Name, config.Name, f.UniqueName)
  2520  	}
  2521  	return f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, config, metav1.CreateOptions{})
  2522  }
  2523  
  2524  // createMutatingWebhookConfiguration ensures the webhook config scopes object or namespace selection
  2525  // to avoid interfering with other tests, then creates the config.
  2526  func createMutatingWebhookConfiguration(ctx context.Context, f *framework.Framework, config *admissionregistrationv1.MutatingWebhookConfiguration) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
  2527  	for _, webhook := range config.Webhooks {
  2528  		if webhook.NamespaceSelector != nil && webhook.NamespaceSelector.MatchLabels[f.UniqueName] == "true" {
  2529  			continue
  2530  		}
  2531  		if webhook.ObjectSelector != nil && webhook.ObjectSelector.MatchLabels[f.UniqueName] == "true" {
  2532  			continue
  2533  		}
  2534  		framework.Failf(`webhook %s in config %s has no namespace or object selector with %s="true", and can interfere with other tests`, webhook.Name, config.Name, f.UniqueName)
  2535  	}
  2536  	return f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, config, metav1.CreateOptions{})
  2537  }
  2538  
  2539  func newDenyPodWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook {
  2540  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2541  	return admissionregistrationv1.ValidatingWebhook{
  2542  		Name: "deny-unwanted-pod-container-name-and-label.k8s.io",
  2543  		Rules: []admissionregistrationv1.RuleWithOperations{{
  2544  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2545  			Rule: admissionregistrationv1.Rule{
  2546  				APIGroups:   []string{""},
  2547  				APIVersions: []string{"v1"},
  2548  				Resources:   []string{"pods"},
  2549  			},
  2550  		}},
  2551  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2552  			Service: &admissionregistrationv1.ServiceReference{
  2553  				Namespace: f.Namespace.Name,
  2554  				Name:      serviceName,
  2555  				Path:      strPtr("/pods"),
  2556  				Port:      pointer.Int32(servicePort),
  2557  			},
  2558  			CABundle: certCtx.signingCert,
  2559  		},
  2560  		SideEffects:             &sideEffectsNone,
  2561  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2562  		// Scope the webhook to just this namespace
  2563  		NamespaceSelector: &metav1.LabelSelector{
  2564  			MatchLabels: map[string]string{f.UniqueName: "true"},
  2565  		},
  2566  	}
  2567  }
  2568  
  2569  func newDenyConfigMapWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook {
  2570  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2571  	return admissionregistrationv1.ValidatingWebhook{
  2572  		Name: "deny-unwanted-configmap-data.k8s.io",
  2573  		Rules: []admissionregistrationv1.RuleWithOperations{{
  2574  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete},
  2575  			Rule: admissionregistrationv1.Rule{
  2576  				APIGroups:   []string{""},
  2577  				APIVersions: []string{"v1"},
  2578  				Resources:   []string{"configmaps"},
  2579  			},
  2580  		}},
  2581  		// The webhook skips the namespace that has label "skip-webhook-admission":"yes"
  2582  		NamespaceSelector: &metav1.LabelSelector{
  2583  			MatchLabels: map[string]string{f.UniqueName: "true"},
  2584  			MatchExpressions: []metav1.LabelSelectorRequirement{
  2585  				{
  2586  					Key:      skipNamespaceLabelKey,
  2587  					Operator: metav1.LabelSelectorOpNotIn,
  2588  					Values:   []string{skipNamespaceLabelValue},
  2589  				},
  2590  			},
  2591  		},
  2592  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2593  			Service: &admissionregistrationv1.ServiceReference{
  2594  				Namespace: f.Namespace.Name,
  2595  				Name:      serviceName,
  2596  				Path:      strPtr("/configmaps"),
  2597  				Port:      pointer.Int32(servicePort),
  2598  			},
  2599  			CABundle: certCtx.signingCert,
  2600  		},
  2601  		SideEffects:             &sideEffectsNone,
  2602  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2603  	}
  2604  }
  2605  
  2606  func newMutateConfigMapWebhookFixture(f *framework.Framework, certCtx *certContext, stage int, servicePort int32) admissionregistrationv1.MutatingWebhook {
  2607  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2608  	return admissionregistrationv1.MutatingWebhook{
  2609  		Name: fmt.Sprintf("adding-configmap-data-stage-%d.k8s.io", stage),
  2610  		Rules: []admissionregistrationv1.RuleWithOperations{{
  2611  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2612  			Rule: admissionregistrationv1.Rule{
  2613  				APIGroups:   []string{""},
  2614  				APIVersions: []string{"v1"},
  2615  				Resources:   []string{"configmaps"},
  2616  			},
  2617  		}},
  2618  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2619  			Service: &admissionregistrationv1.ServiceReference{
  2620  				Namespace: f.Namespace.Name,
  2621  				Name:      serviceName,
  2622  				Path:      strPtr("/mutating-configmaps"),
  2623  				Port:      pointer.Int32(servicePort),
  2624  			},
  2625  			CABundle: certCtx.signingCert,
  2626  		},
  2627  		SideEffects:             &sideEffectsNone,
  2628  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2629  		// Scope the webhook to just this namespace
  2630  		NamespaceSelector: &metav1.LabelSelector{
  2631  			MatchLabels: map[string]string{f.UniqueName: "true"},
  2632  		},
  2633  	}
  2634  }
  2635  
  2636  // createWebhookConfigurationReadyNamespace creates a separate namespace for webhook configuration ready markers to
  2637  // prevent cross-talk with webhook configurations being tested. It returns the name of the created namespace.
  2638  func createWebhookConfigurationReadyNamespace(ctx context.Context, f *framework.Framework) string {
  2639  	baseName := f.BaseName + "-markers"
  2640  	// the framework will taker care of deleting the namespace
  2641  	ns, err := f.CreateNamespace(ctx, baseName, map[string]string{
  2642  		f.UniqueName + "-markers": "true",
  2643  	})
  2644  	framework.ExpectNoError(err, "creating namespace for webhook configuration ready markers")
  2645  	return ns.Name
  2646  }
  2647  
  2648  // waitWebhookConfigurationReady sends "marker" requests until a webhook configuration is ready.
  2649  // A webhook created with newValidatingIsReadyWebhookFixture or newMutatingIsReadyWebhookFixture should first be added to
  2650  // the webhook configuration.
  2651  func waitWebhookConfigurationReady(ctx context.Context, f *framework.Framework, markersNamespaceName string) error {
  2652  	cmClient := f.ClientSet.CoreV1().ConfigMaps(markersNamespaceName)
  2653  	return wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
  2654  		marker := &v1.ConfigMap{
  2655  			ObjectMeta: metav1.ObjectMeta{
  2656  				Name: string(uuid.NewUUID()),
  2657  				Labels: map[string]string{
  2658  					f.UniqueName: "true",
  2659  				},
  2660  			},
  2661  		}
  2662  		_, err := cmClient.Create(ctx, marker, metav1.CreateOptions{})
  2663  		if err != nil {
  2664  			// The always-deny webhook does not provide a reason, so check for the error string we expect
  2665  			if strings.Contains(err.Error(), "denied") {
  2666  				return true, nil
  2667  			}
  2668  			return false, err
  2669  		}
  2670  		// best effort cleanup of markers that are no longer needed
  2671  		_ = cmClient.Delete(ctx, marker.GetName(), metav1.DeleteOptions{})
  2672  		framework.Logf("Waiting for webhook configuration to be ready...")
  2673  		return false, nil
  2674  	})
  2675  }
  2676  
  2677  // newValidatingIsReadyWebhookFixture creates a validating webhook that can be added to a webhook configuration and then probed
  2678  // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
  2679  func newValidatingIsReadyWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook {
  2680  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2681  	failOpen := admissionregistrationv1.Ignore
  2682  	return admissionregistrationv1.ValidatingWebhook{
  2683  		Name: "validating-is-webhook-configuration-ready.k8s.io",
  2684  		Rules: []admissionregistrationv1.RuleWithOperations{{
  2685  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2686  			Rule: admissionregistrationv1.Rule{
  2687  				APIGroups:   []string{""},
  2688  				APIVersions: []string{"v1"},
  2689  				Resources:   []string{"configmaps"},
  2690  			},
  2691  		}},
  2692  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2693  			Service: &admissionregistrationv1.ServiceReference{
  2694  				Namespace: f.Namespace.Name,
  2695  				Name:      serviceName,
  2696  				Path:      strPtr("/always-deny"),
  2697  				Port:      pointer.Int32(servicePort),
  2698  			},
  2699  			CABundle: certCtx.signingCert,
  2700  		},
  2701  		// network failures while the service network routing is being set up should be ignored by the marker
  2702  		FailurePolicy:           &failOpen,
  2703  		SideEffects:             &sideEffectsNone,
  2704  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2705  		// Scope the webhook to just the markers namespace
  2706  		NamespaceSelector: &metav1.LabelSelector{
  2707  			MatchLabels: map[string]string{f.UniqueName + "-markers": "true"},
  2708  		},
  2709  		// appease createValidatingWebhookConfiguration isolation requirements
  2710  		ObjectSelector: &metav1.LabelSelector{
  2711  			MatchLabels: map[string]string{f.UniqueName: "true"},
  2712  		},
  2713  	}
  2714  }
  2715  
  2716  // newMutatingIsReadyWebhookFixture creates a mutating webhook that can be added to a webhook configuration and then probed
  2717  // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
  2718  func newMutatingIsReadyWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook {
  2719  	sideEffectsNone := admissionregistrationv1.SideEffectClassNone
  2720  	failOpen := admissionregistrationv1.Ignore
  2721  	return admissionregistrationv1.MutatingWebhook{
  2722  		Name: "mutating-is-webhook-configuration-ready.k8s.io",
  2723  		Rules: []admissionregistrationv1.RuleWithOperations{{
  2724  			Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
  2725  			Rule: admissionregistrationv1.Rule{
  2726  				APIGroups:   []string{""},
  2727  				APIVersions: []string{"v1"},
  2728  				Resources:   []string{"configmaps"},
  2729  			},
  2730  		}},
  2731  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  2732  			Service: &admissionregistrationv1.ServiceReference{
  2733  				Namespace: f.Namespace.Name,
  2734  				Name:      serviceName,
  2735  				Path:      strPtr("/always-deny"),
  2736  				Port:      pointer.Int32(servicePort),
  2737  			},
  2738  			CABundle: certCtx.signingCert,
  2739  		},
  2740  		// network failures while the service network routing is being set up should be ignored by the marker
  2741  		FailurePolicy:           &failOpen,
  2742  		SideEffects:             &sideEffectsNone,
  2743  		AdmissionReviewVersions: []string{"v1", "v1beta1"},
  2744  		// Scope the webhook to just the markers namespace
  2745  		NamespaceSelector: &metav1.LabelSelector{
  2746  			MatchLabels: map[string]string{f.UniqueName + "-markers": "true"},
  2747  		},
  2748  		// appease createMutatingWebhookConfiguration isolation requirements
  2749  		ObjectSelector: &metav1.LabelSelector{
  2750  			MatchLabels: map[string]string{f.UniqueName: "true"},
  2751  		},
  2752  	}
  2753  }
  2754
View as plain text