Source file src/k8s.io/kubernetes/test/integration/apiserver/admissionwebhook/admission_test.go

Documentation: k8s.io/kubernetes/test/integration/apiserver/admissionwebhook

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package admissionwebhook
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"encoding/json"
    24  	"fmt"
    25  	"io"
    26  	"net/http"
    27  	"net/http/httptest"
    28  	"path"
    29  	"sort"
    30  	"strings"
    31  	"sync"
    32  	"testing"
    33  	"time"
    34  
    35  	clientv3 "go.etcd.io/etcd/client/v3"
    36  	admissionreviewv1 "k8s.io/api/admission/v1"
    37  	"k8s.io/api/admission/v1beta1"
    38  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    39  	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
    40  	appsv1beta1 "k8s.io/api/apps/v1beta1"
    41  	authenticationv1 "k8s.io/api/authentication/v1"
    42  	corev1 "k8s.io/api/core/v1"
    43  	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
    44  	policyv1 "k8s.io/api/policy/v1"
    45  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    46  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    47  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    48  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    49  	"k8s.io/apimachinery/pkg/runtime"
    50  	"k8s.io/apimachinery/pkg/runtime/schema"
    51  	"k8s.io/apimachinery/pkg/types"
    52  	"k8s.io/apimachinery/pkg/util/sets"
    53  	"k8s.io/apimachinery/pkg/util/wait"
    54  	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
    55  	"k8s.io/client-go/dynamic"
    56  	clientset "k8s.io/client-go/kubernetes"
    57  	"k8s.io/client-go/rest"
    58  	"k8s.io/client-go/util/retry"
    59  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    60  	apisv1beta1 "k8s.io/kubernetes/pkg/apis/admissionregistration/v1beta1"
    61  	"k8s.io/kubernetes/test/integration/etcd"
    62  	"k8s.io/kubernetes/test/integration/framework"
    63  )
    64  
    65  const (
    66  	testNamespace      = "webhook-integration"
    67  	testClientUsername = "webhook-integration-client"
    68  
    69  	mutation   = "mutation"
    70  	validation = "validation"
    71  )
    72  
    73  var (
    74  	noSideEffects = admissionregistrationv1.SideEffectClassNone
    75  )
    76  
    77  type testContext struct {
    78  	t *testing.T
    79  
    80  	admissionHolder *holder
    81  
    82  	client    dynamic.Interface
    83  	clientset clientset.Interface
    84  	verb      string
    85  	gvr       schema.GroupVersionResource
    86  	resource  metav1.APIResource
    87  	resources map[schema.GroupVersionResource]metav1.APIResource
    88  }
    89  
    90  type testFunc func(*testContext)
    91  
    92  var (
    93  	// defaultResourceFuncs holds the default test functions.
    94  	// may be overridden for specific resources by customTestFuncs.
    95  	defaultResourceFuncs = map[string]testFunc{
    96  		"create":           testResourceCreate,
    97  		"update":           testResourceUpdate,
    98  		"patch":            testResourcePatch,
    99  		"delete":           testResourceDelete,
   100  		"deletecollection": testResourceDeletecollection,
   101  	}
   102  
   103  	// defaultSubresourceFuncs holds default subresource test functions.
   104  	// may be overridden for specific resources by customTestFuncs.
   105  	defaultSubresourceFuncs = map[string]testFunc{
   106  		"update": testSubresourceUpdate,
   107  		"patch":  testSubresourcePatch,
   108  	}
   109  
   110  	// customTestFuncs holds custom test functions by resource and verb.
   111  	customTestFuncs = map[schema.GroupVersionResource]map[string]testFunc{
   112  		gvr("", "v1", "namespaces"): {"delete": testNamespaceDelete},
   113  
   114  		gvr("apps", "v1beta1", "deployments/rollback"):       {"create": testDeploymentRollback},
   115  		gvr("extensions", "v1beta1", "deployments/rollback"): {"create": testDeploymentRollback},
   116  
   117  		gvr("", "v1", "pods/attach"):      {"create": testPodConnectSubresource},
   118  		gvr("", "v1", "pods/exec"):        {"create": testPodConnectSubresource},
   119  		gvr("", "v1", "pods/portforward"): {"create": testPodConnectSubresource},
   120  
   121  		gvr("", "v1", "bindings"):      {"create": testPodBindingEviction},
   122  		gvr("", "v1", "pods/binding"):  {"create": testPodBindingEviction},
   123  		gvr("", "v1", "pods/eviction"): {"create": testPodBindingEviction},
   124  
   125  		gvr("", "v1", "nodes/proxy"):    {"*": testSubresourceProxy},
   126  		gvr("", "v1", "pods/proxy"):     {"*": testSubresourceProxy},
   127  		gvr("", "v1", "services/proxy"): {"*": testSubresourceProxy},
   128  
   129  		gvr("", "v1", "serviceaccounts/token"): {"create": testTokenCreate},
   130  
   131  		gvr("random.numbers.com", "v1", "integers"): {"create": testPruningRandomNumbers},
   132  		gvr("custom.fancy.com", "v2", "pants"):      {"create": testNoPruningCustomFancy},
   133  	}
   134  
   135  	// admissionExemptResources lists objects which are exempt from admission validation/mutation,
   136  	// only resources exempted from admission processing by API server should be listed here.
   137  	admissionExemptResources = map[schema.GroupVersionResource]bool{
   138  		gvr("admissionregistration.k8s.io", "v1beta1", "mutatingwebhookconfigurations"):       true,
   139  		gvr("admissionregistration.k8s.io", "v1beta1", "validatingwebhookconfigurations"):     true,
   140  		gvr("admissionregistration.k8s.io", "v1", "mutatingwebhookconfigurations"):            true,
   141  		gvr("admissionregistration.k8s.io", "v1", "validatingwebhookconfigurations"):          true,
   142  		gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"):        true,
   143  		gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies/status"): true,
   144  		gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicybindings"):  true,
   145  		gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies"):         true,
   146  		gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies/status"):  true,
   147  		gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicybindings"):   true,
   148  		gvr("admissionregistration.k8s.io", "v1", "validatingadmissionpolicies"):              true,
   149  		gvr("admissionregistration.k8s.io", "v1", "validatingadmissionpolicies/status"):       true,
   150  		gvr("admissionregistration.k8s.io", "v1", "validatingadmissionpolicybindings"):        true,
   151  	}
   152  
   153  	parentResources = map[schema.GroupVersionResource]schema.GroupVersionResource{
   154  		gvr("extensions", "v1beta1", "replicationcontrollers/scale"): gvr("", "v1", "replicationcontrollers"),
   155  	}
   156  
   157  	// stubDataOverrides holds either non persistent resources' definitions or resources where default stub needs to be overridden.
   158  	stubDataOverrides = map[schema.GroupVersionResource]string{
   159  		// Non persistent Reviews resource
   160  		gvr("authentication.k8s.io", "v1", "tokenreviews"):                  `{"metadata": {"name": "tokenreview"}, "spec": {"token": "token", "audience": ["audience1","audience2"]}}`,
   161  		gvr("authentication.k8s.io", "v1beta1", "tokenreviews"):             `{"metadata": {"name": "tokenreview"}, "spec": {"token": "token", "audience": ["audience1","audience2"]}}`,
   162  		gvr("authentication.k8s.io", "v1alpha1", "selfsubjectreviews"):      `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`,
   163  		gvr("authentication.k8s.io", "v1beta1", "selfsubjectreviews"):       `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`,
   164  		gvr("authentication.k8s.io", "v1", "selfsubjectreviews"):            `{"metadata": {"name": "SelfSubjectReview"},"status":{"userInfo":{}}}`,
   165  		gvr("authorization.k8s.io", "v1", "localsubjectaccessreviews"):      `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"uid": "token", "user": "user1","groups": ["group1","group2"],"resourceAttributes": {"name":"name1","namespace":"` + testNamespace + `"}}}`,
   166  		gvr("authorization.k8s.io", "v1", "subjectaccessreviews"):           `{"metadata": {"name": "", "namespace":""}, "spec": {"user":"user1","resourceAttributes": {"name":"name1", "namespace":"` + testNamespace + `"}}}`,
   167  		gvr("authorization.k8s.io", "v1", "selfsubjectaccessreviews"):       `{"metadata": {"name": "", "namespace":""}, "spec": {"resourceAttributes": {"name":"name1", "namespace":""}}}`,
   168  		gvr("authorization.k8s.io", "v1", "selfsubjectrulesreviews"):        `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"namespace":"` + testNamespace + `"}}`,
   169  		gvr("authorization.k8s.io", "v1beta1", "localsubjectaccessreviews"): `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"uid": "token", "user": "user1","groups": ["group1","group2"],"resourceAttributes": {"name":"name1","namespace":"` + testNamespace + `"}}}`,
   170  		gvr("authorization.k8s.io", "v1beta1", "subjectaccessreviews"):      `{"metadata": {"name": "", "namespace":""}, "spec": {"user":"user1","resourceAttributes": {"name":"name1", "namespace":"` + testNamespace + `"}}}`,
   171  		gvr("authorization.k8s.io", "v1beta1", "selfsubjectaccessreviews"):  `{"metadata": {"name": "", "namespace":""}, "spec": {"resourceAttributes": {"name":"name1", "namespace":""}}}`,
   172  		gvr("authorization.k8s.io", "v1beta1", "selfsubjectrulesreviews"):   `{"metadata": {"name": "", "namespace":"` + testNamespace + `"}, "spec": {"namespace":"` + testNamespace + `"}}`,
   173  
   174  		// Other Non persistent resources
   175  	}
   176  )
   177  
   178  type webhookOptions struct {
   179  	version string
   180  
   181  	// phase indicates whether this is a mutating or validating webhook
   182  	phase string
   183  	// converted indicates if this webhook makes use of matchPolicy:equivalent and expects conversion.
   184  	// if true, recordGVR and expectGVK are mapped through gvrToConvertedGVR/gvrToConvertedGVK.
   185  	// if false, recordGVR and expectGVK are compared directly to the admission review.
   186  	converted bool
   187  }
   188  
   189  type holder struct {
   190  	lock sync.RWMutex
   191  
   192  	t *testing.T
   193  
   194  	warningHandler *warningHandler
   195  
   196  	recordGVR       metav1.GroupVersionResource
   197  	recordOperation string
   198  	recordNamespace string
   199  	recordName      string
   200  
   201  	expectGVK        schema.GroupVersionKind
   202  	expectObject     bool
   203  	expectOldObject  bool
   204  	expectOptionsGVK schema.GroupVersionKind
   205  	expectOptions    bool
   206  
   207  	// gvrToConvertedGVR maps the GVR submitted to the API server to the expected GVR when converted to the webhook-recognized resource.
   208  	// When a converted request is recorded, gvrToConvertedGVR[recordGVR] is compared to the GVR seen by the webhook.
   209  	gvrToConvertedGVR map[metav1.GroupVersionResource]metav1.GroupVersionResource
   210  	// gvrToConvertedGVR maps the GVR submitted to the API server to the expected GVK when converted to the webhook-recognized resource.
   211  	// When a converted request is recorded, gvrToConvertedGVR[expectGVK] is compared to the GVK seen by the webhook.
   212  	gvrToConvertedGVK map[metav1.GroupVersionResource]schema.GroupVersionKind
   213  
   214  	recorded map[webhookOptions]*admissionRequest
   215  }
   216  
   217  func (h *holder) reset(t *testing.T) {
   218  	h.lock.Lock()
   219  	defer h.lock.Unlock()
   220  	h.t = t
   221  	h.recordGVR = metav1.GroupVersionResource{}
   222  	h.expectGVK = schema.GroupVersionKind{}
   223  	h.recordOperation = ""
   224  	h.recordName = ""
   225  	h.recordNamespace = ""
   226  	h.expectObject = false
   227  	h.expectOldObject = false
   228  	h.expectOptionsGVK = schema.GroupVersionKind{}
   229  	h.expectOptions = false
   230  	h.warningHandler.reset()
   231  
   232  	// Set up the recorded map with nil records for all combinations
   233  	h.recorded = map[webhookOptions]*admissionRequest{}
   234  	for _, phase := range []string{mutation, validation} {
   235  		for _, converted := range []bool{true, false} {
   236  			for _, version := range []string{"v1", "v1beta1"} {
   237  				h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = nil
   238  			}
   239  		}
   240  	}
   241  }
   242  func (h *holder) expect(gvr schema.GroupVersionResource, gvk, optionsGVK schema.GroupVersionKind, operation v1beta1.Operation, name, namespace string, object, oldObject, options bool) {
   243  	// Special-case namespaces, since the object name shows up in request attributes
   244  	if len(namespace) == 0 && gvk.Group == "" && gvk.Version == "v1" && gvk.Kind == "Namespace" {
   245  		namespace = name
   246  	}
   247  
   248  	h.lock.Lock()
   249  	defer h.lock.Unlock()
   250  	h.recordGVR = metav1.GroupVersionResource{Group: gvr.Group, Version: gvr.Version, Resource: gvr.Resource}
   251  	h.expectGVK = gvk
   252  	h.recordOperation = string(operation)
   253  	h.recordName = name
   254  	h.recordNamespace = namespace
   255  	h.expectObject = object
   256  	h.expectOldObject = oldObject
   257  	h.expectOptionsGVK = optionsGVK
   258  	h.expectOptions = options
   259  	h.warningHandler.reset()
   260  
   261  	// Set up the recorded map with nil records for all combinations
   262  	h.recorded = map[webhookOptions]*admissionRequest{}
   263  	for _, phase := range []string{mutation, validation} {
   264  		for _, converted := range []bool{true, false} {
   265  			for _, version := range []string{"v1", "v1beta1"} {
   266  				h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = nil
   267  			}
   268  		}
   269  	}
   270  }
   271  
   272  type admissionRequest struct {
   273  	Operation   string
   274  	Resource    metav1.GroupVersionResource
   275  	SubResource string
   276  	Namespace   string
   277  	Name        string
   278  	Object      runtime.RawExtension
   279  	OldObject   runtime.RawExtension
   280  	Options     runtime.RawExtension
   281  }
   282  
   283  func (h *holder) record(version string, phase string, converted bool, request *admissionRequest) {
   284  	h.lock.Lock()
   285  	defer h.lock.Unlock()
   286  
   287  	// this is useful to turn on if items aren't getting recorded and you need to figure out why
   288  	debug := false
   289  	if debug {
   290  		h.t.Logf("%s %#v %v", request.Operation, request.Resource, request.SubResource)
   291  	}
   292  
   293  	resource := request.Resource
   294  	if len(request.SubResource) > 0 {
   295  		resource.Resource += "/" + request.SubResource
   296  	}
   297  
   298  	// See if we should record this
   299  	gvrToRecord := h.recordGVR
   300  	if converted {
   301  		// If this is a converted webhook, map to the GVR we expect the webhook to see
   302  		gvrToRecord = h.gvrToConvertedGVR[h.recordGVR]
   303  	}
   304  	if resource != gvrToRecord {
   305  		if debug {
   306  			h.t.Log(resource, "!=", gvrToRecord)
   307  		}
   308  		return
   309  	}
   310  
   311  	if request.Operation != h.recordOperation {
   312  		if debug {
   313  			h.t.Log(request.Operation, "!=", h.recordOperation)
   314  		}
   315  		return
   316  	}
   317  	if request.Namespace != h.recordNamespace {
   318  		if debug {
   319  			h.t.Log(request.Namespace, "!=", h.recordNamespace)
   320  		}
   321  		return
   322  	}
   323  
   324  	name := request.Name
   325  	if name != h.recordName {
   326  		if debug {
   327  			h.t.Log(name, "!=", h.recordName)
   328  		}
   329  		return
   330  	}
   331  
   332  	if debug {
   333  		h.t.Logf("recording: %#v = %s %#v %v", webhookOptions{version: version, phase: phase, converted: converted}, request.Operation, request.Resource, request.SubResource)
   334  	}
   335  	h.recorded[webhookOptions{version: version, phase: phase, converted: converted}] = request
   336  }
   337  
   338  func (h *holder) verify(t *testing.T) {
   339  	h.lock.Lock()
   340  	defer h.lock.Unlock()
   341  
   342  	for options, value := range h.recorded {
   343  		if err := h.verifyRequest(options, value); err != nil {
   344  			t.Errorf("version: %v, phase:%v, converted:%v error: %v", options.version, options.phase, options.converted, err)
   345  		}
   346  	}
   347  }
   348  
   349  func (h *holder) verifyRequest(webhookOptions webhookOptions, request *admissionRequest) error {
   350  	converted := webhookOptions.converted
   351  
   352  	// Check if current resource should be exempted from Admission processing
   353  	if admissionExemptResources[gvr(h.recordGVR.Group, h.recordGVR.Version, h.recordGVR.Resource)] {
   354  		if request == nil {
   355  			return nil
   356  		}
   357  		return fmt.Errorf("admission webhook was called, but not supposed to")
   358  	}
   359  
   360  	if request == nil {
   361  		return fmt.Errorf("no request received")
   362  	}
   363  
   364  	if h.expectObject {
   365  		if err := h.verifyObject(converted, request.Object.Object); err != nil {
   366  			return fmt.Errorf("object error: %v", err)
   367  		}
   368  	} else if request.Object.Object != nil {
   369  		return fmt.Errorf("unexpected object: %#v", request.Object.Object)
   370  	}
   371  
   372  	if h.expectOldObject {
   373  		if err := h.verifyObject(converted, request.OldObject.Object); err != nil {
   374  			return fmt.Errorf("old object error: %v", err)
   375  		}
   376  	} else if request.OldObject.Object != nil {
   377  		return fmt.Errorf("unexpected old object: %#v", request.OldObject.Object)
   378  	}
   379  
   380  	if h.expectOptions {
   381  		if err := h.verifyOptions(request.Options.Object); err != nil {
   382  			return fmt.Errorf("options error: %v", err)
   383  		}
   384  	} else if request.Options.Object != nil {
   385  		return fmt.Errorf("unexpected options: %#v", request.Options.Object)
   386  	}
   387  
   388  	if !h.warningHandler.hasWarning(makeWarning(webhookOptions.version, webhookOptions.phase, webhookOptions.converted)) {
   389  		return fmt.Errorf("no warning received from webhook")
   390  	}
   391  
   392  	return nil
   393  }
   394  
   395  func (h *holder) verifyObject(converted bool, obj runtime.Object) error {
   396  	if obj == nil {
   397  		return fmt.Errorf("no object sent")
   398  	}
   399  	expectGVK := h.expectGVK
   400  	if converted {
   401  		expectGVK = h.gvrToConvertedGVK[h.recordGVR]
   402  	}
   403  	if obj.GetObjectKind().GroupVersionKind() != expectGVK {
   404  		return fmt.Errorf("expected %#v, got %#v", expectGVK, obj.GetObjectKind().GroupVersionKind())
   405  	}
   406  	return nil
   407  }
   408  
   409  func (h *holder) verifyOptions(options runtime.Object) error {
   410  	if options == nil {
   411  		return fmt.Errorf("no options sent")
   412  	}
   413  	if options.GetObjectKind().GroupVersionKind() != h.expectOptionsGVK {
   414  		return fmt.Errorf("expected %#v, got %#v", h.expectOptionsGVK, options.GetObjectKind().GroupVersionKind())
   415  	}
   416  	return nil
   417  }
   418  
   419  type warningHandler struct {
   420  	lock     sync.Mutex
   421  	warnings map[string]bool
   422  }
   423  
   424  func (w *warningHandler) reset() {
   425  	w.lock.Lock()
   426  	defer w.lock.Unlock()
   427  	w.warnings = map[string]bool{}
   428  }
   429  func (w *warningHandler) hasWarning(warning string) bool {
   430  	w.lock.Lock()
   431  	defer w.lock.Unlock()
   432  	return w.warnings[warning]
   433  }
   434  func makeWarning(version string, phase string, converted bool) string {
   435  	return fmt.Sprintf("%v/%v/%v", version, phase, converted)
   436  }
   437  
   438  func (w *warningHandler) HandleWarningHeader(code int, agent string, message string) {
   439  	if code != 299 || len(message) == 0 {
   440  		return
   441  	}
   442  	w.lock.Lock()
   443  	defer w.lock.Unlock()
   444  	w.warnings[message] = true
   445  }
   446  
   447  // TestWebhookAdmissionWithWatchCache tests communication between API server and webhook process.
   448  func TestWebhookAdmissionWithWatchCache(t *testing.T) {
   449  	testWebhookAdmission(t, true)
   450  }
   451  
   452  // TestWebhookAdmissionWithoutWatchCache tests communication between API server and webhook process.
   453  func TestWebhookAdmissionWithoutWatchCache(t *testing.T) {
   454  	testWebhookAdmission(t, false)
   455  }
   456  
   457  // testWebhookAdmission tests communication between API server and webhook process.
   458  func testWebhookAdmission(t *testing.T, watchCache bool) {
   459  	// holder communicates expectations to webhooks, and results from webhooks
   460  	holder := &holder{
   461  		t:                 t,
   462  		warningHandler:    &warningHandler{warnings: map[string]bool{}},
   463  		gvrToConvertedGVR: map[metav1.GroupVersionResource]metav1.GroupVersionResource{},
   464  		gvrToConvertedGVK: map[metav1.GroupVersionResource]schema.GroupVersionKind{},
   465  	}
   466  
   467  	// set up webhook server
   468  	roots := x509.NewCertPool()
   469  	if !roots.AppendCertsFromPEM(localhostCert) {
   470  		t.Fatal("Failed to append Cert from PEM")
   471  	}
   472  	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
   473  	if err != nil {
   474  		t.Fatalf("Failed to build cert with error: %+v", err)
   475  	}
   476  
   477  	webhookMux := http.NewServeMux()
   478  	webhookMux.Handle("/v1beta1/"+mutation, newV1beta1WebhookHandler(t, holder, mutation, false))
   479  	webhookMux.Handle("/v1beta1/convert/"+mutation, newV1beta1WebhookHandler(t, holder, mutation, true))
   480  	webhookMux.Handle("/v1beta1/"+validation, newV1beta1WebhookHandler(t, holder, validation, false))
   481  	webhookMux.Handle("/v1beta1/convert/"+validation, newV1beta1WebhookHandler(t, holder, validation, true))
   482  	webhookMux.Handle("/v1/"+mutation, newV1WebhookHandler(t, holder, mutation, false))
   483  	webhookMux.Handle("/v1/convert/"+mutation, newV1WebhookHandler(t, holder, mutation, true))
   484  	webhookMux.Handle("/v1/"+validation, newV1WebhookHandler(t, holder, validation, false))
   485  	webhookMux.Handle("/v1/convert/"+validation, newV1WebhookHandler(t, holder, validation, true))
   486  	webhookMux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   487  		holder.t.Errorf("unexpected request to %v", req.URL.Path)
   488  	}))
   489  	webhookServer := httptest.NewUnstartedServer(webhookMux)
   490  	webhookServer.TLS = &tls.Config{
   491  		RootCAs:      roots,
   492  		Certificates: []tls.Certificate{cert},
   493  	}
   494  	webhookServer.StartTLS()
   495  	defer webhookServer.Close()
   496  
   497  	// start API server
   498  	etcdConfig := framework.SharedEtcd()
   499  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{
   500  		fmt.Sprintf("--watch-cache=%v", watchCache),
   501  		// turn off admission plugins that add finalizers
   502  		"--disable-admission-plugins=ServiceAccount,StorageObjectInUseProtection",
   503  		// force enable all resources so we can check storage.
   504  		"--runtime-config=api/all=true",
   505  		// enable feature-gates that protect resources to check their storage, too.
   506  		// e.g. "--feature-gates=EphemeralContainers=true",
   507  	}, etcdConfig)
   508  	defer server.TearDownFn()
   509  
   510  	// Configure a client with a distinct user name so that it is easy to distinguish requests
   511  	// made by the client from requests made by controllers. We use this to filter out requests
   512  	// before recording them to ensure we don't accidentally mistake requests from controllers
   513  	// as requests made by the client.
   514  	clientConfig := rest.CopyConfig(server.ClientConfig)
   515  	clientConfig.Impersonate.UserName = testClientUsername
   516  	clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"}
   517  	clientConfig.WarningHandler = holder.warningHandler
   518  	client, err := clientset.NewForConfig(clientConfig)
   519  	if err != nil {
   520  		t.Fatalf("unexpected error: %v", err)
   521  	}
   522  
   523  	// create CRDs
   524  	etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...)
   525  
   526  	if _, err := client.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}, metav1.CreateOptions{}); err != nil {
   527  		t.Fatal(err)
   528  	}
   529  
   530  	// gather resources to test
   531  	dynamicClient, err := dynamic.NewForConfig(clientConfig)
   532  	if err != nil {
   533  		t.Fatal(err)
   534  	}
   535  	_, resources, err := client.Discovery().ServerGroupsAndResources()
   536  	if err != nil {
   537  		t.Fatalf("Failed to get ServerGroupsAndResources with error: %+v", err)
   538  	}
   539  
   540  	gvrsToTest := []schema.GroupVersionResource{}
   541  	resourcesByGVR := map[schema.GroupVersionResource]metav1.APIResource{}
   542  
   543  	for _, list := range resources {
   544  		defaultGroupVersion, err := schema.ParseGroupVersion(list.GroupVersion)
   545  		if err != nil {
   546  			t.Errorf("Failed to get GroupVersion for: %+v", list)
   547  			continue
   548  		}
   549  		for _, resource := range list.APIResources {
   550  			if resource.Group == "" {
   551  				resource.Group = defaultGroupVersion.Group
   552  			}
   553  			if resource.Version == "" {
   554  				resource.Version = defaultGroupVersion.Version
   555  			}
   556  			gvr := defaultGroupVersion.WithResource(resource.Name)
   557  			resourcesByGVR[gvr] = resource
   558  			if shouldTestResource(gvr, resource) {
   559  				gvrsToTest = append(gvrsToTest, gvr)
   560  			}
   561  		}
   562  	}
   563  
   564  	sort.SliceStable(gvrsToTest, func(i, j int) bool {
   565  		if gvrsToTest[i].Group < gvrsToTest[j].Group {
   566  			return true
   567  		}
   568  		if gvrsToTest[i].Group > gvrsToTest[j].Group {
   569  			return false
   570  		}
   571  		if gvrsToTest[i].Version < gvrsToTest[j].Version {
   572  			return true
   573  		}
   574  		if gvrsToTest[i].Version > gvrsToTest[j].Version {
   575  			return false
   576  		}
   577  		if gvrsToTest[i].Resource < gvrsToTest[j].Resource {
   578  			return true
   579  		}
   580  		if gvrsToTest[i].Resource > gvrsToTest[j].Resource {
   581  			return false
   582  		}
   583  		return true
   584  	})
   585  
   586  	// map unqualified resource names to the fully qualified resource we will expect to be converted to
   587  	// Note: this only works because there are no overlapping resource names in-process that are not co-located
   588  	convertedResources := map[string]schema.GroupVersionResource{}
   589  	// build the webhook rules enumerating the specific group/version/resources we want
   590  	convertedV1beta1Rules := []admissionregistrationv1beta1.RuleWithOperations{}
   591  	convertedV1Rules := []admissionregistrationv1.RuleWithOperations{}
   592  	for _, gvr := range gvrsToTest {
   593  		metaGVR := metav1.GroupVersionResource{Group: gvr.Group, Version: gvr.Version, Resource: gvr.Resource}
   594  
   595  		convertedGVR, ok := convertedResources[gvr.Resource]
   596  		if !ok {
   597  			// this is the first time we've seen this resource
   598  			// record the fully qualified resource we expect
   599  			convertedGVR = gvr
   600  			convertedResources[gvr.Resource] = gvr
   601  			// add an admission rule indicating we can receive this version
   602  			convertedV1beta1Rules = append(convertedV1beta1Rules, admissionregistrationv1beta1.RuleWithOperations{
   603  				Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.OperationAll},
   604  				Rule:       admissionregistrationv1beta1.Rule{APIGroups: []string{gvr.Group}, APIVersions: []string{gvr.Version}, Resources: []string{gvr.Resource}},
   605  			})
   606  			convertedV1Rules = append(convertedV1Rules, admissionregistrationv1.RuleWithOperations{
   607  				Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
   608  				Rule:       admissionregistrationv1.Rule{APIGroups: []string{gvr.Group}, APIVersions: []string{gvr.Version}, Resources: []string{gvr.Resource}},
   609  			})
   610  		}
   611  
   612  		// record the expected resource and kind
   613  		holder.gvrToConvertedGVR[metaGVR] = metav1.GroupVersionResource{Group: convertedGVR.Group, Version: convertedGVR.Version, Resource: convertedGVR.Resource}
   614  		holder.gvrToConvertedGVK[metaGVR] = schema.GroupVersionKind{Group: resourcesByGVR[convertedGVR].Group, Version: resourcesByGVR[convertedGVR].Version, Kind: resourcesByGVR[convertedGVR].Kind}
   615  	}
   616  
   617  	if err := createV1beta1MutationWebhook(server.EtcdClient, server.EtcdStoragePrefix, client, webhookServer.URL+"/v1beta1/"+mutation, webhookServer.URL+"/v1beta1/convert/"+mutation, convertedV1beta1Rules); err != nil {
   618  		t.Fatal(err)
   619  	}
   620  	if err := createV1beta1ValidationWebhook(server.EtcdClient, server.EtcdStoragePrefix, client, webhookServer.URL+"/v1beta1/"+validation, webhookServer.URL+"/v1beta1/convert/"+validation, convertedV1beta1Rules); err != nil {
   621  		t.Fatal(err)
   622  	}
   623  	if err := createV1MutationWebhook(client, webhookServer.URL+"/v1/"+mutation, webhookServer.URL+"/v1/convert/"+mutation, convertedV1Rules); err != nil {
   624  		t.Fatal(err)
   625  	}
   626  	if err := createV1ValidationWebhook(client, webhookServer.URL+"/v1/"+validation, webhookServer.URL+"/v1/convert/"+validation, convertedV1Rules); err != nil {
   627  		t.Fatal(err)
   628  	}
   629  
   630  	// Allow the webhook to establish
   631  	time.Sleep(time.Second)
   632  
   633  	start := time.Now()
   634  	count := 0
   635  
   636  	// Test admission on all resources, subresources, and verbs
   637  	for _, gvr := range gvrsToTest {
   638  		resource := resourcesByGVR[gvr]
   639  		t.Run(gvr.Group+"."+gvr.Version+"."+strings.ReplaceAll(resource.Name, "/", "."), func(t *testing.T) {
   640  			for _, verb := range []string{"create", "update", "patch", "connect", "delete", "deletecollection"} {
   641  				if shouldTestResourceVerb(gvr, resource, verb) {
   642  					t.Run(verb, func(t *testing.T) {
   643  						count++
   644  						holder.reset(t)
   645  						testFunc := getTestFunc(gvr, verb)
   646  						testFunc(&testContext{
   647  							t:               t,
   648  							admissionHolder: holder,
   649  							client:          dynamicClient,
   650  							clientset:       client,
   651  							verb:            verb,
   652  							gvr:             gvr,
   653  							resource:        resource,
   654  							resources:       resourcesByGVR,
   655  						})
   656  						holder.verify(t)
   657  					})
   658  				}
   659  			}
   660  		})
   661  	}
   662  
   663  	duration := time.Since(start)
   664  	perResourceDuration := time.Duration(int(duration) / count)
   665  	if perResourceDuration >= 150*time.Millisecond {
   666  		t.Errorf("expected resources to process in < 150ms, average was %v", perResourceDuration)
   667  	}
   668  }
   669  
   670  //
   671  // generic resource testing
   672  //
   673  
   674  func testResourceCreate(c *testContext) {
   675  	stubObj, err := getStubObj(c.gvr, c.resource)
   676  	if err != nil {
   677  		c.t.Error(err)
   678  		return
   679  	}
   680  	ns := ""
   681  	if c.resource.Namespaced {
   682  		ns = testNamespace
   683  	}
   684  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, stubObj.GetName(), ns, true, false, true)
   685  	_, err = c.client.Resource(c.gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{})
   686  	if err != nil {
   687  		c.t.Error(err)
   688  		return
   689  	}
   690  }
   691  
   692  func testResourceUpdate(c *testContext) {
   693  	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
   694  		obj, err := createOrGetResource(c.client, c.gvr, c.resource)
   695  		if err != nil {
   696  			return err
   697  		}
   698  		obj.SetAnnotations(map[string]string{"update": "true"})
   699  		c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true)
   700  		_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Update(context.TODO(), obj, metav1.UpdateOptions{})
   701  		return err
   702  	}); err != nil {
   703  		c.t.Error(err)
   704  		return
   705  	}
   706  }
   707  
   708  func testResourcePatch(c *testContext) {
   709  	obj, err := createOrGetResource(c.client, c.gvr, c.resource)
   710  	if err != nil {
   711  		c.t.Error(err)
   712  		return
   713  	}
   714  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true)
   715  	_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch(
   716  		context.TODO(),
   717  		obj.GetName(),
   718  		types.MergePatchType,
   719  		[]byte(`{"metadata":{"annotations":{"patch":"true"}}}`),
   720  		metav1.PatchOptions{})
   721  	if err != nil {
   722  		c.t.Error(err)
   723  		return
   724  	}
   725  }
   726  
   727  func testResourceDelete(c *testContext) {
   728  	// Verify that an immediate delete triggers the webhook and populates the admisssionRequest.oldObject.
   729  	obj, err := createOrGetResource(c.client, c.gvr, c.resource)
   730  	if err != nil {
   731  		c.t.Error(err)
   732  		return
   733  	}
   734  	background := metav1.DeletePropagationBackground
   735  	zero := int64(0)
   736  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true)
   737  	err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background})
   738  	if err != nil {
   739  		c.t.Error(err)
   740  		return
   741  	}
   742  	c.admissionHolder.verify(c.t)
   743  
   744  	// wait for the item to be gone
   745  	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) {
   746  		obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
   747  		if apierrors.IsNotFound(err) {
   748  			return true, nil
   749  		}
   750  		if err == nil {
   751  			c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers())
   752  			return false, nil
   753  		}
   754  		return false, err
   755  	})
   756  	if err != nil {
   757  		c.t.Error(err)
   758  		return
   759  	}
   760  
   761  	// Verify that an update-on-delete triggers the webhook and populates the admisssionRequest.oldObject.
   762  	obj, err = createOrGetResource(c.client, c.gvr, c.resource)
   763  	if err != nil {
   764  		c.t.Error(err)
   765  		return
   766  	}
   767  	// Adding finalizer to the object, then deleting it.
   768  	// We don't add finalizers by setting DeleteOptions.PropagationPolicy
   769  	// because some resource (e.g., events) do not support garbage
   770  	// collector finalizers.
   771  	_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch(
   772  		context.TODO(),
   773  		obj.GetName(),
   774  		types.MergePatchType,
   775  		[]byte(`{"metadata":{"finalizers":["test/k8s.io"]}}`),
   776  		metav1.PatchOptions{})
   777  	if err != nil {
   778  		c.t.Error(err)
   779  		return
   780  	}
   781  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true)
   782  	err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background})
   783  	if err != nil {
   784  		c.t.Error(err)
   785  		return
   786  	}
   787  	c.admissionHolder.verify(c.t)
   788  
   789  	// wait other finalizers (e.g., crd's customresourcecleanup finalizer) to be removed.
   790  	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) {
   791  		obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
   792  		if err != nil {
   793  			return false, err
   794  		}
   795  		finalizers := obj.GetFinalizers()
   796  		if len(finalizers) != 1 {
   797  			c.t.Logf("waiting for other finalizers on %#v %s to be removed, existing finalizers are %v", c.gvr, obj.GetName(), obj.GetFinalizers())
   798  			return false, nil
   799  		}
   800  		if finalizers[0] != "test/k8s.io" {
   801  			return false, fmt.Errorf("expected the single finalizer on %#v %s to be test/k8s.io, got %v", c.gvr, obj.GetName(), obj.GetFinalizers())
   802  		}
   803  		return true, nil
   804  	})
   805  	if err != nil {
   806  		c.t.Error(err)
   807  		return
   808  	}
   809  
   810  	// remove the finalizer
   811  	_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch(
   812  		context.TODO(),
   813  		obj.GetName(),
   814  		types.MergePatchType,
   815  		[]byte(`{"metadata":{"finalizers":[]}}`),
   816  		metav1.PatchOptions{})
   817  	if err != nil {
   818  		c.t.Error(err)
   819  		return
   820  	}
   821  	// wait for the item to be gone
   822  	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) {
   823  		obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
   824  		if apierrors.IsNotFound(err) {
   825  			return true, nil
   826  		}
   827  		if err == nil {
   828  			c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers())
   829  			return false, nil
   830  		}
   831  		return false, err
   832  	})
   833  	if err != nil {
   834  		c.t.Error(err)
   835  		return
   836  	}
   837  }
   838  
   839  func testResourceDeletecollection(c *testContext) {
   840  	obj, err := createOrGetResource(c.client, c.gvr, c.resource)
   841  	if err != nil {
   842  		c.t.Error(err)
   843  		return
   844  	}
   845  	background := metav1.DeletePropagationBackground
   846  	zero := int64(0)
   847  
   848  	// update the object with a label that matches our selector
   849  	_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Patch(
   850  		context.TODO(),
   851  		obj.GetName(),
   852  		types.MergePatchType,
   853  		[]byte(`{"metadata":{"labels":{"webhooktest":"true"}}}`),
   854  		metav1.PatchOptions{})
   855  	if err != nil {
   856  		c.t.Error(err)
   857  		return
   858  	}
   859  
   860  	// set expectations
   861  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, "", obj.GetNamespace(), false, true, true)
   862  
   863  	// delete
   864  	err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).DeleteCollection(context.TODO(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}, metav1.ListOptions{LabelSelector: "webhooktest=true"})
   865  	if err != nil {
   866  		c.t.Error(err)
   867  		return
   868  	}
   869  
   870  	// wait for the item to be gone
   871  	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) {
   872  		obj, err := c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
   873  		if apierrors.IsNotFound(err) {
   874  			return true, nil
   875  		}
   876  		if err == nil {
   877  			c.t.Logf("waiting for %#v to be deleted (name: %s, finalizers: %v)...\n", c.gvr, obj.GetName(), obj.GetFinalizers())
   878  			return false, nil
   879  		}
   880  		return false, err
   881  	})
   882  	if err != nil {
   883  		c.t.Error(err)
   884  		return
   885  	}
   886  }
   887  
   888  func getParentGVR(gvr schema.GroupVersionResource) schema.GroupVersionResource {
   889  	parentGVR, found := parentResources[gvr]
   890  	// if no special override is found, just drop the subresource
   891  	if !found {
   892  		parentGVR = gvr
   893  		parentGVR.Resource = strings.Split(parentGVR.Resource, "/")[0]
   894  	}
   895  	return parentGVR
   896  }
   897  
   898  func testTokenCreate(c *testContext) {
   899  	saGVR := gvr("", "v1", "serviceaccounts")
   900  	sa, err := createOrGetResource(c.client, saGVR, c.resources[saGVR])
   901  	if err != nil {
   902  		c.t.Error(err)
   903  		return
   904  	}
   905  
   906  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, sa.GetName(), sa.GetNamespace(), true, false, true)
   907  	if err = c.clientset.CoreV1().RESTClient().Post().Namespace(sa.GetNamespace()).Resource("serviceaccounts").Name(sa.GetName()).SubResource("token").Body(&authenticationv1.TokenRequest{
   908  		ObjectMeta: metav1.ObjectMeta{Name: sa.GetName()},
   909  		Spec: authenticationv1.TokenRequestSpec{
   910  			Audiences: []string{"api"},
   911  		},
   912  	}).Do(context.TODO()).Error(); err != nil {
   913  		c.t.Error(err)
   914  		return
   915  	}
   916  	c.admissionHolder.verify(c.t)
   917  }
   918  
   919  func testSubresourceUpdate(c *testContext) {
   920  	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
   921  		parentGVR := getParentGVR(c.gvr)
   922  		parentResource := c.resources[parentGVR]
   923  		obj, err := createOrGetResource(c.client, parentGVR, parentResource)
   924  		if err != nil {
   925  			return err
   926  		}
   927  
   928  		// Save the parent object as what we submit
   929  		submitObj := obj
   930  
   931  		gvrWithoutSubresources := c.gvr
   932  		gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0]
   933  		subresources := strings.Split(c.gvr.Resource, "/")[1:]
   934  
   935  		// If the subresource supports get, fetch that as the object to submit (namespaces/finalize, */scale, etc)
   936  		if sets.NewString(c.resource.Verbs...).Has("get") {
   937  			submitObj, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}, subresources...)
   938  			if err != nil {
   939  				return err
   940  			}
   941  		}
   942  
   943  		// Modify the object
   944  		submitObj.SetAnnotations(map[string]string{"subresourceupdate": "true"})
   945  
   946  		// set expectations
   947  		c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true)
   948  
   949  		_, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Update(
   950  			context.TODO(),
   951  			submitObj,
   952  			metav1.UpdateOptions{},
   953  			subresources...,
   954  		)
   955  		return err
   956  	}); err != nil {
   957  		c.t.Error(err)
   958  	}
   959  }
   960  
   961  func testSubresourcePatch(c *testContext) {
   962  	parentGVR := getParentGVR(c.gvr)
   963  	parentResource := c.resources[parentGVR]
   964  	obj, err := createOrGetResource(c.client, parentGVR, parentResource)
   965  	if err != nil {
   966  		c.t.Error(err)
   967  		return
   968  	}
   969  
   970  	gvrWithoutSubresources := c.gvr
   971  	gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0]
   972  	subresources := strings.Split(c.gvr.Resource, "/")[1:]
   973  
   974  	// set expectations
   975  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkUpdateOptions, v1beta1.Update, obj.GetName(), obj.GetNamespace(), true, true, true)
   976  
   977  	_, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Patch(
   978  		context.TODO(),
   979  		obj.GetName(),
   980  		types.MergePatchType,
   981  		[]byte(`{"metadata":{"annotations":{"subresourcepatch":"true"}}}`),
   982  		metav1.PatchOptions{},
   983  		subresources...,
   984  	)
   985  	if err != nil {
   986  		c.t.Error(err)
   987  		return
   988  	}
   989  }
   990  
   991  func unimplemented(c *testContext) {
   992  	c.t.Errorf("Test function for %+v has not been implemented...", c.gvr)
   993  }
   994  
   995  //
   996  // custom methods
   997  //
   998  
   999  // testNamespaceDelete verifies namespace-specific delete behavior:
  1000  // - ensures admission is called on first delete (which only sets deletionTimestamp and terminating state)
  1001  // - removes finalizer from namespace
  1002  // - ensures admission is called on final delete once finalizers are removed
  1003  func testNamespaceDelete(c *testContext) {
  1004  	obj, err := createOrGetResource(c.client, c.gvr, c.resource)
  1005  	if err != nil {
  1006  		c.t.Error(err)
  1007  		return
  1008  	}
  1009  	background := metav1.DeletePropagationBackground
  1010  	zero := int64(0)
  1011  
  1012  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkDeleteOptions, v1beta1.Delete, obj.GetName(), obj.GetNamespace(), false, true, true)
  1013  	err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background})
  1014  	if err != nil {
  1015  		c.t.Error(err)
  1016  		return
  1017  	}
  1018  	c.admissionHolder.verify(c.t)
  1019  
  1020  	// do the finalization so the namespace can be deleted
  1021  	obj, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
  1022  	if err != nil {
  1023  		c.t.Error(err)
  1024  		return
  1025  	}
  1026  	err = unstructured.SetNestedField(obj.Object, nil, "spec", "finalizers")
  1027  	if err != nil {
  1028  		c.t.Error(err)
  1029  		return
  1030  	}
  1031  	_, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Update(context.TODO(), obj, metav1.UpdateOptions{}, "finalize")
  1032  	if err != nil {
  1033  		c.t.Error(err)
  1034  		return
  1035  	}
  1036  	// verify namespace is gone
  1037  	obj, err = c.client.Resource(c.gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
  1038  	if err == nil || !apierrors.IsNotFound(err) {
  1039  		c.t.Errorf("expected namespace to be gone, got %#v, %v", obj, err)
  1040  	}
  1041  }
  1042  
  1043  // testDeploymentRollback verifies rollback-specific behavior:
  1044  // - creates a parent deployment
  1045  // - creates a rollback object and posts it
  1046  func testDeploymentRollback(c *testContext) {
  1047  	deploymentGVR := gvr("apps", "v1", "deployments")
  1048  	obj, err := createOrGetResource(c.client, deploymentGVR, c.resources[deploymentGVR])
  1049  	if err != nil {
  1050  		c.t.Error(err)
  1051  		return
  1052  	}
  1053  
  1054  	gvrWithoutSubresources := c.gvr
  1055  	gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0]
  1056  	subresources := strings.Split(c.gvr.Resource, "/")[1:]
  1057  
  1058  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, obj.GetName(), obj.GetNamespace(), true, false, true)
  1059  
  1060  	var rollbackObj runtime.Object
  1061  	switch c.gvr {
  1062  	case gvr("apps", "v1beta1", "deployments/rollback"):
  1063  		rollbackObj = &appsv1beta1.DeploymentRollback{
  1064  			TypeMeta:   metav1.TypeMeta{APIVersion: "apps/v1beta1", Kind: "DeploymentRollback"},
  1065  			Name:       obj.GetName(),
  1066  			RollbackTo: appsv1beta1.RollbackConfig{Revision: 0},
  1067  		}
  1068  	case gvr("extensions", "v1beta1", "deployments/rollback"):
  1069  		rollbackObj = &extensionsv1beta1.DeploymentRollback{
  1070  			TypeMeta:   metav1.TypeMeta{APIVersion: "extensions/v1beta1", Kind: "DeploymentRollback"},
  1071  			Name:       obj.GetName(),
  1072  			RollbackTo: extensionsv1beta1.RollbackConfig{Revision: 0},
  1073  		}
  1074  	default:
  1075  		c.t.Errorf("unknown rollback resource %#v", c.gvr)
  1076  		return
  1077  	}
  1078  
  1079  	rollbackUnstructuredBody, err := runtime.DefaultUnstructuredConverter.ToUnstructured(rollbackObj)
  1080  	if err != nil {
  1081  		c.t.Errorf("ToUnstructured failed: %v", err)
  1082  		return
  1083  	}
  1084  	rollbackUnstructuredObj := &unstructured.Unstructured{Object: rollbackUnstructuredBody}
  1085  	rollbackUnstructuredObj.SetName(obj.GetName())
  1086  
  1087  	_, err = c.client.Resource(gvrWithoutSubresources).Namespace(obj.GetNamespace()).Create(context.TODO(), rollbackUnstructuredObj, metav1.CreateOptions{}, subresources...)
  1088  	if err != nil {
  1089  		c.t.Error(err)
  1090  		return
  1091  	}
  1092  }
  1093  
  1094  // testPodConnectSubresource verifies connect subresources
  1095  func testPodConnectSubresource(c *testContext) {
  1096  	podGVR := gvr("", "v1", "pods")
  1097  	pod, err := createOrGetResource(c.client, podGVR, c.resources[podGVR])
  1098  	if err != nil {
  1099  		c.t.Error(err)
  1100  		return
  1101  	}
  1102  
  1103  	// check all upgradeable verbs
  1104  	for _, httpMethod := range []string{"GET", "POST"} {
  1105  		c.t.Logf("verifying %v", httpMethod)
  1106  
  1107  		c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), schema.GroupVersionKind{}, v1beta1.Connect, pod.GetName(), pod.GetNamespace(), true, false, false)
  1108  		var err error
  1109  		switch c.gvr {
  1110  		case gvr("", "v1", "pods/exec"):
  1111  			err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("exec").Do(context.TODO()).Error()
  1112  		case gvr("", "v1", "pods/attach"):
  1113  			err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("attach").Do(context.TODO()).Error()
  1114  		case gvr("", "v1", "pods/portforward"):
  1115  			err = c.clientset.CoreV1().RESTClient().Verb(httpMethod).Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("portforward").Do(context.TODO()).Error()
  1116  		default:
  1117  			c.t.Errorf("unknown subresource %#v", c.gvr)
  1118  			return
  1119  		}
  1120  
  1121  		if err != nil {
  1122  			c.t.Logf("debug: result of subresource connect: %v", err)
  1123  		}
  1124  		c.admissionHolder.verify(c.t)
  1125  
  1126  	}
  1127  }
  1128  
  1129  // testPodBindingEviction verifies pod binding and eviction admission
  1130  func testPodBindingEviction(c *testContext) {
  1131  	podGVR := gvr("", "v1", "pods")
  1132  	pod, err := createOrGetResource(c.client, podGVR, c.resources[podGVR])
  1133  	if err != nil {
  1134  		c.t.Error(err)
  1135  		return
  1136  	}
  1137  
  1138  	background := metav1.DeletePropagationBackground
  1139  	zero := int64(0)
  1140  	forceDelete := metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background}
  1141  	defer func() {
  1142  		err := c.clientset.CoreV1().Pods(pod.GetNamespace()).Delete(context.TODO(), pod.GetName(), forceDelete)
  1143  		if err != nil && !apierrors.IsNotFound(err) {
  1144  			c.t.Error(err)
  1145  			return
  1146  		}
  1147  	}()
  1148  
  1149  	c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, pod.GetName(), pod.GetNamespace(), true, false, true)
  1150  
  1151  	switch c.gvr {
  1152  	case gvr("", "v1", "bindings"):
  1153  		err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("bindings").Body(&corev1.Binding{
  1154  			ObjectMeta: metav1.ObjectMeta{Name: pod.GetName()},
  1155  			Target:     corev1.ObjectReference{Name: "foo", Kind: "Node", APIVersion: "v1"},
  1156  		}).Do(context.TODO()).Error()
  1157  
  1158  	case gvr("", "v1", "pods/binding"):
  1159  		err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("binding").Body(&corev1.Binding{
  1160  			ObjectMeta: metav1.ObjectMeta{Name: pod.GetName()},
  1161  			Target:     corev1.ObjectReference{Name: "foo", Kind: "Node", APIVersion: "v1"},
  1162  		}).Do(context.TODO()).Error()
  1163  
  1164  	case gvr("", "v1", "pods/eviction"):
  1165  		err = c.clientset.CoreV1().RESTClient().Post().Namespace(pod.GetNamespace()).Resource("pods").Name(pod.GetName()).SubResource("eviction").Body(&policyv1.Eviction{
  1166  			ObjectMeta:    metav1.ObjectMeta{Name: pod.GetName()},
  1167  			DeleteOptions: &forceDelete,
  1168  		}).Do(context.TODO()).Error()
  1169  
  1170  	default:
  1171  		c.t.Errorf("unhandled resource %#v", c.gvr)
  1172  		return
  1173  	}
  1174  
  1175  	if err != nil {
  1176  		c.t.Error(err)
  1177  		return
  1178  	}
  1179  }
  1180  
  1181  // testSubresourceProxy verifies proxy subresources
  1182  func testSubresourceProxy(c *testContext) {
  1183  	parentGVR := getParentGVR(c.gvr)
  1184  	parentResource := c.resources[parentGVR]
  1185  	obj, err := createOrGetResource(c.client, parentGVR, parentResource)
  1186  	if err != nil {
  1187  		c.t.Error(err)
  1188  		return
  1189  	}
  1190  
  1191  	gvrWithoutSubresources := c.gvr
  1192  	gvrWithoutSubresources.Resource = strings.Split(gvrWithoutSubresources.Resource, "/")[0]
  1193  	subresources := strings.Split(c.gvr.Resource, "/")[1:]
  1194  
  1195  	verbToHTTPMethods := map[string][]string{
  1196  		"create": {"POST", "GET", "HEAD", "OPTIONS"}, // also test read-only verbs map to Connect admission
  1197  		"update": {"PUT"},
  1198  		"patch":  {"PATCH"},
  1199  		"delete": {"DELETE"},
  1200  	}
  1201  	httpMethodsToTest, ok := verbToHTTPMethods[c.verb]
  1202  	if !ok {
  1203  		c.t.Errorf("unknown verb %v", c.verb)
  1204  		return
  1205  	}
  1206  
  1207  	for _, httpMethod := range httpMethodsToTest {
  1208  		c.t.Logf("testing %v", httpMethod)
  1209  		request := c.clientset.CoreV1().RESTClient().Verb(httpMethod)
  1210  
  1211  		// add the namespace if required
  1212  		if len(obj.GetNamespace()) > 0 {
  1213  			request = request.Namespace(obj.GetNamespace())
  1214  		}
  1215  
  1216  		// set expectations
  1217  		c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), schema.GroupVersionKind{}, v1beta1.Connect, obj.GetName(), obj.GetNamespace(), true, false, false)
  1218  		// run the request. we don't actually care if the request is successful, just that admission gets called as expected
  1219  		err = request.Resource(gvrWithoutSubresources.Resource).Name(obj.GetName()).SubResource(subresources...).Do(context.TODO()).Error()
  1220  		if err != nil {
  1221  			c.t.Logf("debug: result of subresource proxy (error expected): %v", err)
  1222  		}
  1223  		// verify the result
  1224  		c.admissionHolder.verify(c.t)
  1225  	}
  1226  }
  1227  
  1228  func testPruningRandomNumbers(c *testContext) {
  1229  	testResourceCreate(c)
  1230  
  1231  	cr2pant, err := c.client.Resource(c.gvr).Get(context.TODO(), "fortytwo", metav1.GetOptions{})
  1232  	if err != nil {
  1233  		c.t.Error(err)
  1234  		return
  1235  	}
  1236  
  1237  	foo, found, err := unstructured.NestedString(cr2pant.Object, "foo")
  1238  	if err != nil {
  1239  		c.t.Error(err)
  1240  		return
  1241  	}
  1242  	if found {
  1243  		c.t.Errorf("expected .foo to be pruned, but got: %s", foo)
  1244  	}
  1245  }
  1246  
  1247  func testNoPruningCustomFancy(c *testContext) {
  1248  	testResourceCreate(c)
  1249  
  1250  	cr2pant, err := c.client.Resource(c.gvr).Get(context.TODO(), "cr2pant", metav1.GetOptions{})
  1251  	if err != nil {
  1252  		c.t.Error(err)
  1253  		return
  1254  	}
  1255  
  1256  	foo, _, err := unstructured.NestedString(cr2pant.Object, "foo")
  1257  	if err != nil {
  1258  		c.t.Error(err)
  1259  		return
  1260  	}
  1261  
  1262  	// check that no pruning took place
  1263  	if expected, got := "test", foo; expected != got {
  1264  		c.t.Errorf("expected /foo to be %q, got: %q", expected, got)
  1265  	}
  1266  }
  1267  
  1268  //
  1269  // utility methods
  1270  //
  1271  
  1272  func newV1beta1WebhookHandler(t *testing.T, holder *holder, phase string, converted bool) http.Handler {
  1273  	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1274  		defer r.Body.Close()
  1275  		data, err := io.ReadAll(r.Body)
  1276  		if err != nil {
  1277  			t.Error(err)
  1278  			return
  1279  		}
  1280  
  1281  		if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
  1282  			t.Errorf("contentType=%s, expect application/json", contentType)
  1283  			return
  1284  		}
  1285  
  1286  		review := v1beta1.AdmissionReview{}
  1287  		if err := json.Unmarshal(data, &review); err != nil {
  1288  			t.Errorf("Fail to deserialize object: %s with error: %v", string(data), err)
  1289  			http.Error(w, err.Error(), 400)
  1290  			return
  1291  		}
  1292  
  1293  		if review.GetObjectKind().GroupVersionKind() != gvk("admission.k8s.io", "v1beta1", "AdmissionReview") {
  1294  			t.Errorf("Invalid admission review kind: %#v", review.GetObjectKind().GroupVersionKind())
  1295  			http.Error(w, err.Error(), 400)
  1296  			return
  1297  		}
  1298  
  1299  		if len(review.Request.Object.Raw) > 0 {
  1300  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1301  			if err := json.Unmarshal(review.Request.Object.Raw, u); err != nil {
  1302  				t.Errorf("Fail to deserialize object: %s with error: %v", string(review.Request.Object.Raw), err)
  1303  				http.Error(w, err.Error(), 400)
  1304  				return
  1305  			}
  1306  			review.Request.Object.Object = u
  1307  		}
  1308  		if len(review.Request.OldObject.Raw) > 0 {
  1309  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1310  			if err := json.Unmarshal(review.Request.OldObject.Raw, u); err != nil {
  1311  				t.Errorf("Fail to deserialize object: %s with error: %v", string(review.Request.OldObject.Raw), err)
  1312  				http.Error(w, err.Error(), 400)
  1313  				return
  1314  			}
  1315  			review.Request.OldObject.Object = u
  1316  		}
  1317  
  1318  		if len(review.Request.Options.Raw) > 0 {
  1319  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1320  			if err := json.Unmarshal(review.Request.Options.Raw, u); err != nil {
  1321  				t.Errorf("Fail to deserialize options object: %s for admission request %#+v with error: %v", string(review.Request.Options.Raw), review.Request, err)
  1322  				http.Error(w, err.Error(), 400)
  1323  				return
  1324  			}
  1325  			review.Request.Options.Object = u
  1326  		}
  1327  
  1328  		if review.Request.UserInfo.Username == testClientUsername {
  1329  			// only record requests originating from this integration test's client
  1330  			reviewRequest := &admissionRequest{
  1331  				Operation:   string(review.Request.Operation),
  1332  				Resource:    review.Request.Resource,
  1333  				SubResource: review.Request.SubResource,
  1334  				Namespace:   review.Request.Namespace,
  1335  				Name:        review.Request.Name,
  1336  				Object:      review.Request.Object,
  1337  				OldObject:   review.Request.OldObject,
  1338  				Options:     review.Request.Options,
  1339  			}
  1340  			holder.record("v1beta1", phase, converted, reviewRequest)
  1341  		}
  1342  
  1343  		review.Response = &v1beta1.AdmissionResponse{
  1344  			Allowed: true,
  1345  			Result:  &metav1.Status{Message: "admitted"},
  1346  		}
  1347  
  1348  		// v1beta1 webhook handler tolerated these not being set. verify the server continues to accept these as unset.
  1349  		review.APIVersion = ""
  1350  		review.Kind = ""
  1351  		review.Response.UID = ""
  1352  
  1353  		// test plumbing warnings back to the client
  1354  		review.Response.Warnings = []string{makeWarning("v1beta1", phase, converted)}
  1355  
  1356  		// If we're mutating, and have an object, return a patch to exercise conversion
  1357  		if phase == mutation && len(review.Request.Object.Raw) > 0 {
  1358  			review.Response.Patch = []byte(`[{"op":"add","path":"/foo","value":"test"}]`)
  1359  			jsonPatch := v1beta1.PatchTypeJSONPatch
  1360  			review.Response.PatchType = &jsonPatch
  1361  		}
  1362  
  1363  		w.Header().Set("Content-Type", "application/json")
  1364  		if err := json.NewEncoder(w).Encode(review); err != nil {
  1365  			t.Errorf("Marshal of response failed with error: %v", err)
  1366  		}
  1367  	})
  1368  }
  1369  
  1370  func newV1WebhookHandler(t *testing.T, holder *holder, phase string, converted bool) http.Handler {
  1371  	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1372  		defer r.Body.Close()
  1373  		data, err := io.ReadAll(r.Body)
  1374  		if err != nil {
  1375  			t.Error(err)
  1376  			return
  1377  		}
  1378  
  1379  		if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
  1380  			t.Errorf("contentType=%s, expect application/json", contentType)
  1381  			return
  1382  		}
  1383  
  1384  		review := admissionreviewv1.AdmissionReview{}
  1385  		if err := json.Unmarshal(data, &review); err != nil {
  1386  			t.Errorf("Fail to deserialize object: %s with error: %v", string(data), err)
  1387  			http.Error(w, err.Error(), 400)
  1388  			return
  1389  		}
  1390  
  1391  		if review.GetObjectKind().GroupVersionKind() != gvk("admission.k8s.io", "v1", "AdmissionReview") {
  1392  			err := fmt.Errorf("Invalid admission review kind: %#v", review.GetObjectKind().GroupVersionKind())
  1393  			t.Error(err)
  1394  			http.Error(w, err.Error(), 400)
  1395  			return
  1396  		}
  1397  
  1398  		if len(review.Request.Object.Raw) > 0 {
  1399  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1400  			if err := json.Unmarshal(review.Request.Object.Raw, u); err != nil {
  1401  				t.Errorf("Fail to deserialize object: %s with error: %v", string(review.Request.Object.Raw), err)
  1402  				http.Error(w, err.Error(), 400)
  1403  				return
  1404  			}
  1405  			review.Request.Object.Object = u
  1406  		}
  1407  		if len(review.Request.OldObject.Raw) > 0 {
  1408  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1409  			if err := json.Unmarshal(review.Request.OldObject.Raw, u); err != nil {
  1410  				t.Errorf("Fail to deserialize object: %s with error: %v", string(review.Request.OldObject.Raw), err)
  1411  				http.Error(w, err.Error(), 400)
  1412  				return
  1413  			}
  1414  			review.Request.OldObject.Object = u
  1415  		}
  1416  
  1417  		if len(review.Request.Options.Raw) > 0 {
  1418  			u := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1419  			if err := json.Unmarshal(review.Request.Options.Raw, u); err != nil {
  1420  				t.Errorf("Fail to deserialize options object: %s for admission request %#+v with error: %v", string(review.Request.Options.Raw), review.Request, err)
  1421  				http.Error(w, err.Error(), 400)
  1422  				return
  1423  			}
  1424  			review.Request.Options.Object = u
  1425  		}
  1426  
  1427  		if review.Request.UserInfo.Username == testClientUsername {
  1428  			// only record requests originating from this integration test's client
  1429  			reviewRequest := &admissionRequest{
  1430  				Operation:   string(review.Request.Operation),
  1431  				Resource:    review.Request.Resource,
  1432  				SubResource: review.Request.SubResource,
  1433  				Namespace:   review.Request.Namespace,
  1434  				Name:        review.Request.Name,
  1435  				Object:      review.Request.Object,
  1436  				OldObject:   review.Request.OldObject,
  1437  				Options:     review.Request.Options,
  1438  			}
  1439  			holder.record("v1", phase, converted, reviewRequest)
  1440  		}
  1441  
  1442  		review.Response = &admissionreviewv1.AdmissionResponse{
  1443  			Allowed: true,
  1444  			UID:     review.Request.UID,
  1445  			Result:  &metav1.Status{Message: "admitted"},
  1446  
  1447  			// test plumbing warnings back
  1448  			Warnings: []string{makeWarning("v1", phase, converted)},
  1449  		}
  1450  		// If we're mutating, and have an object, return a patch to exercise conversion
  1451  		if phase == mutation && len(review.Request.Object.Raw) > 0 {
  1452  			review.Response.Patch = []byte(`[{"op":"add","path":"/bar","value":"test"}]`)
  1453  			jsonPatch := admissionreviewv1.PatchTypeJSONPatch
  1454  			review.Response.PatchType = &jsonPatch
  1455  		}
  1456  
  1457  		w.Header().Set("Content-Type", "application/json")
  1458  		if err := json.NewEncoder(w).Encode(review); err != nil {
  1459  			t.Errorf("Marshal of response failed with error: %v", err)
  1460  		}
  1461  	})
  1462  }
  1463  
  1464  func getTestFunc(gvr schema.GroupVersionResource, verb string) testFunc {
  1465  	if f, found := customTestFuncs[gvr][verb]; found {
  1466  		return f
  1467  	}
  1468  	if f, found := customTestFuncs[gvr]["*"]; found {
  1469  		return f
  1470  	}
  1471  	if strings.Contains(gvr.Resource, "/") {
  1472  		if f, found := defaultSubresourceFuncs[verb]; found {
  1473  			return f
  1474  		}
  1475  		return unimplemented
  1476  	}
  1477  	if f, found := defaultResourceFuncs[verb]; found {
  1478  		return f
  1479  	}
  1480  	return unimplemented
  1481  }
  1482  
  1483  func getStubObj(gvr schema.GroupVersionResource, resource metav1.APIResource) (*unstructured.Unstructured, error) {
  1484  	stub := ""
  1485  	if data, ok := etcd.GetEtcdStorageDataForNamespace(testNamespace)[gvr]; ok {
  1486  		stub = data.Stub
  1487  	}
  1488  	if data, ok := stubDataOverrides[gvr]; ok {
  1489  		stub = data
  1490  	}
  1491  	if len(stub) == 0 {
  1492  		return nil, fmt.Errorf("no stub data for %#v", gvr)
  1493  	}
  1494  
  1495  	stubObj := &unstructured.Unstructured{Object: map[string]interface{}{}}
  1496  	if err := json.Unmarshal([]byte(stub), &stubObj.Object); err != nil {
  1497  		return nil, fmt.Errorf("error unmarshaling stub for %#v: %v", gvr, err)
  1498  	}
  1499  	return stubObj, nil
  1500  }
  1501  
  1502  func createOrGetResource(client dynamic.Interface, gvr schema.GroupVersionResource, resource metav1.APIResource) (*unstructured.Unstructured, error) {
  1503  	stubObj, err := getStubObj(gvr, resource)
  1504  	if err != nil {
  1505  		return nil, err
  1506  	}
  1507  	ns := ""
  1508  	if resource.Namespaced {
  1509  		ns = testNamespace
  1510  	}
  1511  	obj, err := client.Resource(gvr).Namespace(ns).Get(context.TODO(), stubObj.GetName(), metav1.GetOptions{})
  1512  	if err == nil {
  1513  		return obj, nil
  1514  	}
  1515  	if !apierrors.IsNotFound(err) {
  1516  		return nil, err
  1517  	}
  1518  	return client.Resource(gvr).Namespace(ns).Create(context.TODO(), stubObj, metav1.CreateOptions{})
  1519  }
  1520  
  1521  func gvr(group, version, resource string) schema.GroupVersionResource {
  1522  	return schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
  1523  }
  1524  func gvk(group, version, kind string) schema.GroupVersionKind {
  1525  	return schema.GroupVersionKind{Group: group, Version: version, Kind: kind}
  1526  }
  1527  
  1528  var (
  1529  	gvkCreateOptions = metav1.SchemeGroupVersion.WithKind("CreateOptions")
  1530  	gvkUpdateOptions = metav1.SchemeGroupVersion.WithKind("UpdateOptions")
  1531  	gvkDeleteOptions = metav1.SchemeGroupVersion.WithKind("DeleteOptions")
  1532  )
  1533  
  1534  func shouldTestResource(gvr schema.GroupVersionResource, resource metav1.APIResource) bool {
  1535  	return sets.NewString(resource.Verbs...).HasAny("create", "update", "patch", "connect", "delete", "deletecollection")
  1536  }
  1537  
  1538  func shouldTestResourceVerb(gvr schema.GroupVersionResource, resource metav1.APIResource, verb string) bool {
  1539  	return sets.NewString(resource.Verbs...).Has(verb)
  1540  }
  1541  
  1542  //
  1543  // webhook registration helpers
  1544  //
  1545  
  1546  func createV1beta1ValidationWebhook(etcdClient *clientv3.Client, etcdStoragePrefix string, client clientset.Interface, endpoint, convertedEndpoint string, convertedRules []admissionregistrationv1beta1.RuleWithOperations) error {
  1547  	fail := admissionregistrationv1beta1.Fail
  1548  	equivalent := admissionregistrationv1beta1.Equivalent
  1549  	webhookConfig := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{
  1550  		ObjectMeta: metav1.ObjectMeta{Name: "admission.integration.test"},
  1551  		Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{
  1552  			{
  1553  				Name: "admission.integration.test",
  1554  				ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
  1555  					URL:      &endpoint,
  1556  					CABundle: localhostCert,
  1557  				},
  1558  				Rules: []admissionregistrationv1beta1.RuleWithOperations{{
  1559  					Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.OperationAll},
  1560  					Rule:       admissionregistrationv1beta1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}},
  1561  				}},
  1562  				FailurePolicy:           &fail,
  1563  				AdmissionReviewVersions: []string{"v1beta1"},
  1564  			},
  1565  			{
  1566  				Name: "admission.integration.testconversion",
  1567  				ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
  1568  					URL:      &convertedEndpoint,
  1569  					CABundle: localhostCert,
  1570  				},
  1571  				Rules:                   convertedRules,
  1572  				FailurePolicy:           &fail,
  1573  				MatchPolicy:             &equivalent,
  1574  				AdmissionReviewVersions: []string{"v1beta1"},
  1575  			},
  1576  		},
  1577  	}
  1578  	// run through to get defaulting
  1579  	apisv1beta1.SetObjectDefaults_ValidatingWebhookConfiguration(webhookConfig)
  1580  	webhookConfig.TypeMeta.Kind = "ValidatingWebhookConfiguration"
  1581  	webhookConfig.TypeMeta.APIVersion = "admissionregistration.k8s.io/v1beta1"
  1582  
  1583  	// Attaching Mutation webhook to API server
  1584  	ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceNone)
  1585  	key := path.Join("/", etcdStoragePrefix, "validatingwebhookconfigurations", webhookConfig.Name)
  1586  	val, _ := json.Marshal(webhookConfig)
  1587  	if _, err := etcdClient.Put(ctx, key, string(val)); err != nil {
  1588  		return err
  1589  	}
  1590  
  1591  	// make sure we can get the webhook
  1592  	if _, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.TODO(), webhookConfig.Name, metav1.GetOptions{}); err != nil {
  1593  		return err
  1594  	}
  1595  
  1596  	return nil
  1597  }
  1598  
  1599  func createV1beta1MutationWebhook(etcdClient *clientv3.Client, etcdStoragePrefix string, client clientset.Interface, endpoint, convertedEndpoint string, convertedRules []admissionregistrationv1beta1.RuleWithOperations) error {
  1600  	fail := admissionregistrationv1beta1.Fail
  1601  	equivalent := admissionregistrationv1beta1.Equivalent
  1602  	webhookConfig := &admissionregistrationv1beta1.MutatingWebhookConfiguration{
  1603  		ObjectMeta: metav1.ObjectMeta{Name: "mutation.integration.test"},
  1604  		Webhooks: []admissionregistrationv1beta1.MutatingWebhook{
  1605  			{
  1606  				Name: "mutation.integration.test",
  1607  				ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
  1608  					URL:      &endpoint,
  1609  					CABundle: localhostCert,
  1610  				},
  1611  				Rules: []admissionregistrationv1beta1.RuleWithOperations{{
  1612  					Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.OperationAll},
  1613  					Rule:       admissionregistrationv1beta1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}},
  1614  				}},
  1615  				FailurePolicy:           &fail,
  1616  				AdmissionReviewVersions: []string{"v1beta1"},
  1617  			},
  1618  			{
  1619  				Name: "mutation.integration.testconversion",
  1620  				ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
  1621  					URL:      &convertedEndpoint,
  1622  					CABundle: localhostCert,
  1623  				},
  1624  				Rules:                   convertedRules,
  1625  				FailurePolicy:           &fail,
  1626  				MatchPolicy:             &equivalent,
  1627  				AdmissionReviewVersions: []string{"v1beta1"},
  1628  			},
  1629  		},
  1630  	}
  1631  	// run through to get defaulting
  1632  	apisv1beta1.SetObjectDefaults_MutatingWebhookConfiguration(webhookConfig)
  1633  	webhookConfig.TypeMeta.Kind = "MutatingWebhookConfiguration"
  1634  	webhookConfig.TypeMeta.APIVersion = "admissionregistration.k8s.io/v1beta1"
  1635  
  1636  	// Attaching Mutation webhook to API server
  1637  	ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceNone)
  1638  	key := path.Join("/", etcdStoragePrefix, "mutatingwebhookconfigurations", webhookConfig.Name)
  1639  	val, _ := json.Marshal(webhookConfig)
  1640  	if _, err := etcdClient.Put(ctx, key, string(val)); err != nil {
  1641  		return err
  1642  	}
  1643  
  1644  	// make sure we can get the webhook
  1645  	if _, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), webhookConfig.Name, metav1.GetOptions{}); err != nil {
  1646  		return err
  1647  	}
  1648  
  1649  	return nil
  1650  }
  1651  
  1652  func createV1ValidationWebhook(client clientset.Interface, endpoint, convertedEndpoint string, convertedRules []admissionregistrationv1.RuleWithOperations) error {
  1653  	fail := admissionregistrationv1.Fail
  1654  	equivalent := admissionregistrationv1.Equivalent
  1655  	none := admissionregistrationv1.SideEffectClassNone
  1656  	// Attaching Admission webhook to API server
  1657  	_, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.ValidatingWebhookConfiguration{
  1658  		ObjectMeta: metav1.ObjectMeta{Name: "admissionregistrationv1.integration.test"},
  1659  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
  1660  			{
  1661  				Name: "admissionregistrationv1.integration.test",
  1662  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1663  					URL:      &endpoint,
  1664  					CABundle: localhostCert,
  1665  				},
  1666  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1667  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
  1668  					Rule:       admissionregistrationv1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}},
  1669  				}},
  1670  				FailurePolicy:           &fail,
  1671  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1672  				SideEffects:             &none,
  1673  			},
  1674  			{
  1675  				Name: "admissionregistrationv1.integration.testconversion",
  1676  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1677  					URL:      &convertedEndpoint,
  1678  					CABundle: localhostCert,
  1679  				},
  1680  				Rules:                   convertedRules,
  1681  				FailurePolicy:           &fail,
  1682  				MatchPolicy:             &equivalent,
  1683  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1684  				SideEffects:             &none,
  1685  			},
  1686  		},
  1687  	}, metav1.CreateOptions{})
  1688  	return err
  1689  }
  1690  
  1691  func createV1MutationWebhook(client clientset.Interface, endpoint, convertedEndpoint string, convertedRules []admissionregistrationv1.RuleWithOperations) error {
  1692  	fail := admissionregistrationv1.Fail
  1693  	equivalent := admissionregistrationv1.Equivalent
  1694  	none := admissionregistrationv1.SideEffectClassNone
  1695  	// Attaching Mutation webhook to API server
  1696  	_, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.MutatingWebhookConfiguration{
  1697  		ObjectMeta: metav1.ObjectMeta{Name: "mutationv1.integration.test"},
  1698  		Webhooks: []admissionregistrationv1.MutatingWebhook{
  1699  			{
  1700  				Name: "mutationv1.integration.test",
  1701  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1702  					URL:      &endpoint,
  1703  					CABundle: localhostCert,
  1704  				},
  1705  				Rules: []admissionregistrationv1.RuleWithOperations{{
  1706  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
  1707  					Rule:       admissionregistrationv1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*/*"}},
  1708  				}},
  1709  				FailurePolicy:           &fail,
  1710  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1711  				SideEffects:             &none,
  1712  			},
  1713  			{
  1714  				Name: "mutationv1.integration.testconversion",
  1715  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1716  					URL:      &convertedEndpoint,
  1717  					CABundle: localhostCert,
  1718  				},
  1719  				Rules:                   convertedRules,
  1720  				FailurePolicy:           &fail,
  1721  				MatchPolicy:             &equivalent,
  1722  				AdmissionReviewVersions: []string{"v1", "v1beta1"},
  1723  				SideEffects:             &none,
  1724  			},
  1725  		},
  1726  	}, metav1.CreateOptions{})
  1727  	return err
  1728  }
  1729  
  1730  // localhostCert was generated from crypto/tls/generate_cert.go with the following command:
  1731  //
  1732  //	go run generate_cert.go  --rsa-bits 2048 --host 127.0.0.1,::1,example.com,webhook.test.svc --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h
  1733  var localhostCert = []byte(`-----BEGIN CERTIFICATE-----
  1734  MIIDTDCCAjSgAwIBAgIRAJXp/H5o/ItwCEK9emP3NiMwDQYJKoZIhvcNAQELBQAw
  1735  EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2
  1736  MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEP
  1737  ADCCAQoCggEBAOCyQ/2e9SVZ3QSW1yxe9OoZeyX7N8jRRyRkWlSL/OiEIxGsDJHK
  1738  GcDrGONOm9FeKM73evSiNX+7AZEqdanT37RsvVHTbRKAKsNIilyFTYmSvPHC05iG
  1739  agcIBm/Wt+NvfNb3DFLPhCLZbeuqlKhMzc8NeWHNY6eJj1qqks70PNlcb3Q5Ufa2
  1740  ttxs3N4pUmi7/ntiFE+X42A6IGX94Zyu9E7kH+0/ajvEA0qAyIXp1TneMgybS+ox
  1741  UBLDBQvsOH5lwvVIUfJLI483geXbFaUpHc6fTKE/8/f6EuWWEN3UFvuDM6cqr51e
  1742  MPTziUVUs5NBIeHIGyTKTbF3+gTXFKDf/jECAwEAAaOBmjCBlzAOBgNVHQ8BAf8E
  1743  BAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNV
  1744  HQ4EFgQURFTsa1/pfERE/WJ3YpkbnKI6NkEwQAYDVR0RBDkwN4ILZXhhbXBsZS5j
  1745  b22CEHdlYmhvb2sudGVzdC5zdmOHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJ
  1746  KoZIhvcNAQELBQADggEBAE60cASylHw0DsHtTkQwjhmW0Bd1Dy0+BvGngD9P85tB
  1747  fNHtcurzGG1GSGVX7ClxghDZo84WcV742qenxBlZ37WTqmD5/4pWlEvbrjKmgr3W
  1748  yWM6WJts1W4T5aR6mU2jHz1mxIFq9Fcw2XcdtwHAJKoCKpLv6pYswW4LYODdKNii
  1749  eAKBEcbEBQ3oU4529yeDpkU6ZLBKH+ZVxWI3ZUWbpv5O6vMtSB9nvtTripbWrm1t
  1750  vpCEETNAOP2hbLnPwBXUEN8KBs94UdufOFIhArNgKonY/oZoZnZYWVyRtkex+b+r
  1751  MarmcIKMrgoYweSQiCa+XVWofz2ZSOvzxta6Y9iDI74=
  1752  -----END CERTIFICATE-----`)
  1753  
  1754  // localhostKey is the private key for localhostCert.
  1755  var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
  1756  MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgskP9nvUlWd0E
  1757  ltcsXvTqGXsl+zfI0UckZFpUi/zohCMRrAyRyhnA6xjjTpvRXijO93r0ojV/uwGR
  1758  KnWp09+0bL1R020SgCrDSIpchU2JkrzxwtOYhmoHCAZv1rfjb3zW9wxSz4Qi2W3r
  1759  qpSoTM3PDXlhzWOniY9aqpLO9DzZXG90OVH2trbcbNzeKVJou/57YhRPl+NgOiBl
  1760  /eGcrvRO5B/tP2o7xANKgMiF6dU53jIMm0vqMVASwwUL7Dh+ZcL1SFHySyOPN4Hl
  1761  2xWlKR3On0yhP/P3+hLllhDd1Bb7gzOnKq+dXjD084lFVLOTQSHhyBskyk2xd/oE
  1762  1xSg3/4xAgMBAAECggEAbykB5ejL0oyggPK2xKa9d0rf16xurpSKI4DaB1Wx6r3k
  1763  M4vwM/fNwdkM2Pc8stloSuu4EmplGSnE3rIov7mnxDS/fEmifjKV9UJf4OG5uEO1
  1764  4czGrYBh19Sqio2pL4UqN5bEq/spnav/a0VageBtOO+riyz3Dh1JpEsakfPWXpkk
  1765  gZ7Vl/jZ4zU27/LMfIqngOPeAGiUkLGikM6fPvm/4PbvgnSCZ4mhOSyzgCLmAWKi
  1766  Kr8zCD7BJk62/BUogk3qim+uW4Sf3RvZACTBWq6ZhWNeU2Z3CHI4G8p8sl7jtmPR
  1767  a1BWSV8Lf+83VFCfk/O+oSdb0f2z/RBAZ6uV9ZtHoQKBgQDikFsRxgXPXllSlytI
  1768  QU//19Z4S7dqWqFOX6+ap1aSyj01IsN1kvZzyGZ6ZyyAPUrNheokccijkXgooBHL
  1769  aLMxa4v0i/pHGcXAFbzIlzKwkmi0zIy7nX6cSIg2cg0sKWDGVxxJ4ODxFJRyd6Vq
  1770  Pao4/L+nUPVMRi2ME2iYe/qp/QKBgQD948teuZ4lEGTZx5IhmBpNuj45C8y5sd4W
  1771  vy+oFK8aOoTl4nCscYAAVXnS+CxInpQHI35GYRIDdjk2IL8eFThtsB+wS//Cd7h8
  1772  yY0JZC+XWhWPG5U+dSkSyzVsaK9jDJFRcnfnvHqO2+masyeq9FFTo8gX6KpF8wDL
  1773  97+UFz3xRQKBgQDa7ygx2quOodurBc2bexG1Z3smr/RD3+R0ed6VkhMEsk3HZRqA
  1774  KU3iwMrWiZDlM1VvmXKTWSjLdy0oBNZtO3W90fFilUl7H5qKbfcJ16HyIujvnaJ5
  1775  Qk4w8549DqVQAYQ05cS+V4LHNF3m51t/eKtfek4xfvgrhr1I2RCAGX42eQKBgFOw
  1776  miIgZ4vqKoRLL9VZERqcENS3GgYAJqgy31+1ab7omVQ531BInZv+kQjE+7v4Ye00
  1777  evRyHQD9IIDCLJ2a+x3VF60CcE1HL44a1h3JY5KthDvHKNwMvLxQNc0FeQLaarCB
  1778  XhsKWw/qV8fB1IqavJAohdWzwSULpDCX+xOy0Z1NAoGAPXGRPSw0p0b8zHuJ6SmM
  1779  blkpX9rdFMN08MJYIBG+ZiRobU+OOvClBZiDpYHpBnFCFpsXiStSYKOBrAAypC01
  1780  UFJJZe7Tfz1R4VcexsS3yfXOZV/+9t/PnyFofSBB8wf/dokhgfEOYq8rbiunHFVT
  1781  20/b/zX8pbSiK6Kgy9vIm7w=
  1782  -----END RSA PRIVATE KEY-----`)
  1783  

View as plain text