reinvocation_test.go

Documentation: k8s.io/kubernetes/test/integration/apiserver/admissionwebhook

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package admissionwebhook
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"encoding/json"
    24  	"fmt"
    25  	"io"
    26  	"net/http"
    27  	"net/http/httptest"
    28  	"os"
    29  	"reflect"
    30  	"strconv"
    31  	"strings"
    32  	"sync"
    33  	"testing"
    34  	"time"
    35  
    36  	"k8s.io/api/admission/v1beta1"
    37  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    38  	corev1 "k8s.io/api/core/v1"
    39  	schedulingv1 "k8s.io/api/scheduling/v1"
    40  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    41  	"k8s.io/apimachinery/pkg/types"
    42  	"k8s.io/apimachinery/pkg/util/wait"
    43  	auditinternal "k8s.io/apiserver/pkg/apis/audit"
    44  	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
    45  	clientset "k8s.io/client-go/kubernetes"
    46  	"k8s.io/client-go/rest"
    47  	utiltesting "k8s.io/client-go/util/testing"
    48  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    49  	"k8s.io/kubernetes/test/integration/framework"
    50  	"k8s.io/kubernetes/test/utils"
    51  )
    52  
    53  const (
    54  	testReinvocationClientUsername = "webhook-reinvocation-integration-client"
    55  	auditPolicy                    = `
    56  apiVersion: audit.k8s.io/v1
    57  kind: Policy
    58  rules:
    59    - level: Request
    60      resources:
    61        - group: "" # core
    62          resources: ["pods"]
    63  `
    64  )
    65  
    66  // TestWebhookReinvocationPolicyWithWatchCache ensures that the admission webhook reinvocation policy is applied correctly with the watch cache enabled.
    67  func TestWebhookReinvocationPolicyWithWatchCache(t *testing.T) {
    68  	testWebhookReinvocationPolicy(t, true)
    69  }
    70  
    71  // TestWebhookReinvocationPolicyWithoutWatchCache ensures that the admission webhook reinvocation policy is applied correctly without the watch cache enabled.
    72  func TestWebhookReinvocationPolicyWithoutWatchCache(t *testing.T) {
    73  	testWebhookReinvocationPolicy(t, false)
    74  }
    75  
    76  func mutationAnnotationValue(configuration, webhook string, mutated bool) string {
    77  	return fmt.Sprintf(`{"configuration":"%s","webhook":"%s","mutated":%t}`, configuration, webhook, mutated)
    78  }
    79  
    80  func patchAnnotationValue(configuration, webhook string, patch string) string {
    81  	return strings.Replace(fmt.Sprintf(`{"configuration": "%s", "webhook": "%s", "patch": %s, "patchType": "JSONPatch"}`, configuration, webhook, patch), " ", "", -1)
    82  }
    83  
    84  // testWebhookReinvocationPolicy ensures that the admission webhook reinvocation policy is applied correctly.
    85  func testWebhookReinvocationPolicy(t *testing.T, watchCache bool) {
    86  	reinvokeNever := admissionregistrationv1.NeverReinvocationPolicy
    87  	reinvokeIfNeeded := admissionregistrationv1.IfNeededReinvocationPolicy
    88  
    89  	type testWebhook struct {
    90  		path           string
    91  		policy         *admissionregistrationv1.ReinvocationPolicyType
    92  		objectSelector *metav1.LabelSelector
    93  	}
    94  
    95  	testCases := []struct {
    96  		name                           string
    97  		initialPriorityClass           string
    98  		webhooks                       []testWebhook
    99  		expectLabels                   map[string]string
   100  		expectInvocations              map[string]int
   101  		expectError                    bool
   102  		errorContains                  string
   103  		expectAuditMutationAnnotations map[string]string
   104  		expectAuditPatchAnnotations    map[string]string
   105  	}{
   106  		{ // in-tree (mutation), webhook (no mutation), no reinvocation required
   107  			name:                 "no reinvocation for in-tree only mutation",
   108  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   109  			webhooks: []testWebhook{
   110  				{path: "/noop", policy: &reinvokeIfNeeded},
   111  			},
   112  			expectInvocations: map[string]int{"/noop": 1},
   113  			expectAuditMutationAnnotations: map[string]string{
   114  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-0", "admission.integration.test.0.noop", false),
   115  			},
   116  		},
   117  		{ // in-tree (mutation), webhook (mutation), reinvoke in-tree (no-mutation), no webhook reinvocation required
   118  			name:                 "no webhook reinvocation for webhook when no in-tree reinvocation mutations",
   119  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   120  			webhooks: []testWebhook{
   121  				{path: "/addlabel", policy: &reinvokeIfNeeded},
   122  			},
   123  			expectInvocations: map[string]int{"/addlabel": 1},
   124  			expectAuditPatchAnnotations: map[string]string{
   125  				"patch.webhook.admission.k8s.io/round_0_index_0": patchAnnotationValue("admission.integration.test-1", "admission.integration.test.0.addlabel", `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`),
   126  			},
   127  			expectAuditMutationAnnotations: map[string]string{
   128  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-1", "admission.integration.test.0.addlabel", true),
   129  			},
   130  		},
   131  		{ // in-tree (mutation), webhook (mutation), reinvoke in-tree (mutation), webhook (no-mutation), both reinvoked
   132  			name:                 "webhook is reinvoked after in-tree reinvocation",
   133  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   134  			webhooks: []testWebhook{
   135  				// Priority plugin is ordered to run before mutating webhooks
   136  				{path: "/setpriority", policy: &reinvokeIfNeeded}, // trigger in-tree reinvoke mutation
   137  			},
   138  			expectInvocations: map[string]int{"/setpriority": 2},
   139  			expectAuditPatchAnnotations: map[string]string{
   140  				"patch.webhook.admission.k8s.io/round_0_index_0": patchAnnotationValue("admission.integration.test-2", "admission.integration.test.0.setpriority", `[{"op": "add", "path": "/spec/priorityClassName", "value": "high-priority"},{"op": "remove", "path": "/spec/priority"}]`),
   141  			},
   142  			expectAuditMutationAnnotations: map[string]string{
   143  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-2", "admission.integration.test.0.setpriority", true),
   144  				"mutation.webhook.admission.k8s.io/round_1_index_0": mutationAnnotationValue("admission.integration.test-2", "admission.integration.test.0.setpriority", false),
   145  			},
   146  		},
   147  		{ // in-tree (mutation), webhook A (mutation), webhook B (mutation), reinvoke in-tree (no-mutation), reinvoke webhook A (no-mutation), no reinvocation of webhook B required
   148  			name:                 "no reinvocation of webhook B when in-tree or prior webhook mutations",
   149  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   150  			webhooks: []testWebhook{
   151  				{path: "/addlabel", policy: &reinvokeIfNeeded},
   152  				{path: "/conditionaladdlabel", policy: &reinvokeIfNeeded},
   153  			},
   154  			expectLabels:      map[string]string{"x": "true", "a": "true", "b": "true"},
   155  			expectInvocations: map[string]int{"/addlabel": 2, "/conditionaladdlabel": 1},
   156  			expectAuditPatchAnnotations: map[string]string{
   157  				"patch.webhook.admission.k8s.io/round_0_index_0": patchAnnotationValue("admission.integration.test-3", "admission.integration.test.0.addlabel", `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`),
   158  				"patch.webhook.admission.k8s.io/round_0_index_1": patchAnnotationValue("admission.integration.test-3", "admission.integration.test.1.conditionaladdlabel", `[{"op": "add", "path": "/metadata/labels/b", "value": "true"}]`),
   159  			},
   160  			expectAuditMutationAnnotations: map[string]string{
   161  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-3", "admission.integration.test.0.addlabel", true),
   162  				"mutation.webhook.admission.k8s.io/round_0_index_1": mutationAnnotationValue("admission.integration.test-3", "admission.integration.test.1.conditionaladdlabel", true),
   163  				"mutation.webhook.admission.k8s.io/round_1_index_0": mutationAnnotationValue("admission.integration.test-3", "admission.integration.test.0.addlabel", false),
   164  			},
   165  		},
   166  		{ // in-tree (mutation), webhook A (mutation), webhook B (mutation), reinvoke in-tree (no-mutation), reinvoke webhook A (mutation), reinvoke webhook B (mutation), both webhooks reinvoked
   167  			name:                 "all webhooks reinvoked when any webhook reinvocation causes mutation",
   168  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   169  			webhooks: []testWebhook{
   170  				{path: "/settrue", policy: &reinvokeIfNeeded},
   171  				{path: "/setfalse", policy: &reinvokeIfNeeded},
   172  			},
   173  			expectLabels:      map[string]string{"x": "true", "fight": "false"},
   174  			expectInvocations: map[string]int{"/settrue": 2, "/setfalse": 2},
   175  			expectAuditPatchAnnotations: map[string]string{
   176  				"patch.webhook.admission.k8s.io/round_0_index_0": patchAnnotationValue("admission.integration.test-4", "admission.integration.test.0.settrue", `[{"op": "replace", "path": "/metadata/labels/fight", "value": "true"}]`),
   177  				"patch.webhook.admission.k8s.io/round_0_index_1": patchAnnotationValue("admission.integration.test-4", "admission.integration.test.1.setfalse", `[{"op": "replace", "path": "/metadata/labels/fight", "value": "false"}]`),
   178  				"patch.webhook.admission.k8s.io/round_1_index_0": patchAnnotationValue("admission.integration.test-4", "admission.integration.test.0.settrue", `[{"op": "replace", "path": "/metadata/labels/fight", "value": "true"}]`),
   179  				"patch.webhook.admission.k8s.io/round_1_index_1": patchAnnotationValue("admission.integration.test-4", "admission.integration.test.1.setfalse", `[{"op": "replace", "path": "/metadata/labels/fight", "value": "false"}]`),
   180  			},
   181  			expectAuditMutationAnnotations: map[string]string{
   182  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-4", "admission.integration.test.0.settrue", true),
   183  				"mutation.webhook.admission.k8s.io/round_0_index_1": mutationAnnotationValue("admission.integration.test-4", "admission.integration.test.1.setfalse", true),
   184  				"mutation.webhook.admission.k8s.io/round_1_index_0": mutationAnnotationValue("admission.integration.test-4", "admission.integration.test.0.settrue", true),
   185  				"mutation.webhook.admission.k8s.io/round_1_index_1": mutationAnnotationValue("admission.integration.test-4", "admission.integration.test.1.setfalse", true),
   186  			},
   187  		},
   188  		{ // in-tree (mutation), webhook A is SKIPPED due to objectSelector not matching, webhook B (mutation), reinvoke in-tree (no-mutation), webhook A is SKIPPED even though the labels match now, because it's not called in the first round. No reinvocation of webhook B required
   189  			name:                 "no reinvocation of webhook B when in-tree or prior webhook mutations",
   190  			initialPriorityClass: "low-priority", // trigger initial in-tree mutation
   191  			webhooks: []testWebhook{
   192  				{path: "/conditionaladdlabel", policy: &reinvokeIfNeeded, objectSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   193  				{path: "/addlabel", policy: &reinvokeIfNeeded},
   194  			},
   195  			expectLabels:      map[string]string{"x": "true", "a": "true"},
   196  			expectInvocations: map[string]int{"/addlabel": 1, "/conditionaladdlabel": 0},
   197  			expectAuditPatchAnnotations: map[string]string{
   198  				"patch.webhook.admission.k8s.io/round_0_index_1": patchAnnotationValue("admission.integration.test-5", "admission.integration.test.1.addlabel", `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`),
   199  			},
   200  			expectAuditMutationAnnotations: map[string]string{
   201  				"mutation.webhook.admission.k8s.io/round_0_index_1": mutationAnnotationValue("admission.integration.test-5", "admission.integration.test.1.addlabel", true),
   202  			},
   203  		},
   204  		{
   205  			name: "invalid priority class set by webhook should result in error from in-tree priority plugin",
   206  			webhooks: []testWebhook{
   207  				// Priority plugin is ordered to run before mutating webhooks
   208  				{path: "/setinvalidpriority", policy: &reinvokeIfNeeded},
   209  			},
   210  			expectError:       true,
   211  			errorContains:     "no PriorityClass with name invalid was found",
   212  			expectInvocations: map[string]int{"/setinvalidpriority": 1},
   213  		},
   214  		{
   215  			name: "'reinvoke never' policy respected",
   216  			webhooks: []testWebhook{
   217  				{path: "/conditionaladdlabel", policy: &reinvokeNever},
   218  				{path: "/addlabel", policy: &reinvokeNever},
   219  			},
   220  			expectLabels:      map[string]string{"x": "true", "a": "true"},
   221  			expectInvocations: map[string]int{"/conditionaladdlabel": 1, "/addlabel": 1},
   222  			expectAuditPatchAnnotations: map[string]string{
   223  				"patch.webhook.admission.k8s.io/round_0_index_1": patchAnnotationValue("admission.integration.test-7", "admission.integration.test.1.addlabel", `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`),
   224  			},
   225  			expectAuditMutationAnnotations: map[string]string{
   226  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-7", "admission.integration.test.0.conditionaladdlabel", false),
   227  				"mutation.webhook.admission.k8s.io/round_0_index_1": mutationAnnotationValue("admission.integration.test-7", "admission.integration.test.1.addlabel", true),
   228  			},
   229  		},
   230  		{
   231  			name: "'reinvoke never' (by default) policy respected",
   232  			webhooks: []testWebhook{
   233  				{path: "/conditionaladdlabel", policy: nil},
   234  				{path: "/addlabel", policy: nil},
   235  			},
   236  			expectLabels:      map[string]string{"x": "true", "a": "true"},
   237  			expectInvocations: map[string]int{"/conditionaladdlabel": 1, "/addlabel": 1},
   238  			expectAuditPatchAnnotations: map[string]string{
   239  				"patch.webhook.admission.k8s.io/round_0_index_1": patchAnnotationValue("admission.integration.test-8", "admission.integration.test.1.addlabel", `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`),
   240  			},
   241  			expectAuditMutationAnnotations: map[string]string{
   242  				"mutation.webhook.admission.k8s.io/round_0_index_0": mutationAnnotationValue("admission.integration.test-8", "admission.integration.test.0.conditionaladdlabel", false),
   243  				"mutation.webhook.admission.k8s.io/round_0_index_1": mutationAnnotationValue("admission.integration.test-8", "admission.integration.test.1.addlabel", true),
   244  			},
   245  		},
   246  	}
   247  
   248  	roots := x509.NewCertPool()
   249  	if !roots.AppendCertsFromPEM(localhostCert) {
   250  		t.Fatal("Failed to append Cert from PEM")
   251  	}
   252  	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
   253  	if err != nil {
   254  		t.Fatalf("Failed to build cert with error: %+v", err)
   255  	}
   256  
   257  	recorder := &invocationRecorder{counts: map[string]int{}}
   258  	webhookServer := httptest.NewUnstartedServer(newReinvokeWebhookHandler(recorder))
   259  	webhookServer.TLS = &tls.Config{
   260  
   261  		RootCAs:      roots,
   262  		Certificates: []tls.Certificate{cert},
   263  	}
   264  	webhookServer.StartTLS()
   265  	defer webhookServer.Close()
   266  
   267  	// prepare audit policy file
   268  	policyFile, err := os.CreateTemp("", "audit-policy.yaml")
   269  	if err != nil {
   270  		t.Fatalf("Failed to create audit policy file: %v", err)
   271  	}
   272  	defer os.Remove(policyFile.Name())
   273  	if _, err := policyFile.Write([]byte(auditPolicy)); err != nil {
   274  		t.Fatalf("Failed to write audit policy file: %v", err)
   275  	}
   276  	if err := policyFile.Close(); err != nil {
   277  		t.Fatalf("Failed to close audit policy file: %v", err)
   278  	}
   279  
   280  	// prepare audit log file
   281  	logFile, err := os.CreateTemp("", "audit.log")
   282  	if err != nil {
   283  		t.Fatalf("Failed to create audit log file: %v", err)
   284  	}
   285  	defer utiltesting.CloseAndRemove(t, logFile)
   286  
   287  	s := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{
   288  		"--disable-admission-plugins=ServiceAccount",
   289  		fmt.Sprintf("--watch-cache=%v", watchCache),
   290  		"--audit-policy-file", policyFile.Name(),
   291  		"--audit-log-version", "audit.k8s.io/v1",
   292  		"--audit-log-mode", "blocking",
   293  		"--audit-log-path", logFile.Name(),
   294  	}, framework.SharedEtcd())
   295  	defer s.TearDownFn()
   296  
   297  	// Configure a client with a distinct user name so that it is easy to distinguish requests
   298  	// made by the client from requests made by controllers. We use this to filter out requests
   299  	// before recording them to ensure we don't accidentally mistake requests from controllers
   300  	// as requests made by the client.
   301  	clientConfig := rest.CopyConfig(s.ClientConfig)
   302  	clientConfig.Impersonate.UserName = testReinvocationClientUsername
   303  	clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"}
   304  	client, err := clientset.NewForConfig(clientConfig)
   305  	if err != nil {
   306  		t.Fatalf("unexpected error: %v", err)
   307  	}
   308  
   309  	for priorityClass, priority := range map[string]int{"low-priority": 1, "high-priority": 10} {
   310  		_, err = client.SchedulingV1().PriorityClasses().Create(context.TODO(), &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: priorityClass}, Value: int32(priority)}, metav1.CreateOptions{})
   311  		if err != nil {
   312  			t.Fatal(err)
   313  		}
   314  	}
   315  
   316  	for i, tt := range testCases {
   317  		t.Run(tt.name, func(t *testing.T) {
   318  			upCh := recorder.Reset()
   319  			testCaseID := strconv.Itoa(i)
   320  			ns := "reinvoke-" + testCaseID
   321  			nsLabels := map[string]string{"test-case": testCaseID}
   322  			_, err = client.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns, Labels: nsLabels}}, metav1.CreateOptions{})
   323  			if err != nil {
   324  				t.Fatal(err)
   325  			}
   326  
   327  			// Write markers to a separate namespace to avoid cross-talk
   328  			markerNs := ns + "-markers"
   329  			markerNsLabels := map[string]string{"test-markers": testCaseID}
   330  			_, err = client.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: markerNs, Labels: markerNsLabels}}, metav1.CreateOptions{})
   331  			if err != nil {
   332  				t.Fatal(err)
   333  			}
   334  
   335  			// Create a maker object to use to check for the webhook configurations to be ready.
   336  			marker, err := client.CoreV1().Pods(markerNs).Create(context.TODO(), newReinvocationMarkerFixture(markerNs), metav1.CreateOptions{})
   337  			if err != nil {
   338  				t.Fatal(err)
   339  			}
   340  
   341  			fail := admissionregistrationv1.Fail
   342  			webhooks := []admissionregistrationv1.MutatingWebhook{}
   343  			for j, webhook := range tt.webhooks {
   344  				endpoint := webhookServer.URL + webhook.path
   345  				name := fmt.Sprintf("admission.integration.test.%d.%s", j, strings.TrimPrefix(webhook.path, "/"))
   346  				webhooks = append(webhooks, admissionregistrationv1.MutatingWebhook{
   347  					Name: name,
   348  					ClientConfig: admissionregistrationv1.WebhookClientConfig{
   349  						URL:      &endpoint,
   350  						CABundle: localhostCert,
   351  					},
   352  					Rules: []admissionregistrationv1.RuleWithOperations{{
   353  						Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
   354  						Rule:       admissionregistrationv1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}},
   355  					}},
   356  					ObjectSelector:          webhook.objectSelector,
   357  					NamespaceSelector:       &metav1.LabelSelector{MatchLabels: nsLabels},
   358  					FailurePolicy:           &fail,
   359  					ReinvocationPolicy:      webhook.policy,
   360  					AdmissionReviewVersions: []string{"v1beta1"},
   361  					SideEffects:             &noSideEffects,
   362  				})
   363  			}
   364  			// Register a marker checking webhook with each set of webhook configurations
   365  			markerEndpoint := webhookServer.URL + "/marker"
   366  			webhooks = append(webhooks, admissionregistrationv1.MutatingWebhook{
   367  				Name: "admission.integration.test.marker",
   368  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
   369  					URL:      &markerEndpoint,
   370  					CABundle: localhostCert,
   371  				},
   372  				Rules: []admissionregistrationv1.RuleWithOperations{{
   373  					Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
   374  					Rule:       admissionregistrationv1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}},
   375  				}},
   376  				NamespaceSelector:       &metav1.LabelSelector{MatchLabels: markerNsLabels},
   377  				ObjectSelector:          &metav1.LabelSelector{MatchLabels: map[string]string{"marker": "true"}},
   378  				AdmissionReviewVersions: []string{"v1beta1"},
   379  				SideEffects:             &noSideEffects,
   380  			})
   381  
   382  			cfg, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.MutatingWebhookConfiguration{
   383  				ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("admission.integration.test-%d", i)},
   384  				Webhooks:   webhooks,
   385  			}, metav1.CreateOptions{})
   386  			if err != nil {
   387  				t.Fatal(err)
   388  			}
   389  			defer func() {
   390  				err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), cfg.GetName(), metav1.DeleteOptions{})
   391  				if err != nil {
   392  					t.Fatal(err)
   393  				}
   394  			}()
   395  
   396  			// wait until new webhook is called the first time
   397  			if err := wait.PollImmediate(time.Millisecond*5, wait.ForeverTestTimeout, func() (bool, error) {
   398  				_, err = client.CoreV1().Pods(markerNs).Patch(context.TODO(), marker.Name, types.JSONPatchType, []byte("[]"), metav1.PatchOptions{})
   399  				select {
   400  				case <-upCh:
   401  					return true, nil
   402  				default:
   403  					t.Logf("Waiting for webhook to become effective, getting marker object: %v", err)
   404  					return false, nil
   405  				}
   406  			}); err != nil {
   407  				t.Fatal(err)
   408  			}
   409  
   410  			pod := &corev1.Pod{
   411  				ObjectMeta: metav1.ObjectMeta{
   412  					Namespace: ns,
   413  					Name:      "labeled",
   414  					Labels:    map[string]string{"x": "true"},
   415  				},
   416  				Spec: corev1.PodSpec{
   417  					Containers: []corev1.Container{{
   418  						Name:  "fake-name",
   419  						Image: "fakeimage",
   420  					}},
   421  				},
   422  			}
   423  			if tt.initialPriorityClass != "" {
   424  				pod.Spec.PriorityClassName = tt.initialPriorityClass
   425  			}
   426  			obj, err := client.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
   427  
   428  			if tt.expectError {
   429  				if err == nil {
   430  					t.Fatalf("expected error but got none")
   431  				}
   432  				if tt.errorContains != "" {
   433  					if !strings.Contains(err.Error(), tt.errorContains) {
   434  						t.Errorf("expected an error saying %q, but got: %v", tt.errorContains, err)
   435  					}
   436  				}
   437  				return
   438  			}
   439  
   440  			if err != nil {
   441  				t.Fatal(err)
   442  			}
   443  
   444  			if tt.expectLabels != nil {
   445  				labels := obj.GetLabels()
   446  				if !reflect.DeepEqual(tt.expectLabels, labels) {
   447  					t.Errorf("expected labels '%v', but got '%v'", tt.expectLabels, labels)
   448  				}
   449  			}
   450  
   451  			for k, v := range tt.expectInvocations {
   452  				if recorder.GetCount(k) != v {
   453  					t.Errorf("expected %d invocations of %s, but got %d", v, k, recorder.GetCount(k))
   454  				}
   455  			}
   456  
   457  			stream, err := os.OpenFile(logFile.Name(), os.O_RDWR, 0600)
   458  			if err != nil {
   459  				t.Errorf("unexpected error: %v", err)
   460  			}
   461  			defer stream.Close()
   462  			missing, err := utils.CheckAuditLines(stream, expectedAuditEvents(tt.expectAuditMutationAnnotations, tt.expectAuditPatchAnnotations, ns), auditv1.SchemeGroupVersion)
   463  			if err != nil {
   464  				t.Errorf("unexpected error checking audit lines: %v", err)
   465  			}
   466  			if len(missing.MissingEvents) > 0 {
   467  				t.Errorf("failed to get expected events -- missing: %s", missing)
   468  			}
   469  			if err := stream.Truncate(0); err != nil {
   470  				t.Errorf("unexpected error truncate file: %v", err)
   471  			}
   472  			if _, err := stream.Seek(0, 0); err != nil {
   473  				t.Errorf("unexpected error reset offset: %v", err)
   474  			}
   475  		})
   476  	}
   477  }
   478  
   479  type invocationRecorder struct {
   480  	mu     sync.Mutex
   481  	upCh   chan struct{}
   482  	upOnce sync.Once
   483  	counts map[string]int
   484  }
   485  
   486  // Reset zeros out all counts and returns a channel that is closed when the first admission of the
   487  // marker object is received.
   488  func (i *invocationRecorder) Reset() chan struct{} {
   489  	i.mu.Lock()
   490  	defer i.mu.Unlock()
   491  	i.counts = map[string]int{}
   492  	i.upCh = make(chan struct{})
   493  	i.upOnce = sync.Once{}
   494  	return i.upCh
   495  }
   496  
   497  func (i *invocationRecorder) MarkerReceived() {
   498  	i.mu.Lock()
   499  	defer i.mu.Unlock()
   500  	i.upOnce.Do(func() {
   501  		close(i.upCh)
   502  	})
   503  }
   504  
   505  func (i *invocationRecorder) GetCount(path string) int {
   506  	i.mu.Lock()
   507  	defer i.mu.Unlock()
   508  	return i.counts[path]
   509  }
   510  
   511  func (i *invocationRecorder) IncrementCount(path string) {
   512  	i.mu.Lock()
   513  	defer i.mu.Unlock()
   514  	i.counts[path]++
   515  }
   516  
   517  func newReinvokeWebhookHandler(recorder *invocationRecorder) http.Handler {
   518  	patch := func(w http.ResponseWriter, patch string) {
   519  		w.Header().Set("Content-Type", "application/json")
   520  		pt := v1beta1.PatchTypeJSONPatch
   521  		json.NewEncoder(w).Encode(&v1beta1.AdmissionReview{
   522  			Response: &v1beta1.AdmissionResponse{
   523  				Allowed:   true,
   524  				PatchType: &pt,
   525  				Patch:     []byte(patch),
   526  			},
   527  		})
   528  	}
   529  	allow := func(w http.ResponseWriter) {
   530  		w.Header().Set("Content-Type", "application/json")
   531  		json.NewEncoder(w).Encode(&v1beta1.AdmissionReview{
   532  			Response: &v1beta1.AdmissionResponse{
   533  				Allowed: true,
   534  			},
   535  		})
   536  	}
   537  	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   538  		defer r.Body.Close()
   539  		data, err := io.ReadAll(r.Body)
   540  		if err != nil {
   541  			http.Error(w, err.Error(), 400)
   542  		}
   543  		review := v1beta1.AdmissionReview{}
   544  		if err := json.Unmarshal(data, &review); err != nil {
   545  			http.Error(w, err.Error(), 400)
   546  		}
   547  		if review.Request.UserInfo.Username != testReinvocationClientUsername {
   548  			// skip requests not originating from this integration test's client
   549  			allow(w)
   550  			return
   551  		}
   552  
   553  		if len(review.Request.Object.Raw) == 0 {
   554  			http.Error(w, err.Error(), 400)
   555  		}
   556  		pod := &corev1.Pod{}
   557  		if err := json.Unmarshal(review.Request.Object.Raw, pod); err != nil {
   558  			http.Error(w, err.Error(), 400)
   559  		}
   560  
   561  		recorder.IncrementCount(r.URL.Path)
   562  
   563  		switch r.URL.Path {
   564  		case "/marker":
   565  			// When resetting between tests, a marker object is patched until this webhook
   566  			// observes it, at which point it is considered ready.
   567  			recorder.MarkerReceived()
   568  			allow(w)
   569  			return
   570  		case "/noop":
   571  			allow(w)
   572  		case "/settrue":
   573  			patch(w, `[{"op": "replace", "path": "/metadata/labels/fight", "value": "true"}]`)
   574  		case "/setfalse":
   575  			patch(w, `[{"op": "replace", "path": "/metadata/labels/fight", "value": "false"}]`)
   576  		case "/addlabel":
   577  			labels := pod.GetLabels()
   578  			if a, ok := labels["a"]; !ok || a != "true" {
   579  				patch(w, `[{"op": "add", "path": "/metadata/labels/a", "value": "true"}]`)
   580  				return
   581  			}
   582  			allow(w)
   583  		case "/conditionaladdlabel": // if 'a' is set, set 'b' to true
   584  			labels := pod.GetLabels()
   585  			if _, ok := labels["a"]; ok {
   586  				patch(w, `[{"op": "add", "path": "/metadata/labels/b", "value": "true"}]`)
   587  				return
   588  			}
   589  			allow(w)
   590  		case "/setpriority": // sets /spec/priorityClassName to high-priority if it is not already set
   591  			if pod.Spec.PriorityClassName != "high-priority" {
   592  				if pod.Spec.Priority != nil {
   593  					patch(w, `[{"op": "add", "path": "/spec/priorityClassName", "value": "high-priority"},{"op": "remove", "path": "/spec/priority"}]`)
   594  				} else {
   595  					patch(w, `[{"op": "add", "path": "/spec/priorityClassName", "value": "high-priority"}]`)
   596  				}
   597  				return
   598  			}
   599  			allow(w)
   600  		case "/setinvalidpriority":
   601  			patch(w, `[{"op": "add", "path": "/spec/priorityClassName", "value": "invalid"}]`)
   602  		default:
   603  			http.NotFound(w, r)
   604  		}
   605  	})
   606  }
   607  
   608  func expectedAuditEvents(webhookMutationAnnotations, webhookPatchAnnotations map[string]string, namespace string) []utils.AuditEvent {
   609  	return []utils.AuditEvent{
   610  		{
   611  			Level:                               auditinternal.LevelRequest,
   612  			Stage:                               auditinternal.StageResponseComplete,
   613  			RequestURI:                          fmt.Sprintf("/api/v1/namespaces/%s/pods", namespace),
   614  			Verb:                                "create",
   615  			Code:                                201,
   616  			User:                                "system:apiserver",
   617  			ImpersonatedUser:                    testReinvocationClientUsername,
   618  			ImpersonatedGroups:                  "system:authenticated,system:masters",
   619  			Resource:                            "pods",
   620  			Namespace:                           namespace,
   621  			AuthorizeDecision:                   "allow",
   622  			RequestObject:                       true,
   623  			ResponseObject:                      false,
   624  			AdmissionWebhookMutationAnnotations: webhookMutationAnnotations,
   625  			AdmissionWebhookPatchAnnotations:    webhookPatchAnnotations,
   626  		},
   627  	}
   628  }
   629  
   630  func newReinvocationMarkerFixture(namespace string) *corev1.Pod {
   631  	return &corev1.Pod{
   632  		ObjectMeta: metav1.ObjectMeta{
   633  			Namespace: namespace,
   634  			Name:      "marker",
   635  			Labels: map[string]string{
   636  				"marker": "true",
   637  			},
   638  		},
   639  		Spec: corev1.PodSpec{
   640  			Containers: []corev1.Container{{
   641  				Name:  "fake-name",
   642  				Image: "fakeimage",
   643  			}},
   644  		},
   645  	}
   646  }
   647
View as plain text