/* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package auth import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "regexp" "strconv" "strings" "sync/atomic" "testing" "time" authorizationv1 "k8s.io/api/authorization/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" celmetrics "k8s.io/apiserver/pkg/authorization/cel" authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics" "k8s.io/apiserver/pkg/features" authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics" utilfeature "k8s.io/apiserver/pkg/util/feature" webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/authutil" "k8s.io/kubernetes/test/integration/framework" ) func TestAuthzConfig(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)() dir := t.TempDir() configFileName := filepath.Join(dir, "config.yaml") if err := atomicWriteFile(configFileName, []byte(` apiVersion: apiserver.config.k8s.io/v1alpha1 kind: AuthorizationConfiguration authorizers: - type: RBAC name: rbac `), os.FileMode(0644)); err != nil { t.Fatal(err) } server := kubeapiservertesting.StartTestServerOrDie( t, nil, []string{"--authorization-config=" + configFileName}, framework.SharedEtcd(), ) t.Cleanup(server.TearDownFn) // Make sure anonymous requests work anonymousClient := clientset.NewForConfigOrDie(rest.AnonymousClientConfig(server.ClientConfig)) healthzResult, err := anonymousClient.DiscoveryClient.RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).Raw() if !bytes.Equal(healthzResult, []byte(`ok`)) { t.Fatalf("expected 'ok', got %s", string(healthzResult)) } if err != nil { t.Fatal(err) } adminClient := clientset.NewForConfigOrDie(server.ClientConfig) sar := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Namespace: "foo", Verb: "create", Group: "", Version: "v1", Resource: "configmaps", }, }} result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{}) if err != nil { t.Fatal(err) } if result.Status.Allowed { t.Fatal("expected denied, got allowed") } authutil.GrantUserAuthorization(t, context.TODO(), adminClient, "alice", rbacv1.PolicyRule{ Verbs: []string{"create"}, APIGroups: []string{""}, Resources: []string{"configmaps"}, }, ) result, err = adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{}) if err != nil { t.Fatal(err) } if !result.Status.Allowed { t.Fatal("expected allowed, got denied") } } func TestMultiWebhookAuthzConfig(t *testing.T) { authzmetrics.ResetMetricsForTest() defer authzmetrics.ResetMetricsForTest() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)() dir := t.TempDir() kubeconfigTemplate := ` apiVersion: v1 kind: Config clusters: - name: integration cluster: server: %q insecure-skip-tls-verify: true contexts: - name: default-context context: cluster: integration user: test current-context: default-context users: - name: test ` // returns malformed responses when called errorName := "error.example.com" serverErrorCalled := atomic.Int32{} serverError := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverErrorCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverError", sar) if _, err := w.Write([]byte(`error response`)); err != nil { t.Error(err) } })) defer serverError.Close() serverErrorKubeconfigName := filepath.Join(dir, "serverError.yaml") if err := os.WriteFile(serverErrorKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverError.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // hangs for 2 seconds when called timeoutName := "timeout.example.com" serverTimeoutCalled := atomic.Int32{} serverTimeout := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverTimeoutCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverTimeout", sar) time.Sleep(2 * time.Second) })) defer serverTimeout.Close() serverTimeoutKubeconfigName := filepath.Join(dir, "serverTimeout.yaml") if err := os.WriteFile(serverTimeoutKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverTimeout.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // returns a deny response when called denyName := "deny.example.com" serverDenyCalled := atomic.Int32{} serverDeny := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverDenyCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverDeny", sar) sar.Status.Allowed = false sar.Status.Denied = true sar.Status.Reason = "denied by webhook" if err := json.NewEncoder(w).Encode(sar); err != nil { t.Error(err) } })) defer serverDeny.Close() serverDenyKubeconfigName := filepath.Join(dir, "serverDeny.yaml") if err := os.WriteFile(serverDenyKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverDeny.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // returns a no opinion response when called noOpinionName := "noopinion.example.com" serverNoOpinionCalled := atomic.Int32{} serverNoOpinion := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverNoOpinionCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverNoOpinion", sar) sar.Status.Allowed = false sar.Status.Denied = false if err := json.NewEncoder(w).Encode(sar); err != nil { t.Error(err) } })) defer serverNoOpinion.Close() serverNoOpinionKubeconfigName := filepath.Join(dir, "serverNoOpinion.yaml") if err := os.WriteFile(serverNoOpinionKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverNoOpinion.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // returns malformed responses when called, which is then configured to fail open failOpenName := "failopen.example.com" serverFailOpenCalled := atomic.Int32{} serverFailOpen := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverFailOpenCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverFailOpen", sar) if _, err := w.Write([]byte(`malformed response`)); err != nil { t.Error(err) } })) defer serverFailOpen.Close() serverFailOpenKubeconfigName := filepath.Join(dir, "failOpen.yaml") if err := os.WriteFile(serverFailOpenKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverFailOpen.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // returns an allow response when called allowName := "allow.example.com" serverAllowCalled := atomic.Int32{} serverAllow := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverAllowCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverAllow", sar) sar.Status.Allowed = true sar.Status.Reason = "allowed by webhook" if err := json.NewEncoder(w).Encode(sar); err != nil { t.Error(err) } })) defer serverAllow.Close() serverAllowKubeconfigName := filepath.Join(dir, "serverAllow.yaml") if err := os.WriteFile(serverAllowKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverAllow.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } // returns an allow response when called allowReloadedName := "allowreloaded.example.com" serverAllowReloadedCalled := atomic.Int32{} serverAllowReloaded := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { serverAllowReloadedCalled.Add(1) sar := &authorizationv1.SubjectAccessReview{} if err := json.NewDecoder(req.Body).Decode(sar); err != nil { t.Error(err) } t.Log("serverAllowReloaded", sar) sar.Status.Allowed = true sar.Status.Reason = "allowed2 by webhook" if err := json.NewEncoder(w).Encode(sar); err != nil { t.Error(err) } })) defer serverAllowReloaded.Close() serverAllowReloadedKubeconfigName := filepath.Join(dir, "serverAllowReloaded.yaml") if err := os.WriteFile(serverAllowReloadedKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverAllowReloaded.URL)), os.FileMode(0644)); err != nil { t.Fatal(err) } resetCounts := func() { serverErrorCalled.Store(0) serverTimeoutCalled.Store(0) serverDenyCalled.Store(0) serverNoOpinionCalled.Store(0) serverFailOpenCalled.Store(0) serverAllowCalled.Store(0) serverAllowReloadedCalled.Store(0) authorizationmetrics.ResetMetricsForTest() celmetrics.ResetMetricsForTest() webhookmetrics.ResetMetricsForTest() } var adminClient *clientset.Clientset type counts struct { errorCount, timeoutCount, denyCount, noOpinionCount, failOpenCount, allowCount, allowReloadedCount, webhookExclusionCount, evalErrorsCount int32 } assertCounts := func(c counts) { t.Helper() metrics, err := getMetrics(t, adminClient) if err != nil { t.Fatalf("error getting metrics: %v", err) } assertCount := func(name string, expected int32, serverCalls *atomic.Int32) { t.Helper() if actual := serverCalls.Load(); expected != actual { t.Fatalf("expected %q webhook calls: %d, got %d", name, expected, actual) } if actual := int32(metrics.whTotal[name]); expected != actual { t.Fatalf("expected %q webhook metric call count: %d, got %d (%#v)", name, expected, actual, metrics.whTotal) } if actual := int32(metrics.whDurationCount[name]); expected != actual { t.Fatalf("expected %q webhook metric duration count: %d, got %d (%#v)", name, expected, actual, metrics.whDurationCount) } } assertCount(errorName, c.errorCount, &serverErrorCalled) assertCount(timeoutName, c.timeoutCount, &serverTimeoutCalled) assertCount(denyName, c.denyCount, &serverDenyCalled) if e, a := c.denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) { t.Fatalf("expected deny webhook denied metrics calls: %d, got %d", e, a) } assertCount(noOpinionName, c.noOpinionCount, &serverNoOpinionCalled) assertCount(failOpenName, c.failOpenCount, &serverFailOpenCalled) expectedFailOpenCounts := map[string]int{} if c.failOpenCount > 0 { expectedFailOpenCounts[failOpenName] = int(c.failOpenCount) } if !reflect.DeepEqual(expectedFailOpenCounts, metrics.whFailOpenTotal) { t.Fatalf("expected fail open %#v, got %#v", expectedFailOpenCounts, metrics.whFailOpenTotal) } assertCount(allowName, c.allowCount, &serverAllowCalled) if e, a := c.allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) { t.Fatalf("expected allow webhook allowed metrics calls: %d, got %d", e, a) } assertCount(allowReloadedName, c.allowReloadedCount, &serverAllowReloadedCalled) if e, a := c.allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) { t.Fatalf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a) } if e, a := c.webhookExclusionCount, metrics.exclusions; e != int32(a) { t.Fatalf("expected webhook exclusions due to match conditions: %d, got %d", e, a) } if e, a := c.evalErrorsCount, metrics.evalErrors; e != int32(a) { t.Fatalf("expected webhook match condition eval errors: %d, got %d", e, a) } resetCounts() } configFileName := filepath.Join(dir, "config.yaml") if err := atomicWriteFile(configFileName, []byte(` apiVersion: apiserver.config.k8s.io/v1alpha1 kind: AuthorizationConfiguration authorizers: - type: Webhook name: `+errorName+` webhook: timeout: 5s failurePolicy: Deny subjectAccessReviewVersion: v1 matchConditionSubjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverErrorKubeconfigName+` matchConditions: - expression: has(request.resourceAttributes) - expression: 'request.resourceAttributes.namespace == "fail"' - expression: 'request.resourceAttributes.name == "error"' - type: Webhook name: `+timeoutName+` webhook: timeout: 1s failurePolicy: Deny subjectAccessReviewVersion: v1 matchConditionSubjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverTimeoutKubeconfigName+` matchConditions: # intentionally skip this check so we can trigger an eval error with a non-resource request # - expression: has(request.resourceAttributes) - expression: 'request.resourceAttributes.namespace == "fail"' - expression: 'request.resourceAttributes.name == "timeout"' - type: Webhook name: `+denyName+` webhook: timeout: 5s failurePolicy: NoOpinion subjectAccessReviewVersion: v1 matchConditionSubjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverDenyKubeconfigName+` matchConditions: - expression: has(request.resourceAttributes) - expression: 'request.resourceAttributes.namespace == "fail"' - type: Webhook name: `+noOpinionName+` webhook: timeout: 5s failurePolicy: Deny subjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverNoOpinionKubeconfigName+` - type: Webhook name: `+failOpenName+` webhook: timeout: 5s failurePolicy: NoOpinion subjectAccessReviewVersion: v1 matchConditionSubjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverFailOpenKubeconfigName+` - type: Webhook name: `+allowName+` webhook: timeout: 5s failurePolicy: Deny subjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverAllowKubeconfigName+` `), os.FileMode(0644)); err != nil { t.Fatal(err) } server := kubeapiservertesting.StartTestServerOrDie( t, nil, []string{"--authorization-config=" + configFileName}, framework.SharedEtcd(), ) t.Cleanup(server.TearDownFn) adminClient = clientset.NewForConfigOrDie(server.ClientConfig) // malformed webhook short circuits t.Log("checking error") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "get", Group: "", Version: "v1", Resource: "configmaps", Namespace: "fail", Name: "error", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if result.Status.Allowed { t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) assertCounts(counts{errorCount: 1}) } // timeout webhook short circuits t.Log("checking timeout") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "get", Group: "", Version: "v1", Resource: "configmaps", Namespace: "fail", Name: "timeout", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if result.Status.Allowed { t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) assertCounts(counts{timeoutCount: 1, webhookExclusionCount: 1}) } // deny webhook short circuits t.Log("checking deny") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "list", Group: "", Version: "v1", Resource: "configmaps", Namespace: "fail", Name: "", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if result.Status.Allowed { t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) assertCounts(counts{denyCount: 1, webhookExclusionCount: 2}) } // no-opinion webhook passes through, allow webhook allows t.Log("checking allow") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "list", Group: "", Version: "v1", Resource: "configmaps", Namespace: "allow", Name: "", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if !result.Status.Allowed { t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) assertCounts(counts{noOpinionCount: 1, failOpenCount: 1, allowCount: 1, webhookExclusionCount: 3}) } // the timeout webhook results in match condition eval errors when evaluating a non-resource request // failure policy is deny t.Log("checking match condition eval error") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", NonResourceAttributes: &authorizationv1.NonResourceAttributes{ Verb: "list", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if result.Status.Allowed { t.Fatal("expected denied, got allowed") } else { t.Log(result.Status.Reason) // error webhook matchConditions skip non-resource request // timeout webhook matchConditions error on non-resource request assertCounts(counts{webhookExclusionCount: 1, evalErrorsCount: 1}) } // check last loaded success/failure metric timestamps, ensure success is present, failure is not initialMetrics, err := getMetrics(t, adminClient) if err != nil { t.Fatal(err) } if initialMetrics.reloadSuccess == nil { t.Fatal("expected success timestamp, got none") } if initialMetrics.reloadFailure != nil { t.Fatal("expected no failure timestamp, got one") } // write bogus file if err := atomicWriteFile(configFileName, []byte(`apiVersion: apiserver.config.k8s.io`), os.FileMode(0644)); err != nil { t.Fatal(err) } // wait for failure timestamp > success timestamp var reload1Metrics *metrics err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) { reload1Metrics, err = getMetrics(t, adminClient) if err != nil { t.Fatal(err) } if reload1Metrics.reloadSuccess == nil { t.Fatal("expected success timestamp, got none") } if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) { t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String()) } if reload1Metrics.reloadFailure == nil { t.Log("expected failure timestamp, got nil, retrying") return false, nil } if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) { t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String()) } return true, nil }) if err != nil { t.Fatal(err) } // ensure authz still works t.Log("checking allow") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "list", Group: "", Version: "v1", Resource: "configmaps", Namespace: "allow", Name: "", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if !result.Status.Allowed { t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) assertCounts(counts{noOpinionCount: 1, failOpenCount: 1, allowCount: 1, webhookExclusionCount: 3}) } // write good config with different webhook if err := atomicWriteFile(configFileName, []byte(` apiVersion: apiserver.config.k8s.io/v1beta1 kind: AuthorizationConfiguration authorizers: - type: Webhook name: `+allowReloadedName+` webhook: timeout: 5s failurePolicy: Deny subjectAccessReviewVersion: v1 authorizedTTL: 1ms unauthorizedTTL: 1ms connectionInfo: type: KubeConfigFile kubeConfigFile: `+serverAllowReloadedKubeconfigName+` `), os.FileMode(0644)); err != nil { t.Fatal(err) } // wait for success timestamp > reload1Metrics.reloadFailure timestamp var reload2Metrics *metrics err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) { reload2Metrics, err = getMetrics(t, adminClient) if err != nil { t.Fatal(err) } if reload2Metrics.reloadFailure == nil { t.Log("expected failure timestamp, got nil, retrying") return false, nil } if !reload2Metrics.reloadFailure.Equal(*reload1Metrics.reloadFailure) { t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String()) } if reload2Metrics.reloadSuccess == nil { t.Fatal("expected success timestamp, got none") } if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) { t.Log("success timestamp hasn't updated from initial success, retrying") return false, nil } if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) { t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String()) } return true, nil }) if err != nil { t.Fatal(err) } // ensure authz still works, new webhook is called t.Log("checking allow") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "list", Group: "", Version: "v1", Resource: "configmaps", Namespace: "allow", Name: "", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if !result.Status.Allowed { t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) assertCounts(counts{allowReloadedCount: 1}) } // delete file (do this test last because it makes file watch fall back to one minute poll interval) if err := os.Remove(configFileName); err != nil { t.Fatal(err) } // wait for failure timestamp > success timestamp var reload3Metrics *metrics err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) { reload3Metrics, err = getMetrics(t, adminClient) if err != nil { t.Fatal(err) } if reload3Metrics.reloadSuccess == nil { t.Fatal("expected success timestamp, got none") } if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) { t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String()) } if reload3Metrics.reloadFailure == nil { t.Log("expected failure timestamp, got nil, retrying") return false, nil } if reload3Metrics.reloadFailure.Equal(*reload2Metrics.reloadFailure) { t.Log("failure timestamp hasn't updated, retrying") return false, nil } if !reload3Metrics.reloadFailure.After(*reload3Metrics.reloadSuccess) { t.Fatalf("expected failure timestamp to be more recent than success, got %s <= %s", reload3Metrics.reloadFailure.String(), reload3Metrics.reloadSuccess.String()) } return true, nil }) if err != nil { t.Fatal(err) } // ensure authz still works, new webhook is called t.Log("checking allow") if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{ User: "alice", ResourceAttributes: &authorizationv1.ResourceAttributes{ Verb: "list", Group: "", Version: "v1", Resource: "configmaps", Namespace: "allow", Name: "", }, }}, metav1.CreateOptions{}); err != nil { t.Fatal(err) } else if !result.Status.Allowed { t.Fatal("expected allowed, got denied") } else { t.Log(result.Status.Reason) assertCounts(counts{allowReloadedCount: 1}) } } type metrics struct { reloadSuccess *time.Time reloadFailure *time.Time decisions map[authorizerKey]map[string]int exclusions int evalErrors int whTotal map[string]int whFailOpenTotal map[string]int whDurationCount map[string]int } type authorizerKey struct { authorizerType string authorizerName string } var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`) var webhookExclusionMetric = regexp.MustCompile(`apiserver_authorization_match_condition_exclusions_total\{name="(.*?)",type="(.*?)"\} (\d+)`) var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authorization_match_condition_evaluation_errors_total\{name="(.*?)",type="(.*?)"\} (\d+)`) var whTotalMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_total{name="(.*?)",result="(.*?)"} (\d+)`) var webhookDurationMetric = regexp.MustCompile(`apiserver_authorization_webhook_duration_seconds_count{name="(.*?)",result="(.*?)"} (\d+)`) var webhookFailOpenMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_fail_open_total{name="(.*?)",result="(.*?)"} (\d+)`) func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO()) // apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09 // apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09 // apiserver_authorization_decisions_total{decision="allowed",name="allow.example.com",type="Webhook"} 2 // apiserver_authorization_decisions_total{decision="allowed",name="allowreloaded.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="error.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="timeout.example.com",type="Webhook"} 1 // apiserver_authorization_match_condition_exclusions_total{name="exclusion.example.com",type="webhook"} 1 if err != nil { return nil, err } var m metrics m.whTotal = map[string]int{} m.whFailOpenTotal = map[string]int{} m.whDurationCount = map[string]int{} m.exclusions = 0 for _, line := range strings.Split(string(data), "\n") { if matches := decisionMetric.FindStringSubmatch(line); matches != nil { t.Log(line) if m.decisions == nil { m.decisions = map[authorizerKey]map[string]int{} } key := authorizerKey{authorizerType: matches[3], authorizerName: matches[2]} if m.decisions[key] == nil { m.decisions[key] = map[string]int{} } count, err := strconv.Atoi(matches[4]) if err != nil { return nil, err } m.decisions[key][matches[1]] = count } if matches := webhookExclusionMetric.FindStringSubmatch(line); matches != nil { t.Log(matches) count, err := strconv.Atoi(matches[3]) if err != nil { return nil, err } t.Log(count) m.exclusions += count } if matches := webhookMatchConditionEvalErrorMetric.FindStringSubmatch(line); matches != nil { t.Log(matches) count, err := strconv.Atoi(matches[3]) if err != nil { return nil, err } t.Log(count) m.evalErrors += count } if matches := whTotalMetric.FindStringSubmatch(line); matches != nil { t.Log(matches) count, err := strconv.Atoi(matches[3]) if err != nil { return nil, err } t.Log(count) m.whTotal[matches[1]] += count } if matches := webhookDurationMetric.FindStringSubmatch(line); matches != nil { t.Log(matches) count, err := strconv.Atoi(matches[3]) if err != nil { return nil, err } t.Log(count) m.whDurationCount[matches[1]] += count } if matches := webhookFailOpenMetric.FindStringSubmatch(line); matches != nil { t.Log(matches) count, err := strconv.Atoi(matches[3]) if err != nil { return nil, err } t.Log(count) m.whFailOpenTotal[matches[1]] += count } if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") { t.Log(line) values := strings.Split(line, " ") value, err := strconv.ParseFloat(values[len(values)-1], 64) if err != nil { return nil, err } seconds := int64(value) nanoseconds := int64((value - float64(seconds)) * 1000000000) tm := time.Unix(seconds, nanoseconds) if strings.Contains(line, `"success"`) { m.reloadSuccess = &tm t.Log("success", m.reloadSuccess.String()) } if strings.Contains(line, `"failure"`) { m.reloadFailure = &tm t.Log("failure", m.reloadFailure.String()) } } } return &m, nil } func atomicWriteFile(name string, data []byte, perm os.FileMode) error { tmp := name + ".tmp" if err := os.WriteFile(tmp, data, perm); err != nil { return err } return os.Rename(tmp, name) }