1
16
17 package auth
18
19 import (
20 "bytes"
21 "context"
22 "encoding/json"
23 "fmt"
24 "net/http"
25 "net/http/httptest"
26 "os"
27 "path/filepath"
28 "reflect"
29 "regexp"
30 "strconv"
31 "strings"
32 "sync/atomic"
33 "testing"
34 "time"
35
36 authorizationv1 "k8s.io/api/authorization/v1"
37 rbacv1 "k8s.io/api/rbac/v1"
38 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
39 "k8s.io/apimachinery/pkg/util/wait"
40 celmetrics "k8s.io/apiserver/pkg/authorization/cel"
41 authorizationmetrics "k8s.io/apiserver/pkg/authorization/metrics"
42 "k8s.io/apiserver/pkg/features"
43 authzmetrics "k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics"
44 utilfeature "k8s.io/apiserver/pkg/util/feature"
45 webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
46 clientset "k8s.io/client-go/kubernetes"
47 "k8s.io/client-go/rest"
48 featuregatetesting "k8s.io/component-base/featuregate/testing"
49 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
50 "k8s.io/kubernetes/test/integration/authutil"
51 "k8s.io/kubernetes/test/integration/framework"
52 )
53
54 func TestAuthzConfig(t *testing.T) {
55 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
56
57 dir := t.TempDir()
58 configFileName := filepath.Join(dir, "config.yaml")
59 if err := atomicWriteFile(configFileName, []byte(`
60 apiVersion: apiserver.config.k8s.io/v1alpha1
61 kind: AuthorizationConfiguration
62 authorizers:
63 - type: RBAC
64 name: rbac
65 `), os.FileMode(0644)); err != nil {
66 t.Fatal(err)
67 }
68
69 server := kubeapiservertesting.StartTestServerOrDie(
70 t,
71 nil,
72 []string{"--authorization-config=" + configFileName},
73 framework.SharedEtcd(),
74 )
75 t.Cleanup(server.TearDownFn)
76
77
78 anonymousClient := clientset.NewForConfigOrDie(rest.AnonymousClientConfig(server.ClientConfig))
79 healthzResult, err := anonymousClient.DiscoveryClient.RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).Raw()
80 if !bytes.Equal(healthzResult, []byte(`ok`)) {
81 t.Fatalf("expected 'ok', got %s", string(healthzResult))
82 }
83 if err != nil {
84 t.Fatal(err)
85 }
86
87 adminClient := clientset.NewForConfigOrDie(server.ClientConfig)
88
89 sar := &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
90 User: "alice",
91 ResourceAttributes: &authorizationv1.ResourceAttributes{
92 Namespace: "foo",
93 Verb: "create",
94 Group: "",
95 Version: "v1",
96 Resource: "configmaps",
97 },
98 }}
99 result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{})
100 if err != nil {
101 t.Fatal(err)
102 }
103 if result.Status.Allowed {
104 t.Fatal("expected denied, got allowed")
105 }
106
107 authutil.GrantUserAuthorization(t, context.TODO(), adminClient, "alice",
108 rbacv1.PolicyRule{
109 Verbs: []string{"create"},
110 APIGroups: []string{""},
111 Resources: []string{"configmaps"},
112 },
113 )
114
115 result, err = adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), sar, metav1.CreateOptions{})
116 if err != nil {
117 t.Fatal(err)
118 }
119 if !result.Status.Allowed {
120 t.Fatal("expected allowed, got denied")
121 }
122 }
123
124 func TestMultiWebhookAuthzConfig(t *testing.T) {
125 authzmetrics.ResetMetricsForTest()
126 defer authzmetrics.ResetMetricsForTest()
127 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
128
129 dir := t.TempDir()
130
131 kubeconfigTemplate := `
132 apiVersion: v1
133 kind: Config
134 clusters:
135 - name: integration
136 cluster:
137 server: %q
138 insecure-skip-tls-verify: true
139 contexts:
140 - name: default-context
141 context:
142 cluster: integration
143 user: test
144 current-context: default-context
145 users:
146 - name: test
147 `
148
149
150 errorName := "error.example.com"
151 serverErrorCalled := atomic.Int32{}
152 serverError := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
153 serverErrorCalled.Add(1)
154 sar := &authorizationv1.SubjectAccessReview{}
155 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
156 t.Error(err)
157 }
158 t.Log("serverError", sar)
159 if _, err := w.Write([]byte(`error response`)); err != nil {
160 t.Error(err)
161 }
162 }))
163 defer serverError.Close()
164 serverErrorKubeconfigName := filepath.Join(dir, "serverError.yaml")
165 if err := os.WriteFile(serverErrorKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverError.URL)), os.FileMode(0644)); err != nil {
166 t.Fatal(err)
167 }
168
169
170 timeoutName := "timeout.example.com"
171 serverTimeoutCalled := atomic.Int32{}
172 serverTimeout := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
173 serverTimeoutCalled.Add(1)
174 sar := &authorizationv1.SubjectAccessReview{}
175 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
176 t.Error(err)
177 }
178 t.Log("serverTimeout", sar)
179 time.Sleep(2 * time.Second)
180 }))
181 defer serverTimeout.Close()
182 serverTimeoutKubeconfigName := filepath.Join(dir, "serverTimeout.yaml")
183 if err := os.WriteFile(serverTimeoutKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverTimeout.URL)), os.FileMode(0644)); err != nil {
184 t.Fatal(err)
185 }
186
187
188 denyName := "deny.example.com"
189 serverDenyCalled := atomic.Int32{}
190 serverDeny := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
191 serverDenyCalled.Add(1)
192 sar := &authorizationv1.SubjectAccessReview{}
193 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
194 t.Error(err)
195 }
196 t.Log("serverDeny", sar)
197 sar.Status.Allowed = false
198 sar.Status.Denied = true
199 sar.Status.Reason = "denied by webhook"
200 if err := json.NewEncoder(w).Encode(sar); err != nil {
201 t.Error(err)
202 }
203 }))
204 defer serverDeny.Close()
205 serverDenyKubeconfigName := filepath.Join(dir, "serverDeny.yaml")
206 if err := os.WriteFile(serverDenyKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverDeny.URL)), os.FileMode(0644)); err != nil {
207 t.Fatal(err)
208 }
209
210
211 noOpinionName := "noopinion.example.com"
212 serverNoOpinionCalled := atomic.Int32{}
213 serverNoOpinion := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
214 serverNoOpinionCalled.Add(1)
215 sar := &authorizationv1.SubjectAccessReview{}
216 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
217 t.Error(err)
218 }
219 t.Log("serverNoOpinion", sar)
220 sar.Status.Allowed = false
221 sar.Status.Denied = false
222 if err := json.NewEncoder(w).Encode(sar); err != nil {
223 t.Error(err)
224 }
225 }))
226 defer serverNoOpinion.Close()
227 serverNoOpinionKubeconfigName := filepath.Join(dir, "serverNoOpinion.yaml")
228 if err := os.WriteFile(serverNoOpinionKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverNoOpinion.URL)), os.FileMode(0644)); err != nil {
229 t.Fatal(err)
230 }
231
232
233 failOpenName := "failopen.example.com"
234 serverFailOpenCalled := atomic.Int32{}
235 serverFailOpen := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
236 serverFailOpenCalled.Add(1)
237 sar := &authorizationv1.SubjectAccessReview{}
238 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
239 t.Error(err)
240 }
241 t.Log("serverFailOpen", sar)
242 if _, err := w.Write([]byte(`malformed response`)); err != nil {
243 t.Error(err)
244 }
245 }))
246 defer serverFailOpen.Close()
247 serverFailOpenKubeconfigName := filepath.Join(dir, "failOpen.yaml")
248 if err := os.WriteFile(serverFailOpenKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverFailOpen.URL)), os.FileMode(0644)); err != nil {
249 t.Fatal(err)
250 }
251
252
253 allowName := "allow.example.com"
254 serverAllowCalled := atomic.Int32{}
255 serverAllow := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
256 serverAllowCalled.Add(1)
257 sar := &authorizationv1.SubjectAccessReview{}
258 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
259 t.Error(err)
260 }
261 t.Log("serverAllow", sar)
262 sar.Status.Allowed = true
263 sar.Status.Reason = "allowed by webhook"
264 if err := json.NewEncoder(w).Encode(sar); err != nil {
265 t.Error(err)
266 }
267 }))
268 defer serverAllow.Close()
269 serverAllowKubeconfigName := filepath.Join(dir, "serverAllow.yaml")
270 if err := os.WriteFile(serverAllowKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverAllow.URL)), os.FileMode(0644)); err != nil {
271 t.Fatal(err)
272 }
273
274
275 allowReloadedName := "allowreloaded.example.com"
276 serverAllowReloadedCalled := atomic.Int32{}
277 serverAllowReloaded := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
278 serverAllowReloadedCalled.Add(1)
279 sar := &authorizationv1.SubjectAccessReview{}
280 if err := json.NewDecoder(req.Body).Decode(sar); err != nil {
281 t.Error(err)
282 }
283 t.Log("serverAllowReloaded", sar)
284 sar.Status.Allowed = true
285 sar.Status.Reason = "allowed2 by webhook"
286 if err := json.NewEncoder(w).Encode(sar); err != nil {
287 t.Error(err)
288 }
289 }))
290 defer serverAllowReloaded.Close()
291 serverAllowReloadedKubeconfigName := filepath.Join(dir, "serverAllowReloaded.yaml")
292 if err := os.WriteFile(serverAllowReloadedKubeconfigName, []byte(fmt.Sprintf(kubeconfigTemplate, serverAllowReloaded.URL)), os.FileMode(0644)); err != nil {
293 t.Fatal(err)
294 }
295
296 resetCounts := func() {
297 serverErrorCalled.Store(0)
298 serverTimeoutCalled.Store(0)
299 serverDenyCalled.Store(0)
300 serverNoOpinionCalled.Store(0)
301 serverFailOpenCalled.Store(0)
302 serverAllowCalled.Store(0)
303 serverAllowReloadedCalled.Store(0)
304 authorizationmetrics.ResetMetricsForTest()
305 celmetrics.ResetMetricsForTest()
306 webhookmetrics.ResetMetricsForTest()
307 }
308 var adminClient *clientset.Clientset
309 type counts struct {
310 errorCount, timeoutCount, denyCount, noOpinionCount, failOpenCount, allowCount, allowReloadedCount, webhookExclusionCount, evalErrorsCount int32
311 }
312 assertCounts := func(c counts) {
313 t.Helper()
314 metrics, err := getMetrics(t, adminClient)
315 if err != nil {
316 t.Fatalf("error getting metrics: %v", err)
317 }
318
319 assertCount := func(name string, expected int32, serverCalls *atomic.Int32) {
320 t.Helper()
321 if actual := serverCalls.Load(); expected != actual {
322 t.Fatalf("expected %q webhook calls: %d, got %d", name, expected, actual)
323 }
324 if actual := int32(metrics.whTotal[name]); expected != actual {
325 t.Fatalf("expected %q webhook metric call count: %d, got %d (%#v)", name, expected, actual, metrics.whTotal)
326 }
327 if actual := int32(metrics.whDurationCount[name]); expected != actual {
328 t.Fatalf("expected %q webhook metric duration count: %d, got %d (%#v)", name, expected, actual, metrics.whDurationCount)
329 }
330 }
331
332 assertCount(errorName, c.errorCount, &serverErrorCalled)
333 assertCount(timeoutName, c.timeoutCount, &serverTimeoutCalled)
334 assertCount(denyName, c.denyCount, &serverDenyCalled)
335 if e, a := c.denyCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: denyName}]["denied"]; e != int32(a) {
336 t.Fatalf("expected deny webhook denied metrics calls: %d, got %d", e, a)
337 }
338 assertCount(noOpinionName, c.noOpinionCount, &serverNoOpinionCalled)
339 assertCount(failOpenName, c.failOpenCount, &serverFailOpenCalled)
340 expectedFailOpenCounts := map[string]int{}
341 if c.failOpenCount > 0 {
342 expectedFailOpenCounts[failOpenName] = int(c.failOpenCount)
343 }
344 if !reflect.DeepEqual(expectedFailOpenCounts, metrics.whFailOpenTotal) {
345 t.Fatalf("expected fail open %#v, got %#v", expectedFailOpenCounts, metrics.whFailOpenTotal)
346 }
347 assertCount(allowName, c.allowCount, &serverAllowCalled)
348 if e, a := c.allowCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowName}]["allowed"]; e != int32(a) {
349 t.Fatalf("expected allow webhook allowed metrics calls: %d, got %d", e, a)
350 }
351 assertCount(allowReloadedName, c.allowReloadedCount, &serverAllowReloadedCalled)
352 if e, a := c.allowReloadedCount, metrics.decisions[authorizerKey{authorizerType: "Webhook", authorizerName: allowReloadedName}]["allowed"]; e != int32(a) {
353 t.Fatalf("expected allowReloaded webhook allowed metrics calls: %d, got %d", e, a)
354 }
355 if e, a := c.webhookExclusionCount, metrics.exclusions; e != int32(a) {
356 t.Fatalf("expected webhook exclusions due to match conditions: %d, got %d", e, a)
357 }
358 if e, a := c.evalErrorsCount, metrics.evalErrors; e != int32(a) {
359 t.Fatalf("expected webhook match condition eval errors: %d, got %d", e, a)
360 }
361 resetCounts()
362 }
363
364 configFileName := filepath.Join(dir, "config.yaml")
365 if err := atomicWriteFile(configFileName, []byte(`
366 apiVersion: apiserver.config.k8s.io/v1alpha1
367 kind: AuthorizationConfiguration
368 authorizers:
369 - type: Webhook
370 name: `+errorName+`
371 webhook:
372 timeout: 5s
373 failurePolicy: Deny
374 subjectAccessReviewVersion: v1
375 matchConditionSubjectAccessReviewVersion: v1
376 authorizedTTL: 1ms
377 unauthorizedTTL: 1ms
378 connectionInfo:
379 type: KubeConfigFile
380 kubeConfigFile: `+serverErrorKubeconfigName+`
381 matchConditions:
382 - expression: has(request.resourceAttributes)
383 - expression: 'request.resourceAttributes.namespace == "fail"'
384 - expression: 'request.resourceAttributes.name == "error"'
385
386 - type: Webhook
387 name: `+timeoutName+`
388 webhook:
389 timeout: 1s
390 failurePolicy: Deny
391 subjectAccessReviewVersion: v1
392 matchConditionSubjectAccessReviewVersion: v1
393 authorizedTTL: 1ms
394 unauthorizedTTL: 1ms
395 connectionInfo:
396 type: KubeConfigFile
397 kubeConfigFile: `+serverTimeoutKubeconfigName+`
398 matchConditions:
399 # intentionally skip this check so we can trigger an eval error with a non-resource request
400 # - expression: has(request.resourceAttributes)
401 - expression: 'request.resourceAttributes.namespace == "fail"'
402 - expression: 'request.resourceAttributes.name == "timeout"'
403
404 - type: Webhook
405 name: `+denyName+`
406 webhook:
407 timeout: 5s
408 failurePolicy: NoOpinion
409 subjectAccessReviewVersion: v1
410 matchConditionSubjectAccessReviewVersion: v1
411 authorizedTTL: 1ms
412 unauthorizedTTL: 1ms
413 connectionInfo:
414 type: KubeConfigFile
415 kubeConfigFile: `+serverDenyKubeconfigName+`
416 matchConditions:
417 - expression: has(request.resourceAttributes)
418 - expression: 'request.resourceAttributes.namespace == "fail"'
419
420 - type: Webhook
421 name: `+noOpinionName+`
422 webhook:
423 timeout: 5s
424 failurePolicy: Deny
425 subjectAccessReviewVersion: v1
426 authorizedTTL: 1ms
427 unauthorizedTTL: 1ms
428 connectionInfo:
429 type: KubeConfigFile
430 kubeConfigFile: `+serverNoOpinionKubeconfigName+`
431
432 - type: Webhook
433 name: `+failOpenName+`
434 webhook:
435 timeout: 5s
436 failurePolicy: NoOpinion
437 subjectAccessReviewVersion: v1
438 matchConditionSubjectAccessReviewVersion: v1
439 authorizedTTL: 1ms
440 unauthorizedTTL: 1ms
441 connectionInfo:
442 type: KubeConfigFile
443 kubeConfigFile: `+serverFailOpenKubeconfigName+`
444
445 - type: Webhook
446 name: `+allowName+`
447 webhook:
448 timeout: 5s
449 failurePolicy: Deny
450 subjectAccessReviewVersion: v1
451 authorizedTTL: 1ms
452 unauthorizedTTL: 1ms
453 connectionInfo:
454 type: KubeConfigFile
455 kubeConfigFile: `+serverAllowKubeconfigName+`
456 `), os.FileMode(0644)); err != nil {
457 t.Fatal(err)
458 }
459
460 server := kubeapiservertesting.StartTestServerOrDie(
461 t,
462 nil,
463 []string{"--authorization-config=" + configFileName},
464 framework.SharedEtcd(),
465 )
466 t.Cleanup(server.TearDownFn)
467
468 adminClient = clientset.NewForConfigOrDie(server.ClientConfig)
469
470
471 t.Log("checking error")
472 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
473 User: "alice",
474 ResourceAttributes: &authorizationv1.ResourceAttributes{
475 Verb: "get",
476 Group: "",
477 Version: "v1",
478 Resource: "configmaps",
479 Namespace: "fail",
480 Name: "error",
481 },
482 }}, metav1.CreateOptions{}); err != nil {
483 t.Fatal(err)
484 } else if result.Status.Allowed {
485 t.Fatal("expected denied, got allowed")
486 } else {
487 t.Log(result.Status.Reason)
488 assertCounts(counts{errorCount: 1})
489 }
490
491
492 t.Log("checking timeout")
493 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
494 User: "alice",
495 ResourceAttributes: &authorizationv1.ResourceAttributes{
496 Verb: "get",
497 Group: "",
498 Version: "v1",
499 Resource: "configmaps",
500 Namespace: "fail",
501 Name: "timeout",
502 },
503 }}, metav1.CreateOptions{}); err != nil {
504 t.Fatal(err)
505 } else if result.Status.Allowed {
506 t.Fatal("expected denied, got allowed")
507 } else {
508 t.Log(result.Status.Reason)
509 assertCounts(counts{timeoutCount: 1, webhookExclusionCount: 1})
510 }
511
512
513 t.Log("checking deny")
514 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
515 User: "alice",
516 ResourceAttributes: &authorizationv1.ResourceAttributes{
517 Verb: "list",
518 Group: "",
519 Version: "v1",
520 Resource: "configmaps",
521 Namespace: "fail",
522 Name: "",
523 },
524 }}, metav1.CreateOptions{}); err != nil {
525 t.Fatal(err)
526 } else if result.Status.Allowed {
527 t.Fatal("expected denied, got allowed")
528 } else {
529 t.Log(result.Status.Reason)
530 assertCounts(counts{denyCount: 1, webhookExclusionCount: 2})
531 }
532
533
534 t.Log("checking allow")
535 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
536 User: "alice",
537 ResourceAttributes: &authorizationv1.ResourceAttributes{
538 Verb: "list",
539 Group: "",
540 Version: "v1",
541 Resource: "configmaps",
542 Namespace: "allow",
543 Name: "",
544 },
545 }}, metav1.CreateOptions{}); err != nil {
546 t.Fatal(err)
547 } else if !result.Status.Allowed {
548 t.Fatal("expected allowed, got denied")
549 } else {
550 t.Log(result.Status.Reason)
551 assertCounts(counts{noOpinionCount: 1, failOpenCount: 1, allowCount: 1, webhookExclusionCount: 3})
552 }
553
554
555
556 t.Log("checking match condition eval error")
557 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
558 User: "alice",
559 NonResourceAttributes: &authorizationv1.NonResourceAttributes{
560 Verb: "list",
561 },
562 }}, metav1.CreateOptions{}); err != nil {
563 t.Fatal(err)
564 } else if result.Status.Allowed {
565 t.Fatal("expected denied, got allowed")
566 } else {
567 t.Log(result.Status.Reason)
568
569
570 assertCounts(counts{webhookExclusionCount: 1, evalErrorsCount: 1})
571 }
572
573
574 initialMetrics, err := getMetrics(t, adminClient)
575 if err != nil {
576 t.Fatal(err)
577 }
578 if initialMetrics.reloadSuccess == nil {
579 t.Fatal("expected success timestamp, got none")
580 }
581 if initialMetrics.reloadFailure != nil {
582 t.Fatal("expected no failure timestamp, got one")
583 }
584
585
586 if err := atomicWriteFile(configFileName, []byte(`apiVersion: apiserver.config.k8s.io`), os.FileMode(0644)); err != nil {
587 t.Fatal(err)
588 }
589
590
591 var reload1Metrics *metrics
592 err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
593 reload1Metrics, err = getMetrics(t, adminClient)
594 if err != nil {
595 t.Fatal(err)
596 }
597 if reload1Metrics.reloadSuccess == nil {
598 t.Fatal("expected success timestamp, got none")
599 }
600 if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
601 t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String())
602 }
603 if reload1Metrics.reloadFailure == nil {
604 t.Log("expected failure timestamp, got nil, retrying")
605 return false, nil
606 }
607 if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) {
608 t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String())
609 }
610 return true, nil
611 })
612 if err != nil {
613 t.Fatal(err)
614 }
615
616
617 t.Log("checking allow")
618 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
619 User: "alice",
620 ResourceAttributes: &authorizationv1.ResourceAttributes{
621 Verb: "list",
622 Group: "",
623 Version: "v1",
624 Resource: "configmaps",
625 Namespace: "allow",
626 Name: "",
627 },
628 }}, metav1.CreateOptions{}); err != nil {
629 t.Fatal(err)
630 } else if !result.Status.Allowed {
631 t.Fatal("expected allowed, got denied")
632 } else {
633 t.Log(result.Status.Reason)
634 assertCounts(counts{noOpinionCount: 1, failOpenCount: 1, allowCount: 1, webhookExclusionCount: 3})
635 }
636
637
638 if err := atomicWriteFile(configFileName, []byte(`
639 apiVersion: apiserver.config.k8s.io/v1beta1
640 kind: AuthorizationConfiguration
641 authorizers:
642 - type: Webhook
643 name: `+allowReloadedName+`
644 webhook:
645 timeout: 5s
646 failurePolicy: Deny
647 subjectAccessReviewVersion: v1
648 authorizedTTL: 1ms
649 unauthorizedTTL: 1ms
650 connectionInfo:
651 type: KubeConfigFile
652 kubeConfigFile: `+serverAllowReloadedKubeconfigName+`
653 `), os.FileMode(0644)); err != nil {
654 t.Fatal(err)
655 }
656
657
658 var reload2Metrics *metrics
659 err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
660 reload2Metrics, err = getMetrics(t, adminClient)
661 if err != nil {
662 t.Fatal(err)
663 }
664 if reload2Metrics.reloadFailure == nil {
665 t.Log("expected failure timestamp, got nil, retrying")
666 return false, nil
667 }
668 if !reload2Metrics.reloadFailure.Equal(*reload1Metrics.reloadFailure) {
669 t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String())
670 }
671 if reload2Metrics.reloadSuccess == nil {
672 t.Fatal("expected success timestamp, got none")
673 }
674 if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
675 t.Log("success timestamp hasn't updated from initial success, retrying")
676 return false, nil
677 }
678 if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) {
679 t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String())
680 }
681 return true, nil
682 })
683 if err != nil {
684 t.Fatal(err)
685 }
686
687
688 t.Log("checking allow")
689 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
690 User: "alice",
691 ResourceAttributes: &authorizationv1.ResourceAttributes{
692 Verb: "list",
693 Group: "",
694 Version: "v1",
695 Resource: "configmaps",
696 Namespace: "allow",
697 Name: "",
698 },
699 }}, metav1.CreateOptions{}); err != nil {
700 t.Fatal(err)
701 } else if !result.Status.Allowed {
702 t.Fatal("expected allowed, got denied")
703 } else {
704 t.Log(result.Status.Reason)
705 assertCounts(counts{allowReloadedCount: 1})
706 }
707
708
709 if err := os.Remove(configFileName); err != nil {
710 t.Fatal(err)
711 }
712
713
714 var reload3Metrics *metrics
715 err = wait.PollUntilContextTimeout(context.TODO(), time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
716 reload3Metrics, err = getMetrics(t, adminClient)
717 if err != nil {
718 t.Fatal(err)
719 }
720 if reload3Metrics.reloadSuccess == nil {
721 t.Fatal("expected success timestamp, got none")
722 }
723 if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) {
724 t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String())
725 }
726 if reload3Metrics.reloadFailure == nil {
727 t.Log("expected failure timestamp, got nil, retrying")
728 return false, nil
729 }
730 if reload3Metrics.reloadFailure.Equal(*reload2Metrics.reloadFailure) {
731 t.Log("failure timestamp hasn't updated, retrying")
732 return false, nil
733 }
734 if !reload3Metrics.reloadFailure.After(*reload3Metrics.reloadSuccess) {
735 t.Fatalf("expected failure timestamp to be more recent than success, got %s <= %s", reload3Metrics.reloadFailure.String(), reload3Metrics.reloadSuccess.String())
736 }
737 return true, nil
738 })
739 if err != nil {
740 t.Fatal(err)
741 }
742
743
744 t.Log("checking allow")
745 if result, err := adminClient.AuthorizationV1().SubjectAccessReviews().Create(context.TODO(), &authorizationv1.SubjectAccessReview{Spec: authorizationv1.SubjectAccessReviewSpec{
746 User: "alice",
747 ResourceAttributes: &authorizationv1.ResourceAttributes{
748 Verb: "list",
749 Group: "",
750 Version: "v1",
751 Resource: "configmaps",
752 Namespace: "allow",
753 Name: "",
754 },
755 }}, metav1.CreateOptions{}); err != nil {
756 t.Fatal(err)
757 } else if !result.Status.Allowed {
758 t.Fatal("expected allowed, got denied")
759 } else {
760 t.Log(result.Status.Reason)
761 assertCounts(counts{allowReloadedCount: 1})
762 }
763 }
764
765 type metrics struct {
766 reloadSuccess *time.Time
767 reloadFailure *time.Time
768 decisions map[authorizerKey]map[string]int
769 exclusions int
770 evalErrors int
771
772 whTotal map[string]int
773 whFailOpenTotal map[string]int
774 whDurationCount map[string]int
775 }
776 type authorizerKey struct {
777 authorizerType string
778 authorizerName string
779 }
780
781 var decisionMetric = regexp.MustCompile(`apiserver_authorization_decisions_total\{decision="(.*?)",name="(.*?)",type="(.*?)"\} (\d+)`)
782 var webhookExclusionMetric = regexp.MustCompile(`apiserver_authorization_match_condition_exclusions_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
783 var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authorization_match_condition_evaluation_errors_total\{name="(.*?)",type="(.*?)"\} (\d+)`)
784 var whTotalMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_total{name="(.*?)",result="(.*?)"} (\d+)`)
785 var webhookDurationMetric = regexp.MustCompile(`apiserver_authorization_webhook_duration_seconds_count{name="(.*?)",result="(.*?)"} (\d+)`)
786 var webhookFailOpenMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_fail_open_total{name="(.*?)",result="(.*?)"} (\d+)`)
787
788 func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
789 data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
790
791
792
793
794
795
796
797
798
799 if err != nil {
800 return nil, err
801 }
802
803 var m metrics
804
805 m.whTotal = map[string]int{}
806 m.whFailOpenTotal = map[string]int{}
807 m.whDurationCount = map[string]int{}
808 m.exclusions = 0
809 for _, line := range strings.Split(string(data), "\n") {
810 if matches := decisionMetric.FindStringSubmatch(line); matches != nil {
811 t.Log(line)
812 if m.decisions == nil {
813 m.decisions = map[authorizerKey]map[string]int{}
814 }
815 key := authorizerKey{authorizerType: matches[3], authorizerName: matches[2]}
816 if m.decisions[key] == nil {
817 m.decisions[key] = map[string]int{}
818 }
819 count, err := strconv.Atoi(matches[4])
820 if err != nil {
821 return nil, err
822 }
823 m.decisions[key][matches[1]] = count
824
825 }
826 if matches := webhookExclusionMetric.FindStringSubmatch(line); matches != nil {
827 t.Log(matches)
828 count, err := strconv.Atoi(matches[3])
829 if err != nil {
830 return nil, err
831 }
832 t.Log(count)
833 m.exclusions += count
834 }
835 if matches := webhookMatchConditionEvalErrorMetric.FindStringSubmatch(line); matches != nil {
836 t.Log(matches)
837 count, err := strconv.Atoi(matches[3])
838 if err != nil {
839 return nil, err
840 }
841 t.Log(count)
842 m.evalErrors += count
843 }
844 if matches := whTotalMetric.FindStringSubmatch(line); matches != nil {
845 t.Log(matches)
846 count, err := strconv.Atoi(matches[3])
847 if err != nil {
848 return nil, err
849 }
850 t.Log(count)
851 m.whTotal[matches[1]] += count
852 }
853 if matches := webhookDurationMetric.FindStringSubmatch(line); matches != nil {
854 t.Log(matches)
855 count, err := strconv.Atoi(matches[3])
856 if err != nil {
857 return nil, err
858 }
859 t.Log(count)
860 m.whDurationCount[matches[1]] += count
861 }
862 if matches := webhookFailOpenMetric.FindStringSubmatch(line); matches != nil {
863 t.Log(matches)
864 count, err := strconv.Atoi(matches[3])
865 if err != nil {
866 return nil, err
867 }
868 t.Log(count)
869 m.whFailOpenTotal[matches[1]] += count
870 }
871 if strings.HasPrefix(line, "apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds") {
872 t.Log(line)
873 values := strings.Split(line, " ")
874 value, err := strconv.ParseFloat(values[len(values)-1], 64)
875 if err != nil {
876 return nil, err
877 }
878 seconds := int64(value)
879 nanoseconds := int64((value - float64(seconds)) * 1000000000)
880 tm := time.Unix(seconds, nanoseconds)
881 if strings.Contains(line, `"success"`) {
882 m.reloadSuccess = &tm
883 t.Log("success", m.reloadSuccess.String())
884 }
885 if strings.Contains(line, `"failure"`) {
886 m.reloadFailure = &tm
887 t.Log("failure", m.reloadFailure.String())
888 }
889 }
890 }
891 return &m, nil
892 }
893
894 func atomicWriteFile(name string, data []byte, perm os.FileMode) error {
895 tmp := name + ".tmp"
896 if err := os.WriteFile(tmp, data, perm); err != nil {
897 return err
898 }
899 return os.Rename(tmp, name)
900 }
901
View as plain text