1
16
17 package serviceaccount
18
19
20
21
22
23 import (
24 "context"
25 "fmt"
26 "sync"
27 "testing"
28 "time"
29
30 v1 "k8s.io/api/core/v1"
31 apierrors "k8s.io/apimachinery/pkg/api/errors"
32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33 "k8s.io/apimachinery/pkg/util/wait"
34 serviceaccountapiserver "k8s.io/apiserver/pkg/authentication/serviceaccount"
35 "k8s.io/apiserver/pkg/authorization/authorizer"
36 unionauthz "k8s.io/apiserver/pkg/authorization/union"
37 clientinformers "k8s.io/client-go/informers"
38 clientset "k8s.io/client-go/kubernetes"
39 restclient "k8s.io/client-go/rest"
40 "k8s.io/client-go/util/keyutil"
41 "k8s.io/kubernetes/cmd/kube-apiserver/app/options"
42 "k8s.io/kubernetes/pkg/controller"
43 serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
44 "k8s.io/kubernetes/pkg/controlplane"
45 "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
46 "k8s.io/kubernetes/pkg/serviceaccount"
47 serviceaccountadmission "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
48 "k8s.io/kubernetes/test/integration/framework"
49 "k8s.io/kubernetes/test/utils/ktesting"
50 )
51
52 const (
53 readOnlyServiceAccountName = "ro"
54 readWriteServiceAccountName = "rw"
55 )
56
57 func TestServiceAccountAutoCreate(t *testing.T) {
58 tCtx := ktesting.Init(t)
59 c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(tCtx, t)
60 defer stopFunc()
61 if err != nil {
62 t.Fatalf("failed to setup ServiceAccounts server: %v", err)
63 }
64
65 ns := "test-service-account-creation"
66
67
68 _, err = c.CoreV1().Namespaces().Create(tCtx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{})
69 if err != nil {
70 t.Fatalf("could not create namespace: %v", err)
71 }
72
73
74 defaultUser, err := getServiceAccount(c, ns, "default", true)
75 if err != nil {
76 t.Fatalf("Default serviceaccount not created: %v", err)
77 }
78
79
80 err = c.CoreV1().ServiceAccounts(ns).Delete(tCtx, defaultUser.Name, metav1.DeleteOptions{})
81 if err != nil {
82 t.Fatalf("Could not delete default serviceaccount: %v", err)
83 }
84
85
86 defaultUser2, err := getServiceAccount(c, ns, "default", true)
87 if err != nil {
88 t.Fatalf("Default serviceaccount not created: %v", err)
89 }
90 if defaultUser2.UID == defaultUser.UID {
91 t.Fatalf("Expected different UID with recreated serviceaccount")
92 }
93 }
94
95 func TestServiceAccountTokenAutoMount(t *testing.T) {
96 tCtx := ktesting.Init(t)
97 c, _, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(tCtx, t)
98 defer stopFunc()
99 if err != nil {
100 t.Fatalf("failed to setup ServiceAccounts server: %v", err)
101 }
102
103 ns := "auto-mount-ns"
104
105
106 _, err = c.CoreV1().Namespaces().Create(tCtx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{})
107 if err != nil && !apierrors.IsAlreadyExists(err) {
108 t.Fatalf("could not create namespace: %v", err)
109 }
110
111
112 protoPod := v1.Pod{
113 ObjectMeta: metav1.ObjectMeta{Name: "protopod"},
114 Spec: v1.PodSpec{
115 Containers: []v1.Container{
116 {
117 Name: "container",
118 Image: "container-image",
119 },
120 },
121 },
122 }
123
124 createdPod, err := c.CoreV1().Pods(ns).Create(tCtx, &protoPod, metav1.CreateOptions{})
125 if err != nil {
126 t.Fatal(err)
127 }
128
129 expectedServiceAccount := serviceaccountadmission.DefaultServiceAccountName
130 if createdPod.Spec.ServiceAccountName != expectedServiceAccount {
131 t.Fatalf("Expected %s, got %s", expectedServiceAccount, createdPod.Spec.ServiceAccountName)
132 }
133 if len(createdPod.Spec.Volumes) == 0 || createdPod.Spec.Volumes[0].Projected == nil {
134 t.Fatal("Expected projected volume for service account token inserted")
135 }
136 }
137
138 func TestServiceAccountTokenAuthentication(t *testing.T) {
139 tCtx := ktesting.Init(t)
140 c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(tCtx, t)
141 defer stopFunc()
142 if err != nil {
143 t.Fatalf("failed to setup ServiceAccounts server: %v", err)
144 }
145
146 myns := "auth-ns"
147 otherns := "other-ns"
148
149
150 _, err = c.CoreV1().Namespaces().Create(tCtx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: myns}}, metav1.CreateOptions{})
151 if err != nil && !apierrors.IsAlreadyExists(err) {
152 t.Fatalf("could not create namespace: %v", err)
153 }
154
155
156 _, err = c.CoreV1().Namespaces().Create(tCtx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherns}}, metav1.CreateOptions{})
157 if err != nil && !apierrors.IsAlreadyExists(err) {
158 t.Fatalf("could not create namespace: %v", err)
159 }
160
161
162 roSA, err := c.CoreV1().ServiceAccounts(myns).Create(tCtx, &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readOnlyServiceAccountName}}, metav1.CreateOptions{})
163 if err != nil {
164 t.Fatalf("Service Account not created: %v", err)
165 }
166
167 roTokenName := "ro-test-token"
168 secret, err := createServiceAccountToken(c, roSA, myns, roTokenName)
169 if err != nil {
170 t.Fatalf("Secret not created: %v", err)
171 }
172 roClientConfig := *config
173 roClientConfig.BearerToken = string(secret.Data[v1.ServiceAccountTokenKey])
174 roClient := clientset.NewForConfigOrDie(&roClientConfig)
175 doServiceAccountAPIRequests(t, roClient, myns, true, true, false)
176 doServiceAccountAPIRequests(t, roClient, otherns, true, false, false)
177 err = c.CoreV1().Secrets(myns).Delete(tCtx, roTokenName, metav1.DeleteOptions{})
178 if err != nil {
179 t.Fatalf("could not delete token: %v", err)
180 }
181
182 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
183 _, err := roClient.CoreV1().Secrets(myns).List(tCtx, metav1.ListOptions{})
184 if err == nil {
185 t.Logf("token is still valid, waiting")
186 return false, nil
187 }
188 if !apierrors.IsUnauthorized(err) {
189 t.Logf("expected unauthorized error, got %v", err)
190 return false, nil
191 }
192 return true, nil
193 })
194 if err != nil {
195 t.Fatalf("waiting for token to be invalidated: %v", err)
196 }
197 doServiceAccountAPIRequests(t, roClient, myns, false, false, false)
198
199
200 rwSA, err := c.CoreV1().ServiceAccounts(myns).Create(tCtx, &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readWriteServiceAccountName}}, metav1.CreateOptions{})
201 if err != nil {
202 t.Fatalf("Service Account not created: %v", err)
203 }
204 rwTokenName := "rw-test-token"
205 secret, err = createServiceAccountToken(c, rwSA, myns, rwTokenName)
206 if err != nil {
207 t.Fatalf("Secret not created: %v", err)
208 }
209 rwClientConfig := *config
210 rwClientConfig.BearerToken = string(secret.Data[v1.ServiceAccountTokenKey])
211 rwClient := clientset.NewForConfigOrDie(&rwClientConfig)
212 doServiceAccountAPIRequests(t, rwClient, myns, true, true, true)
213 doServiceAccountAPIRequests(t, rwClient, otherns, true, false, false)
214 }
215
216 func TestLegacyServiceAccountTokenTracking(t *testing.T) {
217 tCtx := ktesting.Init(t)
218 c, config, stopFunc, _, err := startServiceAccountTestServerAndWaitForCaches(tCtx, t)
219 defer stopFunc()
220 if err != nil {
221 t.Fatalf("failed to setup ServiceAccounts server: %v", err)
222 }
223
224
225 myns := "auth-ns"
226 _, err = c.CoreV1().Namespaces().Create(tCtx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: myns}}, metav1.CreateOptions{})
227 if err != nil && !apierrors.IsAlreadyExists(err) {
228 t.Fatalf("could not create namespace: %v", err)
229 }
230 mysa, err := c.CoreV1().ServiceAccounts(myns).Create(tCtx, &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readOnlyServiceAccountName}}, metav1.CreateOptions{})
231 if err != nil {
232 t.Fatalf("Service Account not created: %v", err)
233 }
234 manualSecretName := "manual-token"
235 manualSecret, err := createServiceAccountToken(c, mysa, myns, manualSecretName)
236 if err != nil {
237 t.Fatalf("Secret not created: %v", err)
238 }
239
240
241 autoSecretName := "auto-token"
242 autoSecret, err := createServiceAccountToken(c, mysa, myns, autoSecretName)
243 if err != nil {
244 t.Fatalf("Secret not created: %v", err)
245 }
246 if err := addReferencedServiceAccountToken(c, myns, readOnlyServiceAccountName, autoSecret); err != nil {
247 t.Fatal(err)
248 }
249
250 tests := []struct {
251 name string
252 secretName string
253 secretTokenData string
254
255 expectWarning bool
256 }{
257 {
258 name: "manually created legacy token",
259 secretName: manualSecretName,
260 secretTokenData: string(manualSecret.Data[v1.ServiceAccountTokenKey]),
261 },
262 {
263 name: "auto created legacy token",
264 secretName: autoSecretName,
265 secretTokenData: string(autoSecret.Data[v1.ServiceAccountTokenKey]),
266 expectWarning: true,
267 },
268 }
269 for _, test := range tests {
270 t.Run(test.name, func(t *testing.T) {
271 myConfig := *config
272 wh := &warningHandler{}
273 myConfig.WarningHandler = wh
274 myConfig.BearerToken = string(test.secretTokenData)
275 roClient := clientset.NewForConfigOrDie(&myConfig)
276 dateBefore := time.Now().UTC().Format(dateFormat)
277
278 var wg sync.WaitGroup
279 concurrency := 5
280 for i := 0; i < concurrency; i++ {
281 wg.Add(1)
282 go func() {
283 doServiceAccountAPIRequests(t, roClient, myns, true, true, false)
284 wg.Done()
285 }()
286 }
287 wg.Wait()
288 dateAfter := time.Now().UTC().Format(dateFormat)
289 liveSecret, err := c.CoreV1().Secrets(myns).Get(tCtx, test.secretName, metav1.GetOptions{})
290 if err != nil {
291 t.Fatalf("Could not get secret: %v", err)
292 }
293
294
295 if test.expectWarning && len(wh.warnings) != 4*concurrency {
296 t.Fatalf("Expect %d warnings, got %d", 4*concurrency, len(wh.warnings))
297 }
298 if !test.expectWarning && len(wh.warnings) != 0 {
299 t.Fatalf("Don't expect warnings, got %d", len(wh.warnings))
300 }
301
302
303 date, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
304 if !ok {
305 t.Fatalf("Secret wasn't labeled with %q", serviceaccount.LastUsedLabelKey)
306 }
307 if date != dateBefore || date != dateAfter {
308 t.Fatalf("Secret was labeled with wrong date: %q", date)
309 }
310 })
311 }
312
313
314 if err = wait.PollImmediate(time.Millisecond*10, wait.ForeverTestTimeout, func() (bool, error) {
315 dateBefore := time.Now().UTC().Format("2006-01-02")
316 configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(tCtx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
317 if err != nil {
318 return false, fmt.Errorf("failed to get %q configmap, err %w", legacytokentracking.ConfigMapDataKey, err)
319 }
320 dateAfter := time.Now().UTC().Format("2006-01-02")
321 date, ok := configMap.Data[legacytokentracking.ConfigMapDataKey]
322 if !ok {
323 return false, fmt.Errorf("configMap doesn't contain key %q", legacytokentracking.ConfigMapDataKey)
324 }
325 if date != dateBefore || date != dateAfter {
326 return false, fmt.Errorf("configMap contains a wrong date %q", date)
327 }
328 return true, nil
329 }); err != nil {
330 t.Fatal(err)
331 }
332 }
333
334
335
336 func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testing.T) (clientset.Interface, *restclient.Config, func(), clientinformers.SharedInformerFactory, error) {
337 var serviceAccountKey interface{}
338
339 ctx, cancel := context.WithCancel(ctx)
340
341
342 rootClientset, clientConfig, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{
343 ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
344 var err error
345 serviceAccountKey, err = keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
346 if err != nil {
347 t.Fatal(err)
348 }
349 },
350 ModifyServerConfig: func(config *controlplane.Config) {
351
352
353
354
355 authorizer := authorizer.AuthorizerFunc(func(ctx context.Context, attrs authorizer.Attributes) (authorizer.Decision, string, error) {
356 username := ""
357 if user := attrs.GetUser(); user != nil {
358 username = user.GetName()
359 }
360 ns := attrs.GetNamespace()
361
362
363 if serviceAccountNamespace, serviceAccountName, err := serviceaccountapiserver.SplitUsername(username); err == nil {
364
365 if serviceAccountNamespace == ns {
366 switch serviceAccountName {
367 case readOnlyServiceAccountName:
368 if attrs.IsReadOnly() {
369 return authorizer.DecisionAllow, "", nil
370 }
371 case readWriteServiceAccountName:
372 return authorizer.DecisionAllow, "", nil
373 }
374 }
375 }
376
377 return authorizer.DecisionNoOpinion, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil
378 })
379 config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, authorizer)
380 },
381 })
382
383 stop := func() {
384 cancel()
385 tearDownFn()
386 }
387
388 informers := clientinformers.NewSharedInformerFactory(rootClientset, controller.NoResyncPeriodFunc())
389
390
391 tokenGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, serviceAccountKey)
392 if err != nil {
393 return rootClientset, clientConfig, stop, informers, err
394 }
395 tokenController, err := serviceaccountcontroller.NewTokensController(
396 informers.Core().V1().ServiceAccounts(),
397 informers.Core().V1().Secrets(),
398 rootClientset,
399 serviceaccountcontroller.TokensControllerOptions{
400 TokenGenerator: tokenGenerator,
401 },
402 )
403 if err != nil {
404 return rootClientset, clientConfig, stop, informers, err
405 }
406 go tokenController.Run(ctx, 1)
407
408 serviceAccountController, err := serviceaccountcontroller.NewServiceAccountsController(
409 informers.Core().V1().ServiceAccounts(),
410 informers.Core().V1().Namespaces(),
411 rootClientset,
412 serviceaccountcontroller.DefaultServiceAccountsControllerOptions(),
413 )
414 if err != nil {
415 return rootClientset, clientConfig, stop, informers, err
416 }
417 informers.Start(ctx.Done())
418 go serviceAccountController.Run(ctx, 5)
419
420
421
422
423
424 informers.WaitForCacheSync(ctx.Done())
425
426 return rootClientset, clientConfig, stop, informers, nil
427 }
428
429 func getServiceAccount(c clientset.Interface, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {
430 if !shouldWait {
431 return c.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
432 }
433
434 var user *v1.ServiceAccount
435 var err error
436 err = wait.Poll(time.Second, 10*time.Second, func() (bool, error) {
437 user, err = c.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
438 if apierrors.IsNotFound(err) {
439 return false, nil
440 }
441 if err != nil {
442 return false, err
443 }
444 return true, nil
445 })
446 return user, err
447 }
448
449 func createServiceAccountToken(c clientset.Interface, sa *v1.ServiceAccount, ns string, name string) (*v1.Secret, error) {
450 secret := &v1.Secret{
451 ObjectMeta: metav1.ObjectMeta{
452 Name: name,
453 Namespace: ns,
454 Annotations: map[string]string{
455 v1.ServiceAccountNameKey: sa.GetName(),
456 v1.ServiceAccountUIDKey: string(sa.UID),
457 },
458 },
459 Type: v1.SecretTypeServiceAccountToken,
460 Data: map[string][]byte{},
461 }
462 secret, err := c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{})
463 if err != nil {
464 return nil, fmt.Errorf("failed to create secret (%s:%s) %+v, err: %v", ns, secret.Name, *secret, err)
465 }
466 err = wait.Poll(time.Second, 10*time.Second, func() (bool, error) {
467 if len(secret.Data[v1.ServiceAccountTokenKey]) != 0 {
468 return false, nil
469 }
470 secret, err = c.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{})
471 if err != nil {
472 return true, fmt.Errorf("failed to get secret (%s:%s) %+v, err: %v", ns, secret.Name, *secret, err)
473 }
474 return true, nil
475 })
476 return secret, nil
477 }
478
479 func addReferencedServiceAccountToken(c clientset.Interface, ns string, name string, secret *v1.Secret) error {
480 sa, err := c.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
481 if err != nil {
482 return err
483 }
484 sa.Secrets = append(sa.Secrets, v1.ObjectReference{
485 APIVersion: secret.APIVersion,
486 Kind: secret.Kind,
487 Namespace: secret.Namespace,
488 Name: secret.Name,
489 ResourceVersion: secret.ResourceVersion,
490 })
491 if _, err = c.CoreV1().ServiceAccounts(ns).Update(context.TODO(), sa, metav1.UpdateOptions{}); err != nil {
492 return err
493 }
494 return nil
495 }
496
497 type testOperation func() error
498
499 func doServiceAccountAPIRequests(t *testing.T, c clientset.Interface, ns string, authenticated bool, canRead bool, canWrite bool) {
500 testSecret := &v1.Secret{
501 ObjectMeta: metav1.ObjectMeta{Name: "testSecret"},
502 Data: map[string][]byte{"test": []byte("data")},
503 }
504
505 readOps := []testOperation{
506 func() error {
507 _, err := c.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{})
508 return err
509 },
510 func() error {
511 _, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
512 return err
513 },
514 }
515 writeOps := []testOperation{
516 func() error {
517 _, err := c.CoreV1().Secrets(ns).Create(context.TODO(), testSecret, metav1.CreateOptions{})
518 return err
519 },
520 func() error {
521 return c.CoreV1().Secrets(ns).Delete(context.TODO(), testSecret.Name, metav1.DeleteOptions{})
522 },
523 }
524
525 for _, op := range readOps {
526 err := op()
527 unauthorizedError := apierrors.IsUnauthorized(err)
528 forbiddenError := apierrors.IsForbidden(err)
529
530 switch {
531 case !authenticated && !unauthorizedError:
532 t.Fatalf("expected unauthorized error, got %v", err)
533 case authenticated && unauthorizedError:
534 t.Fatalf("unexpected unauthorized error: %v", err)
535 case authenticated && canRead && forbiddenError:
536 t.Fatalf("unexpected forbidden error: %v", err)
537 case authenticated && !canRead && !forbiddenError:
538 t.Fatalf("expected forbidden error, got: %v", err)
539 }
540 }
541
542 for _, op := range writeOps {
543 err := op()
544 unauthorizedError := apierrors.IsUnauthorized(err)
545 forbiddenError := apierrors.IsForbidden(err)
546
547 switch {
548 case !authenticated && !unauthorizedError:
549 t.Fatalf("expected unauthorized error, got %v", err)
550 case authenticated && unauthorizedError:
551 t.Fatalf("unexpected unauthorized error: %v", err)
552 case authenticated && canWrite && forbiddenError:
553 t.Fatalf("unexpected forbidden error: %v", err)
554 case authenticated && !canWrite && !forbiddenError:
555 t.Fatalf("expected forbidden error, got: %v", err)
556 }
557 }
558 }
559
560 type warningHandler struct {
561 mu sync.Mutex
562 warnings []string
563 }
564
565 func (r *warningHandler) HandleWarningHeader(code int, agent string, message string) {
566 r.mu.Lock()
567 defer r.mu.Unlock()
568 r.warnings = append(r.warnings, message)
569 }
570
View as plain text