1
16
17 package serviceaccount
18
19 import (
20 "bytes"
21 "context"
22 "fmt"
23 "time"
24
25 v1 "k8s.io/api/core/v1"
26 apierrors "k8s.io/apimachinery/pkg/api/errors"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/types"
29 utilerrors "k8s.io/apimachinery/pkg/util/errors"
30 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
31 "k8s.io/apimachinery/pkg/util/sets"
32 "k8s.io/apimachinery/pkg/util/wait"
33 apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
34 informers "k8s.io/client-go/informers/core/v1"
35 clientset "k8s.io/client-go/kubernetes"
36 listersv1 "k8s.io/client-go/listers/core/v1"
37 "k8s.io/client-go/tools/cache"
38 clientretry "k8s.io/client-go/util/retry"
39 "k8s.io/client-go/util/workqueue"
40 "k8s.io/klog/v2"
41 "k8s.io/kubernetes/pkg/serviceaccount"
42 )
43
44
45
46
47 var RemoveTokenBackoff = wait.Backoff{
48 Steps: 10,
49 Duration: 100 * time.Millisecond,
50 Jitter: 1.0,
51 }
52
53
54 type TokensControllerOptions struct {
55
56 TokenGenerator serviceaccount.TokenGenerator
57
58
59 ServiceAccountResync time.Duration
60
61
62 SecretResync time.Duration
63
64 RootCA []byte
65
66
67
68 MaxRetries int
69 }
70
71
72 func NewTokensController(serviceAccounts informers.ServiceAccountInformer, secrets informers.SecretInformer, cl clientset.Interface, options TokensControllerOptions) (*TokensController, error) {
73 maxRetries := options.MaxRetries
74 if maxRetries == 0 {
75 maxRetries = 10
76 }
77
78 e := &TokensController{
79 client: cl,
80 token: options.TokenGenerator,
81 rootCA: options.RootCA,
82
83 syncServiceAccountQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "serviceaccount_tokens_service"),
84 syncSecretQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "serviceaccount_tokens_secret"),
85
86 maxRetries: maxRetries,
87 }
88
89 e.serviceAccounts = serviceAccounts.Lister()
90 e.serviceAccountSynced = serviceAccounts.Informer().HasSynced
91 serviceAccounts.Informer().AddEventHandlerWithResyncPeriod(
92 cache.ResourceEventHandlerFuncs{
93 AddFunc: e.queueServiceAccountSync,
94 UpdateFunc: e.queueServiceAccountUpdateSync,
95 DeleteFunc: e.queueServiceAccountSync,
96 },
97 options.ServiceAccountResync,
98 )
99
100 secretCache := secrets.Informer().GetIndexer()
101 e.updatedSecrets = cache.NewIntegerResourceVersionMutationCache(secretCache, secretCache, 60*time.Second, true)
102 e.secretSynced = secrets.Informer().HasSynced
103 secrets.Informer().AddEventHandlerWithResyncPeriod(
104 cache.FilteringResourceEventHandler{
105 FilterFunc: func(obj interface{}) bool {
106 switch t := obj.(type) {
107 case *v1.Secret:
108 return t.Type == v1.SecretTypeServiceAccountToken
109 default:
110 utilruntime.HandleError(fmt.Errorf("object passed to %T that is not expected: %T", e, obj))
111 return false
112 }
113 },
114 Handler: cache.ResourceEventHandlerFuncs{
115 AddFunc: e.queueSecretSync,
116 UpdateFunc: e.queueSecretUpdateSync,
117 DeleteFunc: e.queueSecretSync,
118 },
119 },
120 options.SecretResync,
121 )
122
123 return e, nil
124 }
125
126
127 type TokensController struct {
128 client clientset.Interface
129 token serviceaccount.TokenGenerator
130
131 rootCA []byte
132
133 serviceAccounts listersv1.ServiceAccountLister
134
135
136
137 updatedSecrets cache.MutationCache
138
139
140 serviceAccountSynced cache.InformerSynced
141 secretSynced cache.InformerSynced
142
143
144
145
146 syncServiceAccountQueue workqueue.RateLimitingInterface
147
148
149
150
151
152
153 syncSecretQueue workqueue.RateLimitingInterface
154
155 maxRetries int
156 }
157
158
159 func (e *TokensController) Run(ctx context.Context, workers int) {
160
161 defer utilruntime.HandleCrash()
162 defer e.syncServiceAccountQueue.ShutDown()
163 defer e.syncSecretQueue.ShutDown()
164
165 if !cache.WaitForNamedCacheSync("tokens", ctx.Done(), e.serviceAccountSynced, e.secretSynced) {
166 return
167 }
168
169 logger := klog.FromContext(ctx)
170 logger.V(5).Info("Starting workers")
171 for i := 0; i < workers; i++ {
172 go wait.UntilWithContext(ctx, e.syncServiceAccount, 0)
173 go wait.UntilWithContext(ctx, e.syncSecret, 0)
174 }
175 <-ctx.Done()
176 logger.V(1).Info("Shutting down")
177 }
178
179 func (e *TokensController) queueServiceAccountSync(obj interface{}) {
180 if serviceAccount, ok := obj.(*v1.ServiceAccount); ok {
181 e.syncServiceAccountQueue.Add(makeServiceAccountKey(serviceAccount))
182 }
183 }
184
185 func (e *TokensController) queueServiceAccountUpdateSync(oldObj interface{}, newObj interface{}) {
186 if serviceAccount, ok := newObj.(*v1.ServiceAccount); ok {
187 e.syncServiceAccountQueue.Add(makeServiceAccountKey(serviceAccount))
188 }
189 }
190
191
192 func (e *TokensController) retryOrForget(logger klog.Logger, queue workqueue.RateLimitingInterface, key interface{}, requeue bool) {
193 if !requeue {
194 queue.Forget(key)
195 return
196 }
197
198 requeueCount := queue.NumRequeues(key)
199 if requeueCount < e.maxRetries {
200 queue.AddRateLimited(key)
201 return
202 }
203
204 logger.V(4).Info("retried several times", "key", key, "count", requeueCount)
205 queue.Forget(key)
206 }
207
208 func (e *TokensController) queueSecretSync(obj interface{}) {
209 if secret, ok := obj.(*v1.Secret); ok {
210 e.syncSecretQueue.Add(makeSecretQueueKey(secret))
211 }
212 }
213
214 func (e *TokensController) queueSecretUpdateSync(oldObj interface{}, newObj interface{}) {
215 if secret, ok := newObj.(*v1.Secret); ok {
216 e.syncSecretQueue.Add(makeSecretQueueKey(secret))
217 }
218 }
219
220 func (e *TokensController) syncServiceAccount(ctx context.Context) {
221 logger := klog.FromContext(ctx)
222 key, quit := e.syncServiceAccountQueue.Get()
223 if quit {
224 return
225 }
226 defer e.syncServiceAccountQueue.Done(key)
227
228 retry := false
229 defer func() {
230 e.retryOrForget(logger, e.syncServiceAccountQueue, key, retry)
231 }()
232
233 saInfo, err := parseServiceAccountKey(key)
234 if err != nil {
235 logger.Error(err, "Parsing service account key")
236 return
237 }
238
239 sa, err := e.getServiceAccount(saInfo.namespace, saInfo.name, saInfo.uid, false)
240 switch {
241 case err != nil:
242 logger.Error(err, "Getting service account")
243 retry = true
244 case sa == nil:
245
246 logger.V(4).Info("Service account deleted, removing tokens", "namespace", saInfo.namespace, "serviceaccount", saInfo.name)
247 sa = &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: saInfo.namespace, Name: saInfo.name, UID: saInfo.uid}}
248 retry, err = e.deleteTokens(sa)
249 if err != nil {
250 logger.Error(err, "Error deleting serviceaccount tokens", "namespace", saInfo.namespace, "serviceaccount", saInfo.name)
251 }
252 }
253 }
254
255 func (e *TokensController) syncSecret(ctx context.Context) {
256 key, quit := e.syncSecretQueue.Get()
257 if quit {
258 return
259 }
260 defer e.syncSecretQueue.Done(key)
261
262 logger := klog.FromContext(ctx)
263
264 retry := false
265 defer func() {
266 e.retryOrForget(logger, e.syncSecretQueue, key, retry)
267 }()
268
269 secretInfo, err := parseSecretQueueKey(key)
270 if err != nil {
271 logger.Error(err, "Parsing secret queue key")
272 return
273 }
274
275 secret, err := e.getSecret(secretInfo.namespace, secretInfo.name, secretInfo.uid, false)
276 switch {
277 case err != nil:
278 logger.Error(err, "Getting secret")
279 retry = true
280 case secret == nil:
281
282 if sa, saErr := e.getServiceAccount(secretInfo.namespace, secretInfo.saName, secretInfo.saUID, false); saErr == nil && sa != nil {
283
284 if err := clientretry.RetryOnConflict(RemoveTokenBackoff, func() error {
285 return e.removeSecretReference(secretInfo.namespace, secretInfo.saName, secretInfo.saUID, secretInfo.name)
286 }); err != nil {
287 logger.Error(err, "Removing secret reference")
288 }
289 }
290 default:
291
292 sa, saErr := e.getServiceAccount(secretInfo.namespace, secretInfo.saName, secretInfo.saUID, true)
293 switch {
294 case saErr != nil:
295 logger.Error(saErr, "Getting service account")
296 retry = true
297 case sa == nil:
298
299 logger.V(4).Info("Service account does not exist, deleting token", "secret", klog.KRef(secretInfo.namespace, secretInfo.name))
300 if retriable, err := e.deleteToken(secretInfo.namespace, secretInfo.name, secretInfo.uid); err != nil {
301 logger.Error(err, "Deleting serviceaccount token", "secret", klog.KRef(secretInfo.namespace, secretInfo.name), "serviceAccount", klog.KRef(secretInfo.namespace, secretInfo.saName))
302 retry = retriable
303 }
304 default:
305
306 if retriable, err := e.generateTokenIfNeeded(logger, sa, secret); err != nil {
307 logger.Error(err, "Populating serviceaccount token", "secret", klog.KRef(secretInfo.namespace, secretInfo.name), "serviceAccount", klog.KRef(secretInfo.namespace, secretInfo.saName))
308 retry = retriable
309 }
310 }
311 }
312 }
313
314 func (e *TokensController) deleteTokens(serviceAccount *v1.ServiceAccount) ( bool, error) {
315 tokens, err := e.listTokenSecrets(serviceAccount)
316 if err != nil {
317
318 return false, err
319 }
320 retry := false
321 errs := []error{}
322 for _, token := range tokens {
323 r, err := e.deleteToken(token.Namespace, token.Name, token.UID)
324 if err != nil {
325 errs = append(errs, err)
326 }
327 if r {
328 retry = true
329 }
330 }
331 return retry, utilerrors.NewAggregate(errs)
332 }
333
334 func (e *TokensController) deleteToken(ns, name string, uid types.UID) ( bool, error) {
335 var opts metav1.DeleteOptions
336 if len(uid) > 0 {
337 opts.Preconditions = &metav1.Preconditions{UID: &uid}
338 }
339 err := e.client.CoreV1().Secrets(ns).Delete(context.TODO(), name, opts)
340
341
342 if err == nil || apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
343 return false, nil
344 }
345
346 return true, err
347 }
348
349 func (e *TokensController) secretUpdateNeeded(secret *v1.Secret) (bool, bool, bool) {
350 caData := secret.Data[v1.ServiceAccountRootCAKey]
351 needsCA := len(e.rootCA) > 0 && !bytes.Equal(caData, e.rootCA)
352
353 needsNamespace := len(secret.Data[v1.ServiceAccountNamespaceKey]) == 0
354
355 tokenData := secret.Data[v1.ServiceAccountTokenKey]
356 needsToken := len(tokenData) == 0
357
358 return needsCA, needsNamespace, needsToken
359 }
360
361
362 func (e *TokensController) generateTokenIfNeeded(logger klog.Logger, serviceAccount *v1.ServiceAccount, cachedSecret *v1.Secret) ( bool, error) {
363
364 if needsCA, needsNamespace, needsToken := e.secretUpdateNeeded(cachedSecret); !needsCA && !needsToken && !needsNamespace {
365 return false, nil
366 }
367
368
369
370 secrets := e.client.CoreV1().Secrets(cachedSecret.Namespace)
371 liveSecret, err := secrets.Get(context.TODO(), cachedSecret.Name, metav1.GetOptions{})
372 if err != nil {
373
374 return !apierrors.IsNotFound(err), err
375 }
376 if liveSecret.ResourceVersion != cachedSecret.ResourceVersion {
377
378
379 logger.V(2).Info("Secret is not up to date, skipping token population", "secret", klog.KRef(liveSecret.Namespace, liveSecret.Name))
380 return false, nil
381 }
382
383 needsCA, needsNamespace, needsToken := e.secretUpdateNeeded(liveSecret)
384 if !needsCA && !needsToken && !needsNamespace {
385 return false, nil
386 }
387
388 if liveSecret.Annotations == nil {
389 liveSecret.Annotations = map[string]string{}
390 }
391 if liveSecret.Data == nil {
392 liveSecret.Data = map[string][]byte{}
393 }
394
395
396 if needsCA {
397 liveSecret.Data[v1.ServiceAccountRootCAKey] = e.rootCA
398 }
399
400 if needsNamespace {
401 liveSecret.Data[v1.ServiceAccountNamespaceKey] = []byte(liveSecret.Namespace)
402 }
403
404
405 if needsToken {
406 token, err := e.token.GenerateToken(serviceaccount.LegacyClaims(*serviceAccount, *liveSecret))
407 if err != nil {
408 return false, err
409 }
410 liveSecret.Data[v1.ServiceAccountTokenKey] = []byte(token)
411 }
412
413
414 liveSecret.Annotations[v1.ServiceAccountNameKey] = serviceAccount.Name
415 liveSecret.Annotations[v1.ServiceAccountUIDKey] = string(serviceAccount.UID)
416
417
418 _, err = secrets.Update(context.TODO(), liveSecret, metav1.UpdateOptions{})
419 if apierrors.IsConflict(err) || apierrors.IsNotFound(err) {
420
421
422 return false, nil
423 }
424 if err != nil {
425 return true, err
426 }
427 return false, nil
428 }
429
430
431 func (e *TokensController) removeSecretReference(saNamespace string, saName string, saUID types.UID, secretName string) error {
432
433
434 serviceAccounts := e.client.CoreV1().ServiceAccounts(saNamespace)
435 serviceAccount, err := serviceAccounts.Get(context.TODO(), saName, metav1.GetOptions{})
436
437 if apierrors.IsNotFound(err) {
438 return nil
439 }
440 if err != nil {
441 return err
442 }
443
444
445 if len(saUID) > 0 && saUID != serviceAccount.UID {
446 return nil
447 }
448
449
450 if !getSecretReferences(serviceAccount).Has(secretName) {
451 return nil
452 }
453
454
455 secrets := []v1.ObjectReference{}
456 for _, s := range serviceAccount.Secrets {
457 if s.Name != secretName {
458 secrets = append(secrets, s)
459 }
460 }
461 serviceAccount.Secrets = secrets
462 _, err = serviceAccounts.Update(context.TODO(), serviceAccount, metav1.UpdateOptions{})
463
464 if apierrors.IsNotFound(err) {
465 return nil
466 }
467 return err
468 }
469
470 func (e *TokensController) getServiceAccount(ns string, name string, uid types.UID, fetchOnCacheMiss bool) (*v1.ServiceAccount, error) {
471
472 sa, err := e.serviceAccounts.ServiceAccounts(ns).Get(name)
473 if err != nil && !apierrors.IsNotFound(err) {
474 return nil, err
475 }
476 if sa != nil {
477
478 if len(uid) == 0 || uid == sa.UID {
479 return sa, nil
480 }
481 }
482
483 if !fetchOnCacheMiss {
484 return nil, nil
485 }
486
487
488 sa, err = e.client.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
489 if apierrors.IsNotFound(err) {
490 return nil, nil
491 }
492 if err != nil {
493 return nil, err
494 }
495
496 if len(uid) == 0 || uid == sa.UID {
497 return sa, nil
498 }
499 return nil, nil
500 }
501
502 func (e *TokensController) getSecret(ns string, name string, uid types.UID, fetchOnCacheMiss bool) (*v1.Secret, error) {
503
504 obj, exists, err := e.updatedSecrets.GetByKey(makeCacheKey(ns, name))
505 if err != nil {
506 return nil, err
507 }
508 if exists {
509 secret, ok := obj.(*v1.Secret)
510 if !ok {
511 return nil, fmt.Errorf("expected *v1.Secret, got %#v", secret)
512 }
513
514 if len(uid) == 0 || uid == secret.UID {
515 return secret, nil
516 }
517 }
518
519 if !fetchOnCacheMiss {
520 return nil, nil
521 }
522
523
524 secret, err := e.client.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{})
525 if apierrors.IsNotFound(err) {
526 return nil, nil
527 }
528 if err != nil {
529 return nil, err
530 }
531
532 if len(uid) == 0 || uid == secret.UID {
533 return secret, nil
534 }
535 return nil, nil
536 }
537
538
539
540 func (e *TokensController) listTokenSecrets(serviceAccount *v1.ServiceAccount) ([]*v1.Secret, error) {
541 namespaceSecrets, err := e.updatedSecrets.ByIndex("namespace", serviceAccount.Namespace)
542 if err != nil {
543 return nil, err
544 }
545
546 items := []*v1.Secret{}
547 for _, obj := range namespaceSecrets {
548 secret := obj.(*v1.Secret)
549
550 if apiserverserviceaccount.IsServiceAccountToken(secret, serviceAccount) {
551 items = append(items, secret)
552 }
553 }
554 return items, nil
555 }
556
557 func getSecretReferences(serviceAccount *v1.ServiceAccount) sets.String {
558 references := sets.NewString()
559 for _, secret := range serviceAccount.Secrets {
560 references.Insert(secret.Name)
561 }
562 return references
563 }
564
565
566
567
568 type serviceAccountQueueKey struct {
569 namespace string
570 name string
571 uid types.UID
572 }
573
574 func makeServiceAccountKey(sa *v1.ServiceAccount) interface{} {
575 return serviceAccountQueueKey{
576 namespace: sa.Namespace,
577 name: sa.Name,
578 uid: sa.UID,
579 }
580 }
581
582 func parseServiceAccountKey(key interface{}) (serviceAccountQueueKey, error) {
583 queueKey, ok := key.(serviceAccountQueueKey)
584 if !ok || len(queueKey.namespace) == 0 || len(queueKey.name) == 0 || len(queueKey.uid) == 0 {
585 return serviceAccountQueueKey{}, fmt.Errorf("invalid serviceaccount key: %#v", key)
586 }
587 return queueKey, nil
588 }
589
590
591
592
593 type secretQueueKey struct {
594 namespace string
595 name string
596 uid types.UID
597 saName string
598
599 saUID types.UID
600 }
601
602 func makeSecretQueueKey(secret *v1.Secret) interface{} {
603 return secretQueueKey{
604 namespace: secret.Namespace,
605 name: secret.Name,
606 uid: secret.UID,
607 saName: secret.Annotations[v1.ServiceAccountNameKey],
608 saUID: types.UID(secret.Annotations[v1.ServiceAccountUIDKey]),
609 }
610 }
611
612 func parseSecretQueueKey(key interface{}) (secretQueueKey, error) {
613 queueKey, ok := key.(secretQueueKey)
614 if !ok || len(queueKey.namespace) == 0 || len(queueKey.name) == 0 || len(queueKey.uid) == 0 || len(queueKey.saName) == 0 {
615 return secretQueueKey{}, fmt.Errorf("invalid secret key: %#v", key)
616 }
617 return queueKey, nil
618 }
619
620
621 func makeCacheKey(namespace, name string) string {
622 return namespace + "/" + name
623 }
624
View as plain text