1
16
17 package serviceaccount
18
19
20
21 import (
22 "context"
23 "encoding/json"
24 "fmt"
25 "testing"
26 "time"
27
28 v1 "k8s.io/api/core/v1"
29 apierrors "k8s.io/apimachinery/pkg/api/errors"
30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31 "k8s.io/apimachinery/pkg/types"
32 "k8s.io/apimachinery/pkg/util/wait"
33 applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
34 clientinformers "k8s.io/client-go/informers"
35 clientset "k8s.io/client-go/kubernetes"
36 listersv1 "k8s.io/client-go/listers/core/v1"
37 serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
38 "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking"
39 "k8s.io/kubernetes/pkg/serviceaccount"
40 "k8s.io/utils/clock"
41 testingclock "k8s.io/utils/clock/testing"
42 )
43
44 const (
45 dateFormat = "2006-01-02"
46 cleanUpPeriod = 24 * time.Hour
47 syncInterval = 5 * time.Second
48 pollTimeout = 15 * time.Second
49 pollInterval = time.Second
50 )
51
52 func TestLegacyServiceAccountTokenCleanUp(t *testing.T) {
53 ctx, cancel := context.WithCancel(context.Background())
54 defer cancel()
55 c, config, stopFunc, informers, err := startServiceAccountTestServerAndWaitForCaches(ctx, t)
56 defer stopFunc()
57 if err != nil {
58 t.Fatalf("failed to setup ServiceAccounts server: %v", err)
59 }
60
61
62 waitConfigmapToBeLabeled(ctx, t, c)
63
64 tests := []struct {
65 name string
66 secretName string
67 secretTokenData string
68 namespace string
69 expectCleanedUp bool
70 expectInvalidLabel bool
71 lastUsedLabel bool
72 isPodMounted bool
73 isManual bool
74 }{
75 {
76 name: "auto created legacy token without pod binding",
77 secretName: "auto-token-without-pod-mounting-a",
78 namespace: "clean-ns-1",
79 lastUsedLabel: true,
80 isManual: false,
81 isPodMounted: false,
82 expectCleanedUp: true,
83 expectInvalidLabel: true,
84 },
85 {
86 name: "manually created legacy token",
87 secretName: "manual-token",
88 namespace: "clean-ns-2",
89 lastUsedLabel: true,
90 isManual: true,
91 isPodMounted: false,
92 expectCleanedUp: false,
93 expectInvalidLabel: false,
94 },
95 {
96 name: "auto created legacy token with pod binding",
97 secretName: "auto-token-with-pod-mounting",
98 namespace: "clean-ns-3",
99 lastUsedLabel: true,
100 isManual: false,
101 isPodMounted: true,
102 expectCleanedUp: false,
103 expectInvalidLabel: false,
104 },
105 {
106 name: "auto created legacy token without pod binding, secret has not been used after tracking",
107 secretName: "auto-token-without-pod-mounting-b",
108 namespace: "clean-ns-4",
109 lastUsedLabel: false,
110 isManual: false,
111 isPodMounted: false,
112 expectCleanedUp: true,
113 expectInvalidLabel: true,
114 },
115 }
116 for _, test := range tests {
117 t.Run(test.name, func(t *testing.T) {
118
119 fakeClock := testingclock.NewFakeClock(time.Now().UTC())
120
121
122 ctxForCleaner, cancelFunc := context.WithCancel(context.Background())
123 startLegacyServiceAccountTokenCleaner(ctxForCleaner, c, fakeClock, informers)
124 informers.Start(ctx.Done())
125 defer cancelFunc()
126
127
128 _, err = c.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: test.namespace}}, metav1.CreateOptions{})
129 if err != nil && !apierrors.IsAlreadyExists(err) {
130 t.Fatalf("could not create namespace: %v", err)
131 }
132 mysa, err := c.CoreV1().ServiceAccounts(test.namespace).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: readOnlyServiceAccountName}}, metav1.CreateOptions{})
133 if err != nil {
134 t.Fatalf("Service Account not created: %v", err)
135 }
136
137
138 secret, err := createServiceAccountToken(c, mysa, test.namespace, test.secretName)
139 if err != nil {
140 t.Fatalf("Secret not created: %v", err)
141 }
142 if !test.isManual {
143 if err := addReferencedServiceAccountToken(c, test.namespace, readOnlyServiceAccountName, secret); err != nil {
144 t.Fatal(err)
145 }
146 }
147 podLister := informers.Core().V1().Pods().Lister()
148 if test.isPodMounted {
149 createAutotokenMountedPod(ctx, t, c, test.namespace, test.secretName, podLister)
150 }
151
152 myConfig := *config
153 wh := &warningHandler{}
154 myConfig.WarningHandler = wh
155 myConfig.BearerToken = string(string(secret.Data[v1.ServiceAccountTokenKey]))
156 roClient := clientset.NewForConfigOrDie(&myConfig)
157
158
159 checkLastUsedLabel(ctx, t, c, secret, false)
160
161 if test.lastUsedLabel {
162 doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, true)
163
164
165 checkLastUsedLabel(ctx, t, c, secret, true)
166 }
167
168
169 fakeClock.Step(cleanUpPeriod + 24*time.Hour)
170 checkInvalidSinceLabel(ctx, t, c, secret, fakeClock, test.expectInvalidLabel)
171
172
173 if test.expectInvalidLabel {
174 t.Logf("Check the invalid token cannot authenticate request.")
175 doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, false)
176
177
178 if !test.lastUsedLabel {
179 checkLastUsedLabel(ctx, t, c, secret, true)
180 }
181
182
183 removeInvalidLabel(ctx, c, t, secret)
184
185 t.Logf("Check the token can authenticate request after patching the secret by removing the invalid label.")
186 doServiceAccountAPIReadRequest(ctx, t, roClient, test.namespace, true)
187
188
189 patchSecret(ctx, c, t, fakeClock.Now().UTC().Format(dateFormat), secret)
190
191
192 fakeClock.Step(cleanUpPeriod + 24*time.Hour)
193 checkInvalidSinceLabel(ctx, t, c, secret, fakeClock, true)
194 }
195
196 fakeClock.Step(cleanUpPeriod + 24*time.Hour)
197 checkSecretCleanUp(ctx, t, c, secret, test.expectCleanedUp)
198 })
199 }
200 }
201
202 func waitConfigmapToBeLabeled(ctx context.Context, t *testing.T, c clientset.Interface) {
203 if err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
204 configMap, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, legacytokentracking.ConfigMapName, metav1.GetOptions{})
205 if err != nil {
206 return false, err
207 }
208 _, exist := configMap.Data[legacytokentracking.ConfigMapDataKey]
209 if !exist {
210 return false, fmt.Errorf("configMap does not have since label")
211 }
212 return true, nil
213 }); err != nil {
214 t.Fatalf("failed to wait configmap starts to track: %v", err)
215 }
216 }
217
218 func checkSecretCleanUp(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, shouldCleanUp bool) {
219 err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
220 _, err := c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
221 if shouldCleanUp {
222 if err == nil {
223 return false, nil
224 } else if !apierrors.IsNotFound(err) {
225 t.Fatalf("Failed to get secret %s, err: %v", secret.Name, err)
226 }
227 return true, nil
228 }
229 if err != nil {
230 if apierrors.IsNotFound(err) {
231 t.Fatalf("The secret %s should not be cleaned up, err: %v", secret.Name, err)
232 } else {
233 t.Fatalf("Failed to get secret %s, err: %v", secret.Name, err)
234 }
235 }
236 return true, nil
237 })
238 if err != nil {
239 t.Fatalf("Failed to check the existence for secret: %s, shouldCleanUp: %v, error: %v", secret.Name, shouldCleanUp, err)
240 }
241 }
242
243 func checkInvalidSinceLabel(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, fakeClock *testingclock.FakeClock, shouldLabel bool) {
244 err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
245 liveSecret, err := c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
246 if err != nil {
247 t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
248 }
249 invalidSince, ok := liveSecret.GetLabels()[serviceaccount.InvalidSinceLabelKey]
250 if shouldLabel {
251 if !ok || invalidSince != fakeClock.Now().UTC().Format(dateFormat) {
252 return false, nil
253 }
254 return true, nil
255 }
256 if invalidSince != "" {
257 return false, nil
258 }
259 return true, nil
260 })
261
262 if err != nil {
263 t.Fatalf("Failed to check secret invalid since label for secret: %s, shouldLabel: %v, error: %v", secret.Name, shouldLabel, err)
264 }
265 }
266
267 func checkLastUsedLabel(ctx context.Context, t *testing.T, c clientset.Interface, secret *v1.Secret, shouldLabel bool) {
268 err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
269 liveSecret, err := c.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
270 if err != nil {
271 t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
272 }
273 lastUsed, ok := liveSecret.GetLabels()[serviceaccount.LastUsedLabelKey]
274 if shouldLabel {
275 if !ok || lastUsed != time.Now().UTC().Format(dateFormat) {
276 return false, nil
277 }
278 t.Logf("The secret %s has been labeled with %s", secret.Name, lastUsed)
279 return true, nil
280 }
281 if ok {
282 t.Fatalf("Secret %s should not have the lastUsed label", secret.Name)
283 }
284 return true, nil
285 })
286 if err != nil {
287 t.Fatalf("Failed to check secret last used label for secret: %s, shouldLabel: %v, error: %v", secret.Name, shouldLabel, err)
288 }
289 }
290
291 func removeInvalidLabel(ctx context.Context, c clientset.Interface, t *testing.T, secret *v1.Secret) {
292 lastUsed := secret.GetLabels()[serviceaccount.LastUsedLabelKey]
293 patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: "", serviceaccount.LastUsedLabelKey: lastUsed}))
294 if err != nil {
295 t.Fatalf("Failed to marshal invalid since label, err: %v", err)
296 }
297 t.Logf("Patch the secret by removing the invalid label.")
298 if _, err := c.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
299 t.Fatalf("Failed to remove invalid since label, err: %v", err)
300 }
301 err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
302 secret, err = c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
303 if err != nil {
304 t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
305 }
306 invalidSince := secret.GetLabels()[serviceaccount.InvalidSinceLabelKey]
307 if invalidSince != "" {
308 t.Log("Patch has not completed.")
309 return false, nil
310 }
311 return true, nil
312 })
313 if err != nil {
314 t.Fatalf("Failed to patch secret: %s, err: %v", secret.Name, err)
315 }
316 }
317
318 func patchSecret(ctx context.Context, c clientset.Interface, t *testing.T, lastUsed string, secret *v1.Secret) {
319 patchContent, err := json.Marshal(applyv1.Secret(secret.Name, secret.Namespace).WithUID(secret.UID).WithLabels(map[string]string{serviceaccount.InvalidSinceLabelKey: "", serviceaccount.LastUsedLabelKey: lastUsed}))
320 if err != nil {
321 t.Fatalf("Failed to marshal invalid since label, err: %v", err)
322 }
323 t.Logf("Patch the secret by removing the invalid label.")
324 if _, err := c.CoreV1().Secrets(secret.Namespace).Patch(ctx, secret.Name, types.MergePatchType, patchContent, metav1.PatchOptions{}); err != nil {
325 t.Fatalf("Failed to remove invalid since label, err: %v", err)
326 }
327 err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
328 secret, err = c.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
329 if err != nil {
330 t.Fatalf("Failed to get secret: %s, err: %v", secret.Name, err)
331 }
332 lastUsedString := secret.GetLabels()[serviceaccount.LastUsedLabelKey]
333 if lastUsedString != lastUsed {
334 t.Log("Patch has not completed.")
335 return false, nil
336 }
337 return true, nil
338 })
339 if err != nil {
340 t.Fatalf("Failed to patch secret: %s, err: %v", secret.Name, err)
341 }
342 }
343
344 func startLegacyServiceAccountTokenCleaner(ctx context.Context, client clientset.Interface, fakeClock clock.Clock, informers clientinformers.SharedInformerFactory) {
345 legacySATokenCleaner, _ := serviceaccountcontroller.NewLegacySATokenCleaner(
346 informers.Core().V1().ServiceAccounts(),
347 informers.Core().V1().Secrets(),
348 informers.Core().V1().Pods(),
349 client,
350 fakeClock,
351 serviceaccountcontroller.LegacySATokenCleanerOptions{
352 SyncInterval: syncInterval,
353 CleanUpPeriod: cleanUpPeriod,
354 })
355 go legacySATokenCleaner.Run(ctx)
356 }
357
358 func doServiceAccountAPIReadRequest(ctx context.Context, t *testing.T, c clientset.Interface, ns string, authenticated bool) {
359 readOps := []testOperation{
360 func() error {
361 _, err := c.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{})
362 return err
363 },
364 func() error {
365 _, err := c.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{})
366 return err
367 },
368 }
369
370 for _, op := range readOps {
371 err := wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
372 err := op()
373 if authenticated && err != nil || !authenticated && err == nil {
374 return false, nil
375 }
376 return true, nil
377 })
378 if err != nil {
379 t.Fatalf("Failed to check secret token authentication: error: %v", err)
380 }
381 }
382 }
383
384 func createAutotokenMountedPod(ctx context.Context, t *testing.T, c clientset.Interface, ns, secretName string, podLister listersv1.PodLister) *v1.Pod {
385 pod := &v1.Pod{
386 ObjectMeta: metav1.ObjectMeta{
387 Name: "token-bound-pod",
388 Namespace: ns,
389 },
390 Spec: v1.PodSpec{
391 Containers: []v1.Container{
392 {Name: "name", Image: "image"},
393 },
394 Volumes: []v1.Volume{{Name: "foo", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: secretName}}}},
395 },
396 }
397 pod, err := c.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
398 if err != nil {
399 t.Fatalf("Failed to create pod with token (%s:%s) bound, err: %v", ns, secretName, err)
400 }
401 err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) {
402 pod, err = podLister.Pods(ns).Get("token-bound-pod")
403 if err != nil {
404 return false, nil
405 }
406 return true, nil
407 })
408 if err != nil {
409 t.Fatalf("Failed to wait auto-token mounted pod: err: %v", err)
410 }
411 return pod
412 }
413
View as plain text