1
16
17 package controllers
18
19 import (
20 "context"
21 "fmt"
22 "sync"
23 "testing"
24 "time"
25
26 cmutil "github.com/cert-manager/cert-manager/pkg/api/util"
27 cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
28 cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
29 cmgen "github.com/cert-manager/cert-manager/test/unit/gen"
30 "github.com/stretchr/testify/assert"
31 "github.com/stretchr/testify/require"
32 "k8s.io/apimachinery/pkg/runtime"
33 "k8s.io/apimachinery/pkg/runtime/schema"
34 "k8s.io/apimachinery/pkg/watch"
35 "k8s.io/client-go/tools/record"
36 "k8s.io/client-go/util/retry"
37 "k8s.io/client-go/util/workqueue"
38 "k8s.io/utils/clock"
39 ctrl "sigs.k8s.io/controller-runtime"
40 "sigs.k8s.io/controller-runtime/pkg/builder"
41 "sigs.k8s.io/controller-runtime/pkg/client"
42 "sigs.k8s.io/controller-runtime/pkg/controller"
43
44 v1alpha1 "github.com/cert-manager/issuer-lib/api/v1alpha1"
45 "github.com/cert-manager/issuer-lib/conditions"
46 "github.com/cert-manager/issuer-lib/controllers/signer"
47 "github.com/cert-manager/issuer-lib/internal/testapi/api"
48 "github.com/cert-manager/issuer-lib/internal/testapi/testutil"
49 "github.com/cert-manager/issuer-lib/internal/tests/testcontext"
50 "github.com/cert-manager/issuer-lib/internal/tests/testresource"
51 )
52
53
54
55 func TestCombinedControllerTemporaryFailedCertificateRequestRetrigger(t *testing.T) {
56 t.Parallel()
57
58 t.Log(
59 "Tests to show that the CertificateRequest controller handles IssuerErrors from the Sign function correctly",
60 "i.e. that it updates the CertificateRequest status to Ready=false with a Pending reason",
61 "and that it updates the Issuer status to Ready=false with a Pending reason or Ready=false with a Failed reason if the IssuerError wraps a PermanentError",
62 "Additionally, it tests that the Issuer Controller is able to recover from a temporary IssuerError",
63 )
64
65 fieldOwner := "failed-certificate-request-should-retrigger-issuer"
66
67 ctx := testcontext.ForTest(t)
68 kubeClients := testresource.KubeClients(t, nil)
69
70 checkResult, signResult := make(chan error, 10), make(chan error, 10)
71 ctx = setupControllersAPIServerAndClient(t, ctx, kubeClients,
72 func(mgr ctrl.Manager) controllerInterface {
73 return &CombinedController{
74 IssuerTypes: []v1alpha1.Issuer{&api.TestIssuer{}},
75 ClusterIssuerTypes: []v1alpha1.Issuer{&api.TestClusterIssuer{}},
76 FieldOwner: fieldOwner,
77 MaxRetryDuration: time.Minute,
78 Check: func(_ context.Context, _ v1alpha1.Issuer) error {
79 select {
80 case err := <-checkResult:
81 return err
82 case <-ctx.Done():
83 return ctx.Err()
84 }
85 },
86 Sign: func(_ context.Context, _ signer.CertificateRequestObject, _ v1alpha1.Issuer) (signer.PEMBundle, error) {
87 select {
88 case err := <-signResult:
89 return signer.PEMBundle{}, err
90 case <-ctx.Done():
91 return signer.PEMBundle{}, ctx.Err()
92 }
93 },
94 EventRecorder: record.NewFakeRecorder(100),
95 }
96 },
97 )
98
99 type testcase struct {
100 name string
101 issuerError error
102 issuerReadyCondition *cmapi.IssuerCondition
103 certificateReadyCondition *cmapi.CertificateRequestCondition
104 checkAutoRecovery bool
105 }
106
107 testcases := []testcase{
108 {
109 name: "test-normal-error",
110 issuerError: fmt.Errorf("[error message]"),
111 issuerReadyCondition: &cmapi.IssuerCondition{
112 Type: cmapi.IssuerConditionReady,
113 Status: cmmeta.ConditionFalse,
114 Reason: v1alpha1.IssuerConditionReasonPending,
115 Message: "Not ready yet: [error message]",
116 },
117 certificateReadyCondition: &cmapi.CertificateRequestCondition{
118 Type: cmapi.CertificateRequestConditionReady,
119 Status: cmmeta.ConditionFalse,
120 Reason: cmapi.CertificateRequestReasonPending,
121 Message: "Waiting for issuer to become ready. Current issuer ready condition is \"Pending\": Not ready yet: [error message].",
122 },
123 checkAutoRecovery: true,
124 },
125 {
126 name: "test-permanent-error",
127 issuerError: signer.PermanentError{Err: fmt.Errorf("[error message]")},
128 issuerReadyCondition: &cmapi.IssuerCondition{
129 Type: cmapi.IssuerConditionReady,
130 Status: cmmeta.ConditionFalse,
131 Reason: v1alpha1.IssuerConditionReasonFailed,
132 Message: "Failed permanently: [error message]",
133 },
134 certificateReadyCondition: &cmapi.CertificateRequestCondition{
135 Type: cmapi.CertificateRequestConditionReady,
136 Status: cmmeta.ConditionFalse,
137 Reason: cmapi.CertificateRequestReasonPending,
138 Message: "Waiting for issuer to become ready. Current issuer ready condition is \"Failed\": Failed permanently: [error message].",
139 },
140 checkAutoRecovery: false,
141 },
142 }
143
144
145 for _, tc := range testcases {
146 tc := tc
147 t.Run(tc.name, func(t *testing.T) {
148 t.Logf("Creating a namespace")
149 namespace, cleanup := kubeClients.SetupNamespace(t, ctx)
150 defer cleanup()
151
152 issuer := testutil.TestIssuer(
153 "issuer-1",
154 testutil.SetTestIssuerNamespace(namespace),
155 testutil.SetTestIssuerGeneration(70),
156 testutil.SetTestIssuerStatusCondition(
157 clock.RealClock{},
158 cmapi.IssuerConditionReady,
159 cmmeta.ConditionTrue,
160 v1alpha1.IssuerConditionReasonChecked,
161 "Succeeded checking the issuer",
162 ),
163 )
164
165 cr := cmgen.CertificateRequest(
166 "certificate-request-1",
167 cmgen.SetCertificateRequestNamespace(namespace),
168 cmgen.SetCertificateRequestCSR([]byte("doo")),
169 cmgen.SetCertificateRequestIssuer(cmmeta.ObjectReference{
170 Name: issuer.Name,
171 Kind: issuer.Kind,
172 Group: api.SchemeGroupVersion.Group,
173 }),
174 )
175
176 checkComplete := kubeClients.StartObjectWatch(t, ctx, issuer)
177 t.Log("Creating the TestIssuer")
178 require.NoError(t, kubeClients.Client.Create(ctx, issuer))
179 checkResult <- error(nil)
180 t.Log("Waiting for the TestIssuer to be Ready")
181 err := checkComplete(func(obj runtime.Object) error {
182 readyCondition := conditions.GetIssuerStatusCondition(obj.(*api.TestIssuer).Status.Conditions, cmapi.IssuerConditionReady)
183
184 if (readyCondition == nil) ||
185 (readyCondition.ObservedGeneration != issuer.Generation) ||
186 (readyCondition.Status != cmmeta.ConditionTrue) ||
187 (readyCondition.Reason != v1alpha1.IssuerConditionReasonChecked) ||
188 (readyCondition.Message != "Succeeded checking the issuer") {
189 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
190 }
191
192 return nil
193 }, watch.Added, watch.Modified)
194 require.NoError(t, err)
195
196 createApprovedCR(t, ctx, kubeClients.Client, cr)
197
198 checkCr1Complete := kubeClients.StartObjectWatch(t, ctx, cr)
199 checkCr2Complete := kubeClients.StartObjectWatch(t, ctx, cr)
200 checkIssuerComplete := kubeClients.StartObjectWatch(t, ctx, issuer)
201
202 signResult <- error(signer.IssuerError{Err: tc.issuerError})
203
204 t.Log("Waiting for CertificateRequest to have a Pending IssuerOutdated condition")
205 err = checkCr1Complete(func(obj runtime.Object) error {
206 readyCondition := cmutil.GetCertificateRequestCondition(obj.(*cmapi.CertificateRequest), cmapi.CertificateRequestConditionReady)
207
208 if (readyCondition == nil) ||
209 (readyCondition.Status != cmmeta.ConditionFalse) ||
210 (readyCondition.Reason != cmapi.CertificateRequestReasonPending) ||
211 (readyCondition.Message != "Waiting for issuer to become ready. Current issuer ready condition is outdated.") {
212 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
213 }
214
215 return nil
216 }, watch.Added, watch.Modified)
217 require.NoError(t, err)
218
219 t.Log("Waiting for Issuer to have a Pending IssuerFailedWillRetry condition")
220 err = checkIssuerComplete(func(obj runtime.Object) error {
221 readyCondition := conditions.GetIssuerStatusCondition(obj.(*api.TestIssuer).Status.Conditions, cmapi.IssuerConditionReady)
222
223 if (readyCondition == nil) ||
224 (readyCondition.ObservedGeneration != issuer.Generation) ||
225 (readyCondition.Status != tc.issuerReadyCondition.Status) ||
226 (readyCondition.Reason != tc.issuerReadyCondition.Reason) ||
227 (readyCondition.Message != tc.issuerReadyCondition.Message) {
228 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
229 }
230
231 return nil
232 }, watch.Added, watch.Modified)
233 require.NoError(t, err)
234
235 t.Log("Waiting for CertificateRequest to have a Pending IssuerNotReady condition")
236 err = checkCr2Complete(func(obj runtime.Object) error {
237 readyCondition := cmutil.GetCertificateRequestCondition(obj.(*cmapi.CertificateRequest), cmapi.CertificateRequestConditionReady)
238
239 if (readyCondition == nil) ||
240 (readyCondition.Status != tc.certificateReadyCondition.Status) ||
241 (readyCondition.Reason != tc.certificateReadyCondition.Reason) ||
242 (readyCondition.Message != tc.certificateReadyCondition.Message) {
243 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
244 }
245
246 return nil
247 }, watch.Added, watch.Modified)
248 require.NoError(t, err)
249
250 if tc.checkAutoRecovery {
251 t.Log("Waiting for Issuer to have a Ready Checked condition")
252 checkComplete = kubeClients.StartObjectWatch(t, ctx, issuer)
253 checkResult <- error(nil)
254 err = checkComplete(func(obj runtime.Object) error {
255 readyCondition := conditions.GetIssuerStatusCondition(obj.(*api.TestIssuer).Status.Conditions, cmapi.IssuerConditionReady)
256
257 if (readyCondition == nil) ||
258 (readyCondition.ObservedGeneration != issuer.Generation) ||
259 (readyCondition.Status != cmmeta.ConditionTrue) ||
260 (readyCondition.Reason != v1alpha1.IssuerConditionReasonChecked) ||
261 (readyCondition.Message != "Succeeded checking the issuer") {
262 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
263 }
264
265 return nil
266 }, watch.Added, watch.Modified)
267 require.NoError(t, err)
268
269 t.Log("Waiting for CertificateRequest to have a Ready Issued condition")
270 checkComplete = kubeClients.StartObjectWatch(t, ctx, cr)
271 signResult <- error(nil)
272 err = checkComplete(func(obj runtime.Object) error {
273 readyCondition := cmutil.GetCertificateRequestCondition(obj.(*cmapi.CertificateRequest), cmapi.CertificateRequestConditionReady)
274
275 if (readyCondition == nil) ||
276 (readyCondition.Status != cmmeta.ConditionTrue) ||
277 (readyCondition.Reason != cmapi.CertificateRequestReasonIssued) ||
278 (readyCondition.Message != "Succeeded signing the CertificateRequest") {
279 return fmt.Errorf("incorrect ready condition: %v", readyCondition)
280 }
281
282 return nil
283 }, watch.Added, watch.Modified)
284 require.NoError(t, err)
285 }
286 })
287 }
288 }
289
290 func TestCombinedControllerTiming(t *testing.T) {
291 t.Parallel()
292
293 t.Log(
294 "Tests to show that the CertificateRequest controller and Issuer controller call the Check and Sign functions at the correct times",
295 )
296
297 fieldOwner := "failed-certificate-request-should-retrigger-issuer"
298
299 ctx := testcontext.ForTest(t)
300 kubeClients := testresource.KubeClients(t, nil)
301
302 type simulatedCheckResult struct {
303 err error
304 }
305 type simulatedSignResult struct {
306 cert []byte
307 err error
308 }
309
310 type simulatedResult struct {
311 *simulatedCheckResult
312 *simulatedSignResult
313 expectedSinceLastResult time.Duration
314 }
315
316 type testcase struct {
317 name string
318 maxRetryDuration time.Duration
319 results []simulatedResult
320 }
321
322 testcases := []testcase{
323 {
324 name: "single-error-for-issuer-and-certificate-request",
325 maxRetryDuration: 1 * time.Hour,
326 results: []simulatedResult{
327 {
328 simulatedCheckResult: &simulatedCheckResult{err: fmt.Errorf("[error message]")},
329 expectedSinceLastResult: 0,
330 },
331 {
332 simulatedCheckResult: &simulatedCheckResult{err: nil},
333 expectedSinceLastResult: 200 * time.Millisecond,
334 },
335 {
336 simulatedSignResult: &simulatedSignResult{cert: nil, err: fmt.Errorf("[error message]")},
337 expectedSinceLastResult: 0,
338 },
339 {
340 simulatedSignResult: &simulatedSignResult{cert: []byte("cert"), err: nil},
341 expectedSinceLastResult: 200 * time.Millisecond,
342 },
343 },
344 },
345 {
346 name: "double-error-for-issuer-and-certificate-request",
347 maxRetryDuration: 1 * time.Hour,
348 results: []simulatedResult{
349 {
350 simulatedCheckResult: &simulatedCheckResult{err: fmt.Errorf("[error message]")},
351 expectedSinceLastResult: 0,
352 },
353 {
354 simulatedCheckResult: &simulatedCheckResult{err: fmt.Errorf("[error message]")},
355 expectedSinceLastResult: 200 * time.Millisecond,
356 },
357 {
358 simulatedCheckResult: &simulatedCheckResult{err: nil},
359 expectedSinceLastResult: 400 * time.Millisecond,
360 },
361 {
362 simulatedSignResult: &simulatedSignResult{cert: nil, err: fmt.Errorf("[error message]")},
363 expectedSinceLastResult: 0,
364 },
365 {
366 simulatedSignResult: &simulatedSignResult{cert: nil, err: fmt.Errorf("[error message]")},
367 expectedSinceLastResult: 200 * time.Millisecond,
368 },
369 {
370 simulatedSignResult: &simulatedSignResult{cert: []byte("cert"), err: nil},
371 expectedSinceLastResult: 400 * time.Millisecond,
372 },
373 },
374 },
375 {
376 name: "single-error-for-issuer-and-certificate-request-reaching-max-retry-duration",
377 maxRetryDuration: 300 * time.Millisecond,
378 results: []simulatedResult{
379 {
380 simulatedCheckResult: &simulatedCheckResult{err: fmt.Errorf("[error message]")},
381 expectedSinceLastResult: 0,
382 },
383 {
384 simulatedCheckResult: &simulatedCheckResult{err: nil},
385 expectedSinceLastResult: 200 * time.Millisecond,
386 },
387 {
388 simulatedSignResult: &simulatedSignResult{cert: nil, err: fmt.Errorf("[error message]")},
389 expectedSinceLastResult: 0,
390 },
391 },
392 },
393 {
394 name: "single-pending-error-for-issuer-and-certificate-request-reaching-max-retry-duration",
395 maxRetryDuration: 300 * time.Millisecond,
396 results: []simulatedResult{
397 {
398 simulatedCheckResult: &simulatedCheckResult{err: fmt.Errorf("[error message]")},
399 expectedSinceLastResult: 0,
400 },
401 {
402 simulatedCheckResult: &simulatedCheckResult{err: nil},
403 expectedSinceLastResult: 200 * time.Millisecond,
404 },
405 {
406 simulatedSignResult: &simulatedSignResult{cert: nil, err: signer.PendingError{Err: fmt.Errorf("[error message]")}},
407 expectedSinceLastResult: 0,
408 },
409 {
410 simulatedSignResult: &simulatedSignResult{cert: []byte("ok"), err: nil},
411 expectedSinceLastResult: 200 * time.Millisecond,
412 },
413 },
414 },
415 {
416 name: "fail-issuer-permanently",
417 maxRetryDuration: 1 * time.Hour,
418 results: []simulatedResult{
419 {
420 simulatedCheckResult: &simulatedCheckResult{err: signer.PermanentError{Err: fmt.Errorf("[error message]")}},
421 expectedSinceLastResult: 0,
422 },
423 },
424 },
425 {
426 name: "trigger-issuer-error-then-recover",
427 maxRetryDuration: 1 * time.Hour,
428 results: []simulatedResult{
429 {
430 simulatedCheckResult: &simulatedCheckResult{err: nil},
431 expectedSinceLastResult: 0,
432 },
433 {
434 simulatedSignResult: &simulatedSignResult{cert: nil, err: signer.IssuerError{Err: fmt.Errorf("[error message]")}},
435 expectedSinceLastResult: 0,
436 },
437 {
438 simulatedCheckResult: &simulatedCheckResult{err: nil},
439 expectedSinceLastResult: 200 * time.Millisecond,
440 },
441 {
442 simulatedSignResult: &simulatedSignResult{cert: []byte("ok"), err: nil},
443 expectedSinceLastResult: 0,
444 },
445 },
446 },
447 }
448
449 for _, tc := range testcases {
450 tc := tc
451
452 t.Run(tc.name, func(t *testing.T) {
453 resultsMutex := sync.Mutex{}
454 resultsIndex := 0
455 results := tc.results
456 durations := make([]time.Time, len(results))
457 errorCh := make(chan error)
458 done := make(chan struct{})
459
460 ctx := setupControllersAPIServerAndClient(t, ctx, kubeClients,
461 func(mgr ctrl.Manager) controllerInterface {
462 return &CombinedController{
463 IssuerTypes: []v1alpha1.Issuer{&api.TestIssuer{}},
464 ClusterIssuerTypes: []v1alpha1.Issuer{&api.TestClusterIssuer{}},
465 FieldOwner: fieldOwner,
466 MaxRetryDuration: tc.maxRetryDuration,
467 Check: func(_ context.Context, _ v1alpha1.Issuer) error {
468 resultsMutex.Lock()
469 defer resultsMutex.Unlock()
470 defer func() { resultsIndex++ }()
471
472 if resultsIndex >= len(results)-1 {
473 if resultsIndex > len(results)-1 {
474 errorCh <- fmt.Errorf("too many calls to Check")
475 return nil
476 }
477 defer close(done)
478 }
479 durations[resultsIndex] = time.Now()
480 if results[resultsIndex].simulatedCheckResult == nil {
481 errorCh <- fmt.Errorf("unexpected call to Check")
482 return nil
483 }
484 return results[resultsIndex].simulatedCheckResult.err
485 },
486 Sign: func(_ context.Context, _ signer.CertificateRequestObject, _ v1alpha1.Issuer) (signer.PEMBundle, error) {
487 resultsMutex.Lock()
488 defer resultsMutex.Unlock()
489 defer func() { resultsIndex++ }()
490
491 if resultsIndex >= len(results)-1 {
492 if resultsIndex > len(results)-1 {
493 errorCh <- fmt.Errorf("too many calls to Sign")
494 return signer.PEMBundle{}, nil
495 }
496 defer close(done)
497 }
498 durations[resultsIndex] = time.Now()
499 if results[resultsIndex].simulatedSignResult == nil {
500 errorCh <- fmt.Errorf("unexpected call to Sign")
501 return signer.PEMBundle{}, nil
502 }
503 result := results[resultsIndex].simulatedSignResult
504 return signer.PEMBundle{
505 ChainPEM: result.cert,
506 }, result.err
507 },
508 EventRecorder: record.NewFakeRecorder(100),
509
510 PreSetupWithManager: func(ctx context.Context, gvk schema.GroupVersionKind, mgr ctrl.Manager, b *builder.Builder) error {
511 b.WithOptions(controller.Options{
512 RateLimiter: workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 5*time.Second),
513 })
514 return nil
515 },
516 }
517 },
518 )
519
520 t.Logf("Creating a namespace")
521 namespace, cleanup := kubeClients.SetupNamespace(t, ctx)
522 defer cleanup()
523
524 issuer := testutil.TestIssuer(
525 "issuer-1",
526 testutil.SetTestIssuerNamespace(namespace),
527 )
528
529 cr := cmgen.CertificateRequest(
530 "certificate-request-1",
531 cmgen.SetCertificateRequestNamespace(namespace),
532 cmgen.SetCertificateRequestCSR([]byte("doo")),
533 cmgen.SetCertificateRequestIssuer(cmmeta.ObjectReference{
534 Name: issuer.Name,
535 Kind: issuer.Kind,
536 Group: api.SchemeGroupVersion.Group,
537 }),
538 )
539
540 require.NoError(t, kubeClients.Client.Create(ctx, issuer))
541 createApprovedCR(t, ctx, kubeClients.Client, cr)
542
543 <-done
544 time.Sleep(1 * time.Second)
545 select {
546 case err := <-errorCh:
547 assert.NoError(t, err)
548 default:
549 }
550
551 require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error {
552 err := kubeClients.Client.Get(ctx, client.ObjectKeyFromObject(cr), cr)
553 if err != nil {
554 return err
555 }
556 return kubeClients.Client.Delete(ctx, cr)
557 }))
558 require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error {
559 err := kubeClients.Client.Get(ctx, client.ObjectKeyFromObject(issuer), issuer)
560 if err != nil {
561 return err
562 }
563 return kubeClients.Client.Delete(ctx, issuer)
564 }))
565
566 for i := 1; i < len(results); i++ {
567 measuredDuration := durations[i].Sub(durations[i-1])
568 expectedDuration := results[i].expectedSinceLastResult
569
570 require.True(t, expectedDuration-150*time.Millisecond < measuredDuration, "result %d: expected %v, got %v", i, expectedDuration, measuredDuration)
571 require.True(t, expectedDuration+150*time.Millisecond > measuredDuration, "result %d: expected %v, got %v", i, expectedDuration, measuredDuration)
572 }
573 })
574 }
575 }
576
View as plain text