1
16
17 package upgrade
18
19 import (
20 "crypto/sha256"
21 "crypto/x509"
22 "fmt"
23 "math/big"
24 "os"
25 "path/filepath"
26 "strings"
27 "testing"
28 "time"
29
30 "github.com/pkg/errors"
31 "go.etcd.io/etcd/client/pkg/v3/transport"
32
33 "k8s.io/client-go/tools/clientcmd"
34 certutil "k8s.io/client-go/util/cert"
35
36 kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
37 kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
38 "k8s.io/kubernetes/cmd/kubeadm/app/constants"
39 certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
40 "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
41 controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
42 etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
43 kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
44 "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
45 certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs"
46 configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
47 etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
48 "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
49 pkiutiltesting "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil/testing"
50 testutil "k8s.io/kubernetes/cmd/kubeadm/test"
51 )
52
53 const (
54 waitForHashes = "wait-for-hashes"
55 waitForHashChange = "wait-for-hash-change"
56 waitForPodsWithLabel = "wait-for-pods-with-label"
57 )
58
59 var testConfiguration = fmt.Sprintf(`
60 apiVersion: %s
61 kind: InitConfiguration
62 nodeRegistration:
63 name: foo
64 localAPIEndpoint:
65 advertiseAddress: 192.168.2.2
66 bindPort: 6443
67 bootstrapTokens:
68 - token: ce3aa5.5ec8455bb76b379f
69 ttl: 24h
70 ---
71 apiVersion: %[1]s
72 kind: ClusterConfiguration
73
74 apiServer:
75 certSANs: null
76 extraArgs: null
77 certificatesDir: %%s
78 etcd:
79 local:
80 dataDir: %%s
81 image: ""
82 imageRepository: registry.k8s.io
83 kubernetesVersion: %%s
84 networking:
85 dnsDomain: cluster.local
86 podSubnet: ""
87 serviceSubnet: 10.96.0.0/12
88 `, kubeadmapiv1.SchemeGroupVersion.String())
89
90
91 type fakeWaiter struct {
92 errsToReturn map[string]error
93 }
94
95 func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter {
96 return &fakeWaiter{
97 errsToReturn: errsToReturn,
98 }
99 }
100
101
102 func (w *fakeWaiter) WaitForControlPlaneComponents(cfg *kubeadmapi.ClusterConfiguration) error {
103 return nil
104 }
105
106
107 func (w *fakeWaiter) WaitForAPI() error {
108 return nil
109 }
110
111
112 func (w *fakeWaiter) WaitForPodsWithLabel(kvLabel string) error {
113 return w.errsToReturn[waitForPodsWithLabel]
114 }
115
116
117 func (w *fakeWaiter) WaitForPodToDisappear(podName string) error {
118 return nil
119 }
120
121
122 func (w *fakeWaiter) SetTimeout(_ time.Duration) {}
123
124
125 func (w *fakeWaiter) WaitForStaticPodControlPlaneHashes(_ string) (map[string]string, error) {
126 return map[string]string{}, w.errsToReturn[waitForHashes]
127 }
128
129
130 func (w *fakeWaiter) WaitForStaticPodSingleHash(_ string, _ string) (string, error) {
131 return "", w.errsToReturn[waitForHashes]
132 }
133
134
135 func (w *fakeWaiter) WaitForStaticPodHashChange(_, _, _ string) error {
136 return w.errsToReturn[waitForHashChange]
137 }
138
139
140 func (w *fakeWaiter) WaitForKubelet() error {
141 return nil
142 }
143
144 type fakeStaticPodPathManager struct {
145 kubernetesDir string
146 patchesDir string
147 realManifestDir string
148 tempManifestDir string
149 backupManifestDir string
150 backupEtcdDir string
151 MoveFileFunc func(string, string) error
152 }
153
154 func NewFakeStaticPodPathManager(moveFileFunc func(string, string) error) (StaticPodPathManager, error) {
155 kubernetesDir, err := os.MkdirTemp("", "kubeadm-pathmanager-")
156 if err != nil {
157 return nil, errors.Wrapf(err, "couldn't create a temporary directory for the upgrade")
158 }
159
160 realManifestDir := filepath.Join(kubernetesDir, constants.ManifestsSubDirName)
161 if err := os.Mkdir(realManifestDir, 0700); err != nil {
162 return nil, errors.Wrapf(err, "couldn't create a realManifestDir for the upgrade")
163 }
164
165 upgradedManifestDir := filepath.Join(kubernetesDir, "upgraded-manifests")
166 if err := os.Mkdir(upgradedManifestDir, 0700); err != nil {
167 return nil, errors.Wrapf(err, "couldn't create a upgradedManifestDir for the upgrade")
168 }
169
170 backupManifestDir := filepath.Join(kubernetesDir, "backup-manifests")
171 if err := os.Mkdir(backupManifestDir, 0700); err != nil {
172 return nil, errors.Wrap(err, "couldn't create a backupManifestDir for the upgrade")
173 }
174
175 backupEtcdDir := filepath.Join(kubernetesDir, "kubeadm-backup-etcd")
176 if err := os.Mkdir(backupEtcdDir, 0700); err != nil {
177 return nil, err
178 }
179
180 return &fakeStaticPodPathManager{
181 kubernetesDir: kubernetesDir,
182 realManifestDir: realManifestDir,
183 tempManifestDir: upgradedManifestDir,
184 backupManifestDir: backupManifestDir,
185 backupEtcdDir: backupEtcdDir,
186 MoveFileFunc: moveFileFunc,
187 }, nil
188 }
189
190 func (spm *fakeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
191 return spm.MoveFileFunc(oldPath, newPath)
192 }
193
194 func (spm *fakeStaticPodPathManager) KubernetesDir() string {
195 return spm.kubernetesDir
196 }
197
198 func (spm *fakeStaticPodPathManager) PatchesDir() string {
199 return spm.patchesDir
200 }
201
202 func (spm *fakeStaticPodPathManager) RealManifestPath(component string) string {
203 return constants.GetStaticPodFilepath(component, spm.realManifestDir)
204 }
205 func (spm *fakeStaticPodPathManager) RealManifestDir() string {
206 return spm.realManifestDir
207 }
208
209 func (spm *fakeStaticPodPathManager) TempManifestPath(component string) string {
210 return constants.GetStaticPodFilepath(component, spm.tempManifestDir)
211 }
212 func (spm *fakeStaticPodPathManager) TempManifestDir() string {
213 return spm.tempManifestDir
214 }
215
216 func (spm *fakeStaticPodPathManager) BackupManifestPath(component string) string {
217 return constants.GetStaticPodFilepath(component, spm.backupManifestDir)
218 }
219 func (spm *fakeStaticPodPathManager) BackupManifestDir() string {
220 return spm.backupManifestDir
221 }
222
223 func (spm *fakeStaticPodPathManager) BackupEtcdDir() string {
224 return spm.backupEtcdDir
225 }
226
227 func (spm *fakeStaticPodPathManager) CleanupDirs() error {
228 if err := os.RemoveAll(spm.TempManifestDir()); err != nil {
229 return err
230 }
231 if err := os.RemoveAll(spm.BackupManifestDir()); err != nil {
232 return err
233 }
234 return os.RemoveAll(spm.BackupEtcdDir())
235 }
236
237 type fakeTLSEtcdClient struct{ TLS bool }
238
239 func (c fakeTLSEtcdClient) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) {
240 return true, nil
241 }
242
243 func (c fakeTLSEtcdClient) CheckClusterHealth() error {
244 return nil
245 }
246
247 func (c fakeTLSEtcdClient) Sync() error { return nil }
248
249 func (c fakeTLSEtcdClient) ListMembers() ([]etcdutil.Member, error) {
250 return []etcdutil.Member{}, nil
251 }
252
253 func (c fakeTLSEtcdClient) AddMemberAsLearner(name string, peerAddrs string) ([]etcdutil.Member, error) {
254 return []etcdutil.Member{}, nil
255 }
256
257 func (c fakeTLSEtcdClient) AddMember(name string, peerAddrs string) ([]etcdutil.Member, error) {
258 return []etcdutil.Member{}, nil
259 }
260
261 func (c fakeTLSEtcdClient) MemberPromote(learnerID uint64) error {
262 return nil
263 }
264
265 func (c fakeTLSEtcdClient) GetMemberID(peerURL string) (uint64, error) {
266 return 0, nil
267 }
268
269 func (c fakeTLSEtcdClient) RemoveMember(id uint64) ([]etcdutil.Member, error) {
270 return []etcdutil.Member{}, nil
271 }
272
273 type fakePodManifestEtcdClient struct{ ManifestDir, CertificatesDir string }
274
275 func (c fakePodManifestEtcdClient) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) {
276 return true, nil
277 }
278
279 func (c fakePodManifestEtcdClient) CheckClusterHealth() error {
280
281 tlsInfo := transport.TLSInfo{
282 CertFile: filepath.Join(c.CertificatesDir, constants.EtcdCACertName),
283 KeyFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientCertName),
284 TrustedCAFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientKeyName),
285 }
286 _, err := tlsInfo.ClientConfig()
287 return err
288 }
289
290 func (c fakePodManifestEtcdClient) Sync() error { return nil }
291
292 func (c fakePodManifestEtcdClient) ListMembers() ([]etcdutil.Member, error) {
293 return []etcdutil.Member{}, nil
294 }
295
296 func (c fakePodManifestEtcdClient) AddMemberAsLearner(name string, peerAddrs string) ([]etcdutil.Member, error) {
297 return []etcdutil.Member{}, nil
298 }
299
300 func (c fakePodManifestEtcdClient) AddMember(name string, peerAddrs string) ([]etcdutil.Member, error) {
301 return []etcdutil.Member{}, nil
302 }
303
304 func (c fakePodManifestEtcdClient) MemberPromote(learnerID uint64) error {
305 return nil
306 }
307
308 func (c fakePodManifestEtcdClient) GetMemberID(peerURL string) (uint64, error) {
309 return 0, nil
310 }
311
312 func (c fakePodManifestEtcdClient) RemoveMember(id uint64) ([]etcdutil.Member, error) {
313 return []etcdutil.Member{}, nil
314 }
315
316 func TestStaticPodControlPlane(t *testing.T) {
317 tests := []struct {
318 description string
319 waitErrsToReturn map[string]error
320 moveFileFunc func(string, string) error
321 skipKubeConfig string
322 expectedErr bool
323 manifestShouldChange bool
324 }{
325 {
326 description: "error-free case should succeed",
327 waitErrsToReturn: map[string]error{
328 waitForHashes: nil,
329 waitForHashChange: nil,
330 waitForPodsWithLabel: nil,
331 },
332 moveFileFunc: os.Rename,
333 expectedErr: false,
334 manifestShouldChange: true,
335 },
336 {
337 description: "any wait error should result in a rollback and an abort 1",
338 waitErrsToReturn: map[string]error{
339 waitForHashes: errors.New("boo! failed"),
340 waitForHashChange: nil,
341 waitForPodsWithLabel: nil,
342 },
343 moveFileFunc: os.Rename,
344 expectedErr: true,
345 manifestShouldChange: false,
346 },
347 {
348 description: "any wait error should result in a rollback and an abort 2",
349 waitErrsToReturn: map[string]error{
350 waitForHashes: nil,
351 waitForHashChange: errors.New("boo! failed"),
352 waitForPodsWithLabel: nil,
353 },
354 moveFileFunc: os.Rename,
355 expectedErr: true,
356 manifestShouldChange: false,
357 },
358 {
359 description: "any wait error should result in a rollback and an abort 3",
360 waitErrsToReturn: map[string]error{
361 waitForHashes: nil,
362 waitForHashChange: nil,
363 waitForPodsWithLabel: errors.New("boo! failed"),
364 },
365 moveFileFunc: os.Rename,
366 expectedErr: true,
367 manifestShouldChange: false,
368 },
369 {
370 description: "any path-moving error should result in a rollback and an abort 1",
371 waitErrsToReturn: map[string]error{
372 waitForHashes: nil,
373 waitForHashChange: nil,
374 waitForPodsWithLabel: nil,
375 },
376 moveFileFunc: func(oldPath, newPath string) error {
377
378 if strings.Contains(newPath, "kube-apiserver") {
379 return errors.New("moving the kube-apiserver file failed")
380 }
381 return os.Rename(oldPath, newPath)
382 },
383 expectedErr: true,
384 manifestShouldChange: false,
385 },
386 {
387 description: "any path-moving error should result in a rollback and an abort 2",
388 waitErrsToReturn: map[string]error{
389 waitForHashes: nil,
390 waitForHashChange: nil,
391 waitForPodsWithLabel: nil,
392 },
393 moveFileFunc: func(oldPath, newPath string) error {
394
395 if strings.Contains(newPath, "kube-controller-manager") {
396 return errors.New("moving the kube-apiserver file failed")
397 }
398 return os.Rename(oldPath, newPath)
399 },
400 expectedErr: true,
401 manifestShouldChange: false,
402 },
403 {
404 description: "any path-moving error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)",
405 waitErrsToReturn: map[string]error{
406 waitForHashes: nil,
407 waitForHashChange: nil,
408 waitForPodsWithLabel: nil,
409 },
410 moveFileFunc: func(oldPath, newPath string) error {
411
412 if strings.Contains(newPath, "kube-scheduler") {
413 return errors.New("moving the kube-apiserver file failed")
414 }
415 return os.Rename(oldPath, newPath)
416 },
417 expectedErr: true,
418 manifestShouldChange: false,
419 },
420 {
421 description: "any cert renew error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)",
422 waitErrsToReturn: map[string]error{
423 waitForHashes: nil,
424 waitForHashChange: nil,
425 waitForPodsWithLabel: nil,
426 },
427 moveFileFunc: os.Rename,
428 skipKubeConfig: constants.SchedulerKubeConfigFileName,
429 expectedErr: true,
430 manifestShouldChange: false,
431 },
432 {
433 description: "any cert renew error should result in a rollback and an abort; even though this is admin.conf (kube-apiserver and kube-controller-manager and kube-scheduler healthy)",
434 waitErrsToReturn: map[string]error{
435 waitForHashes: nil,
436 waitForHashChange: nil,
437 waitForPodsWithLabel: nil,
438 },
439 moveFileFunc: os.Rename,
440 skipKubeConfig: constants.AdminKubeConfigFileName,
441 expectedErr: true,
442 manifestShouldChange: false,
443 },
444 {
445 description: "super-admin.conf is renewed if it exists",
446 waitErrsToReturn: map[string]error{
447 waitForHashes: nil,
448 waitForHashChange: nil,
449 waitForPodsWithLabel: nil,
450 },
451 moveFileFunc: os.Rename,
452 expectedErr: false,
453 manifestShouldChange: true,
454 },
455 {
456 description: "no error is thrown if super-admin.conf does not exist",
457 waitErrsToReturn: map[string]error{
458 waitForHashes: nil,
459 waitForHashChange: nil,
460 waitForPodsWithLabel: nil,
461 },
462 moveFileFunc: os.Rename,
463 skipKubeConfig: constants.SuperAdminKubeConfigFileName,
464 expectedErr: false,
465 manifestShouldChange: true,
466 },
467 }
468
469 for i := range tests {
470 rt := tests[i]
471 t.Run(rt.description, func(t *testing.T) {
472 pkiutiltesting.Reset()
473 waiter := NewFakeStaticPodWaiter(rt.waitErrsToReturn)
474 pathMgr, err := NewFakeStaticPodPathManager(rt.moveFileFunc)
475 if err != nil {
476 t.Fatalf("couldn't run NewFakeStaticPodPathManager: %v", err)
477 }
478 defer os.RemoveAll(pathMgr.(*fakeStaticPodPathManager).KubernetesDir())
479 tmpKubernetesDir := pathMgr.(*fakeStaticPodPathManager).KubernetesDir()
480
481 tempCertsDir, err := os.MkdirTemp("", "kubeadm-certs")
482 if err != nil {
483 t.Fatalf("couldn't create temporary certificates directory: %v", err)
484 }
485 defer os.RemoveAll(tempCertsDir)
486 tmpEtcdDataDir, err := os.MkdirTemp("", "kubeadm-etcd-data")
487 if err != nil {
488 t.Fatalf("couldn't create temporary etcd data directory: %v", err)
489 }
490 defer os.RemoveAll(tmpEtcdDataDir)
491
492 oldcfg, err := getConfig("v1.3.0", tempCertsDir, tmpEtcdDataDir)
493 if err != nil {
494 t.Fatalf("couldn't create config: %v", err)
495 }
496
497 tree, err := certsphase.GetCertsWithoutEtcd().AsMap().CertTree()
498 if err != nil {
499 t.Fatalf("couldn't get cert tree: %v", err)
500 }
501
502 if err := tree.CreateTree(oldcfg); err != nil {
503 t.Fatalf("couldn't get create cert tree: %v", err)
504 }
505
506 for _, kubeConfig := range []string{
507 constants.AdminKubeConfigFileName,
508 constants.SuperAdminKubeConfigFileName,
509 constants.SchedulerKubeConfigFileName,
510 constants.ControllerManagerKubeConfigFileName,
511 } {
512 if rt.skipKubeConfig == kubeConfig {
513 continue
514 }
515 if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpKubernetesDir, oldcfg); err != nil {
516 t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
517 }
518 }
519
520
521 err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), pathMgr.PatchesDir(), oldcfg, false )
522 if err != nil {
523 t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
524 }
525 err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), pathMgr.PatchesDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint, false )
526 if err != nil {
527 t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
528 }
529
530 oldHash, err := getAPIServerHash(pathMgr.RealManifestDir())
531 if err != nil {
532 t.Fatalf("couldn't read temp file: %v", err)
533 }
534
535 newcfg, err := getConfig(constants.CurrentKubernetesVersion.String(), tempCertsDir, tmpEtcdDataDir)
536 if err != nil {
537 t.Fatalf("couldn't create config: %v", err)
538 }
539
540
541 caCert, caKey, err := certsphase.KubeadmCertEtcdCA().CreateAsCA(newcfg)
542 if err != nil {
543 t.Fatalf("couldn't create new CA certificate: %v", err)
544 }
545 for _, cert := range []*certsphase.KubeadmCert{
546 certsphase.KubeadmCertEtcdServer(),
547 certsphase.KubeadmCertEtcdPeer(),
548 certsphase.KubeadmCertEtcdHealthcheck(),
549 certsphase.KubeadmCertEtcdAPIClient(),
550 } {
551 if err := cert.CreateFromCA(newcfg, caCert, caKey); err != nil {
552 t.Fatalf("couldn't create certificate %s: %v", cert.Name, err)
553 }
554 }
555
556 actualErr := StaticPodControlPlane(
557 nil,
558 waiter,
559 pathMgr,
560 newcfg,
561 true,
562 true,
563 fakeTLSEtcdClient{
564 TLS: false,
565 },
566 fakePodManifestEtcdClient{
567 ManifestDir: pathMgr.RealManifestDir(),
568 CertificatesDir: newcfg.CertificatesDir,
569 },
570 )
571 if (actualErr != nil) != rt.expectedErr {
572 t.Errorf(
573 "failed UpgradeStaticPodControlPlane\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v",
574 rt.description,
575 rt.expectedErr,
576 (actualErr != nil),
577 actualErr,
578 )
579 }
580
581 newHash, err := getAPIServerHash(pathMgr.RealManifestDir())
582 if err != nil {
583 t.Fatalf("couldn't read temp file: %v", err)
584 }
585
586 if (oldHash != newHash) != rt.manifestShouldChange {
587 t.Errorf(
588 "failed StaticPodControlPlane\n%s\n\texpected manifest change: %t\n\tgot: %t\n\tnewHash: %v",
589 rt.description,
590 rt.manifestShouldChange,
591 (oldHash != newHash),
592 newHash,
593 )
594 }
595 })
596 }
597 }
598
599 func getAPIServerHash(dir string) (string, error) {
600 manifestPath := constants.GetStaticPodFilepath(constants.KubeAPIServer, dir)
601
602 fileBytes, err := os.ReadFile(manifestPath)
603 if err != nil {
604 return "", err
605 }
606
607 return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil
608 }
609
610 func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.InitConfiguration, error) {
611 configBytes := []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version))
612
613
614 return configutil.BytesToInitConfiguration(configBytes, true )
615 }
616
617 func getTempDir(t *testing.T, name string) (string, func()) {
618 dir, err := os.MkdirTemp(os.TempDir(), name)
619 if err != nil {
620 t.Fatalf("couldn't make temporary directory: %v", err)
621 }
622
623 return dir, func() {
624 os.RemoveAll(dir)
625 }
626 }
627
628 func TestCleanupDirs(t *testing.T) {
629 tests := []struct {
630 name string
631 keepManifest, keepEtcd bool
632 }{
633 {
634 name: "save manifest backup",
635 keepManifest: true,
636 },
637 {
638 name: "save both etcd and manifest",
639 keepManifest: true,
640 keepEtcd: true,
641 },
642 {
643 name: "save nothing",
644 },
645 }
646
647 for _, test := range tests {
648 t.Run(test.name, func(t *testing.T) {
649 realKubernetesDir, cleanup := getTempDir(t, "realKubernetesDir")
650 defer cleanup()
651
652 tempManifestDir, cleanup := getTempDir(t, "tempManifestDir")
653 defer cleanup()
654
655 backupManifestDir, cleanup := getTempDir(t, "backupManifestDir")
656 defer cleanup()
657
658 backupEtcdDir, cleanup := getTempDir(t, "backupEtcdDir")
659 defer cleanup()
660
661 mgr := NewKubeStaticPodPathManager(realKubernetesDir, "", tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd)
662 err := mgr.CleanupDirs()
663 if err != nil {
664 t.Errorf("unexpected error cleaning up: %v", err)
665 }
666
667 if _, err := os.Stat(tempManifestDir); !os.IsNotExist(err) {
668 t.Errorf("%q should not have existed", tempManifestDir)
669 }
670 _, err = os.Stat(backupManifestDir)
671 if test.keepManifest {
672 if err != nil {
673 t.Errorf("unexpected error getting backup manifest dir")
674 }
675 } else {
676 if !os.IsNotExist(err) {
677 t.Error("expected backup manifest to not exist")
678 }
679 }
680
681 _, err = os.Stat(backupEtcdDir)
682 if test.keepEtcd {
683 if err != nil {
684 t.Errorf("unexpected error getting backup etcd dir")
685 }
686 } else {
687 if !os.IsNotExist(err) {
688 t.Error("expected backup etcd dir to not exist")
689 }
690 }
691 })
692 }
693 }
694
695 func TestRenewCertsByComponent(t *testing.T) {
696 caCert, caKey := certstestutil.SetupCertificateAuthority(t)
697
698 tests := []struct {
699 name string
700 component string
701 externalCA bool
702 externalFrontProxyCA bool
703 skipCreateEtcdCA bool
704 shouldErrorOnRenew bool
705 certsShouldExist []*certsphase.KubeadmCert
706 certsShouldBeRenewed []*certsphase.KubeadmCert
707 kubeConfigShouldExist []string
708 }{
709 {
710 name: "all CA exist, all certs should be rotated for etcd",
711 component: constants.Etcd,
712 certsShouldExist: []*certsphase.KubeadmCert{
713 certsphase.KubeadmCertEtcdServer(),
714 certsphase.KubeadmCertEtcdPeer(),
715 certsphase.KubeadmCertEtcdHealthcheck(),
716 },
717 },
718 {
719 name: "all CA exist, all certs should be rotated for apiserver",
720 component: constants.KubeAPIServer,
721 certsShouldExist: []*certsphase.KubeadmCert{
722 certsphase.KubeadmCertEtcdAPIClient(),
723 certsphase.KubeadmCertAPIServer(),
724 certsphase.KubeadmCertKubeletClient(),
725 certsphase.KubeadmCertFrontProxyClient(),
726 },
727 },
728 {
729 name: "external CA, renew only certificates not signed by CA for apiserver",
730 component: constants.KubeAPIServer,
731 certsShouldExist: []*certsphase.KubeadmCert{
732 certsphase.KubeadmCertEtcdAPIClient(),
733 certsphase.KubeadmCertFrontProxyClient(),
734 certsphase.KubeadmCertAPIServer(),
735 certsphase.KubeadmCertKubeletClient(),
736 },
737 certsShouldBeRenewed: []*certsphase.KubeadmCert{
738 certsphase.KubeadmCertEtcdAPIClient(),
739 certsphase.KubeadmCertFrontProxyClient(),
740 },
741 externalCA: true,
742 },
743 {
744 name: "external front-proxy-CA, renew only certificates not signed by front-proxy-CA for apiserver",
745 component: constants.KubeAPIServer,
746 certsShouldExist: []*certsphase.KubeadmCert{
747 certsphase.KubeadmCertEtcdAPIClient(),
748 certsphase.KubeadmCertFrontProxyClient(),
749 certsphase.KubeadmCertAPIServer(),
750 certsphase.KubeadmCertKubeletClient(),
751 },
752 certsShouldBeRenewed: []*certsphase.KubeadmCert{
753 certsphase.KubeadmCertEtcdAPIClient(),
754 certsphase.KubeadmCertAPIServer(),
755 certsphase.KubeadmCertKubeletClient(),
756 },
757 externalFrontProxyCA: true,
758 },
759 {
760 name: "all CA exist, should be rotated for scheduler",
761 component: constants.KubeScheduler,
762 kubeConfigShouldExist: []string{
763 constants.SchedulerKubeConfigFileName,
764 },
765 },
766 {
767 name: "all CA exist, should be rotated for controller manager",
768 component: constants.KubeControllerManager,
769 kubeConfigShouldExist: []string{
770 constants.ControllerManagerKubeConfigFileName,
771 },
772 },
773 {
774 name: "missing a cert to renew",
775 component: constants.Etcd,
776 shouldErrorOnRenew: true,
777 certsShouldExist: []*certsphase.KubeadmCert{
778 certsphase.KubeadmCertEtcdServer(),
779 certsphase.KubeadmCertEtcdPeer(),
780 },
781 },
782 {
783 name: "no CA, cannot continue",
784 component: constants.Etcd,
785 skipCreateEtcdCA: true,
786 shouldErrorOnRenew: true,
787 },
788 }
789
790 for i := range tests {
791 test := tests[i]
792 t.Run(test.name, func(t *testing.T) {
793 pkiutiltesting.Reset()
794
795
796 tmpDir := testutil.SetupTempDir(t)
797 defer os.RemoveAll(tmpDir)
798
799 cfg := testutil.GetDefaultInternalConfig(t)
800 cfg.CertificatesDir = tmpDir
801
802 if err := pkiutil.WriteCertAndKey(tmpDir, constants.CACertAndKeyBaseName, caCert, caKey); err != nil {
803 t.Fatalf("couldn't write out CA: %v", err)
804 }
805 if test.externalCA {
806 os.Remove(filepath.Join(tmpDir, constants.CAKeyName))
807 }
808 if err := pkiutil.WriteCertAndKey(tmpDir, constants.FrontProxyCACertAndKeyBaseName, caCert, caKey); err != nil {
809 t.Fatalf("couldn't write out front-proxy-CA: %v", err)
810 }
811 if test.externalFrontProxyCA {
812 os.Remove(filepath.Join(tmpDir, constants.FrontProxyCAKeyName))
813 }
814 if !test.skipCreateEtcdCA {
815 if err := pkiutil.WriteCertAndKey(tmpDir, constants.EtcdCACertAndKeyBaseName, caCert, caKey); err != nil {
816 t.Fatalf("couldn't write out etcd-CA: %v", err)
817 }
818 }
819
820 certMaps := make(map[string]big.Int)
821
822
823 for _, kubeCert := range test.certsShouldExist {
824 if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil {
825 t.Fatalf("couldn't create certificate %q: %v", kubeCert.Name, err)
826 }
827
828 cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
829 if err != nil {
830 t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err)
831 }
832 certMaps[kubeCert.Name] = *cert.SerialNumber
833 }
834
835
836 for _, kubeConfig := range test.kubeConfigShouldExist {
837 if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpDir, cfg); err != nil {
838 t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err)
839 }
840
841 newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig)
842 if err != nil {
843 t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err)
844 }
845 certMaps[kubeConfig] = *newCerts[0].SerialNumber
846 }
847
848
849 rm, err := renewal.NewManager(&cfg.ClusterConfiguration, tmpDir)
850 if err != nil {
851 t.Fatalf("Failed to create the certificate renewal manager: %v", err)
852 }
853
854 err = renewCertsByComponent(cfg, test.component, rm)
855 if test.shouldErrorOnRenew {
856 if err == nil {
857 t.Fatal("expected renewal error, got nothing")
858 }
859
860 return
861 }
862 if err != nil {
863 t.Fatalf("couldn't renew certificates: %v", err)
864 }
865
866
867 for _, kubeCert := range test.certsShouldExist {
868 newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
869 if err != nil {
870 t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err)
871 continue
872 }
873 oldSerial := certMaps[kubeCert.Name]
874
875 shouldBeRenewed := true
876 if test.certsShouldBeRenewed != nil {
877 shouldBeRenewed = false
878 for _, x := range test.certsShouldBeRenewed {
879 if x.Name == kubeCert.Name {
880 shouldBeRenewed = true
881 }
882 }
883 }
884
885 if shouldBeRenewed && oldSerial.Cmp(newCert.SerialNumber) == 0 {
886 t.Errorf("certifitate %v was not reissued when expected", kubeCert.Name)
887 }
888 if !shouldBeRenewed && oldSerial.Cmp(newCert.SerialNumber) != 0 {
889 t.Errorf("certifitate %v was reissued when not expected", kubeCert.Name)
890 }
891 }
892
893
894 for _, kubeConfig := range test.kubeConfigShouldExist {
895 newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig)
896 if err != nil {
897 t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err)
898 }
899 oldSerial := certMaps[kubeConfig]
900 if oldSerial.Cmp(newCerts[0].SerialNumber) == 0 {
901 t.Errorf("certifitate %v was not reissued", kubeConfig)
902 }
903 }
904 })
905
906 }
907 }
908
909 func getEmbeddedCerts(tmpDir, kubeConfig string) ([]*x509.Certificate, error) {
910 kubeconfigPath := filepath.Join(tmpDir, kubeConfig)
911 newConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
912 if err != nil {
913 return nil, errors.Wrapf(err, "failed to load kubeconfig file %s", kubeconfigPath)
914 }
915
916 authInfoName := newConfig.Contexts[newConfig.CurrentContext].AuthInfo
917 authInfo := newConfig.AuthInfos[authInfoName]
918
919 return certutil.ParseCertsPEM(authInfo.ClientCertificateData)
920 }
921
922 func TestGetPathManagerForUpgrade(t *testing.T) {
923
924 externalEtcd := &kubeadmapi.InitConfiguration{
925 ClusterConfiguration: kubeadmapi.ClusterConfiguration{
926 Etcd: kubeadmapi.Etcd{
927 External: &kubeadmapi.ExternalEtcd{
928 Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"},
929 },
930 },
931 },
932 }
933
934 stackedEtcd := &kubeadmapi.InitConfiguration{}
935
936 tests := []struct {
937 name string
938 cfg *kubeadmapi.InitConfiguration
939 etcdUpgrade bool
940 shouldDeleteEtcd bool
941 }{
942 {
943 name: "external etcd but no etcd upgrade",
944 cfg: externalEtcd,
945 etcdUpgrade: false,
946 shouldDeleteEtcd: true,
947 },
948 {
949 name: "external etcd with etcd upgrade",
950 cfg: externalEtcd,
951 etcdUpgrade: true,
952 shouldDeleteEtcd: true,
953 },
954 {
955 name: "stacked etcd but no etcd upgrade",
956 cfg: stackedEtcd,
957 etcdUpgrade: false,
958 shouldDeleteEtcd: true,
959 },
960 {
961 name: "stacked etcd with etcd upgrade",
962 cfg: stackedEtcd,
963 etcdUpgrade: true,
964 shouldDeleteEtcd: false,
965 },
966 }
967
968 for _, test := range tests {
969 t.Run(test.name, func(t *testing.T) {
970
971 tmpdir, err := os.MkdirTemp("", "TestGetPathManagerForUpgrade")
972 if err != nil {
973 t.Fatalf("unexpected error making temporary directory: %v", err)
974 }
975 defer func() {
976 os.RemoveAll(tmpdir)
977 }()
978
979 pathmgr, err := GetPathManagerForUpgrade(tmpdir, "", test.cfg, test.etcdUpgrade)
980 if err != nil {
981 t.Fatalf("unexpected error creating path manager: %v", err)
982 }
983
984 if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
985 t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
986 }
987
988 if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
989 t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err)
990 }
991
992 if err := pathmgr.CleanupDirs(); err != nil {
993 t.Fatalf("unexpected error cleaning up directories: %v", err)
994 }
995
996 if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) {
997 t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err)
998 }
999
1000 if test.shouldDeleteEtcd {
1001 if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) {
1002 t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err)
1003 }
1004 } else {
1005 if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) {
1006 t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir())
1007 }
1008 }
1009 })
1010 }
1011
1012 }
1013
1014 func TestGetEtcdImageTagFromStaticPod(t *testing.T) {
1015 const expectedEtcdVersion = "3.1.12"
1016 const etcdStaticPod = `apiVersion: v1
1017 kind: Pod
1018 metadata:
1019 labels:
1020 component: etcd
1021 tier: control-plane
1022 name: etcd
1023 namespace: kube-system
1024 spec:
1025 containers:
1026 - name: etcd
1027 image: registry.k8s.io/etcd:` + expectedEtcdVersion
1028
1029 manifestsDir, err := os.MkdirTemp("", "GetEtcdImageTagFromStaticPod-test-manifests")
1030 if err != nil {
1031 t.Fatalf("Unable to create temporary directory: %v", err)
1032 }
1033 defer os.RemoveAll(manifestsDir)
1034
1035 if err = os.WriteFile(constants.GetStaticPodFilepath(constants.Etcd, manifestsDir), []byte(etcdStaticPod), 0644); err != nil {
1036 t.Fatalf("Unable to create test static pod manifest: %v", err)
1037 }
1038
1039 got, err := GetEtcdImageTagFromStaticPod(manifestsDir)
1040 if err != nil {
1041 t.Errorf("unexpected error: %v", err)
1042 } else if got != expectedEtcdVersion {
1043 t.Errorf("unexpected result:\n\tgot: %q\n\texpected: %q", got, expectedEtcdVersion)
1044 }
1045 }
1046
View as plain text