staticpods.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package upgrade
    18  
    19  import (
    20  	"fmt"
    21  	"os"
    22  	"path/filepath"
    23  	"strings"
    24  	"time"
    25  
    26  	"github.com/pkg/errors"
    27  
    28  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    29  	"k8s.io/apimachinery/pkg/util/version"
    30  	clientset "k8s.io/client-go/kubernetes"
    31  	"k8s.io/klog/v2"
    32  
    33  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    34  	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
    35  	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
    36  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
    37  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
    38  	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
    39  	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
    40  	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
    41  	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
    42  	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
    43  	"k8s.io/kubernetes/cmd/kubeadm/app/util/image"
    44  	"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
    45  )
    46  
    47  // StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
    48  type StaticPodPathManager interface {
    49  	// MoveFile should move a file from oldPath to newPath
    50  	MoveFile(oldPath, newPath string) error
    51  	// KubernetesDir is the directory Kubernetes owns for storing various configuration files
    52  	KubernetesDir() string
    53  	// PatchesDir should point to the folder where patches for components are stored
    54  	PatchesDir() string
    55  	// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
    56  	RealManifestPath(component string) string
    57  	// RealManifestDir should point to the static pod manifest directory used by the kubelet
    58  	RealManifestDir() string
    59  	// TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade
    60  	TempManifestPath(component string) string
    61  	// TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade
    62  	TempManifestDir() string
    63  	// BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition
    64  	BackupManifestPath(component string) string
    65  	// BackupManifestDir should point to the backup directory used for backuping manifests during the transition
    66  	BackupManifestDir() string
    67  	// BackupEtcdDir should point to the backup directory used for backuping manifests during the transition
    68  	BackupEtcdDir() string
    69  	// CleanupDirs cleans up all temporary directories
    70  	CleanupDirs() error
    71  }
    72  
    73  // KubeStaticPodPathManager is a real implementation of StaticPodPathManager that is used when upgrading a static pod cluster
    74  type KubeStaticPodPathManager struct {
    75  	kubernetesDir     string
    76  	patchesDir        string
    77  	realManifestDir   string
    78  	tempManifestDir   string
    79  	backupManifestDir string
    80  	backupEtcdDir     string
    81  
    82  	keepManifestDir bool
    83  	keepEtcdDir     bool
    84  }
    85  
    86  // NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager
    87  func NewKubeStaticPodPathManager(kubernetesDir, patchesDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager {
    88  	return &KubeStaticPodPathManager{
    89  		kubernetesDir:     kubernetesDir,
    90  		patchesDir:        patchesDir,
    91  		realManifestDir:   filepath.Join(kubernetesDir, constants.ManifestsSubDirName),
    92  		tempManifestDir:   tempDir,
    93  		backupManifestDir: backupDir,
    94  		backupEtcdDir:     backupEtcdDir,
    95  		keepManifestDir:   keepManifestDir,
    96  		keepEtcdDir:       keepEtcdDir,
    97  	}
    98  }
    99  
   100  // NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it
   101  func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) {
   102  
   103  	upgradedManifestsDir, err := constants.CreateTempDirForKubeadm(kubernetesDir, "kubeadm-upgraded-manifests")
   104  	if err != nil {
   105  		return nil, err
   106  	}
   107  	backupManifestsDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-manifests")
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	backupEtcdDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-etcd")
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  
   116  	return NewKubeStaticPodPathManager(kubernetesDir, patchesDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil
   117  }
   118  
   119  // MoveFile should move a file from oldPath to newPath
   120  func (spm *KubeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
   121  	return kubeadmutil.MoveFile(oldPath, newPath)
   122  }
   123  
   124  // KubernetesDir should point to the directory Kubernetes owns for storing various configuration files
   125  func (spm *KubeStaticPodPathManager) KubernetesDir() string {
   126  	return spm.kubernetesDir
   127  }
   128  
   129  // PatchesDir should point to the folder where patches for components are stored
   130  func (spm *KubeStaticPodPathManager) PatchesDir() string {
   131  	return spm.patchesDir
   132  }
   133  
   134  // RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
   135  func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string {
   136  	return constants.GetStaticPodFilepath(component, spm.realManifestDir)
   137  }
   138  
   139  // RealManifestDir should point to the static pod manifest directory used by the kubelet
   140  func (spm *KubeStaticPodPathManager) RealManifestDir() string {
   141  	return spm.realManifestDir
   142  }
   143  
   144  // TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade
   145  func (spm *KubeStaticPodPathManager) TempManifestPath(component string) string {
   146  	return constants.GetStaticPodFilepath(component, spm.tempManifestDir)
   147  }
   148  
   149  // TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade
   150  func (spm *KubeStaticPodPathManager) TempManifestDir() string {
   151  	return spm.tempManifestDir
   152  }
   153  
   154  // BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition
   155  func (spm *KubeStaticPodPathManager) BackupManifestPath(component string) string {
   156  	return constants.GetStaticPodFilepath(component, spm.backupManifestDir)
   157  }
   158  
   159  // BackupManifestDir should point to the backup directory used for backuping manifests during the transition
   160  func (spm *KubeStaticPodPathManager) BackupManifestDir() string {
   161  	return spm.backupManifestDir
   162  }
   163  
   164  // BackupEtcdDir should point to the backup directory used for backuping manifests during the transition
   165  func (spm *KubeStaticPodPathManager) BackupEtcdDir() string {
   166  	return spm.backupEtcdDir
   167  }
   168  
   169  // CleanupDirs cleans up all temporary directories except those the user has requested to keep around
   170  func (spm *KubeStaticPodPathManager) CleanupDirs() error {
   171  	var errlist []error
   172  	if err := os.RemoveAll(spm.TempManifestDir()); err != nil {
   173  		errlist = append(errlist, err)
   174  	}
   175  	if !spm.keepManifestDir {
   176  		if err := os.RemoveAll(spm.BackupManifestDir()); err != nil {
   177  			errlist = append(errlist, err)
   178  		}
   179  	}
   180  
   181  	if !spm.keepEtcdDir {
   182  		if err := os.RemoveAll(spm.BackupEtcdDir()); err != nil {
   183  			errlist = append(errlist, err)
   184  		}
   185  	}
   186  
   187  	return utilerrors.NewAggregate(errlist)
   188  }
   189  
   190  func upgradeComponent(component string, certsRenewMgr *renewal.Manager, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string) error {
   191  	// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
   192  	// manifests only for the case when component is Etcd
   193  	recoverEtcd := false
   194  	if component == constants.Etcd {
   195  		recoverEtcd = true
   196  	}
   197  
   198  	fmt.Printf("[upgrade/staticpods] Preparing for %q upgrade\n", component)
   199  
   200  	// The old manifest is here; in the /etc/kubernetes/manifests/
   201  	currentManifestPath := pathMgr.RealManifestPath(component)
   202  	// The new, upgraded manifest will be written here
   203  	newManifestPath := pathMgr.TempManifestPath(component)
   204  	// The old manifest will be moved here; into a subfolder of the temporary directory
   205  	// If a rollback is needed, these manifests will be put back to where they where initially
   206  	backupManifestPath := pathMgr.BackupManifestPath(component)
   207  
   208  	// Store the backup path in the recover list. If something goes wrong now, this component will be rolled back.
   209  	recoverManifests[component] = backupManifestPath
   210  
   211  	// Skip upgrade if current and new manifests are equal
   212  	equal, diff, err := staticpod.ManifestFilesAreEqual(currentManifestPath, newManifestPath)
   213  	if err != nil {
   214  		return err
   215  	}
   216  	if equal {
   217  		fmt.Printf("[upgrade/staticpods] Current and new manifests of %s are equal, skipping upgrade\n", component)
   218  		return nil
   219  	} else {
   220  		klog.V(4).Infof("Pod manifest files diff:\n%s\n", diff)
   221  	}
   222  
   223  	// if certificate renewal should be performed
   224  	if certsRenewMgr != nil {
   225  		// renew all the certificates used by the current component
   226  		if err := renewCertsByComponent(cfg, component, certsRenewMgr); err != nil {
   227  			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to renew certificates for component %q", component), pathMgr, recoverEtcd)
   228  		}
   229  	}
   230  
   231  	// Move the old manifest into the old-manifests directory
   232  	if err := pathMgr.MoveFile(currentManifestPath, backupManifestPath); err != nil {
   233  		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
   234  	}
   235  
   236  	// Move the new manifest into the manifests directory
   237  	if err := pathMgr.MoveFile(newManifestPath, currentManifestPath); err != nil {
   238  		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
   239  	}
   240  
   241  	fmt.Printf("[upgrade/staticpods] Moved new manifest to %q and backed up old manifest to %q\n", currentManifestPath, backupManifestPath)
   242  
   243  	fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
   244  	fmt.Printf("[upgrade/staticpods] This can take up to %v\n", kubeadmapi.GetActiveTimeouts().UpgradeManifests.Duration)
   245  
   246  	// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
   247  	// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy
   248  	// If we don't do this, there is a case where we remove the Static Pod manifest, kubelet is slow to react, kubeadm checks the
   249  	// API endpoint below of the OLD Static Pod component and proceeds quickly enough, which might lead to unexpected results.
   250  	if err := waiter.WaitForStaticPodHashChange(cfg.NodeRegistration.Name, component, beforePodHash); err != nil {
   251  		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
   252  	}
   253  
   254  	// Wait for the static pod component to come up and register itself as a mirror pod
   255  	if err := waiter.WaitForPodsWithLabel("component=" + component); err != nil {
   256  		return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
   257  	}
   258  
   259  	fmt.Printf("[upgrade/staticpods] Component %q upgraded successfully!\n", component)
   260  
   261  	return nil
   262  }
   263  
   264  // performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
   265  func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
   266  	// Add etcd static pod spec only if external etcd is not configured
   267  	if cfg.Etcd.External != nil {
   268  		return false, errors.New("external etcd detected, won't try to change any etcd state")
   269  	}
   270  
   271  	// Checking health state of etcd before proceeding with the upgrade
   272  	err := oldEtcdClient.CheckClusterHealth()
   273  	if err != nil {
   274  		return true, errors.Wrap(err, "etcd cluster is not healthy")
   275  	}
   276  
   277  	// Backing up etcd data store
   278  	backupEtcdDir := pathMgr.BackupEtcdDir()
   279  	runningEtcdDir := cfg.Etcd.Local.DataDir
   280  	output, err := kubeadmutil.CopyDir(runningEtcdDir, backupEtcdDir)
   281  	if err != nil {
   282  		return true, errors.Wrapf(err, "failed to back up etcd data, output: %q", output)
   283  	}
   284  
   285  	// Get the desired etcd version. That's either the one specified by the user in cfg.Etcd.Local.ImageTag
   286  	// or the kubeadm preferred one for the desired Kubernetes version
   287  	var desiredEtcdVersion *version.Version
   288  	if cfg.Etcd.Local.ImageTag != "" {
   289  		desiredEtcdVersion, err = version.ParseSemantic(
   290  			convertImageTagMetadataToSemver(cfg.Etcd.Local.ImageTag))
   291  		if err != nil {
   292  			return true, errors.Wrapf(err, "failed to parse tag %q as a semantic version", cfg.Etcd.Local.ImageTag)
   293  		}
   294  	} else {
   295  		// Need to check currently used version and version from constants, if differs then upgrade
   296  		var warning error
   297  		desiredEtcdVersion, warning, err = constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, cfg.KubernetesVersion)
   298  		if err != nil {
   299  			return true, errors.Wrap(err, "failed to retrieve an etcd version for the target Kubernetes version")
   300  		}
   301  		if warning != nil {
   302  			klog.V(1).Infof("[upgrade/etcd] WARNING: %v", warning)
   303  		}
   304  	}
   305  
   306  	// Get the etcd version of the local/stacked etcd member running on the current machine
   307  	currentEtcdVersionStr, err := GetEtcdImageTagFromStaticPod(pathMgr.RealManifestDir())
   308  	if err != nil {
   309  		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
   310  	}
   311  
   312  	cmpResult, err := desiredEtcdVersion.Compare(currentEtcdVersionStr)
   313  	if err != nil {
   314  		return true, errors.Wrapf(err, "failed comparing the current etcd version %q to the desired one %q", currentEtcdVersionStr, desiredEtcdVersion)
   315  	}
   316  	if cmpResult < 0 {
   317  		return false, errors.Errorf("the desired etcd version %q is older than the currently installed %q. Skipping etcd upgrade", desiredEtcdVersion, currentEtcdVersionStr)
   318  	}
   319  
   320  	beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd)
   321  	if err != nil {
   322  		return true, err
   323  	}
   324  
   325  	// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
   326  	// has occurred in any aspects.
   327  	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, false /* isDryRun */); err != nil {
   328  		return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
   329  	}
   330  
   331  	retries := 10
   332  	retryInterval := 15 * time.Second
   333  
   334  	// Perform etcd upgrade using common to all control plane components function
   335  	if err := upgradeComponent(constants.Etcd, certsRenewMgr, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil {
   336  		fmt.Printf("[upgrade/etcd] Failed to upgrade etcd: %v\n", err)
   337  		// Since upgrade component failed, the old etcd manifest has either been restored or was never touched
   338  		// Now we need to check the health of etcd cluster if it is up with old manifest
   339  		fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
   340  		if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
   341  			fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
   342  
   343  			// At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest
   344  			fmt.Println("[upgrade/etcd] Rolling back etcd data")
   345  			if err := rollbackEtcdData(cfg, pathMgr); err != nil {
   346  				// Even copying back datastore failed, no options for recovery left, bailing out
   347  				return true, errors.Errorf("fatal error rolling back local etcd cluster datadir: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
   348  			}
   349  			fmt.Println("[upgrade/etcd] Etcd data rollback successful")
   350  
   351  			// Now that we've rolled back the data, let's check if the cluster comes up
   352  			fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
   353  			if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
   354  				fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
   355  				// Nothing else left to try to recover etcd cluster
   356  				return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir)
   357  			}
   358  
   359  			// We've recovered to the previous etcd from this case
   360  		}
   361  		fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")
   362  
   363  		// Since etcd cluster came back up with the old manifest
   364  		return true, errors.Wrap(err, "fatal error when trying to upgrade the etcd cluster, rolled the state back to pre-upgrade state")
   365  	}
   366  
   367  	// Initialize the new etcd client if it wasn't pre-initialized
   368  	if newEtcdClient == nil {
   369  		etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
   370  		if err != nil {
   371  			return true, errors.Wrap(err, "fatal error creating etcd client")
   372  		}
   373  		newEtcdClient = etcdClient
   374  	}
   375  
   376  	// Checking health state of etcd after the upgrade
   377  	fmt.Println("[upgrade/etcd] Waiting for etcd to become available")
   378  	if _, err = newEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
   379  		fmt.Printf("[upgrade/etcd] Failed to healthcheck etcd: %v\n", err)
   380  		// Despite the fact that upgradeComponent was successful, there is something wrong with the etcd cluster
   381  		// First step is to restore back up of datastore
   382  		fmt.Println("[upgrade/etcd] Rolling back etcd data")
   383  		if err := rollbackEtcdData(cfg, pathMgr); err != nil {
   384  			// Even copying back datastore failed, no options for recovery left, bailing out
   385  			return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster datadir, the backup of etcd database is stored here:(%s)", backupEtcdDir)
   386  		}
   387  		fmt.Println("[upgrade/etcd] Etcd data rollback successful")
   388  
   389  		// Old datastore has been copied, rolling back old manifests
   390  		fmt.Println("[upgrade/etcd] Rolling back etcd manifest")
   391  		rollbackOldManifests(recoverManifests, err, pathMgr, true)
   392  		// rollbackOldManifests() always returns an error -- ignore it and continue
   393  
   394  		// Assuming rollback of the old etcd manifest was successful, check the status of etcd cluster again
   395  		fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
   396  		if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil {
   397  			fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
   398  			// Nothing else left to try to recover etcd cluster
   399  			return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir)
   400  		}
   401  		fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")
   402  
   403  		// We've successfully rolled back etcd, and now return an error describing that the upgrade failed
   404  		return true, errors.Wrap(err, "fatal error upgrading local etcd cluster, rolled the state back to pre-upgrade state")
   405  	}
   406  
   407  	return false, nil
   408  }
   409  
   410  // StaticPodControlPlane upgrades a static pod-hosted control plane
   411  func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
   412  	recoverManifests := map[string]string{}
   413  	var isExternalEtcd bool
   414  
   415  	beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeRegistration.Name)
   416  	if err != nil {
   417  		return err
   418  	}
   419  
   420  	if oldEtcdClient == nil {
   421  		if cfg.Etcd.External != nil {
   422  			// External etcd
   423  			isExternalEtcd = true
   424  			etcdClient, err := etcdutil.New(
   425  				cfg.Etcd.External.Endpoints,
   426  				cfg.Etcd.External.CAFile,
   427  				cfg.Etcd.External.CertFile,
   428  				cfg.Etcd.External.KeyFile,
   429  			)
   430  			if err != nil {
   431  				return errors.Wrap(err, "failed to create etcd client for external etcd")
   432  			}
   433  			oldEtcdClient = etcdClient
   434  			// Since etcd is managed externally, the new etcd client will be the same as the old client
   435  			if newEtcdClient == nil {
   436  				newEtcdClient = etcdClient
   437  			}
   438  		} else {
   439  			// etcd Static Pod
   440  			etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
   441  			if err != nil {
   442  				return errors.Wrap(err, "failed to create etcd client")
   443  			}
   444  			oldEtcdClient = etcdClient
   445  		}
   446  	}
   447  
   448  	var certsRenewMgr *renewal.Manager
   449  	if renewCerts {
   450  		certsRenewMgr, err = renewal.NewManager(&cfg.ClusterConfiguration, pathMgr.KubernetesDir())
   451  		if err != nil {
   452  			return errors.Wrap(err, "failed to create the certificate renewal manager")
   453  		}
   454  	}
   455  
   456  	// etcd upgrade is done prior to other control plane components
   457  	if !isExternalEtcd && etcdUpgrade {
   458  		// set the TLS upgrade flag for all components
   459  		fmt.Printf("[upgrade/etcd] Upgrading to TLS for %s\n", constants.Etcd)
   460  
   461  		// Perform etcd upgrade using common to all control plane components function
   462  		fatal, err := performEtcdStaticPodUpgrade(certsRenewMgr, client, waiter, pathMgr, cfg, recoverManifests, oldEtcdClient, newEtcdClient)
   463  		if err != nil {
   464  			if fatal {
   465  				return err
   466  			}
   467  			fmt.Printf("[upgrade/etcd] Non fatal issue encountered during upgrade: %v\n", err)
   468  		}
   469  	}
   470  
   471  	// Write the updated static Pod manifests into the temporary directory
   472  	fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
   473  	err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg, false /* isDryRun */)
   474  	if err != nil {
   475  		return errors.Wrap(err, "error creating init static pod manifest files")
   476  	}
   477  
   478  	for _, component := range constants.ControlPlaneComponents {
   479  		if err = upgradeComponent(component, certsRenewMgr, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests); err != nil {
   480  			return err
   481  		}
   482  	}
   483  
   484  	if renewCerts {
   485  		// renew the certificate embedded in the admin.conf file
   486  		renewed, err := certsRenewMgr.RenewUsingLocalCA(constants.AdminKubeConfigFileName)
   487  		if err != nil {
   488  			return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.AdminKubeConfigFileName), pathMgr, false)
   489  		}
   490  
   491  		if !renewed {
   492  			// if not error, but not renewed because of external CA detected, inform the user
   493  			fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.AdminKubeConfigFileName)
   494  		}
   495  
   496  		// Do the same for super-admin.conf, but only if it exists
   497  		if _, err := os.Stat(filepath.Join(pathMgr.KubernetesDir(), constants.SuperAdminKubeConfigFileName)); err == nil {
   498  			// renew the certificate embedded in the super-admin.conf file
   499  			renewed, err := certsRenewMgr.RenewUsingLocalCA(constants.SuperAdminKubeConfigFileName)
   500  			if err != nil {
   501  				return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.SuperAdminKubeConfigFileName), pathMgr, false)
   502  			}
   503  
   504  			if !renewed {
   505  				// if not error, but not renewed because of external CA detected, inform the user
   506  				fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.SuperAdminKubeConfigFileName)
   507  			}
   508  		}
   509  	}
   510  
   511  	// Remove the temporary directories used on a best-effort (don't fail if the calls error out)
   512  	// The calls are set here by design; we should _not_ use "defer" above as that would remove the directories
   513  	// even in the "fail and rollback" case, where we want the directories preserved for the user.
   514  	return pathMgr.CleanupDirs()
   515  }
   516  
   517  // rollbackOldManifests rolls back the backed-up manifests if something went wrong.
   518  // It always returns an error to the caller.
   519  func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr StaticPodPathManager, restoreEtcd bool) error {
   520  	errs := []error{origErr}
   521  	for component, backupPath := range oldManifests {
   522  		// Will restore etcd manifest only if it was explicitly requested by setting restoreEtcd to True
   523  		if component == constants.Etcd && !restoreEtcd {
   524  			continue
   525  		}
   526  		// Where we should put back the backed up manifest
   527  		realManifestPath := pathMgr.RealManifestPath(component)
   528  
   529  		// Move the backup manifest back into the manifests directory
   530  		err := pathMgr.MoveFile(backupPath, realManifestPath)
   531  		if err != nil {
   532  			errs = append(errs, err)
   533  		}
   534  	}
   535  	// Let the user know there were problems, but we tried to recover
   536  	return errors.Wrap(utilerrors.NewAggregate(errs),
   537  		"couldn't upgrade control plane. kubeadm has tried to recover everything into the earlier state. Errors faced")
   538  }
   539  
   540  // rollbackEtcdData rolls back the content of etcd folder if something went wrong.
   541  // When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned.
   542  func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathManager) error {
   543  	backupEtcdDir := pathMgr.BackupEtcdDir()
   544  	runningEtcdDir := cfg.Etcd.Local.DataDir
   545  
   546  	output, err := kubeadmutil.CopyDir(backupEtcdDir, runningEtcdDir)
   547  	if err != nil {
   548  		// Let the user know there we're problems, but we tried to reçover
   549  		return errors.Wrapf(err, "couldn't recover etcd database with error, the location of etcd backup: %s, output: %q", backupEtcdDir, output)
   550  	}
   551  
   552  	return nil
   553  }
   554  
   555  // renewCertsByComponent takes charge of renewing certificates used by a specific component before
   556  // the static pod of the component is upgraded
   557  func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string, certsRenewMgr *renewal.Manager) error {
   558  	var certificates []string
   559  
   560  	// if etcd, only in case of local etcd, renew server, peer and health check certificate
   561  	if component == constants.Etcd {
   562  		if cfg.Etcd.Local != nil {
   563  			certificates = []string{
   564  				certsphase.KubeadmCertEtcdServer().Name,
   565  				certsphase.KubeadmCertEtcdPeer().Name,
   566  				certsphase.KubeadmCertEtcdHealthcheck().Name,
   567  			}
   568  		}
   569  	}
   570  
   571  	// if apiserver, renew apiserver serving certificate, kubelet and front-proxy client certificate.
   572  	//if local etcd, renew also the etcd client certificate
   573  	if component == constants.KubeAPIServer {
   574  		certificates = []string{
   575  			certsphase.KubeadmCertAPIServer().Name,
   576  			certsphase.KubeadmCertKubeletClient().Name,
   577  			certsphase.KubeadmCertFrontProxyClient().Name,
   578  		}
   579  		if cfg.Etcd.Local != nil {
   580  			certificates = append(certificates, certsphase.KubeadmCertEtcdAPIClient().Name)
   581  		}
   582  	}
   583  
   584  	// if controller-manager, renew the certificate embedded in the controller-manager kubeConfig file
   585  	if component == constants.KubeControllerManager {
   586  		certificates = []string{
   587  			constants.ControllerManagerKubeConfigFileName,
   588  		}
   589  	}
   590  
   591  	// if scheduler, renew the certificate embedded in the scheduler kubeConfig file
   592  	if component == constants.KubeScheduler {
   593  		certificates = []string{
   594  			constants.SchedulerKubeConfigFileName,
   595  		}
   596  	}
   597  
   598  	// renew the selected components
   599  	for _, cert := range certificates {
   600  		fmt.Printf("[upgrade/staticpods] Renewing %s certificate\n", cert)
   601  		renewed, err := certsRenewMgr.RenewUsingLocalCA(cert)
   602  		if err != nil {
   603  			return err
   604  		}
   605  		if !renewed {
   606  			// if not error, but not renewed because of external CA detected, inform the user
   607  			fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", cert)
   608  		}
   609  	}
   610  
   611  	return nil
   612  }
   613  
   614  // GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration.
   615  func GetPathManagerForUpgrade(kubernetesDir, patchesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) {
   616  	isExternalEtcd := internalcfg.Etcd.External != nil
   617  	return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir, true, etcdUpgrade && !isExternalEtcd)
   618  }
   619  
   620  // PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster
   621  func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, patchesDir string) error {
   622  	pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, patchesDir, internalcfg, etcdUpgrade)
   623  	if err != nil {
   624  		return err
   625  	}
   626  
   627  	// The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing
   628  	return StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil)
   629  }
   630  
   631  // DryRunStaticPodUpgrade fakes an upgrade of the control plane
   632  func DryRunStaticPodUpgrade(patchesDir string, internalcfg *kubeadmapi.InitConfiguration) error {
   633  
   634  	dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
   635  	if err != nil {
   636  		return err
   637  	}
   638  	defer os.RemoveAll(dryRunManifestDir)
   639  	if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, patchesDir, internalcfg, true /* isDryRun */); err != nil {
   640  		return err
   641  	}
   642  
   643  	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
   644  	files := []dryrunutil.FileToPrint{}
   645  	for _, component := range constants.ControlPlaneComponents {
   646  		realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir)
   647  		outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
   648  		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
   649  	}
   650  
   651  	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
   652  }
   653  
   654  // GetEtcdImageTagFromStaticPod returns the image tag of the local etcd static pod
   655  func GetEtcdImageTagFromStaticPod(manifestDir string) (string, error) {
   656  	realPath := constants.GetStaticPodFilepath(constants.Etcd, manifestDir)
   657  	pod, err := staticpod.ReadStaticPodFromDisk(realPath)
   658  	if err != nil {
   659  		return "", err
   660  	}
   661  
   662  	return convertImageTagMetadataToSemver(image.TagFromImage(pod.Spec.Containers[0].Image)), nil
   663  }
   664  
   665  // convertImageTagMetadataToSemver converts imagetag in the format of semver_metadata to semver+metadata
   666  func convertImageTagMetadataToSemver(tag string) string {
   667  	// Container registries do not support `+` characters in tag names. This prevents imagetags from
   668  	// correctly representing semantic versions which use the plus symbol to delimit build metadata.
   669  	// Kubernetes uses the convention of using an underscore in image registries to preserve
   670  	// build metadata information in imagetags.
   671  	return strings.Replace(tag, "_", "+", 1)
   672  }
   673
View as plain text