postupgrade.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package upgrade
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"io"
    23  	"os"
    24  	"path/filepath"
    25  
    26  	"github.com/pkg/errors"
    27  
    28  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/labels"
    31  	errorsutil "k8s.io/apimachinery/pkg/util/errors"
    32  	"k8s.io/apimachinery/pkg/util/sets"
    33  	clientset "k8s.io/client-go/kubernetes"
    34  	"k8s.io/klog/v2"
    35  
    36  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    37  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    38  	"k8s.io/kubernetes/cmd/kubeadm/app/features"
    39  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
    40  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
    41  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
    42  	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
    43  	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
    44  	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
    45  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
    46  	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
    47  	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
    48  )
    49  
    50  // PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
    51  // Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
    52  func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
    53  	var errs []error
    54  
    55  	// Upload currently used configuration to the cluster
    56  	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
    57  	// depend on centralized information from this source in the future
    58  	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
    59  		errs = append(errs, err)
    60  	}
    61  
    62  	// Create the new, version-branched kubelet ComponentConfig ConfigMap
    63  	if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil {
    64  		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
    65  	}
    66  
    67  	// Write the new kubelet config down to disk and the env file if needed
    68  	if err := WriteKubeletConfigFiles(cfg, patchesDir, dryRun, out); err != nil {
    69  		errs = append(errs, err)
    70  	}
    71  
    72  	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
    73  	// --cri-socket.
    74  	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
    75  	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
    76  		errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
    77  	}
    78  
    79  	// Create RBAC rules that makes the bootstrap tokens able to get nodes
    80  	if err := nodebootstraptoken.AllowBoostrapTokensToGetNodes(client); err != nil {
    81  		errs = append(errs, err)
    82  	}
    83  
    84  	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
    85  	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
    86  		errs = append(errs, err)
    87  	}
    88  
    89  	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
    90  	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
    91  		errs = append(errs, err)
    92  	}
    93  
    94  	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
    95  	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
    96  		errs = append(errs, err)
    97  	}
    98  
    99  	// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
   100  	// Create the cluster-info ConfigMap with the associated RBAC rules
   101  	// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
   102  	// 	return err
   103  	//}
   104  	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
   105  	if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
   106  		errs = append(errs, err)
   107  	}
   108  
   109  	if err := PerformAddonsUpgrade(client, cfg, out); err != nil {
   110  		errs = append(errs, err)
   111  	}
   112  
   113  	return errorsutil.NewAggregate(errs)
   114  }
   115  
   116  // PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons.
   117  // When UpgradeAddonsBeforeControlPlane feature gate is enabled, the addons will be upgraded immediately.
   118  // When UpgradeAddonsBeforeControlPlane feature gate is disabled, the addons will only get updated after all the control plane instances have been upgraded.
   119  func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) error {
   120  	unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
   121  	if err != nil {
   122  		err = errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
   123  		if !features.Enabled(cfg.FeatureGates, features.UpgradeAddonsBeforeControlPlane) {
   124  			return err
   125  		}
   126  
   127  		// when UpgradeAddonsBeforeControlPlane feature gate is enabled, just throw a warning
   128  		klog.V(1).Info(err)
   129  	}
   130  	if len(unupgradedControlPlanes) > 0 {
   131  		if !features.Enabled(cfg.FeatureGates, features.UpgradeAddonsBeforeControlPlane) {
   132  			fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
   133  			return nil
   134  		}
   135  
   136  		// when UpgradeAddonsBeforeControlPlane feature gate is enabled, just throw a warning
   137  		klog.V(1).Infof("upgrading addons when control plane instances %v have not been upgraded "+
   138  			"may lead to incompatibility problems. You can disable the UpgradeAddonsBeforeControlPlane feature gate to "+
   139  			"ensure that the addons upgrade is executed only when all the control plane instances have been upgraded.", unupgradedControlPlanes)
   140  	}
   141  
   142  	var errs []error
   143  
   144  	// If the coredns ConfigMap is missing, show a warning and assume that the
   145  	// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
   146  	//
   147  	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
   148  	//   https://github.com/kubernetes/kubeadm/issues/1318
   149  	var missingCoreDNSConfigMap bool
   150  	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
   151  		context.TODO(),
   152  		kubeadmconstants.CoreDNSConfigMap,
   153  		metav1.GetOptions{},
   154  	); err != nil && apierrors.IsNotFound(err) {
   155  		missingCoreDNSConfigMap = true
   156  	}
   157  	if missingCoreDNSConfigMap {
   158  		klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
   159  			"Assuming that a DNS server was not deployed for this cluster. "+
   160  			"Note that once 'kubeadm upgrade apply' supports phases you "+
   161  			"will have to skip the DNS upgrade manually",
   162  			kubeadmconstants.CoreDNSConfigMap,
   163  			metav1.NamespaceSystem)
   164  	} else {
   165  		// Upgrade CoreDNS
   166  		if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client, out, false); err != nil {
   167  			errs = append(errs, err)
   168  		}
   169  	}
   170  
   171  	// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
   172  	// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
   173  	//
   174  	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
   175  	//   https://github.com/kubernetes/kubeadm/issues/1318
   176  	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
   177  		context.TODO(),
   178  		kubeadmconstants.KubeProxyConfigMap,
   179  		metav1.GetOptions{},
   180  	); err != nil && apierrors.IsNotFound(err) {
   181  		klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
   182  			"Assuming that kube-proxy was not deployed for this cluster. "+
   183  			"Note that once 'kubeadm upgrade apply' supports phases you "+
   184  			"will have to skip the kube-proxy upgrade manually",
   185  			kubeadmconstants.KubeProxyConfigMap,
   186  			metav1.NamespaceSystem)
   187  	} else {
   188  		// Upgrade kube-proxy
   189  		if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, false); err != nil {
   190  			errs = append(errs, err)
   191  		}
   192  	}
   193  
   194  	return errorsutil.NewAggregate(errs)
   195  }
   196  
   197  // unupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded.
   198  //
   199  // NB. This function can only be called after the current control plane instance has been upgraded already.
   200  // Because it determines whether the other control plane instances have been upgraded by checking whether
   201  // the kube-apiserver image of other control plane instance is the same as that of this instance.
   202  func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
   203  	selector := labels.SelectorFromSet(labels.Set(map[string]string{
   204  		"component": kubeadmconstants.KubeAPIServer,
   205  	}))
   206  	pods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{
   207  		LabelSelector: selector.String(),
   208  	})
   209  	if err != nil {
   210  		return nil, errors.Wrap(err, "failed to list kube-apiserver Pod from cluster")
   211  	}
   212  	if len(pods.Items) == 0 {
   213  		return nil, errors.Errorf("cannot find kube-apiserver Pod by label selector: %v", selector.String())
   214  	}
   215  
   216  	nodeImageMap := map[string]string{}
   217  
   218  	for _, pod := range pods.Items {
   219  		found := false
   220  		for _, c := range pod.Spec.Containers {
   221  			if c.Name == kubeadmconstants.KubeAPIServer {
   222  				nodeImageMap[pod.Spec.NodeName] = c.Image
   223  				found = true
   224  				break
   225  			}
   226  		}
   227  		if !found {
   228  			return nil, errors.Errorf("cannot find container by name %q for Pod %v", kubeadmconstants.KubeAPIServer, klog.KObj(&pod))
   229  		}
   230  	}
   231  
   232  	upgradedImage, ok := nodeImageMap[nodeName]
   233  	if !ok {
   234  		return nil, errors.Errorf("cannot find kube-apiserver image for current control plane instance %v", nodeName)
   235  	}
   236  
   237  	unupgradedNodes := sets.New[string]()
   238  	for node, image := range nodeImageMap {
   239  		if image != upgradedImage {
   240  			unupgradedNodes.Insert(node)
   241  		}
   242  	}
   243  
   244  	if len(unupgradedNodes) > 0 {
   245  		return sets.List(unupgradedNodes), nil
   246  	}
   247  
   248  	return nil, nil
   249  }
   250  
   251  // WriteKubeletConfigFiles writes the kubelet config file to disk, but first creates a backup of any existing one.
   252  func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
   253  	// Set up the kubelet directory to use. If dry-running, this will return a fake directory
   254  	kubeletDir, err := GetKubeletDir(dryRun)
   255  	if err != nil {
   256  		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
   257  		return err
   258  	}
   259  
   260  	// Create a copy of the kubelet config file in the /etc/kubernetes/tmp/ folder.
   261  	backupDir, err := kubeadmconstants.CreateTempDirForKubeadm(kubeadmconstants.KubernetesDir, "kubeadm-kubelet-config")
   262  	if err != nil {
   263  		return err
   264  	}
   265  	src := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)
   266  	dest := filepath.Join(backupDir, kubeadmconstants.KubeletConfigurationFileName)
   267  
   268  	if !dryRun {
   269  		fmt.Printf("[upgrade] Backing up kubelet config file to %s\n", dest)
   270  		err := kubeadmutil.CopyFile(src, dest)
   271  		if err != nil {
   272  			return errors.Wrap(err, "error backing up the kubelet config file")
   273  		}
   274  	} else {
   275  		fmt.Printf("[dryrun] Would back up kubelet config file to %s\n", dest)
   276  	}
   277  
   278  	errs := []error{}
   279  	// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
   280  	if err := kubeletphase.WriteConfigToDisk(&cfg.ClusterConfiguration, kubeletDir, patchesDir, out); err != nil {
   281  		errs = append(errs, errors.Wrap(err, "error writing kubelet configuration to file"))
   282  	}
   283  
   284  	if dryRun { // Print what contents would be written
   285  		err := dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
   286  		if err != nil {
   287  			errs = append(errs, errors.Wrap(err, "error printing files on dryrun"))
   288  		}
   289  	}
   290  	return errorsutil.NewAggregate(errs)
   291  }
   292  
   293  // GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
   294  func GetKubeletDir(dryRun bool) (string, error) {
   295  	if dryRun {
   296  		return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun")
   297  	}
   298  	return kubeadmconstants.KubeletRunDirectory, nil
   299  }
   300
View as plain text