manifests.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"fmt"
    21  	"net"
    22  	"os"
    23  	"path/filepath"
    24  	"strconv"
    25  	"strings"
    26  
    27  	"github.com/pkg/errors"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/klog/v2"
    31  	utilsnet "k8s.io/utils/net"
    32  
    33  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    34  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    35  	"k8s.io/kubernetes/cmd/kubeadm/app/features"
    36  	"k8s.io/kubernetes/cmd/kubeadm/app/images"
    37  	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
    38  	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
    39  	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
    40  	"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
    41  )
    42  
    43  // CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
    44  func CreateInitStaticPodManifestFiles(manifestDir, patchesDir string, cfg *kubeadmapi.InitConfiguration, isDryRun bool) error {
    45  	klog.V(1).Infoln("[control-plane] creating static Pod files")
    46  	return CreateStaticPodFiles(manifestDir, patchesDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, isDryRun, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
    47  }
    48  
    49  // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration
    50  // NB. this method holds the information about how kubeadm creates static pod manifests.
    51  func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, proxyEnvs []kubeadmapi.EnvVar) map[string]v1.Pod {
    52  	// Get the required hostpath mounts
    53  	mounts := getHostPathVolumesForTheControlPlane(cfg)
    54  	if proxyEnvs == nil {
    55  		proxyEnvs = kubeadmutil.GetProxyEnvVars()
    56  	}
    57  	componentHealthCheckTimeout := kubeadmapi.GetActiveTimeouts().ControlPlaneComponentHealthCheck
    58  
    59  	// Prepare static pod specs
    60  	staticPodSpecs := map[string]v1.Pod{
    61  		kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{
    62  			Name:            kubeadmconstants.KubeAPIServer,
    63  			Image:           images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, cfg),
    64  			ImagePullPolicy: v1.PullIfNotPresent,
    65  			Command:         getAPIServerCommand(cfg, endpoint),
    66  			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
    67  			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS),
    68  			ReadinessProbe:  staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", endpoint.BindPort, v1.URISchemeHTTPS),
    69  			StartupProbe:    staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", endpoint.BindPort, v1.URISchemeHTTPS, componentHealthCheckTimeout),
    70  			Resources:       staticpodutil.ComponentResources("250m"),
    71  			Env:             kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.APIServer.ExtraEnvs),
    72  		}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
    73  			map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
    74  		kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{
    75  			Name:            kubeadmconstants.KubeControllerManager,
    76  			Image:           images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, cfg),
    77  			ImagePullPolicy: v1.PullIfNotPresent,
    78  			Command:         getControllerManagerCommand(cfg),
    79  			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
    80  			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
    81  			StartupProbe:    staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout),
    82  			Resources:       staticpodutil.ComponentResources("200m"),
    83  			Env:             kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.ControllerManager.ExtraEnvs),
    84  		}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
    85  		kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
    86  			Name:            kubeadmconstants.KubeScheduler,
    87  			Image:           images.GetKubernetesImage(kubeadmconstants.KubeScheduler, cfg),
    88  			ImagePullPolicy: v1.PullIfNotPresent,
    89  			Command:         getSchedulerCommand(cfg),
    90  			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
    91  			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
    92  			StartupProbe:    staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, componentHealthCheckTimeout),
    93  			Resources:       staticpodutil.ComponentResources("100m"),
    94  			Env:             kubeadmutil.MergeKubeadmEnvVars(proxyEnvs, cfg.Scheduler.ExtraEnvs),
    95  		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
    96  	}
    97  	return staticPodSpecs
    98  }
    99  
   100  // CreateStaticPodFiles creates all the requested static pod files.
   101  func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool, componentNames ...string) error {
   102  	// gets the StaticPodSpecs, actualized for the current ClusterConfiguration
   103  	klog.V(1).Infoln("[control-plane] getting StaticPodSpecs")
   104  	specs := GetStaticPodSpecs(cfg, endpoint, nil)
   105  
   106  	var usersAndGroups *users.UsersAndGroups
   107  	var err error
   108  	if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
   109  		if isDryRun {
   110  			fmt.Printf("[control-plane] Would create users and groups for %+v to run as non-root\n", componentNames)
   111  		} else {
   112  			usersAndGroups, err = staticpodutil.GetUsersAndGroups()
   113  			if err != nil {
   114  				return errors.Wrap(err, "failed to create users and groups")
   115  			}
   116  		}
   117  	}
   118  
   119  	// creates required static pod specs
   120  	for _, componentName := range componentNames {
   121  		// retrieves the StaticPodSpec for given component
   122  		spec, exists := specs[componentName]
   123  		if !exists {
   124  			return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName)
   125  		}
   126  
   127  		// print all volumes that are mounted
   128  		for _, v := range spec.Spec.Volumes {
   129  			klog.V(2).Infof("[control-plane] adding volume %q for component %q", v.Name, componentName)
   130  		}
   131  
   132  		if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
   133  			if isDryRun {
   134  				fmt.Printf("[control-plane] Would update static pod manifest for %q to run run as non-root\n", componentName)
   135  			} else {
   136  				if usersAndGroups != nil {
   137  					if err := staticpodutil.RunComponentAsNonRoot(componentName, &spec, usersAndGroups, cfg); err != nil {
   138  						return errors.Wrapf(err, "failed to run component %q as non-root", componentName)
   139  					}
   140  				}
   141  			}
   142  		}
   143  
   144  		// if patchesDir is defined, patch the static Pod manifest
   145  		if patchesDir != "" {
   146  			patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
   147  			if err != nil {
   148  				return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", componentName)
   149  			}
   150  			spec = *patchedSpec
   151  		}
   152  
   153  		// writes the StaticPodSpec to disk
   154  		if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil {
   155  			return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName)
   156  		}
   157  
   158  		klog.V(1).Infof("[control-plane] wrote static Pod manifest for component %q to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir))
   159  	}
   160  
   161  	return nil
   162  }
   163  
   164  // getAPIServerCommand builds the right API server command from the given config object and version
   165  func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint) []string {
   166  	defaultArguments := []kubeadmapi.Arg{
   167  		{Name: "advertise-address", Value: localAPIEndpoint.AdvertiseAddress},
   168  		{Name: "enable-admission-plugins", Value: "NodeRestriction"},
   169  		{Name: "service-cluster-ip-range", Value: cfg.Networking.ServiceSubnet},
   170  		{Name: "service-account-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)},
   171  		{Name: "service-account-signing-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)},
   172  		{Name: "service-account-issuer", Value: fmt.Sprintf("https://kubernetes.default.svc.%s", cfg.Networking.DNSDomain)},
   173  		{Name: "client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)},
   174  		{Name: "tls-cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName)},
   175  		{Name: "tls-private-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName)},
   176  		{Name: "kubelet-client-certificate", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName)},
   177  		{Name: "kubelet-client-key", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName)},
   178  		{Name: "enable-bootstrap-token-auth", Value: "true"},
   179  		{Name: "secure-port", Value: fmt.Sprintf("%d", localAPIEndpoint.BindPort)},
   180  		{Name: "allow-privileged", Value: "true"},
   181  		{Name: "kubelet-preferred-address-types", Value: "InternalIP,ExternalIP,Hostname"},
   182  		// add options to configure the front proxy.  Without the generated client cert, this will never be useable
   183  		// so add it unconditionally with recommended values
   184  		{Name: "requestheader-username-headers", Value: "X-Remote-User"},
   185  		{Name: "requestheader-group-headers", Value: "X-Remote-Group"},
   186  		{Name: "requestheader-extra-headers-prefix", Value: "X-Remote-Extra-"},
   187  		{Name: "requestheader-client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)},
   188  		{Name: "requestheader-allowed-names", Value: "front-proxy-client"},
   189  		{Name: "proxy-client-cert-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)},
   190  		{Name: "proxy-client-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)},
   191  	}
   192  
   193  	command := []string{"kube-apiserver"}
   194  
   195  	// If the user set endpoints for an external etcd cluster
   196  	if cfg.Etcd.External != nil {
   197  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", strings.Join(cfg.Etcd.External.Endpoints, ","), 1)
   198  
   199  		// Use any user supplied etcd certificates
   200  		if cfg.Etcd.External.CAFile != "" {
   201  			defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-cafile", cfg.Etcd.External.CAFile, 1)
   202  		}
   203  		if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" {
   204  			defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-certfile", cfg.Etcd.External.CertFile, 1)
   205  			defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-keyfile", cfg.Etcd.External.KeyFile, 1)
   206  
   207  		}
   208  	} else {
   209  		// Default to etcd static pod on localhost
   210  		// localhost IP family should be the same that the AdvertiseAddress
   211  		etcdLocalhostAddress := "127.0.0.1"
   212  		if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) {
   213  			etcdLocalhostAddress = "::1"
   214  		}
   215  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))), 1)
   216  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-cafile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), 1)
   217  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-certfile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName), 1)
   218  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-keyfile", filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName), 1)
   219  
   220  		// Apply user configurations for local etcd
   221  		if cfg.Etcd.Local != nil {
   222  			if value, idx := kubeadmapi.GetArgValue(cfg.Etcd.Local.ExtraArgs, "advertise-client-urls", -1); idx > -1 {
   223  				defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "etcd-servers", value, 1)
   224  			}
   225  		}
   226  	}
   227  
   228  	if cfg.APIServer.ExtraArgs == nil {
   229  		cfg.APIServer.ExtraArgs = []kubeadmapi.Arg{}
   230  	}
   231  	authzVal, _ := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, "authorization-mode", -1)
   232  	_, hasStructuredAuthzVal := kubeadmapi.GetArgValue(cfg.APIServer.ExtraArgs, "authorization-config", -1)
   233  	if hasStructuredAuthzVal == -1 {
   234  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "authorization-mode", getAuthzModes(authzVal), 1)
   235  	}
   236  	command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.APIServer.ExtraArgs)...)
   237  
   238  	return command
   239  }
   240  
   241  // getAuthzModes gets the authorization-related parameters to the api server
   242  // Node,RBAC is the default mode if nothing is passed to kubeadm. User provided modes override
   243  // the default.
   244  func getAuthzModes(authzModeExtraArgs string) string {
   245  	defaultMode := []string{
   246  		kubeadmconstants.ModeNode,
   247  		kubeadmconstants.ModeRBAC,
   248  	}
   249  
   250  	if len(authzModeExtraArgs) > 0 {
   251  		mode := []string{}
   252  		for _, requested := range strings.Split(authzModeExtraArgs, ",") {
   253  			if isValidAuthzMode(requested) {
   254  				mode = append(mode, requested)
   255  			} else {
   256  				klog.Warningf("ignoring unknown kube-apiserver authorization-mode %q", requested)
   257  			}
   258  		}
   259  
   260  		// only return the user provided mode if at least one was valid
   261  		if len(mode) > 0 {
   262  			if !compareAuthzModes(defaultMode, mode) {
   263  				klog.Warningf("the default kube-apiserver authorization-mode is %q; using %q",
   264  					strings.Join(defaultMode, ","),
   265  					strings.Join(mode, ","),
   266  				)
   267  			}
   268  			return strings.Join(mode, ",")
   269  		}
   270  	}
   271  	return strings.Join(defaultMode, ",")
   272  }
   273  
   274  // compareAuthzModes compares two given authz modes and returns false if they do not match
   275  func compareAuthzModes(a, b []string) bool {
   276  	if len(a) != len(b) {
   277  		return false
   278  	}
   279  	for i, m := range a {
   280  		if m != b[i] {
   281  			return false
   282  		}
   283  	}
   284  	return true
   285  }
   286  
   287  func isValidAuthzMode(authzMode string) bool {
   288  	allModes := []string{
   289  		kubeadmconstants.ModeNode,
   290  		kubeadmconstants.ModeRBAC,
   291  		kubeadmconstants.ModeWebhook,
   292  		kubeadmconstants.ModeABAC,
   293  		kubeadmconstants.ModeAlwaysAllow,
   294  		kubeadmconstants.ModeAlwaysDeny,
   295  	}
   296  
   297  	for _, mode := range allModes {
   298  		if authzMode == mode {
   299  			return true
   300  		}
   301  	}
   302  	return false
   303  }
   304  
   305  // getControllerManagerCommand builds the right controller manager command from the given config object and version
   306  func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string {
   307  
   308  	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
   309  	caFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)
   310  
   311  	defaultArguments := []kubeadmapi.Arg{
   312  		{Name: "bind-address", Value: "127.0.0.1"},
   313  		{Name: "leader-elect", Value: "true"},
   314  		{Name: "kubeconfig", Value: kubeconfigFile},
   315  		{Name: "authentication-kubeconfig", Value: kubeconfigFile},
   316  		{Name: "authorization-kubeconfig", Value: kubeconfigFile},
   317  		{Name: "client-ca-file", Value: caFile},
   318  		{Name: "requestheader-client-ca-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)},
   319  		{Name: "root-ca-file", Value: caFile},
   320  		{Name: "service-account-private-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName)},
   321  		{Name: "cluster-signing-cert-file", Value: caFile},
   322  		{Name: "cluster-signing-key-file", Value: filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)},
   323  		{Name: "use-service-account-credentials", Value: "true"},
   324  		{Name: "controllers", Value: "*,bootstrapsigner,tokencleaner"},
   325  	}
   326  
   327  	// If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path,
   328  	// so that the csrsigning controller fails to start
   329  	if res, _ := certphase.UsingExternalCA(cfg); res {
   330  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-signing-key-file", "", 1)
   331  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-signing-cert-file", "", 1)
   332  	}
   333  
   334  	// Let the controller-manager allocate Node CIDRs for the Pod network.
   335  	// Each node will get a subspace of the address CIDR provided with --pod-network-cidr.
   336  	if cfg.Networking.PodSubnet != "" {
   337  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "allocate-node-cidrs", "true", 1)
   338  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-cidr", cfg.Networking.PodSubnet, 1)
   339  		if cfg.Networking.ServiceSubnet != "" {
   340  			defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "service-cluster-ip-range", cfg.Networking.ServiceSubnet, 1)
   341  		}
   342  	}
   343  
   344  	// Set cluster name
   345  	if cfg.ClusterName != "" {
   346  		defaultArguments = kubeadmapi.SetArgValues(defaultArguments, "cluster-name", cfg.ClusterName, 1)
   347  	}
   348  
   349  	command := []string{"kube-controller-manager"}
   350  	command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.ControllerManager.ExtraArgs)...)
   351  
   352  	return command
   353  }
   354  
   355  // getSchedulerCommand builds the right scheduler command from the given config object and version
   356  func getSchedulerCommand(cfg *kubeadmapi.ClusterConfiguration) []string {
   357  	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
   358  	defaultArguments := []kubeadmapi.Arg{
   359  		{Name: "bind-address", Value: "127.0.0.1"},
   360  		{Name: "leader-elect", Value: "true"},
   361  		{Name: "kubeconfig", Value: kubeconfigFile},
   362  		{Name: "authentication-kubeconfig", Value: kubeconfigFile},
   363  		{Name: "authorization-kubeconfig", Value: kubeconfigFile},
   364  	}
   365  
   366  	command := []string{"kube-scheduler"}
   367  	command = append(command, kubeadmutil.ArgumentsToCommand(defaultArguments, cfg.Scheduler.ExtraArgs)...)
   368  	return command
   369  }
   370
View as plain text