volumes.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"fmt"
    21  	"os"
    22  	"path/filepath"
    23  	"strings"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	"k8s.io/apimachinery/pkg/util/sets"
    27  
    28  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    29  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    30  	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
    31  )
    32  
    33  const (
    34  	caCertsVolumeName              = "ca-certs"
    35  	caCertsVolumePath              = "/etc/ssl/certs"
    36  	flexvolumeDirVolumeName        = "flexvolume-dir"
    37  	defaultFlexvolumeDirVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
    38  )
    39  
    40  // caCertsExtraVolumePaths specifies the paths that can be conditionally mounted into the apiserver and controller-manager containers
    41  // as /etc/ssl/certs might be or contain a symlink to them. It's a variable since it may be changed in unit testing. This var MUST
    42  // NOT be changed in normal codepaths during runtime.
    43  var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", "/usr/local/share/ca-certificates", "/etc/ca-certificates"}
    44  
    45  // getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane
    46  func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.ClusterConfiguration) controlPlaneHostPathMounts {
    47  	hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
    48  	hostPathFileOrCreate := v1.HostPathFileOrCreate
    49  	mounts := newControlPlaneHostPathMounts()
    50  
    51  	// HostPath volumes for the API Server
    52  	// Read-only mount for the certificates directory
    53  	// TODO: Always mount the K8s Certificates directory to a static path inside of the container
    54  	mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
    55  	// Read-only mount for the ca certs (/etc/ssl/certs) directory
    56  	mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
    57  
    58  	// If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key
    59  	if cfg.Etcd.External != nil {
    60  		etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd.External, cfg.CertificatesDir)
    61  		mounts.AddHostPathMounts(kubeadmconstants.KubeAPIServer, etcdVols, etcdVolMounts)
    62  	}
    63  
    64  	// HostPath volumes for the controller manager
    65  	// Read-only mount for the certificates directory
    66  	// TODO: Always mount the K8s Certificates directory to a static path inside of the container
    67  	mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
    68  	// Read-only mount for the ca certs (/etc/ssl/certs) directory
    69  	mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
    70  	// Read-only mount for the controller manager kubeconfig file
    71  	controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
    72  	mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate)
    73  	// Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec by default)
    74  	// Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket.
    75  	flexvolumeDirVolumePath, idx := kubeadmapi.GetArgValue(cfg.ControllerManager.ExtraArgs, "flex-volume-plugin-dir", -1)
    76  	if idx == -1 {
    77  		flexvolumeDirVolumePath = defaultFlexvolumeDirVolumePath
    78  	}
    79  	mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate)
    80  
    81  	// HostPath volumes for the scheduler
    82  	// Read-only mount for the scheduler kubeconfig file
    83  	schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
    84  	mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeadmconstants.KubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true, &hostPathFileOrCreate)
    85  
    86  	// On some systems were we host-mount /etc/ssl/certs, it is also required to mount additional directories.
    87  	// This is needed due to symlinks pointing from files in /etc/ssl/certs to these directories.
    88  	for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths {
    89  		if isExtraVolumeMountNeeded(caCertsExtraVolumePath) {
    90  			caCertsExtraVolumeName := strings.Replace(caCertsExtraVolumePath, "/", "-", -1)[1:]
    91  			mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate)
    92  			mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate)
    93  		}
    94  	}
    95  
    96  	// Merge user defined mounts and ensure unique volume and volume mount
    97  	// names
    98  	mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServer.ExtraVolumes)
    99  	mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManager.ExtraVolumes)
   100  	mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.Scheduler.ExtraVolumes)
   101  
   102  	return mounts
   103  }
   104  
   105  // controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way
   106  type controlPlaneHostPathMounts struct {
   107  	// volumes is a nested map that forces a unique volumes. The outer map's
   108  	// keys are a string that should specify the target component to add the
   109  	// volume to. The values (inner map) of the outer map are maps with string
   110  	// keys and v1.Volume values. The inner map's key should specify the volume
   111  	// name.
   112  	volumes map[string]map[string]v1.Volume
   113  	// volumeMounts is a nested map that forces a unique volume mounts. The
   114  	// outer map's keys are a string that should specify the target component
   115  	// to add the volume mount to. The values (inner map) of the outer map are
   116  	// maps with string keys and v1.VolumeMount values. The inner map's key
   117  	// should specify the volume mount name.
   118  	volumeMounts map[string]map[string]v1.VolumeMount
   119  }
   120  
   121  func newControlPlaneHostPathMounts() controlPlaneHostPathMounts {
   122  	return controlPlaneHostPathMounts{
   123  		volumes:      map[string]map[string]v1.Volume{},
   124  		volumeMounts: map[string]map[string]v1.VolumeMount{},
   125  	}
   126  }
   127  
   128  func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) {
   129  	vol := staticpodutil.NewVolume(mountName, hostPath, hostPathType)
   130  	c.addComponentVolume(component, vol)
   131  	volMount := staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)
   132  	c.addComponentVolumeMount(component, volMount)
   133  }
   134  
   135  func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) {
   136  	for _, v := range vols {
   137  		c.addComponentVolume(component, v)
   138  	}
   139  	for _, v := range volMounts {
   140  		c.addComponentVolumeMount(component, v)
   141  	}
   142  }
   143  
   144  // AddExtraHostPathMounts adds host path mounts and overwrites the default
   145  // paths in the case that a user specifies the same volume/volume mount name.
   146  func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount) {
   147  	for _, extraVol := range extraVols {
   148  		hostPathType := extraVol.PathType
   149  		c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, extraVol.ReadOnly, &hostPathType)
   150  	}
   151  }
   152  
   153  func (c *controlPlaneHostPathMounts) GetVolumes(component string) map[string]v1.Volume {
   154  	return c.volumes[component]
   155  }
   156  
   157  func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) map[string]v1.VolumeMount {
   158  	return c.volumeMounts[component]
   159  }
   160  
   161  func (c *controlPlaneHostPathMounts) addComponentVolume(component string, vol v1.Volume) {
   162  	if _, ok := c.volumes[component]; !ok {
   163  		c.volumes[component] = map[string]v1.Volume{}
   164  	}
   165  	c.volumes[component][vol.Name] = vol
   166  }
   167  
   168  func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, volMount v1.VolumeMount) {
   169  	if _, ok := c.volumeMounts[component]; !ok {
   170  		c.volumeMounts[component] = map[string]v1.VolumeMount{}
   171  	}
   172  	c.volumeMounts[component][volMount.Name] = volMount
   173  }
   174  
   175  // getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster
   176  func getEtcdCertVolumes(etcdCfg *kubeadmapi.ExternalEtcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) {
   177  	certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile}
   178  	certDirs := sets.New[string]()
   179  	for _, certPath := range certPaths {
   180  		certDir := filepath.ToSlash(filepath.Dir(certPath))
   181  		// Ignore ".", which is the result of passing an empty path.
   182  		// Also ignore the cert directories that already may be mounted; /etc/ssl/certs, /etc/pki or Kubernetes CertificatesDir
   183  		// If the etcd certs are in there, it's okay, we don't have to do anything
   184  		extraVolumePath := false
   185  		for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths {
   186  			if strings.HasPrefix(certDir, caCertsExtraVolumePath) {
   187  				extraVolumePath = true
   188  				break
   189  			}
   190  		}
   191  		if certDir == "." || extraVolumePath || strings.HasPrefix(certDir, caCertsVolumePath) || strings.HasPrefix(certDir, k8sCertificatesDir) {
   192  			continue
   193  		}
   194  		// Filter out any existing hostpath mounts in the list that contains a subset of the path
   195  		alreadyExists := false
   196  		for _, existingCertDir := range sets.List(certDirs) {
   197  			// If the current directory is a parent of an existing one, remove the already existing one
   198  			if strings.HasPrefix(existingCertDir, certDir) {
   199  				certDirs.Delete(existingCertDir)
   200  			} else if strings.HasPrefix(certDir, existingCertDir) {
   201  				// If an existing directory is a parent of the current one, don't add the current one
   202  				alreadyExists = true
   203  			}
   204  		}
   205  		if alreadyExists {
   206  			continue
   207  		}
   208  		certDirs.Insert(certDir)
   209  	}
   210  
   211  	volumes := []v1.Volume{}
   212  	volumeMounts := []v1.VolumeMount{}
   213  	pathType := v1.HostPathDirectoryOrCreate
   214  	for i, certDir := range sets.List(certDirs) {
   215  		name := fmt.Sprintf("etcd-certs-%d", i)
   216  		volumes = append(volumes, staticpodutil.NewVolume(name, certDir, &pathType))
   217  		volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true))
   218  	}
   219  	return volumes, volumeMounts
   220  }
   221  
   222  // isExtraVolumeMountNeeded specifies whether /etc/pki should be host-mounted into the containers
   223  // On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed
   224  // due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/
   225  func isExtraVolumeMountNeeded(caCertsExtraVolumePath string) bool {
   226  	if _, err := os.Stat(caCertsExtraVolumePath); err == nil {
   227  		return true
   228  	}
   229  	return false
   230  }
   231
View as plain text