kubelet_volumes.go

Documentation: k8s.io/kubernetes/pkg/kubelet

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubelet
    18  
    19  import (
    20  	"fmt"
    21  	"os"
    22  	"path/filepath"
    23  	"syscall"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	"k8s.io/apimachinery/pkg/types"
    27  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    28  	"k8s.io/apimachinery/pkg/util/sets"
    29  	"k8s.io/klog/v2"
    30  	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
    31  	"k8s.io/kubernetes/pkg/kubelet/metrics"
    32  	"k8s.io/kubernetes/pkg/util/removeall"
    33  	"k8s.io/kubernetes/pkg/volume"
    34  	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
    35  )
    36  
    37  // ListVolumesForPod returns a map of the mounted volumes for the given pod.
    38  // The key in the map is the OuterVolumeSpecName (i.e. pod.Spec.Volumes[x].Name)
    39  func (kl *Kubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
    40  	volumesToReturn := make(map[string]volume.Volume)
    41  	podVolumes := kl.volumeManager.GetMountedVolumesForPod(
    42  		volumetypes.UniquePodName(podUID))
    43  	for outerVolumeSpecName, volume := range podVolumes {
    44  		// TODO: volume.Mounter could be nil if volume object is recovered
    45  		// from reconciler's sync state process. PR 33616 will fix this problem
    46  		// to create Mounter object when recovering volume state.
    47  		if volume.Mounter == nil {
    48  			continue
    49  		}
    50  		volumesToReturn[outerVolumeSpecName] = volume.Mounter
    51  	}
    52  
    53  	return volumesToReturn, len(volumesToReturn) > 0
    54  }
    55  
    56  // ListBlockVolumesForPod returns a map of the mounted volumes for the given
    57  // pod. The key in the map is the OuterVolumeSpecName (i.e.
    58  // pod.Spec.Volumes[x].Name)
    59  func (kl *Kubelet) ListBlockVolumesForPod(podUID types.UID) (map[string]volume.BlockVolume, bool) {
    60  	volumesToReturn := make(map[string]volume.BlockVolume)
    61  	podVolumes := kl.volumeManager.GetMountedVolumesForPod(
    62  		volumetypes.UniquePodName(podUID))
    63  	for outerVolumeSpecName, volume := range podVolumes {
    64  		// TODO: volume.Mounter could be nil if volume object is recovered
    65  		// from reconciler's sync state process. PR 33616 will fix this problem
    66  		// to create Mounter object when recovering volume state.
    67  		if volume.BlockVolumeMapper == nil {
    68  			continue
    69  		}
    70  		volumesToReturn[outerVolumeSpecName] = volume.BlockVolumeMapper
    71  	}
    72  
    73  	return volumesToReturn, len(volumesToReturn) > 0
    74  }
    75  
    76  // podVolumesExist checks with the volume manager and returns true any of the
    77  // pods for the specified volume are mounted or are uncertain.
    78  func (kl *Kubelet) podVolumesExist(podUID types.UID) bool {
    79  	if mountedVolumes :=
    80  		kl.volumeManager.GetPossiblyMountedVolumesForPod(
    81  			volumetypes.UniquePodName(podUID)); len(mountedVolumes) > 0 {
    82  		return true
    83  	}
    84  	// TODO: This checks pod volume paths and whether they are mounted. If checking returns error, podVolumesExist will return true
    85  	// which means we consider volumes might exist and requires further checking.
    86  	// There are some volume plugins such as flexvolume might not have mounts. See issue #61229
    87  	volumePaths, err := kl.getMountedVolumePathListFromDisk(podUID)
    88  	if err != nil {
    89  		klog.ErrorS(err, "Pod found, but error occurred during checking mounted volumes from disk", "podUID", podUID)
    90  		return true
    91  	}
    92  	if len(volumePaths) > 0 {
    93  		klog.V(4).InfoS("Pod found, but volumes are still mounted on disk", "podUID", podUID, "paths", volumePaths)
    94  		return true
    95  	}
    96  
    97  	return false
    98  }
    99  
   100  // newVolumeMounterFromPlugins attempts to find a plugin by volume spec, pod
   101  // and volume options and then creates a Mounter.
   102  // Returns a valid mounter or an error.
   103  func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
   104  	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
   105  	if err != nil {
   106  		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
   107  	}
   108  	physicalMounter, err := plugin.NewMounter(spec, pod, opts)
   109  	if err != nil {
   110  		return nil, fmt.Errorf("failed to instantiate mounter for volume: %s using plugin: %s with a root cause: %v", spec.Name(), plugin.GetPluginName(), err)
   111  	}
   112  	klog.V(10).InfoS("Using volume plugin for mount", "volumePluginName", plugin.GetPluginName(), "volumeName", spec.Name())
   113  	return physicalMounter, nil
   114  }
   115  
   116  // removeOrphanedPodVolumeDirs attempts to remove the pod volumes directory and
   117  // its subdirectories. There should be no files left under normal conditions
   118  // when this is called, so it effectively does a recursive rmdir instead of
   119  // RemoveAll to ensure it only removes empty directories and files that were
   120  // used as mount points, but not content of the mount points.
   121  func (kl *Kubelet) removeOrphanedPodVolumeDirs(uid types.UID) []error {
   122  	orphanVolumeErrors := []error{}
   123  
   124  	// If there are still volume directories, attempt to rmdir them
   125  	volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
   126  	if err != nil {
   127  		orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("orphaned pod %q found, but error occurred during reading volume dir from disk: %v", uid, err))
   128  		return orphanVolumeErrors
   129  	}
   130  	if len(volumePaths) > 0 {
   131  		for _, volumePath := range volumePaths {
   132  			if err := syscall.Rmdir(volumePath); err != nil {
   133  				orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("orphaned pod %q found, but failed to rmdir() volume at path %v: %v", uid, volumePath, err))
   134  			} else {
   135  				klog.InfoS("Cleaned up orphaned volume from pod", "podUID", uid, "path", volumePath)
   136  			}
   137  		}
   138  	}
   139  
   140  	// If there are any volume-subpaths, attempt to remove them
   141  	subpathVolumePaths, err := kl.getPodVolumeSubpathListFromDisk(uid)
   142  	if err != nil {
   143  		orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("orphaned pod %q found, but error occurred during reading of volume-subpaths dir from disk: %v", uid, err))
   144  		return orphanVolumeErrors
   145  	}
   146  	if len(subpathVolumePaths) > 0 {
   147  		for _, subpathVolumePath := range subpathVolumePaths {
   148  			// Remove both files and empty directories here, as the subpath may have been a bind-mount of a file or a directory.
   149  			if err := os.Remove(subpathVolumePath); err != nil {
   150  				orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("orphaned pod %q found, but failed to rmdir() subpath at path %v: %v", uid, subpathVolumePath, err))
   151  			} else {
   152  				klog.InfoS("Cleaned up orphaned volume subpath from pod", "podUID", uid, "path", subpathVolumePath)
   153  			}
   154  		}
   155  	}
   156  
   157  	// Remove any remaining subdirectories along with the volumes directory itself.
   158  	// Fail if any regular files are encountered.
   159  	podVolDir := kl.getPodVolumesDir(uid)
   160  	if err := removeall.RemoveDirsOneFilesystem(kl.mounter, podVolDir); err != nil {
   161  		orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("orphaned pod %q found, but error occurred when trying to remove the volumes dir: %v", uid, err))
   162  	} else {
   163  		klog.InfoS("Cleaned up orphaned pod volumes dir", "podUID", uid, "path", podVolDir)
   164  	}
   165  
   166  	return orphanVolumeErrors
   167  }
   168  
   169  // cleanupOrphanedPodDirs removes the volumes of pods that should not be
   170  // running and that have no containers running.  Note that we roll up logs here since it runs in the main loop.
   171  func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
   172  	allPods := sets.NewString()
   173  	for _, pod := range pods {
   174  		allPods.Insert(string(pod.UID))
   175  	}
   176  	for _, pod := range runningPods {
   177  		allPods.Insert(string(pod.ID))
   178  	}
   179  
   180  	found, err := kl.listPodsFromDisk()
   181  	if err != nil {
   182  		return err
   183  	}
   184  
   185  	orphanRemovalErrors := []error{}
   186  	orphanVolumeErrors := []error{}
   187  	var totalPods, errorPods int
   188  
   189  	for _, uid := range found {
   190  		if allPods.Has(string(uid)) {
   191  			continue
   192  		}
   193  
   194  		totalPods++
   195  
   196  		// If volumes have not been unmounted/detached, do not delete directory.
   197  		// Doing so may result in corruption of data.
   198  		// TODO: getMountedVolumePathListFromDisk() call may be redundant with
   199  		// kl.getPodVolumePathListFromDisk(). Can this be cleaned up?
   200  		if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist {
   201  			errorPods++
   202  			klog.V(3).InfoS("Orphaned pod found, but volumes are not cleaned up", "podUID", uid)
   203  			continue
   204  		}
   205  
   206  		// Attempt to remove the pod volumes directory and its subdirs
   207  		podVolumeErrors := kl.removeOrphanedPodVolumeDirs(uid)
   208  		if len(podVolumeErrors) > 0 {
   209  			errorPods++
   210  			orphanVolumeErrors = append(orphanVolumeErrors, podVolumeErrors...)
   211  			// Not all volumes were removed, so don't clean up the pod directory yet. It is likely
   212  			// that there are still mountpoints or files left which could cause removal of the pod
   213  			// directory to fail below.
   214  			// Errors for all removal operations have already been recorded, so don't add another
   215  			// one here.
   216  			continue
   217  		}
   218  
   219  		// Call RemoveAllOneFilesystem for remaining subdirs under the pod directory
   220  		podDir := kl.getPodDir(uid)
   221  		podSubdirs, err := os.ReadDir(podDir)
   222  		if err != nil {
   223  			errorPods++
   224  			klog.ErrorS(err, "Could not read directory", "path", podDir)
   225  			orphanRemovalErrors = append(orphanRemovalErrors, fmt.Errorf("orphaned pod %q found, but error occurred during reading the pod dir from disk: %v", uid, err))
   226  			continue
   227  		}
   228  
   229  		var cleanupFailed bool
   230  		for _, podSubdir := range podSubdirs {
   231  			podSubdirName := podSubdir.Name()
   232  			podSubdirPath := filepath.Join(podDir, podSubdirName)
   233  			// Never attempt RemoveAllOneFilesystem on the volumes directory,
   234  			// as this could lead to data loss in some situations. The volumes
   235  			// directory should have been removed by removeOrphanedPodVolumeDirs.
   236  			if podSubdirName == "volumes" {
   237  				cleanupFailed = true
   238  				err := fmt.Errorf("volumes subdir was found after it was removed")
   239  				klog.ErrorS(err, "Orphaned pod found, but failed to remove volumes subdir", "podUID", uid, "path", podSubdirPath)
   240  				continue
   241  			}
   242  			if err := removeall.RemoveAllOneFilesystem(kl.mounter, podSubdirPath); err != nil {
   243  				cleanupFailed = true
   244  				klog.ErrorS(err, "Failed to remove orphaned pod subdir", "podUID", uid, "path", podSubdirPath)
   245  				orphanRemovalErrors = append(orphanRemovalErrors, fmt.Errorf("orphaned pod %q found, but error occurred when trying to remove subdir %q: %v", uid, podSubdirPath, err))
   246  			}
   247  		}
   248  
   249  		// Rmdir the pod dir, which should be empty if everything above was successful
   250  		klog.V(3).InfoS("Orphaned pod found, removing", "podUID", uid)
   251  		if err := syscall.Rmdir(podDir); err != nil {
   252  			cleanupFailed = true
   253  			klog.ErrorS(err, "Failed to remove orphaned pod dir", "podUID", uid)
   254  			orphanRemovalErrors = append(orphanRemovalErrors, fmt.Errorf("orphaned pod %q found, but error occurred when trying to remove the pod directory: %v", uid, err))
   255  		}
   256  		if cleanupFailed {
   257  			errorPods++
   258  		}
   259  	}
   260  
   261  	logSpew := func(errs []error) {
   262  		if len(errs) > 0 {
   263  			klog.ErrorS(errs[0], "There were many similar errors. Turn up verbosity to see them.", "numErrs", len(errs))
   264  			for _, err := range errs {
   265  				klog.V(5).InfoS("Orphan pod", "err", err)
   266  			}
   267  		}
   268  	}
   269  	logSpew(orphanVolumeErrors)
   270  	logSpew(orphanRemovalErrors)
   271  	metrics.OrphanPodCleanedVolumes.Set(float64(totalPods))
   272  	metrics.OrphanPodCleanedVolumesErrors.Set(float64(errorPods))
   273  	return utilerrors.NewAggregate(orphanRemovalErrors)
   274  }
   275
View as plain text