reconstruct.go

Documentation: k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package reconciler
    18  
    19  import (
    20  	"context"
    21  
    22  	v1 "k8s.io/api/core/v1"
    23  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    24  	"k8s.io/apimachinery/pkg/types"
    25  	"k8s.io/klog/v2"
    26  	"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
    27  )
    28  
    29  // TODO: move to reconstruct.go and remove old code there.
    30  
    31  // readyToUnmount returns true when reconciler can start unmounting volumes.
    32  func (rc *reconciler) readyToUnmount() bool {
    33  	// During kubelet startup, all volumes present on disk are added as uncertain to ASW.
    34  	// Allow unmount only when DSW is fully populated to prevent unmounting volumes that
    35  	// did not reach DSW yet.
    36  	if !rc.populatorHasAddedPods() {
    37  		return false
    38  	}
    39  
    40  	// Allow unmount only when ASW device paths were corrected from node.status to prevent
    41  	// calling unmount with a wrong devicePath.
    42  	if len(rc.volumesNeedUpdateFromNodeStatus) != 0 {
    43  		return false
    44  	}
    45  	return true
    46  }
    47  
    48  // reconstructVolumes tries to reconstruct the actual state of world by scanning all pods' volume
    49  // directories from the disk. For the volumes that cannot support or fail reconstruction, it will
    50  // put the volumes to volumesFailedReconstruction to be cleaned up later when DesiredStateOfWorld
    51  // is populated.
    52  func (rc *reconciler) reconstructVolumes() {
    53  	// Get volumes information by reading the pod's directory
    54  	podVolumes, err := getVolumesFromPodDir(rc.kubeletPodsDir)
    55  	if err != nil {
    56  		klog.ErrorS(err, "Cannot get volumes from disk, skip sync states for volume reconstruction")
    57  		return
    58  	}
    59  	reconstructedVolumes := make(map[v1.UniqueVolumeName]*globalVolumeInfo)
    60  	reconstructedVolumeNames := []v1.UniqueVolumeName{}
    61  	for _, volume := range podVolumes {
    62  		if rc.actualStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
    63  			klog.V(4).InfoS("Volume exists in actual state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
    64  			// There is nothing to reconstruct
    65  			continue
    66  		}
    67  		reconstructedVolume, err := rc.reconstructVolume(volume)
    68  		if err != nil {
    69  			klog.InfoS("Could not construct volume information", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName, "err", err)
    70  			// We can't reconstruct the volume. Remember to check DSW after it's fully populated and force unmount the volume when it's orphaned.
    71  			rc.volumesFailedReconstruction = append(rc.volumesFailedReconstruction, volume)
    72  			continue
    73  		}
    74  		klog.V(4).InfoS("Adding reconstructed volume to actual state and node status", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
    75  		gvl := &globalVolumeInfo{
    76  			volumeName:        reconstructedVolume.volumeName,
    77  			volumeSpec:        reconstructedVolume.volumeSpec,
    78  			devicePath:        reconstructedVolume.devicePath,
    79  			deviceMounter:     reconstructedVolume.deviceMounter,
    80  			blockVolumeMapper: reconstructedVolume.blockVolumeMapper,
    81  			mounter:           reconstructedVolume.mounter,
    82  		}
    83  		if cachedInfo, ok := reconstructedVolumes[reconstructedVolume.volumeName]; ok {
    84  			gvl = cachedInfo
    85  		}
    86  		gvl.addPodVolume(reconstructedVolume)
    87  
    88  		reconstructedVolumeNames = append(reconstructedVolumeNames, reconstructedVolume.volumeName)
    89  		reconstructedVolumes[reconstructedVolume.volumeName] = gvl
    90  	}
    91  
    92  	if len(reconstructedVolumes) > 0 {
    93  		// Add the volumes to ASW
    94  		rc.updateStatesNew(reconstructedVolumes)
    95  
    96  		// The reconstructed volumes are mounted, hence a previous kubelet must have already put it into node.status.volumesInUse.
    97  		// Remember to update DSW with this information.
    98  		rc.volumesNeedReportedInUse = reconstructedVolumeNames
    99  		// Remember to update devicePath from node.status.volumesAttached
   100  		rc.volumesNeedUpdateFromNodeStatus = reconstructedVolumeNames
   101  	}
   102  	klog.V(2).InfoS("Volume reconstruction finished")
   103  }
   104  
   105  func (rc *reconciler) updateStatesNew(reconstructedVolumes map[v1.UniqueVolumeName]*globalVolumeInfo) {
   106  	for _, gvl := range reconstructedVolumes {
   107  		err := rc.actualStateOfWorld.AddAttachUncertainReconstructedVolume(
   108  			//TODO: the devicePath might not be correct for some volume plugins: see issue #54108
   109  			gvl.volumeName, gvl.volumeSpec, rc.nodeName, gvl.devicePath)
   110  		if err != nil {
   111  			klog.ErrorS(err, "Could not add volume information to actual state of world", "volumeName", gvl.volumeName)
   112  			continue
   113  		}
   114  		var seLinuxMountContext string
   115  		for _, volume := range gvl.podVolumes {
   116  			markVolumeOpts := operationexecutor.MarkVolumeOpts{
   117  				PodName:             volume.podName,
   118  				PodUID:              types.UID(volume.podName),
   119  				VolumeName:          volume.volumeName,
   120  				Mounter:             volume.mounter,
   121  				BlockVolumeMapper:   volume.blockVolumeMapper,
   122  				OuterVolumeSpecName: volume.outerVolumeSpecName,
   123  				VolumeGidVolume:     volume.volumeGidValue,
   124  				VolumeSpec:          volume.volumeSpec,
   125  				VolumeMountState:    operationexecutor.VolumeMountUncertain,
   126  				SELinuxMountContext: volume.seLinuxMountContext,
   127  			}
   128  
   129  			_, err = rc.actualStateOfWorld.CheckAndMarkVolumeAsUncertainViaReconstruction(markVolumeOpts)
   130  			if err != nil {
   131  				klog.ErrorS(err, "Could not add pod to volume information to actual state of world", "pod", klog.KObj(volume.pod))
   132  				continue
   133  			}
   134  			seLinuxMountContext = volume.seLinuxMountContext
   135  			klog.V(2).InfoS("Volume is marked as uncertain and added into the actual state", "pod", klog.KObj(volume.pod), "podName", volume.podName, "volumeName", volume.volumeName, "seLinuxMountContext", volume.seLinuxMountContext)
   136  		}
   137  		// If the volume has device to mount, we mark its device as uncertain.
   138  		if gvl.deviceMounter != nil || gvl.blockVolumeMapper != nil {
   139  			deviceMountPath, err := getDeviceMountPath(gvl)
   140  			if err != nil {
   141  				klog.ErrorS(err, "Could not find device mount path for volume", "volumeName", gvl.volumeName)
   142  				continue
   143  			}
   144  			err = rc.actualStateOfWorld.MarkDeviceAsUncertain(gvl.volumeName, gvl.devicePath, deviceMountPath, seLinuxMountContext)
   145  			if err != nil {
   146  				klog.ErrorS(err, "Could not mark device is uncertain to actual state of world", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
   147  				continue
   148  			}
   149  			klog.V(2).InfoS("Volume is marked device as uncertain and added into the actual state", "volumeName", gvl.volumeName, "deviceMountPath", deviceMountPath)
   150  		}
   151  	}
   152  }
   153  
   154  // cleanOrphanVolumes tries to clean up all volumes that failed reconstruction.
   155  func (rc *reconciler) cleanOrphanVolumes() {
   156  	if len(rc.volumesFailedReconstruction) == 0 {
   157  		return
   158  	}
   159  
   160  	for _, volume := range rc.volumesFailedReconstruction {
   161  		if rc.desiredStateOfWorld.VolumeExistsWithSpecName(volume.podName, volume.volumeSpecName) {
   162  			// Some pod needs the volume, don't clean it up and hope that
   163  			// reconcile() calls SetUp and reconstructs the volume in ASW.
   164  			klog.V(4).InfoS("Volume exists in desired state, skip cleaning up mounts", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
   165  			continue
   166  		}
   167  		klog.InfoS("Cleaning up mounts for volume that could not be reconstructed", "podName", volume.podName, "volumeSpecName", volume.volumeSpecName)
   168  		rc.cleanupMounts(volume)
   169  	}
   170  
   171  	klog.V(2).InfoS("Orphan volume cleanup finished")
   172  	// Clean the cache, cleanup is one shot operation.
   173  	rc.volumesFailedReconstruction = make([]podVolume, 0)
   174  }
   175  
   176  // updateReconstructedFromNodeStatus tries to file devicePaths of reconstructed volumes from
   177  // node.Status.VolumesAttached. This can be done only after connection to the API
   178  // server is established, i.e. it can't be part of reconstructVolumes().
   179  func (rc *reconciler) updateReconstructedFromNodeStatus() {
   180  	klog.V(4).InfoS("Updating reconstructed devicePaths")
   181  
   182  	if rc.kubeClient == nil {
   183  		// Skip reconstructing devicePath from node objects if kubelet is in standalone mode.
   184  		// Such kubelet is not expected to mount any attachable volume or Secrets / ConfigMap.
   185  		klog.V(2).InfoS("Skipped reconstruction of DevicePaths from node.status in standalone mode")
   186  		rc.volumesNeedUpdateFromNodeStatus = nil
   187  		return
   188  	}
   189  
   190  	node, fetchErr := rc.kubeClient.CoreV1().Nodes().Get(context.TODO(), string(rc.nodeName), metav1.GetOptions{})
   191  	if fetchErr != nil {
   192  		// This may repeat few times per second until kubelet is able to read its own status for the first time.
   193  		klog.V(4).ErrorS(fetchErr, "Failed to get Node status to reconstruct device paths")
   194  		return
   195  	}
   196  
   197  	for _, volumeID := range rc.volumesNeedUpdateFromNodeStatus {
   198  		attachable := false
   199  		for _, attachedVolume := range node.Status.VolumesAttached {
   200  			if volumeID != attachedVolume.Name {
   201  				continue
   202  			}
   203  			rc.actualStateOfWorld.UpdateReconstructedDevicePath(volumeID, attachedVolume.DevicePath)
   204  			attachable = true
   205  			klog.V(4).InfoS("Updated devicePath from node status for volume", "volumeName", attachedVolume.Name, "path", attachedVolume.DevicePath)
   206  		}
   207  		rc.actualStateOfWorld.UpdateReconstructedVolumeAttachability(volumeID, attachable)
   208  	}
   209  
   210  	klog.V(2).InfoS("DevicePaths of reconstructed volumes updated")
   211  	rc.volumesNeedUpdateFromNodeStatus = nil
   212  }
   213
View as plain text