node_expander.go

Documentation: k8s.io/kubernetes/pkg/volume/util/operationexecutor

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package operationexecutor
    18  
    19  import (
    20  	"fmt"
    21  
    22  	v1 "k8s.io/api/core/v1"
    23  	"k8s.io/apimachinery/pkg/api/resource"
    24  	clientset "k8s.io/client-go/kubernetes"
    25  	"k8s.io/client-go/tools/record"
    26  	"k8s.io/klog/v2"
    27  	kevents "k8s.io/kubernetes/pkg/kubelet/events"
    28  	"k8s.io/kubernetes/pkg/volume/util"
    29  	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
    30  )
    31  
    32  type NodeExpander struct {
    33  	nodeResizeOperationOpts
    34  	kubeClient clientset.Interface
    35  	recorder   record.EventRecorder
    36  
    37  	// computed via precheck
    38  	pvcStatusCap resource.Quantity
    39  	pvCap        resource.Quantity
    40  	resizeStatus v1.ClaimResourceStatus
    41  
    42  	// pvcAlreadyUpdated if true indicates that although we are calling NodeExpandVolume on the kubelet
    43  	// PVC has already been updated - possibly because expansion already succeeded on different node.
    44  	// This can happen when a RWX PVC is expanded.
    45  	pvcAlreadyUpdated bool
    46  }
    47  
    48  func newNodeExpander(resizeOp nodeResizeOperationOpts, client clientset.Interface, recorder record.EventRecorder) *NodeExpander {
    49  	return &NodeExpander{
    50  		kubeClient:              client,
    51  		nodeResizeOperationOpts: resizeOp,
    52  		recorder:                recorder,
    53  	}
    54  }
    55  
    56  // testResponseData is merely used for doing sanity checks in unit tests
    57  type testResponseData struct {
    58  	// indicates that resize operation was called on underlying volume driver
    59  	// mainly useful for testing.
    60  	resizeCalledOnPlugin bool
    61  
    62  	// Indicates whether kubelet should assume resize operation as finished.
    63  	// For kubelet - resize operation could be assumed as finished even if
    64  	// actual resizing is *not* finished. This can happen, because certain prechecks
    65  	// are failing and kubelet should not retry expansion, or it could happen
    66  	// because resize operation is genuinely finished.
    67  	assumeResizeFinished bool
    68  }
    69  
    70  // runPreCheck performs some sanity checks before expansion can be performed on the PVC.
    71  // This function returns true only if node expansion is allowed to proceed otherwise
    72  // it returns false.
    73  func (ne *NodeExpander) runPreCheck() bool {
    74  	ne.pvcStatusCap = ne.pvc.Status.Capacity[v1.ResourceStorage]
    75  	ne.pvCap = ne.pv.Spec.Capacity[v1.ResourceStorage]
    76  
    77  	allocatedResourceStatus := ne.pvc.Status.AllocatedResourceStatuses
    78  	if currentStatus, ok := allocatedResourceStatus[v1.ResourceStorage]; ok {
    79  		ne.resizeStatus = currentStatus
    80  	}
    81  
    82  	// PVC is already expanded but we are still trying to expand the volume because
    83  	// last recorded size in ASOW is older. This can happen for RWX volume types.
    84  	if ne.pvcStatusCap.Cmp(ne.pluginResizeOpts.NewSize) >= 0 && ne.resizeStatus == "" {
    85  		ne.pvcAlreadyUpdated = true
    86  		return true
    87  	}
    88  
    89  	// recovery features will only work for newer version of resize controller
    90  	if ne.resizeStatus == "" {
    91  		return false
    92  	}
    93  
    94  	resizeStatusVal := ne.resizeStatus
    95  
    96  	// if resizestatus is nil or NodeExpansionInProgress or NodeExpansionPending then we
    97  	// should allow volume expansion on the node to proceed.
    98  	if resizeStatusVal == v1.PersistentVolumeClaimNodeResizePending ||
    99  		resizeStatusVal == v1.PersistentVolumeClaimNodeResizeInProgress {
   100  		return true
   101  	}
   102  	return false
   103  }
   104  
   105  func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
   106  	allowExpansion := ne.runPreCheck()
   107  	if !allowExpansion {
   108  		return false, nil, testResponseData{false, true}
   109  	}
   110  
   111  	var err error
   112  	nodeName := ne.vmt.Pod.Spec.NodeName
   113  
   114  	if !ne.pvcAlreadyUpdated {
   115  		ne.pvc, err = util.MarkNodeExpansionInProgress(ne.pvc, ne.kubeClient)
   116  
   117  		if err != nil {
   118  			msg := ne.vmt.GenerateErrorDetailed("MountVolume.NodeExpandVolume failed to mark node expansion in progress: %v", err)
   119  			klog.Errorf(msg.Error())
   120  			return false, err, testResponseData{}
   121  		}
   122  	}
   123  	_, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts)
   124  	if resizeErr != nil {
   125  		if volumetypes.IsOperationFinishedError(resizeErr) {
   126  			var markFailedError error
   127  			ne.pvc, markFailedError = util.MarkNodeExpansionFailed(ne.pvc, ne.kubeClient)
   128  			if markFailedError != nil {
   129  				klog.Errorf(ne.vmt.GenerateErrorDetailed("MountMount.NodeExpandVolume failed to mark node expansion as failed: %v", err).Error())
   130  			}
   131  		}
   132  
   133  		// if driver returned FailedPrecondition error that means
   134  		// volume expansion should not be retried on this node but
   135  		// expansion operation should not block mounting
   136  		if volumetypes.IsFailedPreconditionError(resizeErr) {
   137  			ne.actualStateOfWorld.MarkForInUseExpansionError(ne.vmt.VolumeName)
   138  			klog.Errorf(ne.vmt.GenerateErrorDetailed("MountVolume.NodeExapndVolume failed with %v", resizeErr).Error())
   139  			return false, nil, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
   140  		}
   141  		return false, resizeErr, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
   142  	}
   143  	simpleMsg, detailedMsg := ne.vmt.GenerateMsg("MountVolume.NodeExpandVolume succeeded", nodeName)
   144  	ne.recorder.Eventf(ne.vmt.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
   145  	ne.recorder.Eventf(ne.pvc, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
   146  	klog.InfoS(detailedMsg, "pod", klog.KObj(ne.vmt.Pod))
   147  
   148  	// no need to update PVC object if we already updated it
   149  	if ne.pvcAlreadyUpdated {
   150  		return true, nil, testResponseData{true, true}
   151  	}
   152  
   153  	// File system resize succeeded, now update the PVC's Capacity to match the PV's
   154  	ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient)
   155  	if err != nil {
   156  		return true, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %v", err), testResponseData{true, true}
   157  	}
   158  	return true, nil, testResponseData{true, true}
   159  }
   160
View as plain text