fc_util.go

Documentation: k8s.io/kubernetes/pkg/volume/fc

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package fc
    18  
    19  import (
    20  	"fmt"
    21  	"io/ioutil"
    22  	"os"
    23  	"path/filepath"
    24  	"regexp"
    25  	"strconv"
    26  	"strings"
    27  
    28  	"k8s.io/klog/v2"
    29  	"k8s.io/mount-utils"
    30  	utilexec "k8s.io/utils/exec"
    31  
    32  	"k8s.io/kubernetes/pkg/volume"
    33  	volumeutil "k8s.io/kubernetes/pkg/volume/util"
    34  )
    35  
    36  type ioHandler interface {
    37  	ReadDir(dirname string) ([]os.FileInfo, error)
    38  	Lstat(name string) (os.FileInfo, error)
    39  	EvalSymlinks(path string) (string, error)
    40  	WriteFile(filename string, data []byte, perm os.FileMode) error
    41  }
    42  
    43  type osIOHandler struct{}
    44  
    45  const (
    46  	byPath = "/dev/disk/by-path/"
    47  	byID   = "/dev/disk/by-id/"
    48  )
    49  
    50  func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
    51  	return ioutil.ReadDir(dirname)
    52  }
    53  func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
    54  	return os.Lstat(name)
    55  }
    56  func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
    57  	return filepath.EvalSymlinks(path)
    58  }
    59  func (handler *osIOHandler) WriteFile(filename string, data []byte, perm os.FileMode) error {
    60  	return ioutil.WriteFile(filename, data, perm)
    61  }
    62  
    63  // given a wwn and lun, find the device and associated devicemapper parent
    64  func findDisk(wwn, lun string, io ioHandler, deviceUtil volumeutil.DeviceUtil) (string, string) {
    65  	fcPathExp := "^(pci-.*-fc|fc)-0x" + wwn + "-lun-" + lun + "$"
    66  	r := regexp.MustCompile(fcPathExp)
    67  	devPath := byPath
    68  	if dirs, err := io.ReadDir(devPath); err == nil {
    69  		for _, f := range dirs {
    70  			name := f.Name()
    71  			if r.MatchString(name) {
    72  				if disk, err1 := io.EvalSymlinks(devPath + name); err1 == nil {
    73  					dm := deviceUtil.FindMultipathDeviceForDevice(disk)
    74  					klog.Infof("fc: find disk: %v, dm: %v, fc path: %v", disk, dm, name)
    75  					return disk, dm
    76  				}
    77  			}
    78  		}
    79  	}
    80  	return "", ""
    81  }
    82  
    83  // given a wwid, find the device and associated devicemapper parent
    84  func findDiskWWIDs(wwid string, io ioHandler, deviceUtil volumeutil.DeviceUtil) (string, string) {
    85  	// Example wwid format:
    86  	//   3600508b400105e210000900000490000
    87  	//   <VENDOR NAME> <IDENTIFIER NUMBER>
    88  	// Example of symlink under by-id:
    89  	//   /dev/by-id/scsi-3600508b400105e210000900000490000
    90  	//   /dev/by-id/scsi-<VENDOR NAME>_<IDENTIFIER NUMBER>
    91  	// The wwid could contain white space and it will be replaced
    92  	// underscore when wwid is exposed under /dev/by-id.
    93  
    94  	fcPath := "scsi-" + wwid
    95  	devID := byID
    96  	if dirs, err := io.ReadDir(devID); err == nil {
    97  		for _, f := range dirs {
    98  			name := f.Name()
    99  			if name == fcPath {
   100  				disk, err := io.EvalSymlinks(devID + name)
   101  				if err != nil {
   102  					klog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", devID+name, err)
   103  					return "", ""
   104  				}
   105  				dm := deviceUtil.FindMultipathDeviceForDevice(disk)
   106  				klog.Infof("fc: find disk: %v, dm: %v", disk, dm)
   107  				return disk, dm
   108  			}
   109  		}
   110  	}
   111  	klog.V(2).Infof("fc: failed to find a disk [%s]", devID+fcPath)
   112  	return "", ""
   113  }
   114  
   115  // Flushes any outstanding I/O to the device
   116  func flushDevice(deviceName string, exec utilexec.Interface) {
   117  	out, err := exec.Command("blockdev", "--flushbufs", deviceName).CombinedOutput()
   118  	if err != nil {
   119  		// Ignore the error and continue deleting the device. There is will be no retry on error.
   120  		klog.Warningf("Failed to flush device %s: %s\n%s", deviceName, err, string(out))
   121  	}
   122  	klog.V(4).Infof("Flushed device %s", deviceName)
   123  }
   124  
   125  // Removes a scsi device based upon /dev/sdX name
   126  func removeFromScsiSubsystem(deviceName string, io ioHandler) {
   127  	fileName := "/sys/block/" + deviceName + "/device/delete"
   128  	klog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName)
   129  	data := []byte("1")
   130  	io.WriteFile(fileName, data, 0666)
   131  }
   132  
   133  // rescan scsi bus
   134  func scsiHostRescan(io ioHandler) {
   135  	scsiPath := "/sys/class/scsi_host/"
   136  	if dirs, err := io.ReadDir(scsiPath); err == nil {
   137  		for _, f := range dirs {
   138  			name := scsiPath + f.Name() + "/scan"
   139  			data := []byte("- - -")
   140  			io.WriteFile(name, data, 0666)
   141  		}
   142  	}
   143  }
   144  
   145  // make a directory like /var/lib/kubelet/plugins/kubernetes.io/fc/target1-target2-lun-0
   146  func makePDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
   147  	if len(wwns) != 0 {
   148  		w := strings.Join(wwns, "-")
   149  		return filepath.Join(host.GetPluginDir(fcPluginName), w+"-lun-"+lun)
   150  	}
   151  	return filepath.Join(host.GetPluginDir(fcPluginName), strings.Join(wwids, "-"))
   152  }
   153  
   154  // make a directory like /var/lib/kubelet/plugins/kubernetes.io/fc/volumeDevices/target-lun-0
   155  func makeVDPDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
   156  	if len(wwns) != 0 {
   157  		w := strings.Join(wwns, "-")
   158  		return filepath.Join(host.GetVolumeDevicePluginDir(fcPluginName), w+"-lun-"+lun)
   159  	}
   160  	return filepath.Join(host.GetVolumeDevicePluginDir(fcPluginName), strings.Join(wwids, "-"))
   161  }
   162  
   163  func parsePDName(path string) (wwns []string, lun int32, wwids []string, err error) {
   164  	// parse directory name created by makePDNameInternal or makeVDPDNameInternal
   165  	dirname := filepath.Base(path)
   166  	components := strings.Split(dirname, "-")
   167  	l := len(components)
   168  	if l == 1 {
   169  		// No '-', it must be single WWID
   170  		return nil, 0, components, nil
   171  	}
   172  	if components[l-2] == "lun" {
   173  		// it has -lun-, it's list of WWNs + lun number as the last component
   174  		if l == 2 {
   175  			return nil, 0, nil, fmt.Errorf("no wwn in: %s", dirname)
   176  		}
   177  		lun, err := strconv.Atoi(components[l-1])
   178  		if err != nil {
   179  			return nil, 0, nil, err
   180  		}
   181  
   182  		return components[:l-2], int32(lun), nil, nil
   183  	}
   184  	// no -lun-, it's just list of WWIDs
   185  	return nil, 0, components, nil
   186  }
   187  
   188  type fcUtil struct{}
   189  
   190  func (util *fcUtil) MakeGlobalPDName(fc fcDisk) string {
   191  	return makePDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
   192  }
   193  
   194  // Global volume device plugin dir
   195  func (util *fcUtil) MakeGlobalVDPDName(fc fcDisk) string {
   196  	return makeVDPDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
   197  }
   198  
   199  func searchDisk(b fcDiskMounter) (string, error) {
   200  	var diskIDs []string
   201  	var disk string
   202  	var dm string
   203  	io := b.io
   204  	wwids := b.wwids
   205  	wwns := b.wwns
   206  	lun := b.lun
   207  
   208  	if len(wwns) != 0 {
   209  		diskIDs = wwns
   210  	} else {
   211  		diskIDs = wwids
   212  	}
   213  
   214  	rescanned := false
   215  	// two-phase search:
   216  	// first phase, search existing device path, if a multipath dm is found, exit loop
   217  	// otherwise, in second phase, rescan scsi bus and search again, return with any findings
   218  	for true {
   219  		for _, diskID := range diskIDs {
   220  			if len(wwns) != 0 {
   221  				disk, dm = findDisk(diskID, lun, io, b.deviceUtil)
   222  			} else {
   223  				disk, dm = findDiskWWIDs(diskID, io, b.deviceUtil)
   224  			}
   225  			// if multipath device is found, break
   226  			if dm != "" {
   227  				break
   228  			}
   229  		}
   230  		// if a dm is found, exit loop
   231  		if rescanned || dm != "" {
   232  			break
   233  		}
   234  		// rescan and search again
   235  		// rescan scsi bus
   236  		scsiHostRescan(io)
   237  		rescanned = true
   238  	}
   239  	// if no disk matches input wwn and lun, exit
   240  	if disk == "" && dm == "" {
   241  		return "", fmt.Errorf("no fc disk found")
   242  	}
   243  
   244  	// if multipath devicemapper device is found, use it; otherwise use raw disk
   245  	if dm != "" {
   246  		return dm, nil
   247  	}
   248  	return disk, nil
   249  }
   250  
   251  func (util *fcUtil) AttachDisk(b fcDiskMounter) (string, error) {
   252  	devicePath, err := searchDisk(b)
   253  	if err != nil {
   254  		return "", err
   255  	}
   256  
   257  	exists, err := mount.PathExists(devicePath)
   258  	if exists && err == nil {
   259  		return devicePath, nil
   260  	}
   261  	if exists == false {
   262  		return "", fmt.Errorf("device %s does not exist", devicePath)
   263  	} else {
   264  		return "", err
   265  	}
   266  }
   267  
   268  // DetachDisk removes scsi device file such as /dev/sdX from the node.
   269  func (util *fcUtil) DetachDisk(c fcDiskUnmounter, devicePath string) error {
   270  	var devices []string
   271  	// devicePath might be like /dev/mapper/mpathX. Find destination.
   272  	dstPath, err := c.io.EvalSymlinks(devicePath)
   273  	if err != nil {
   274  		return err
   275  	}
   276  	// Find slave
   277  	if strings.HasPrefix(dstPath, "/dev/dm-") {
   278  		devices = c.deviceUtil.FindSlaveDevicesOnMultipath(dstPath)
   279  		if err := util.deleteMultipathDevice(c.exec, dstPath); err != nil {
   280  			return err
   281  		}
   282  	} else {
   283  		// Add single devicepath to devices
   284  		devices = append(devices, dstPath)
   285  	}
   286  	klog.V(4).Infof("fc: DetachDisk devicePath: %v, dstPath: %v, devices: %v", devicePath, dstPath, devices)
   287  	var lastErr error
   288  	for _, device := range devices {
   289  		err := util.detachFCDisk(c.io, c.exec, device)
   290  		if err != nil {
   291  			klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
   292  			lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
   293  		}
   294  	}
   295  	if lastErr != nil {
   296  		klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr)
   297  		return lastErr
   298  	}
   299  	return nil
   300  }
   301  
   302  // detachFCDisk removes scsi device file such as /dev/sdX from the node.
   303  func (util *fcUtil) detachFCDisk(io ioHandler, exec utilexec.Interface, devicePath string) error {
   304  	// Remove scsi device from the node.
   305  	if !strings.HasPrefix(devicePath, "/dev/") {
   306  		return fmt.Errorf("fc detach disk: invalid device name: %s", devicePath)
   307  	}
   308  	flushDevice(devicePath, exec)
   309  	arr := strings.Split(devicePath, "/")
   310  	dev := arr[len(arr)-1]
   311  	removeFromScsiSubsystem(dev, io)
   312  	return nil
   313  }
   314  
   315  // DetachBlockFCDisk detaches a volume from kubelet node, removes scsi device file
   316  // such as /dev/sdX from the node, and then removes loopback for the scsi device.
   317  func (util *fcUtil) DetachBlockFCDisk(c fcDiskUnmapper, mapPath, devicePath string) error {
   318  	// Check if devicePath is valid
   319  	if len(devicePath) != 0 {
   320  		if pathExists, pathErr := checkPathExists(devicePath); !pathExists || pathErr != nil {
   321  			return pathErr
   322  		}
   323  	} else {
   324  		// TODO: FC plugin can't obtain the devicePath from kubelet because devicePath
   325  		// in volume object isn't updated when volume is attached to kubelet node.
   326  		klog.Infof("fc: devicePath is empty. Try to retrieve FC configuration from global map path: %v", mapPath)
   327  	}
   328  
   329  	// Check if global map path is valid
   330  	// global map path examples:
   331  	//   wwn+lun: plugins/kubernetes.io/fc/volumeDevices/50060e801049cfd1-lun-0/
   332  	//   wwid: plugins/kubernetes.io/fc/volumeDevices/3600508b400105e210000900000490000/
   333  	if pathExists, pathErr := checkPathExists(mapPath); !pathExists || pathErr != nil {
   334  		return pathErr
   335  	}
   336  
   337  	// Retrieve volume plugin dependent path like '50060e801049cfd1-lun-0' from global map path
   338  	arr := strings.Split(mapPath, "/")
   339  	if len(arr) < 1 {
   340  		return fmt.Errorf("failed to retrieve volume plugin information from global map path: %v", mapPath)
   341  	}
   342  	volumeInfo := arr[len(arr)-1]
   343  
   344  	// Search symbolic link which matches volumeInfo under /dev/disk/by-path or /dev/disk/by-id
   345  	// then find destination device path from the link
   346  	searchPath := byID
   347  	if strings.Contains(volumeInfo, "-lun-") {
   348  		searchPath = byPath
   349  	}
   350  	fis, err := ioutil.ReadDir(searchPath)
   351  	if err != nil {
   352  		return err
   353  	}
   354  	for _, fi := range fis {
   355  		if strings.Contains(fi.Name(), volumeInfo) {
   356  			devicePath = filepath.Join(searchPath, fi.Name())
   357  			klog.V(5).Infof("fc: updated devicePath: %s", devicePath)
   358  			break
   359  		}
   360  	}
   361  	if len(devicePath) == 0 {
   362  		return fmt.Errorf("fc: failed to find corresponding device from searchPath: %v", searchPath)
   363  	}
   364  	dstPath, err := c.io.EvalSymlinks(devicePath)
   365  	if err != nil {
   366  		return err
   367  	}
   368  	klog.V(4).Infof("fc: find destination device path from symlink: %v", dstPath)
   369  
   370  	var devices []string
   371  	dm := c.deviceUtil.FindMultipathDeviceForDevice(dstPath)
   372  	if len(dm) != 0 {
   373  		dstPath = dm
   374  	}
   375  
   376  	// Detach volume from kubelet node
   377  	if len(dm) != 0 {
   378  		// Find all devices which are managed by multipath
   379  		devices = c.deviceUtil.FindSlaveDevicesOnMultipath(dm)
   380  		if err := util.deleteMultipathDevice(c.exec, dm); err != nil {
   381  			return err
   382  		}
   383  	} else {
   384  		// Add single device path to devices
   385  		devices = append(devices, dstPath)
   386  	}
   387  	var lastErr error
   388  	for _, device := range devices {
   389  		err = util.detachFCDisk(c.io, c.exec, device)
   390  		if err != nil {
   391  			klog.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
   392  			lastErr = fmt.Errorf("fc: detachFCDisk failed. device: %v err: %v", device, err)
   393  		}
   394  	}
   395  	if lastErr != nil {
   396  		klog.Errorf("fc: last error occurred during detach disk:\n%v", lastErr)
   397  		return lastErr
   398  	}
   399  	return nil
   400  }
   401  
   402  func (util *fcUtil) deleteMultipathDevice(exec utilexec.Interface, dmDevice string) error {
   403  	out, err := exec.Command("multipath", "-f", dmDevice).CombinedOutput()
   404  	if err != nil {
   405  		return fmt.Errorf("failed to flush multipath device %s: %s\n%s", dmDevice, err, string(out))
   406  	}
   407  	klog.V(4).Infof("Flushed multipath device: %s", dmDevice)
   408  	return nil
   409  }
   410  
   411  func checkPathExists(path string) (bool, error) {
   412  	if pathExists, pathErr := mount.PathExists(path); pathErr != nil {
   413  		return pathExists, fmt.Errorf("error checking if path exists: %w", pathErr)
   414  	} else if !pathExists {
   415  		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", path)
   416  		return pathExists, nil
   417  	}
   418  	return true, nil
   419  }
   420
View as plain text