Source file src/k8s.io/kubernetes/pkg/volume/iscsi/iscsi_util.go

Documentation: k8s.io/kubernetes/pkg/volume/iscsi

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package iscsi
    18  
    19  import (
    20  	"encoding/json"
    21  	"errors"
    22  	"fmt"
    23  	"io/ioutil"
    24  	"os"
    25  	"path/filepath"
    26  	"regexp"
    27  	"sort"
    28  	"strconv"
    29  	"strings"
    30  	"time"
    31  
    32  	"k8s.io/klog/v2"
    33  	"k8s.io/mount-utils"
    34  	utilexec "k8s.io/utils/exec"
    35  
    36  	v1 "k8s.io/api/core/v1"
    37  	"k8s.io/kubernetes/pkg/kubelet/config"
    38  	"k8s.io/kubernetes/pkg/volume"
    39  	volumeutil "k8s.io/kubernetes/pkg/volume/util"
    40  	"k8s.io/kubernetes/pkg/volume/util/types"
    41  )
    42  
    43  const (
    44  	// Minimum number of paths that the volume plugin considers enough when a multipath volume is requested.
    45  	minMultipathCount = 2
    46  
    47  	// Minimal number of attempts to attach all paths of a multipath volumes. If at least minMultipathCount paths
    48  	// are available after this nr. of attempts, the volume plugin continues with mounting the volume.
    49  	minAttachAttempts = 2
    50  
    51  	// Total number of attempts to attach at least minMultipathCount paths. If there are less than minMultipathCount,
    52  	// the volume plugin tries to attach the remaining paths at least this number of times in total. After
    53  	// maxAttachAttempts attempts, it mounts even a single path.
    54  	maxAttachAttempts = 5
    55  
    56  	// How many seconds to wait for a multipath device if at least two paths are available.
    57  	multipathDeviceTimeout = 10
    58  
    59  	// How many seconds to wait for a device/path to appear before giving up.
    60  	deviceDiscoveryTimeout = 30
    61  
    62  	// 'iscsiadm' error code stating that a session is logged in
    63  	// See https://github.com/open-iscsi/open-iscsi/blob/7d121d12ad6ba7783308c25ffd338a9fa0cc402b/include/iscsi_err.h#L37-L38
    64  	iscsiadmErrorSessExists = 15
    65  
    66  	// iscsiadm exit code for "session could not be found"
    67  	exit_ISCSI_ERR_SESS_NOT_FOUND = 2
    68  	// iscsiadm exit code for "no records/targets/sessions/portals found to execute operation on."
    69  	exit_ISCSI_ERR_NO_OBJS_FOUND = 21
    70  )
    71  
    72  var (
    73  	chapSt = []string{
    74  		"discovery.sendtargets.auth.username",
    75  		"discovery.sendtargets.auth.password",
    76  		"discovery.sendtargets.auth.username_in",
    77  		"discovery.sendtargets.auth.password_in"}
    78  	chapSess = []string{
    79  		"node.session.auth.username",
    80  		"node.session.auth.password",
    81  		"node.session.auth.username_in",
    82  		"node.session.auth.password_in"}
    83  	ifaceTransportNameRe = regexp.MustCompile(`iface.transport_name = (.*)\n`)
    84  	ifaceRe              = regexp.MustCompile(`.+/iface-([^/]+)/.+`)
    85  )
    86  
    87  func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error {
    88  	if !b.chapDiscovery {
    89  		return nil
    90  	}
    91  	out, err := execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP")
    92  	if err != nil {
    93  		return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", out)
    94  	}
    95  
    96  	for _, k := range chapSt {
    97  		v := b.secret[k]
    98  		if len(v) > 0 {
    99  			// explicitly not using execWithLog so secrets are not logged
   100  			out, err := b.exec.Command("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
   101  			if err != nil {
   102  				return fmt.Errorf("iscsi: failed to update discoverydb key %q error: %v", k, string(out))
   103  			}
   104  		}
   105  	}
   106  	return nil
   107  }
   108  
   109  func updateISCSINode(b iscsiDiskMounter, tp string) error {
   110  	// setting node.session.scan to manual to handle https://github.com/kubernetes/kubernetes/issues/90982
   111  	out, err := execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.scan", "-v", "manual")
   112  	if err != nil {
   113  		// don't fail if iscsiadm fails or the version does not support node.session.scan - log a warning to highlight the potential exposure
   114  		klog.Warningf("iscsi: failed to update node with node.session.scan=manual, possible exposure to issue 90982: %v", out)
   115  	}
   116  
   117  	if !b.chapSession {
   118  		return nil
   119  	}
   120  
   121  	out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP")
   122  	if err != nil {
   123  		return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", out)
   124  	}
   125  
   126  	for _, k := range chapSess {
   127  		v := b.secret[k]
   128  		if len(v) > 0 {
   129  			// explicitly not using execWithLog so secrets are not logged
   130  			out, err := b.exec.Command("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v).CombinedOutput()
   131  			if err != nil {
   132  				return fmt.Errorf("iscsi: failed to update node session key %q error: %v", k, string(out))
   133  			}
   134  		}
   135  	}
   136  	return nil
   137  }
   138  
   139  // stat a path, if not exists, retry maxRetries times
   140  // when iscsi transports other than default are used,  use glob instead as pci id of device is unknown
   141  type StatFunc func(string) (os.FileInfo, error)
   142  type GlobFunc func(string) ([]string, error)
   143  
   144  func waitForPathToExist(devicePath *string, maxRetries int, deviceTransport string) bool {
   145  	// This makes unit testing a lot easier
   146  	return waitForPathToExistInternal(devicePath, maxRetries, deviceTransport, os.Stat, filepath.Glob)
   147  }
   148  
   149  func waitForPathToExistInternal(devicePath *string, maxRetries int, deviceTransport string, osStat StatFunc, filepathGlob GlobFunc) bool {
   150  	if devicePath == nil {
   151  		return false
   152  	}
   153  
   154  	for i := 0; i < maxRetries; i++ {
   155  		var err error
   156  		if deviceTransport == "tcp" {
   157  			_, err = osStat(*devicePath)
   158  		} else {
   159  			fpath, _ := filepathGlob(*devicePath)
   160  			if fpath == nil {
   161  				err = os.ErrNotExist
   162  			} else {
   163  				// There might be a case that fpath contains multiple device paths if
   164  				// multiple PCI devices connect to same iscsi target. We handle this
   165  				// case at subsequent logic. Pick up only first path here.
   166  				*devicePath = fpath[0]
   167  			}
   168  		}
   169  		if err == nil {
   170  			return true
   171  		}
   172  		if !os.IsNotExist(err) {
   173  			return false
   174  		}
   175  		if i == maxRetries-1 {
   176  			break
   177  		}
   178  		time.Sleep(time.Second)
   179  	}
   180  	return false
   181  }
   182  
   183  // make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/iface_name/portal-some_iqn-lun-lun_id
   184  func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
   185  	return filepath.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
   186  }
   187  
   188  // make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/iface_name/portal-some_iqn-lun-lun_id
   189  func makeVDPDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
   190  	return filepath.Join(host.GetVolumeDevicePluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
   191  }
   192  
   193  type ISCSIUtil struct{}
   194  
   195  // MakeGlobalPDName returns path of global plugin dir
   196  func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
   197  	return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
   198  }
   199  
   200  // MakeGlobalVDPDName returns path of global volume device plugin dir
   201  func (util *ISCSIUtil) MakeGlobalVDPDName(iscsi iscsiDisk) string {
   202  	return makeVDPDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
   203  }
   204  
   205  // persistISCSIFile saves iSCSI volume configuration for DetachDisk
   206  // into given directory.
   207  func (util *ISCSIUtil) persistISCSIFile(conf iscsiDisk, mnt string) error {
   208  	file := filepath.Join(mnt, "iscsi.json")
   209  	fp, err := os.Create(file)
   210  	if err != nil {
   211  		return fmt.Errorf("iscsi: create %s err %s", file, err)
   212  	}
   213  	defer fp.Close()
   214  	encoder := json.NewEncoder(fp)
   215  	if err = encoder.Encode(conf); err != nil {
   216  		return fmt.Errorf("iscsi: encode err: %v", err)
   217  	}
   218  	return nil
   219  }
   220  
   221  func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
   222  	file := filepath.Join(mnt, "iscsi.json")
   223  	fp, err := os.Open(file)
   224  	if err != nil {
   225  		return fmt.Errorf("iscsi: open %s err %s", file, err)
   226  	}
   227  	defer fp.Close()
   228  	decoder := json.NewDecoder(fp)
   229  	if err = decoder.Decode(conf); err != nil {
   230  		return fmt.Errorf("iscsi: decode err: %v", err)
   231  	}
   232  	return nil
   233  }
   234  
   235  // scanOneLun scans a single LUN on one SCSI bus
   236  // Use this to avoid scanning the whole SCSI bus for all of the LUNs, which
   237  // would result in the kernel on this node discovering LUNs that it shouldn't
   238  // know about. Extraneous LUNs cause problems because they may get deleted
   239  // without us getting notified, since we were never supposed to know about
   240  // them. When LUNs are deleted without proper cleanup in the kernel, I/O errors
   241  // and timeouts result, which can noticeably degrade performance of future
   242  // operations.
   243  func scanOneLun(hostNumber int, lunNumber int) error {
   244  	filename := fmt.Sprintf("/sys/class/scsi_host/host%d/scan", hostNumber)
   245  	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
   246  	if err != nil {
   247  		return err
   248  	}
   249  	defer fd.Close()
   250  
   251  	// Channel/Target are always 0 for iSCSI
   252  	scanCmd := fmt.Sprintf("0 0 %d", lunNumber)
   253  	if written, err := fd.WriteString(scanCmd); err != nil {
   254  		return err
   255  	} else if 0 == written {
   256  		return fmt.Errorf("no data written to file: %s", filename)
   257  	}
   258  
   259  	klog.V(3).Infof("Scanned SCSI host %d LUN %d", hostNumber, lunNumber)
   260  	return nil
   261  }
   262  
   263  func waitForMultiPathToExist(devicePaths []string, maxRetries int, deviceUtil volumeutil.DeviceUtil) string {
   264  	if 0 == len(devicePaths) {
   265  		return ""
   266  	}
   267  
   268  	for i := 0; i < maxRetries; i++ {
   269  		for _, path := range devicePaths {
   270  			// There shouldn't be any empty device paths. However adding this check
   271  			// for safer side to avoid the possibility of an empty entry.
   272  			if path == "" {
   273  				continue
   274  			}
   275  			// check if the dev is using mpio and if so mount it via the dm-XX device
   276  			if mappedDevicePath := deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
   277  				return mappedDevicePath
   278  			}
   279  		}
   280  		if i == maxRetries-1 {
   281  			break
   282  		}
   283  		time.Sleep(time.Second)
   284  	}
   285  	return ""
   286  }
   287  
   288  // AttachDisk returns devicePath of volume if attach succeeded otherwise returns error
   289  func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
   290  	var devicePath string
   291  	devicePaths := map[string]string{}
   292  	var iscsiTransport string
   293  	var lastErr error
   294  
   295  	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
   296  	if err != nil {
   297  		klog.Errorf("iscsi: could not read iface %s error: %s", b.InitIface, out)
   298  		return "", err
   299  	}
   300  
   301  	iscsiTransport = extractTransportname(out)
   302  
   303  	bkpPortal := b.Portals
   304  
   305  	// If the initiator name was set, the iface isn't created yet,
   306  	// so create it and copy parameters from the pre-configured one
   307  	if b.InitiatorName != "" {
   308  		if err = cloneIface(b); err != nil {
   309  			klog.Errorf("iscsi: failed to clone iface: %s error: %v", b.InitIface, err)
   310  			return "", err
   311  		}
   312  	}
   313  
   314  	// Lock the target while we login to avoid races between 2 volumes that share the same
   315  	// target both logging in or one logging out while another logs in.
   316  	b.plugin.targetLocks.LockKey(b.Iqn)
   317  	defer b.plugin.targetLocks.UnlockKey(b.Iqn)
   318  
   319  	// Build a map of SCSI hosts for each target portal. We will need this to
   320  	// issue the bus rescans.
   321  	portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
   322  	if err != nil {
   323  		return "", err
   324  	}
   325  	klog.V(4).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)
   326  
   327  	for i := 1; i <= maxAttachAttempts; i++ {
   328  		for _, tp := range bkpPortal {
   329  			if _, found := devicePaths[tp]; found {
   330  				klog.V(4).Infof("Device for portal %q already known", tp)
   331  				continue
   332  			}
   333  
   334  			hostNumber, loggedIn := portalHostMap[tp]
   335  			if !loggedIn {
   336  				klog.V(4).Infof("Could not get SCSI host number for portal %s, will attempt login", tp)
   337  
   338  				// build discoverydb and discover iscsi target
   339  				execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new")
   340  
   341  				// update discoverydb with CHAP secret
   342  				err = updateISCSIDiscoverydb(b, tp)
   343  				if err != nil {
   344  					lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err)
   345  					continue
   346  				}
   347  
   348  				out, err = execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover")
   349  				if err != nil {
   350  					// delete discoverydb record
   351  					execWithLog(b, "iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete")
   352  					lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, out, err)
   353  					continue
   354  				}
   355  
   356  				err = updateISCSINode(b, tp)
   357  				if err != nil {
   358  					// failure to update node db is rare. But deleting record will likely impact those who already start using it.
   359  					lastErr = fmt.Errorf("iscsi: failed to update iscsi node to portal %s error: %v", tp, err)
   360  					continue
   361  				}
   362  
   363  				// login to iscsi target
   364  				out, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login")
   365  				if err != nil {
   366  					// delete the node record from database
   367  					execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete")
   368  					lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", out, err)
   369  					continue
   370  				}
   371  
   372  				// in case of node failure/restart, explicitly set to manual login so it doesn't hang on boot
   373  				_, err = execWithLog(b, "iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-o", "update", "-n", "node.startup", "-v", "manual")
   374  				if err != nil {
   375  					// don't fail if we can't set startup mode, but log warning so there is a clue
   376  					klog.Warningf("Warning: Failed to set iSCSI login mode to manual. Error: %v", err)
   377  				}
   378  
   379  				// Rebuild the host map after logging in
   380  				portalHostMap, err := b.deviceUtil.GetISCSIPortalHostMapForTarget(b.Iqn)
   381  				if err != nil {
   382  					return "", err
   383  				}
   384  				klog.V(6).Infof("AttachDisk portal->host map for %s is %v", b.Iqn, portalHostMap)
   385  
   386  				hostNumber, loggedIn = portalHostMap[tp]
   387  				if !loggedIn {
   388  					klog.Warningf("Could not get SCSI host number for portal %s after logging in", tp)
   389  					continue
   390  				}
   391  			}
   392  
   393  			klog.V(5).Infof("AttachDisk: scanning SCSI host %d LUN %s", hostNumber, b.Lun)
   394  			lunNumber, err := strconv.Atoi(b.Lun)
   395  			if err != nil {
   396  				return "", fmt.Errorf("AttachDisk: lun is not a number: %s\nError: %v", b.Lun, err)
   397  			}
   398  
   399  			// Scan the iSCSI bus for the LUN
   400  			err = scanOneLun(hostNumber, lunNumber)
   401  			if err != nil {
   402  				return "", err
   403  			}
   404  
   405  			if iscsiTransport == "" {
   406  				klog.Errorf("iscsi: could not find transport name in iface %s", b.Iface)
   407  				return "", fmt.Errorf("could not parse iface file for %s", b.Iface)
   408  			}
   409  
   410  			addr := tp
   411  			if strings.HasPrefix(tp, "[") {
   412  				// Delete [] from IP address, links in /dev/disk/by-path do not have it.
   413  				addr = strings.NewReplacer("[", "", "]", "").Replace(tp)
   414  			}
   415  			if iscsiTransport == "tcp" {
   416  				devicePath = strings.Join([]string{"/dev/disk/by-path/ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
   417  			} else {
   418  				devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", addr, "iscsi", b.Iqn, "lun", b.Lun}, "-")
   419  			}
   420  
   421  			if exist := waitForPathToExist(&devicePath, deviceDiscoveryTimeout, iscsiTransport); !exist {
   422  				msg := fmt.Sprintf("Timed out waiting for device at path %s after %ds", devicePath, deviceDiscoveryTimeout)
   423  				klog.Error(msg)
   424  				// update last error
   425  				lastErr = errors.New(msg)
   426  				continue
   427  			} else {
   428  				devicePaths[tp] = devicePath
   429  			}
   430  		}
   431  		klog.V(4).Infof("iscsi: tried all devices for %q %d times, %d paths found", b.Iqn, i, len(devicePaths))
   432  		if len(devicePaths) == 0 {
   433  			// No path attached, report error and stop trying. kubelet will try again in a short while
   434  			// delete cloned iface
   435  			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
   436  			klog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
   437  			return "", fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr)
   438  		}
   439  		if len(devicePaths) == len(bkpPortal) {
   440  			// We have all paths
   441  			klog.V(4).Infof("iscsi: all devices for %q found", b.Iqn)
   442  			break
   443  		}
   444  		if len(devicePaths) >= minMultipathCount && i >= minAttachAttempts {
   445  			// We have at least two paths for multipath and we tried the other paths long enough
   446  			klog.V(4).Infof("%d devices found for %q", len(devicePaths), b.Iqn)
   447  			break
   448  		}
   449  	}
   450  
   451  	if lastErr != nil {
   452  		klog.Errorf("iscsi: last error occurred during iscsi init:\n%v", lastErr)
   453  	}
   454  
   455  	devicePathList := []string{}
   456  	for _, path := range devicePaths {
   457  		devicePathList = append(devicePathList, path)
   458  	}
   459  	// Try to find a multipath device for the volume
   460  	if len(bkpPortal) > 1 {
   461  		// Multipath volume was requested. Wait up to multipathDeviceTimeout seconds for the multipath device to appear.
   462  		devicePath = waitForMultiPathToExist(devicePathList, multipathDeviceTimeout, b.deviceUtil)
   463  	} else {
   464  		// For PVs with 1 portal, just try one time to find the multipath device. This
   465  		// avoids a long pause when the multipath device will never get created, and
   466  		// matches legacy behavior.
   467  		devicePath = waitForMultiPathToExist(devicePathList, 1, b.deviceUtil)
   468  	}
   469  
   470  	// When no multipath device is found, just use the first (and presumably only) device
   471  	if devicePath == "" {
   472  		devicePath = devicePathList[0]
   473  	}
   474  
   475  	klog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath)
   476  
   477  	if err = util.persistISCSI(b); err != nil {
   478  		// Return uncertain error so kubelet calls Unmount / Unmap when the pod
   479  		// is deleted.
   480  		return "", types.NewUncertainProgressError(err.Error())
   481  	}
   482  	return devicePath, nil
   483  }
   484  
   485  // persistISCSI saves iSCSI volume configuration for DetachDisk into global
   486  // mount / map directory.
   487  func (util *ISCSIUtil) persistISCSI(b iscsiDiskMounter) error {
   488  	klog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode)
   489  	var globalPDPath string
   490  	if b.volumeMode == v1.PersistentVolumeBlock {
   491  		globalPDPath = b.manager.MakeGlobalVDPDName(*b.iscsiDisk)
   492  	} else {
   493  		globalPDPath = b.manager.MakeGlobalPDName(*b.iscsiDisk)
   494  	}
   495  
   496  	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
   497  		klog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
   498  		return err
   499  	}
   500  
   501  	if b.volumeMode == v1.PersistentVolumeFilesystem {
   502  		notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
   503  		if err != nil {
   504  			return err
   505  		}
   506  		if !notMnt {
   507  			// The volume is already mounted, therefore the previous WaitForAttach must have
   508  			// persisted the volume metadata. In addition, the metadata is actually *inside*
   509  			// globalPDPath and we can't write it here, because it was shadowed by the volume
   510  			// mount.
   511  			klog.V(4).Infof("Skipping persistISCSI, the volume is already mounted at %s", globalPDPath)
   512  			return nil
   513  		}
   514  	}
   515  
   516  	// Persist iscsi disk config to json file for DetachDisk path
   517  	return util.persistISCSIFile(*(b.iscsiDisk), globalPDPath)
   518  }
   519  
   520  // Delete 1 block device of the form "sd*"
   521  func deleteDevice(deviceName string) error {
   522  	filename := fmt.Sprintf("/sys/block/%s/device/delete", deviceName)
   523  	fd, err := os.OpenFile(filename, os.O_WRONLY, 0)
   524  	if err != nil {
   525  		// The file was not present, so just return without error
   526  		return nil
   527  	}
   528  	defer fd.Close()
   529  
   530  	if written, err := fd.WriteString("1"); err != nil {
   531  		return err
   532  	} else if 0 == written {
   533  		return fmt.Errorf("no data written to file: %s", filename)
   534  	}
   535  	klog.V(4).Infof("Deleted block device: %s", deviceName)
   536  	return nil
   537  }
   538  
   539  // deleteDevices tries to remove all the block devices and multipath map devices
   540  // associated with a given iscsi device
   541  func deleteDevices(c iscsiDiskUnmounter) error {
   542  	lunNumber, err := strconv.Atoi(c.iscsiDisk.Lun)
   543  	if err != nil {
   544  		klog.Errorf("iscsi delete devices: lun is not a number: %s\nError: %v", c.iscsiDisk.Lun, err)
   545  		return err
   546  	}
   547  	// Enumerate the devices so we can delete them
   548  	deviceNames, err := c.deviceUtil.FindDevicesForISCSILun(c.iscsiDisk.Iqn, lunNumber)
   549  	if err != nil {
   550  		klog.Errorf("iscsi delete devices: could not get devices associated with LUN %d on target %s\nError: %v",
   551  			lunNumber, c.iscsiDisk.Iqn, err)
   552  		return err
   553  	}
   554  	// Find the multipath device path(s)
   555  	mpathDevices := make(map[string]bool)
   556  	for _, deviceName := range deviceNames {
   557  		path := "/dev/" + deviceName
   558  		// check if the dev is using mpio and if so mount it via the dm-XX device
   559  		if mappedDevicePath := c.deviceUtil.FindMultipathDeviceForDevice(path); mappedDevicePath != "" {
   560  			mpathDevices[mappedDevicePath] = true
   561  		}
   562  	}
   563  	// Flush any multipath device maps
   564  	for mpathDevice := range mpathDevices {
   565  		_, err = c.exec.Command("multipath", "-f", mpathDevice).CombinedOutput()
   566  		if err != nil {
   567  			klog.Warningf("Warning: Failed to flush multipath device map: %s\nError: %v", mpathDevice, err)
   568  			// Fall through -- keep deleting the block devices
   569  		}
   570  		klog.V(4).Infof("Flushed multipath device: %s", mpathDevice)
   571  	}
   572  	for _, deviceName := range deviceNames {
   573  		err = deleteDevice(deviceName)
   574  		if err != nil {
   575  			klog.Warningf("Warning: Failed to delete block device: %s\nError: %v", deviceName, err)
   576  			// Fall through -- keep deleting other block devices
   577  		}
   578  	}
   579  	return nil
   580  }
   581  
   582  // DetachDisk unmounts and detaches a volume from node
   583  func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
   584  	if pathExists, pathErr := mount.PathExists(mntPath); pathErr != nil {
   585  		return fmt.Errorf("error checking if path exists: %w", pathErr)
   586  	} else if !pathExists {
   587  		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath)
   588  		return nil
   589  	}
   590  
   591  	notMnt, err := c.mounter.IsLikelyNotMountPoint(mntPath)
   592  	if err != nil {
   593  		return err
   594  	}
   595  	if !notMnt {
   596  		if err := c.mounter.Unmount(mntPath); err != nil {
   597  			klog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err)
   598  			return err
   599  		}
   600  	}
   601  
   602  	// if device is no longer used, see if need to logout the target
   603  	device, _, err := extractDeviceAndPrefix(mntPath)
   604  	if err != nil {
   605  		return err
   606  	}
   607  
   608  	var bkpPortal []string
   609  	var volName, iqn, iface, initiatorName string
   610  	found := true
   611  
   612  	// load iscsi disk config from json file
   613  	if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil {
   614  		bkpPortal, iqn, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface, c.iscsiDisk.VolName
   615  		initiatorName = c.iscsiDisk.InitiatorName
   616  	} else {
   617  		// If the iscsi disk config is not found, fall back to the original behavior.
   618  		// This portal/iqn/iface is no longer referenced, log out.
   619  		// Extract the portal and iqn from device path.
   620  		bkpPortal = make([]string, 1)
   621  		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
   622  		if err != nil {
   623  			return err
   624  		}
   625  		// Extract the iface from the mountPath and use it to log out. If the iface
   626  		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
   627  		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
   628  		iface, found = extractIface(mntPath)
   629  	}
   630  
   631  	// Delete all the scsi devices and any multipath devices after unmounting
   632  	if err = deleteDevices(c); err != nil {
   633  		klog.Warningf("iscsi detach disk: failed to delete devices\nError: %v", err)
   634  		// Fall through -- even if deleting fails, a logout may fix problems
   635  	}
   636  
   637  	// Lock the target while we determine if we can safely log out or not
   638  	c.plugin.targetLocks.LockKey(iqn)
   639  	defer c.plugin.targetLocks.UnlockKey(iqn)
   640  
   641  	portals := removeDuplicate(bkpPortal)
   642  	if len(portals) == 0 {
   643  		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
   644  	}
   645  
   646  	// If device is no longer used, see if need to logout the target
   647  	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
   648  		return nil
   649  	}
   650  
   651  	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
   652  	if err != nil {
   653  		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
   654  	}
   655  	return nil
   656  }
   657  
   658  // DetachBlockISCSIDisk removes loopback device for a volume and detaches a volume from node
   659  func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) error {
   660  	if pathExists, pathErr := mount.PathExists(mapPath); pathErr != nil {
   661  		return fmt.Errorf("error checking if path exists: %w", pathErr)
   662  	} else if !pathExists {
   663  		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
   664  		return nil
   665  	}
   666  	// If we arrive here, device is no longer used, see if need to logout the target
   667  	// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
   668  	device, _, err := extractDeviceAndPrefix(mapPath)
   669  	if err != nil {
   670  		return err
   671  	}
   672  	var bkpPortal []string
   673  	var volName, iqn, lun, iface, initiatorName string
   674  	found := true
   675  	// load iscsi disk config from json file
   676  	if err := util.loadISCSI(c.iscsiDisk, mapPath); err == nil {
   677  		bkpPortal, iqn, lun, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Lun, c.iscsiDisk.Iface, c.iscsiDisk.VolName
   678  		initiatorName = c.iscsiDisk.InitiatorName
   679  	} else {
   680  		// If the iscsi disk config is not found, fall back to the original behavior.
   681  		// This portal/iqn/iface is no longer referenced, log out.
   682  		// Extract the portal and iqn from device path.
   683  		bkpPortal = make([]string, 1)
   684  		bkpPortal[0], iqn, err = extractPortalAndIqn(device)
   685  		if err != nil {
   686  			return err
   687  		}
   688  		arr := strings.Split(device, "-lun-")
   689  		if len(arr) < 2 {
   690  			return fmt.Errorf("failed to retrieve lun from mapPath: %v", mapPath)
   691  		}
   692  		lun = arr[1]
   693  		// Extract the iface from the mountPath and use it to log out. If the iface
   694  		// is not found, maintain the previous behavior to facilitate kubelet upgrade.
   695  		// Logout may fail as no session may exist for the portal/IQN on the specified interface.
   696  		iface, found = extractIface(mapPath)
   697  	}
   698  	portals := removeDuplicate(bkpPortal)
   699  	if len(portals) == 0 {
   700  		return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
   701  	}
   702  
   703  	devicePath := getDevByPath(portals[0], iqn, lun)
   704  	klog.V(5).Infof("iscsi: devicePath: %s", devicePath)
   705  	if _, err = os.Stat(devicePath); err != nil {
   706  		return fmt.Errorf("failed to validate devicePath: %s", devicePath)
   707  	}
   708  
   709  	// Lock the target while we determine if we can safely log out or not
   710  	c.plugin.targetLocks.LockKey(iqn)
   711  	defer c.plugin.targetLocks.UnlockKey(iqn)
   712  
   713  	// If device is no longer used, see if need to logout the target
   714  	if isSessionBusy(c.iscsiDisk.plugin.host, portals[0], iqn) {
   715  		return nil
   716  	}
   717  
   718  	// Detach a volume from kubelet node
   719  	err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
   720  	if err != nil {
   721  		return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
   722  	}
   723  	return nil
   724  }
   725  
   726  func (util *ISCSIUtil) detachISCSIDisk(exec utilexec.Interface, portals []string, iqn, iface, volName, initiatorName string, found bool) error {
   727  	for _, portal := range portals {
   728  		logoutArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}
   729  		deleteArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}
   730  		if found {
   731  			logoutArgs = append(logoutArgs, []string{"-I", iface}...)
   732  			deleteArgs = append(deleteArgs, []string{"-I", iface}...)
   733  		}
   734  		klog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
   735  		out, err := exec.Command("iscsiadm", logoutArgs...).CombinedOutput()
   736  		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
   737  		if err != nil {
   738  			klog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
   739  			return err
   740  		}
   741  		// Delete the node record
   742  		klog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
   743  		out, err = exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
   744  		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
   745  		if err != nil {
   746  			klog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
   747  			return err
   748  		}
   749  	}
   750  	// Delete the iface after all sessions have logged out
   751  	// If the iface is not created via iscsi plugin, skip to delete
   752  	if initiatorName != "" && found && iface == (portals[0]+":"+volName) {
   753  		deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"}
   754  		out, err := exec.Command("iscsiadm", deleteArgs...).CombinedOutput()
   755  		err = ignoreExitCodes(err, exit_ISCSI_ERR_NO_OBJS_FOUND, exit_ISCSI_ERR_SESS_NOT_FOUND)
   756  		if err != nil {
   757  			klog.Errorf("iscsi: failed to delete iface Error: %s", string(out))
   758  			return err
   759  		}
   760  	}
   761  
   762  	return nil
   763  }
   764  
   765  func getDevByPath(portal, iqn, lun string) string {
   766  	return "/dev/disk/by-path/ip-" + portal + "-iscsi-" + iqn + "-lun-" + lun
   767  }
   768  
   769  func extractTransportname(ifaceOutput string) (iscsiTransport string) {
   770  	rexOutput := ifaceTransportNameRe.FindStringSubmatch(ifaceOutput)
   771  	if rexOutput == nil {
   772  		return ""
   773  	}
   774  	iscsiTransport = rexOutput[1]
   775  
   776  	// While iface.transport_name is a required parameter, handle it being unspecified anyways
   777  	if iscsiTransport == "<empty>" {
   778  		iscsiTransport = "tcp"
   779  	}
   780  	return iscsiTransport
   781  }
   782  
   783  func extractDeviceAndPrefix(mntPath string) (string, string, error) {
   784  	ind := strings.LastIndex(mntPath, "/")
   785  	if ind < 0 {
   786  		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
   787  	}
   788  	device := mntPath[(ind + 1):]
   789  	// strip -lun- from mount path
   790  	ind = strings.LastIndex(mntPath, "-lun-")
   791  	if ind < 0 {
   792  		return "", "", fmt.Errorf("iscsi detach disk: malformatted mnt path: %s", mntPath)
   793  	}
   794  	prefix := mntPath[:ind]
   795  	return device, prefix, nil
   796  }
   797  
   798  func extractIface(mntPath string) (string, bool) {
   799  	reOutput := ifaceRe.FindStringSubmatch(mntPath)
   800  	if len(reOutput) > 1 {
   801  		return reOutput[1], true
   802  	}
   803  
   804  	return "", false
   805  }
   806  
   807  func extractPortalAndIqn(device string) (string, string, error) {
   808  	ind1 := strings.Index(device, "-")
   809  	if ind1 < 0 {
   810  		return "", "", fmt.Errorf("iscsi detach disk: no portal in %s", device)
   811  	}
   812  	portal := device[0:ind1]
   813  	ind2 := strings.Index(device, "iqn.")
   814  	if ind2 < 0 {
   815  		ind2 = strings.Index(device, "eui.")
   816  	}
   817  	if ind2 < 0 {
   818  		return "", "", fmt.Errorf("iscsi detach disk: no iqn in %s", device)
   819  	}
   820  	ind := strings.LastIndex(device, "-lun-")
   821  	iqn := device[ind2:ind]
   822  	return portal, iqn, nil
   823  }
   824  
   825  // Remove duplicates or string
   826  func removeDuplicate(s []string) []string {
   827  	m := map[string]bool{}
   828  	for _, v := range s {
   829  		if v != "" && !m[v] {
   830  			s[len(m)] = v
   831  			m[v] = true
   832  		}
   833  	}
   834  	s = s[:len(m)]
   835  	return s
   836  }
   837  
   838  func parseIscsiadmShow(output string) (map[string]string, error) {
   839  	params := make(map[string]string)
   840  	slice := strings.Split(output, "\n")
   841  	for _, line := range slice {
   842  		if !strings.HasPrefix(line, "iface.") || strings.Contains(line, "<empty>") {
   843  			continue
   844  		}
   845  		iface := strings.Fields(line)
   846  		if len(iface) != 3 || iface[1] != "=" {
   847  			return nil, fmt.Errorf("error: invalid iface setting: %v", iface)
   848  		}
   849  		// iscsi_ifacename is immutable once the iface is created
   850  		if iface[0] == "iface.iscsi_ifacename" {
   851  			continue
   852  		}
   853  		params[iface[0]] = iface[2]
   854  	}
   855  	return params, nil
   856  }
   857  
   858  func cloneIface(b iscsiDiskMounter) error {
   859  	var lastErr error
   860  	if b.InitIface == b.Iface {
   861  		return fmt.Errorf("iscsi: cannot clone iface with same name: %s", b.InitIface)
   862  	}
   863  	// get pre-configured iface records
   864  	out, err := execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.InitIface, "-o", "show")
   865  	if err != nil {
   866  		lastErr = fmt.Errorf("iscsi: failed to show iface records: %s (%v)", out, err)
   867  		return lastErr
   868  	}
   869  	// parse obtained records
   870  	params, err := parseIscsiadmShow(out)
   871  	if err != nil {
   872  		lastErr = fmt.Errorf("iscsi: failed to parse iface records: %s (%v)", out, err)
   873  		return lastErr
   874  	}
   875  	// update initiatorname
   876  	params["iface.initiatorname"] = b.InitiatorName
   877  	// create new iface
   878  	out, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "new")
   879  	if err != nil {
   880  		exit, ok := err.(utilexec.ExitError)
   881  		if ok && exit.ExitStatus() == iscsiadmErrorSessExists {
   882  			klog.Infof("iscsi: there is a session already logged in with iface %s", b.Iface)
   883  		} else {
   884  			lastErr = fmt.Errorf("iscsi: failed to create new iface: %s (%v)", out, err)
   885  			return lastErr
   886  		}
   887  	}
   888  	// Get and sort keys to maintain a stable iteration order
   889  	var keys []string
   890  	for k := range params {
   891  		keys = append(keys, k)
   892  	}
   893  	sort.Strings(keys)
   894  	// update new iface records
   895  	for _, key := range keys {
   896  		_, err = execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "update", "-n", key, "-v", params[key])
   897  		if err != nil {
   898  			execWithLog(b, "iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete")
   899  			lastErr = fmt.Errorf("iscsi: failed to update iface records: %s (%v). iface(%s) will be used", out, err, b.InitIface)
   900  			break
   901  		}
   902  	}
   903  	return lastErr
   904  }
   905  
   906  // isSessionBusy determines if the iSCSI session is busy by counting both FS and block volumes in use.
   907  func isSessionBusy(host volume.VolumeHost, portal, iqn string) bool {
   908  	fsDir := host.GetPluginDir(iscsiPluginName)
   909  	countFS, err := getVolCount(fsDir, portal, iqn)
   910  	if err != nil {
   911  		klog.Errorf("iscsi: could not determine FS volumes in use: %v", err)
   912  		return true
   913  	}
   914  
   915  	blockDir := host.GetVolumeDevicePluginDir(iscsiPluginName)
   916  	countBlock, err := getVolCount(blockDir, portal, iqn)
   917  	if err != nil {
   918  		klog.Errorf("iscsi: could not determine block volumes in use: %v", err)
   919  		return true
   920  	}
   921  
   922  	return countFS+countBlock > 1
   923  }
   924  
   925  // getVolCount returns the number of volumes in use by the kubelet.
   926  // It does so by counting the number of directories prefixed by the given portal and IQN.
   927  func getVolCount(dir, portal, iqn string) (int, error) {
   928  	// For FileSystem volumes, the topmost dirs are named after the ifaces, e.g., iface-default or iface-127.0.0.1:3260:pv0.
   929  	// For Block volumes, the default topmost dir is volumeDevices.
   930  	contents, err := ioutil.ReadDir(dir)
   931  	if err != nil {
   932  		if os.IsNotExist(err) {
   933  			return 0, nil
   934  		}
   935  		return 0, err
   936  	}
   937  
   938  	// Inside each iface dir, we look for volume dirs prefixed by the given
   939  	// portal + iqn, e.g., 127.0.0.1:3260-iqn.2003-01.io.k8s:e2e.volume-1-lun-2
   940  	var counter int
   941  	for _, c := range contents {
   942  		if !c.IsDir() || c.Name() == config.DefaultKubeletVolumeDevicesDirName {
   943  			continue
   944  		}
   945  
   946  		mounts, err := ioutil.ReadDir(filepath.Join(dir, c.Name()))
   947  		if err != nil {
   948  			return 0, err
   949  		}
   950  
   951  		for _, m := range mounts {
   952  			volumeMount := m.Name()
   953  			prefix := portal + "-" + iqn
   954  			if strings.HasPrefix(volumeMount, prefix) {
   955  				counter++
   956  			}
   957  		}
   958  	}
   959  
   960  	return counter, nil
   961  }
   962  
   963  func ignoreExitCodes(err error, ignoredExitCodes ...int) error {
   964  	exitError, ok := err.(utilexec.ExitError)
   965  	if !ok {
   966  		return err
   967  	}
   968  	for _, code := range ignoredExitCodes {
   969  		if exitError.ExitStatus() == code {
   970  			klog.V(4).Infof("ignored iscsiadm exit code %d", code)
   971  			return nil
   972  		}
   973  	}
   974  	return err
   975  }
   976  
   977  func execWithLog(b iscsiDiskMounter, cmd string, args ...string) (string, error) {
   978  	start := time.Now()
   979  	out, err := b.exec.Command(cmd, args...).CombinedOutput()
   980  	if klogV := klog.V(5); klogV.Enabled() {
   981  		d := time.Since(start)
   982  		klogV.Infof("Executed %s %v in %v, err: %v", cmd, args, d, err)
   983  		klogV.Infof("Output: %s", string(out))
   984  	}
   985  	return string(out), err
   986  }
   987  

View as plain text