rbd_util.go

Documentation: k8s.io/kubernetes/pkg/volume/rbd

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  //
    18  // utility functions to setup rbd volume
    19  // mainly implement diskManager interface
    20  //
    21  
    22  package rbd
    23  
    24  import (
    25  	"encoding/json"
    26  	"fmt"
    27  	"io/ioutil"
    28  	"os"
    29  	"os/exec"
    30  	"path/filepath"
    31  	"strconv"
    32  	"strings"
    33  	"time"
    34  
    35  	"k8s.io/klog/v2"
    36  	"k8s.io/mount-utils"
    37  	utilexec "k8s.io/utils/exec"
    38  	utilpath "k8s.io/utils/path"
    39  
    40  	v1 "k8s.io/api/core/v1"
    41  	"k8s.io/apimachinery/pkg/api/resource"
    42  	"k8s.io/apimachinery/pkg/util/errors"
    43  	"k8s.io/apimachinery/pkg/util/wait"
    44  	volumehelpers "k8s.io/cloud-provider/volume/helpers"
    45  	nodeutil "k8s.io/component-helpers/node/util"
    46  	"k8s.io/kubernetes/pkg/volume"
    47  	volutil "k8s.io/kubernetes/pkg/volume/util"
    48  )
    49  
    50  const (
    51  	imageWatcherStr = "watcher="
    52  	kubeLockMagic   = "kubelet_lock_magic_"
    53  	// The following three values are used for 30 seconds timeout
    54  	// while waiting for RBD Watcher to expire.
    55  	rbdImageWatcherInitDelay = 1 * time.Second
    56  	rbdImageWatcherFactor    = 1.4
    57  	rbdImageWatcherSteps     = 10
    58  	rbdImageSizeUnitMiB      = 1024 * 1024
    59  )
    60  
    61  // A struct contains rbd image info.
    62  type rbdImageInfo struct {
    63  	pool string
    64  	name string
    65  }
    66  
    67  func getDevFromImageAndPool(pool, image string) (string, bool) {
    68  	device, found := getRbdDevFromImageAndPool(pool, image)
    69  	if found {
    70  		return device, true
    71  	}
    72  	device, found = getNbdDevFromImageAndPool(pool, image)
    73  	if found {
    74  		return device, true
    75  	}
    76  	return "", false
    77  }
    78  
    79  // Search /sys/bus for rbd device that matches given pool and image.
    80  func getRbdDevFromImageAndPool(pool string, image string) (string, bool) {
    81  	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
    82  	sysPath := "/sys/bus/rbd/devices"
    83  	if dirs, err := ioutil.ReadDir(sysPath); err == nil {
    84  		for _, f := range dirs {
    85  			// Pool and name format:
    86  			// see rbd_pool_show() and rbd_name_show() at
    87  			// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
    88  			name := f.Name()
    89  			// First match pool, then match name.
    90  			poolFile := filepath.Join(sysPath, name, "pool")
    91  			poolBytes, err := ioutil.ReadFile(poolFile)
    92  			if err != nil {
    93  				klog.V(4).Infof("error reading %s: %v", poolFile, err)
    94  				continue
    95  			}
    96  			if strings.TrimSpace(string(poolBytes)) != pool {
    97  				klog.V(4).Infof("device %s is not %q: %q", name, pool, string(poolBytes))
    98  				continue
    99  			}
   100  			imgFile := filepath.Join(sysPath, name, "name")
   101  			imgBytes, err := ioutil.ReadFile(imgFile)
   102  			if err != nil {
   103  				klog.V(4).Infof("error reading %s: %v", imgFile, err)
   104  				continue
   105  			}
   106  			if strings.TrimSpace(string(imgBytes)) != image {
   107  				klog.V(4).Infof("device %s is not %q: %q", name, image, string(imgBytes))
   108  				continue
   109  			}
   110  			// Found a match, check if device exists.
   111  			devicePath := "/dev/rbd" + name
   112  			if _, err := os.Lstat(devicePath); err == nil {
   113  				return devicePath, true
   114  			}
   115  		}
   116  	}
   117  	return "", false
   118  }
   119  
   120  func getMaxNbds() (int, error) {
   121  
   122  	// the max number of nbd devices may be found in maxNbdsPath
   123  	// we will check sysfs for possible nbd devices even if this is not available
   124  	maxNbdsPath := "/sys/module/nbd/parameters/nbds_max"
   125  	_, err := os.Lstat(maxNbdsPath)
   126  	if err != nil {
   127  		return 0, fmt.Errorf("rbd-nbd: failed to retrieve max_nbds from %s err: %q", maxNbdsPath, err)
   128  	}
   129  
   130  	klog.V(4).Infof("found nbds max parameters file at %s", maxNbdsPath)
   131  
   132  	maxNbdBytes, err := ioutil.ReadFile(maxNbdsPath)
   133  	if err != nil {
   134  		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds from %s err: %q", maxNbdsPath, err)
   135  	}
   136  
   137  	maxNbds, err := strconv.Atoi(strings.TrimSpace(string(maxNbdBytes)))
   138  	if err != nil {
   139  		return 0, fmt.Errorf("rbd-nbd: failed to read max_nbds err: %q", err)
   140  	}
   141  
   142  	klog.V(4).Infof("rbd-nbd: max_nbds: %d", maxNbds)
   143  	return maxNbds, nil
   144  }
   145  
   146  // Locate any existing rbd-nbd process mapping given a <pool, image>.
   147  // Recent versions of rbd-nbd tool can correctly provide this info using list-mapped
   148  // but older versions of list-mapped don't.
   149  // The implementation below peeks at the command line of nbd bound processes
   150  // to figure out any mapped images.
   151  func getNbdDevFromImageAndPool(pool string, image string) (string, bool) {
   152  	// nbd module exports the pid of serving process in sysfs
   153  	basePath := "/sys/block/nbd"
   154  	// Do not change imgPath format - some tools like rbd-nbd are strict about it.
   155  	imgPath := fmt.Sprintf("%s/%s", pool, image)
   156  
   157  	maxNbds, maxNbdsErr := getMaxNbds()
   158  	if maxNbdsErr != nil {
   159  		klog.V(4).Infof("error reading nbds_max %v", maxNbdsErr)
   160  		return "", false
   161  	}
   162  
   163  	for i := 0; i < maxNbds; i++ {
   164  		nbdPath := basePath + strconv.Itoa(i)
   165  		_, err := os.Lstat(nbdPath)
   166  		if err != nil {
   167  			klog.V(4).Infof("error reading nbd info directory %s: %v", nbdPath, err)
   168  			continue
   169  		}
   170  		pidBytes, err := ioutil.ReadFile(filepath.Join(nbdPath, "pid"))
   171  		if err != nil {
   172  			klog.V(5).Infof("did not find valid pid file in dir %s: %v", nbdPath, err)
   173  			continue
   174  		}
   175  		cmdlineFileName := filepath.Join("/proc", strings.TrimSpace(string(pidBytes)), "cmdline")
   176  		rawCmdline, err := ioutil.ReadFile(cmdlineFileName)
   177  		if err != nil {
   178  			klog.V(4).Infof("failed to read cmdline file %s: %v", cmdlineFileName, err)
   179  			continue
   180  		}
   181  		cmdlineArgs := strings.FieldsFunc(string(rawCmdline), func(r rune) bool {
   182  			return r == '\u0000'
   183  		})
   184  		// Check if this process is mapping a rbd device.
   185  		// Only accepted pattern of cmdline is from execRbdMap:
   186  		// rbd-nbd map pool/image ...
   187  		if len(cmdlineArgs) < 3 || cmdlineArgs[0] != "rbd-nbd" || cmdlineArgs[1] != "map" {
   188  			klog.V(4).Infof("nbd device %s is not used by rbd", nbdPath)
   189  			continue
   190  		}
   191  		if cmdlineArgs[2] != imgPath {
   192  			klog.V(4).Infof("rbd-nbd device %s did not match expected image path: %s with path found: %s",
   193  				nbdPath, imgPath, cmdlineArgs[2])
   194  			continue
   195  		}
   196  		devicePath := filepath.Join("/dev", "nbd"+strconv.Itoa(i))
   197  		if _, err := os.Lstat(devicePath); err != nil {
   198  			klog.Warningf("Stat device %s for imgpath %s failed %v", devicePath, imgPath, err)
   199  			continue
   200  		}
   201  		return devicePath, true
   202  	}
   203  	return "", false
   204  }
   205  
   206  // Stat a path, if it doesn't exist, retry maxRetries times.
   207  func waitForPath(pool, image string, maxRetries int, useNbdDriver bool) (string, bool) {
   208  	for i := 0; i < maxRetries; i++ {
   209  		if i != 0 {
   210  			time.Sleep(time.Second)
   211  		}
   212  		if useNbdDriver {
   213  			if devicePath, found := getNbdDevFromImageAndPool(pool, image); found {
   214  				return devicePath, true
   215  			}
   216  		} else {
   217  			if devicePath, found := getRbdDevFromImageAndPool(pool, image); found {
   218  				return devicePath, true
   219  			}
   220  		}
   221  	}
   222  	return "", false
   223  }
   224  
   225  // Execute command to map a rbd device for mounter.
   226  // rbdCmd is driver dependent and either "rbd" or "rbd-nbd".
   227  func execRbdMap(b rbdMounter, rbdCmd string, mon string) ([]byte, error) {
   228  	// Commandline: rbdCmd map imgPath ...
   229  	// do not change this format - some tools like rbd-nbd are strict about it.
   230  	imgPath := fmt.Sprintf("%s/%s", b.Pool, b.Image)
   231  	if b.Secret != "" {
   232  		return b.exec.Command(rbdCmd,
   233  			"map", imgPath, "--id", b.ID, "-m", mon, "--key="+b.Secret).CombinedOutput()
   234  	}
   235  	return b.exec.Command(rbdCmd,
   236  		"map", imgPath, "--id", b.ID, "-m", mon, "-k", b.Keyring).CombinedOutput()
   237  }
   238  
   239  // Check if rbd-nbd tools are installed.
   240  func checkRbdNbdTools(e utilexec.Interface) bool {
   241  	_, err := e.Command("modprobe", "nbd").CombinedOutput()
   242  	if err != nil {
   243  		klog.V(5).Infof("rbd-nbd: nbd modprobe failed with error %v", err)
   244  		return false
   245  	}
   246  	if _, err := e.Command("rbd-nbd", "--version").CombinedOutput(); err != nil {
   247  		klog.V(5).Infof("rbd-nbd: getting rbd-nbd version failed with error %v", err)
   248  		return false
   249  	}
   250  	klog.V(3).Infof("rbd-nbd tools were found.")
   251  	return true
   252  }
   253  
   254  // Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/mounts/pool-image-image.
   255  func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
   256  	// Backward compatibility for the deprecated format: /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/pool-image-image.
   257  	deprecatedDir := filepath.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
   258  	info, err := os.Stat(deprecatedDir)
   259  	if err == nil && info.IsDir() {
   260  		// The device mount path has already been created with the deprecated format, return it.
   261  		klog.V(5).Infof("Deprecated format path %s found", deprecatedDir)
   262  		return deprecatedDir
   263  	}
   264  	// Return the canonical format path.
   265  	return filepath.Join(host.GetPluginDir(rbdPluginName), volutil.MountsInGlobalPDPath, pool+"-image-"+image)
   266  }
   267  
   268  // Make a directory like /var/lib/kubelet/plugins/kubernetes.io/rbd/volumeDevices/pool-image-image.
   269  func makeVDPDNameInternal(host volume.VolumeHost, pool string, image string) string {
   270  	return filepath.Join(host.GetVolumeDevicePluginDir(rbdPluginName), pool+"-image-"+image)
   271  }
   272  
   273  // rbdUtil implements diskManager interface.
   274  type rbdUtil struct{}
   275  
   276  var _ diskManager = &rbdUtil{}
   277  
   278  // MakeGlobalPDName makes a plugin directory.
   279  func (util *rbdUtil) MakeGlobalPDName(rbd rbd) string {
   280  	return makePDNameInternal(rbd.plugin.host, rbd.Pool, rbd.Image)
   281  }
   282  
   283  // MakeGlobalVDPDName makes a volume device plugin directory.
   284  func (util *rbdUtil) MakeGlobalVDPDName(rbd rbd) string {
   285  	return makeVDPDNameInternal(rbd.plugin.host, rbd.Pool, rbd.Image)
   286  }
   287  
   288  func rbdErrors(runErr, resultErr error) error {
   289  	if err, ok := runErr.(*exec.Error); ok {
   290  		if err.Err == exec.ErrNotFound {
   291  			return fmt.Errorf("rbd: rbd cmd not found")
   292  		}
   293  	}
   294  	return resultErr
   295  }
   296  
   297  // 'rbd' utility builds a comma-separated list of monitor addresses from '-m' /
   298  // '--mon_host` parameter (comma, semi-colon, or white-space delimited monitor
   299  // addresses) and send it to kernel rbd/libceph modules, which can accept
   300  // comma-separated list of monitor addresses (e.g. ip1[:port1][,ip2[:port2]...])
   301  // in their first version in linux (see
   302  // https://github.com/torvalds/linux/blob/602adf400201636e95c3fed9f31fba54a3d7e844/net/ceph/ceph_common.c#L239).
   303  // Also, libceph module chooses monitor randomly, so we can simply pass all
   304  // addresses without randomization (see
   305  // https://github.com/torvalds/linux/blob/602adf400201636e95c3fed9f31fba54a3d7e844/net/ceph/mon_client.c#L132).
   306  func (util *rbdUtil) kernelRBDMonitorsOpt(mons []string) string {
   307  	return strings.Join(mons, ",")
   308  }
   309  
   310  // rbdUnlock releases a lock on image if found.
   311  func (util *rbdUtil) rbdUnlock(b rbdMounter) error {
   312  	var err error
   313  	var output, locker string
   314  	var cmd []byte
   315  	var secretOpt []string
   316  
   317  	if b.Secret != "" {
   318  		secretOpt = []string{"--key=" + b.Secret}
   319  	} else {
   320  		secretOpt = []string{"-k", b.Keyring}
   321  	}
   322  	if len(b.adminID) == 0 {
   323  		b.adminID = b.ID
   324  	}
   325  	if len(b.adminSecret) == 0 {
   326  		b.adminSecret = b.Secret
   327  	}
   328  
   329  	// Construct lock id using host name and a magic prefix.
   330  	hostName, err := nodeutil.GetHostname("")
   331  	if err != nil {
   332  		return err
   333  	}
   334  	lockID := kubeLockMagic + hostName
   335  
   336  	mon := util.kernelRBDMonitorsOpt(b.Mon)
   337  
   338  	// Get the locker name, something like "client.1234".
   339  	args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.ID, "-m", mon}
   340  	args = append(args, secretOpt...)
   341  	cmd, err = b.exec.Command("rbd", args...).CombinedOutput()
   342  	output = string(cmd)
   343  	klog.V(4).Infof("lock list output %q", output)
   344  	if err != nil {
   345  		return err
   346  	}
   347  	ind := strings.LastIndex(output, lockID) - 1
   348  	for i := ind; i >= 0; i-- {
   349  		if output[i] == '\n' {
   350  			locker = output[(i + 1):ind]
   351  			break
   352  		}
   353  	}
   354  
   355  	// Remove a lock if found: rbd lock remove.
   356  	if len(locker) > 0 {
   357  		args := []string{"lock", "remove", b.Image, lockID, locker, "--pool", b.Pool, "--id", b.ID, "-m", mon}
   358  		args = append(args, secretOpt...)
   359  		_, err = b.exec.Command("rbd", args...).CombinedOutput()
   360  		if err == nil {
   361  			klog.V(4).Infof("rbd: successfully remove lock (locker_id: %s) on image: %s/%s with id %s mon %s", lockID, b.Pool, b.Image, b.ID, mon)
   362  		} else {
   363  			klog.Warningf("rbd: failed to remove lock (lockID: %s) on image: %s/%s with id %s mon %s: %v", lockID, b.Pool, b.Image, b.ID, mon, err)
   364  		}
   365  	}
   366  
   367  	return err
   368  }
   369  
   370  // AttachDisk attaches the disk on the node.
   371  func (util *rbdUtil) AttachDisk(b rbdMounter) (string, error) {
   372  	var output []byte
   373  
   374  	globalPDPath := util.MakeGlobalPDName(*b.rbd)
   375  	if pathExists, pathErr := mount.PathExists(globalPDPath); pathErr != nil {
   376  		return "", fmt.Errorf("error checking if path exists: %v", pathErr)
   377  	} else if !pathExists {
   378  		if err := os.MkdirAll(globalPDPath, 0750); err != nil {
   379  			return "", err
   380  		}
   381  	}
   382  
   383  	// Evaluate whether this device was mapped with rbd.
   384  	devicePath, mapped := waitForPath(b.Pool, b.Image, 1 /*maxRetries*/, false /*useNbdDriver*/)
   385  
   386  	// If rbd-nbd tools are found, we will fallback to it should the default krbd driver fail.
   387  	nbdToolsFound := false
   388  
   389  	if !mapped {
   390  		nbdToolsFound = checkRbdNbdTools(b.exec)
   391  		if nbdToolsFound {
   392  			devicePath, mapped = waitForPath(b.Pool, b.Image, 1 /*maxRetries*/, true /*useNbdDriver*/)
   393  		}
   394  	}
   395  
   396  	if !mapped {
   397  		// Currently, we don't acquire advisory lock on image, but for backward
   398  		// compatibility, we need to check if the image is being used by nodes running old kubelet.
   399  		// osd_client_watch_timeout defaults to 30 seconds, if the watcher stays active longer than 30 seconds,
   400  		// rbd image does not get mounted and failure message gets generated.
   401  		backoff := wait.Backoff{
   402  			Duration: rbdImageWatcherInitDelay,
   403  			Factor:   rbdImageWatcherFactor,
   404  			Steps:    rbdImageWatcherSteps,
   405  		}
   406  		needValidUsed := true
   407  		if b.accessModes != nil {
   408  			// If accessModes only contains ReadOnlyMany, we don't need check rbd status of being used.
   409  			if len(b.accessModes) == 1 && b.accessModes[0] == v1.ReadOnlyMany {
   410  				needValidUsed = false
   411  			}
   412  		}
   413  		// If accessModes is nil, the volume is referenced by in-line volume.
   414  		// We can assume the AccessModes to be {"RWO" and "ROX"}, which is what the volume plugin supports.
   415  		// We do not need to consider ReadOnly here, because it is used for VolumeMounts.
   416  
   417  		if needValidUsed {
   418  			err := wait.ExponentialBackoff(backoff, func() (bool, error) {
   419  				used, rbdOutput, err := util.rbdStatus(&b)
   420  				if err != nil {
   421  					return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
   422  				}
   423  				return !used, nil
   424  			})
   425  			// Return error if rbd image has not become available for the specified timeout.
   426  			if err == wait.ErrWaitTimeout {
   427  				return "", fmt.Errorf("rbd image %s/%s is still being used", b.Pool, b.Image)
   428  			}
   429  			// Return error if any other errors were encountered during waiting for the image to become available.
   430  			if err != nil {
   431  				return "", err
   432  			}
   433  		}
   434  
   435  		mon := util.kernelRBDMonitorsOpt(b.Mon)
   436  		klog.V(1).Infof("rbd: map mon %s", mon)
   437  
   438  		_, err := b.exec.Command("modprobe", "rbd").CombinedOutput()
   439  		if err != nil {
   440  			klog.Warningf("rbd: failed to load rbd kernel module:%v", err)
   441  		}
   442  		output, err = execRbdMap(b, "rbd", mon)
   443  		if err != nil {
   444  			if !nbdToolsFound {
   445  				klog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
   446  				return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
   447  			}
   448  			klog.V(3).Infof("rbd: map failed with %v, %s. Retrying with rbd-nbd", err, string(output))
   449  			errList := []error{err}
   450  			outputList := output
   451  			output, err = execRbdMap(b, "rbd-nbd", mon)
   452  			if err != nil {
   453  				errList = append(errList, err)
   454  				outputList = append(outputList, output...)
   455  				return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", errors.NewAggregate(errList), string(outputList))
   456  			}
   457  			devicePath, mapped = waitForPath(b.Pool, b.Image, 10 /*maxRetries*/, true /*useNbdDrive*/)
   458  		} else {
   459  			devicePath, mapped = waitForPath(b.Pool, b.Image, 10 /*maxRetries*/, false /*useNbdDriver*/)
   460  		}
   461  		if !mapped {
   462  			return "", fmt.Errorf("could not map image %s/%s, Timeout after 10s", b.Pool, b.Image)
   463  		}
   464  	}
   465  	return devicePath, nil
   466  }
   467  
   468  // DetachDisk detaches the disk from the node.
   469  // It detaches device from the node if device is provided, and removes the lock
   470  // if there is persisted RBD info under deviceMountPath.
   471  func (util *rbdUtil) DetachDisk(plugin *rbdPlugin, deviceMountPath string, device string) error {
   472  	if len(device) == 0 {
   473  		return fmt.Errorf("DetachDisk failed , device is empty")
   474  	}
   475  
   476  	exec := plugin.host.GetExec(plugin.GetPluginName())
   477  
   478  	var rbdCmd string
   479  
   480  	// Unlike map, we cannot fallthrough for unmap
   481  	// the tool to unmap is based on device type
   482  	if strings.HasPrefix(device, "/dev/nbd") {
   483  		rbdCmd = "rbd-nbd"
   484  	} else {
   485  		rbdCmd = "rbd"
   486  	}
   487  
   488  	// rbd unmap
   489  	output, err := exec.Command(rbdCmd, "unmap", device).CombinedOutput()
   490  	if err != nil {
   491  		return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %s", device, err, string(output)))
   492  	}
   493  	klog.V(3).Infof("rbd: successfully unmap device %s", device)
   494  
   495  	// Currently, we don't persist rbd info on the disk, but for backward
   496  	// compatibility, we need to clean it if found.
   497  	rbdFile := filepath.Join(deviceMountPath, "rbd.json")
   498  	exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, rbdFile)
   499  	if err != nil {
   500  		return err
   501  	}
   502  	if exists {
   503  		klog.V(3).Infof("rbd: old rbd.json is found under %s, cleaning it", deviceMountPath)
   504  		err = util.cleanOldRBDFile(plugin, rbdFile)
   505  		if err != nil {
   506  			klog.Errorf("rbd: failed to clean %s", rbdFile)
   507  			return err
   508  		}
   509  		klog.V(3).Infof("rbd: successfully remove %s", rbdFile)
   510  	}
   511  	return nil
   512  }
   513  
   514  // DetachBlockDisk detaches the disk from the node.
   515  func (util *rbdUtil) DetachBlockDisk(disk rbdDiskUnmapper, mapPath string) error {
   516  
   517  	if pathExists, pathErr := mount.PathExists(mapPath); pathErr != nil {
   518  		return fmt.Errorf("error checking if path exists: %v", pathErr)
   519  	} else if !pathExists {
   520  		klog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
   521  		return nil
   522  	}
   523  	// If we arrive here, device is no longer used, see if we need to logout of the target
   524  	device, err := getBlockVolumeDevice(mapPath)
   525  	if err != nil {
   526  		return err
   527  	}
   528  
   529  	if len(device) == 0 {
   530  		return fmt.Errorf("DetachDisk failed , device is empty")
   531  	}
   532  
   533  	exec := disk.plugin.host.GetExec(disk.plugin.GetPluginName())
   534  
   535  	var rbdCmd string
   536  
   537  	// Unlike map, we cannot fallthrough here.
   538  	// Any nbd device must be unmapped by rbd-nbd
   539  	if strings.HasPrefix(device, "/dev/nbd") {
   540  		rbdCmd = "rbd-nbd"
   541  		klog.V(4).Infof("rbd: using rbd-nbd for unmap function")
   542  	} else {
   543  		rbdCmd = "rbd"
   544  		klog.V(4).Infof("rbd: using rbd for unmap function")
   545  	}
   546  
   547  	// rbd unmap
   548  	output, err := exec.Command(rbdCmd, "unmap", device).CombinedOutput()
   549  	if err != nil {
   550  		return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s, error %v, rbd output: %s", device, err, string(output)))
   551  	}
   552  	klog.V(3).Infof("rbd: successfully unmap device %s", device)
   553  
   554  	return nil
   555  }
   556  
   557  // cleanOldRBDFile read rbd info from rbd.json file and removes lock if found.
   558  // At last, it removes rbd.json file.
   559  func (util *rbdUtil) cleanOldRBDFile(plugin *rbdPlugin, rbdFile string) error {
   560  	mounter := &rbdMounter{
   561  		// util.rbdUnlock needs it to run command.
   562  		rbd: newRBD("", "", "", "", false, plugin, util),
   563  	}
   564  	fp, err := os.Open(rbdFile)
   565  	if err != nil {
   566  		return fmt.Errorf("rbd: open err %s/%s", rbdFile, err)
   567  	}
   568  	defer fp.Close()
   569  
   570  	decoder := json.NewDecoder(fp)
   571  	if err = decoder.Decode(mounter); err != nil {
   572  		return fmt.Errorf("rbd: decode err: %v", err)
   573  	}
   574  
   575  	// Remove rbd lock if found.
   576  	// The disk is not attached to this node anymore, so the lock on image
   577  	// for this node can be removed safely.
   578  	err = util.rbdUnlock(*mounter)
   579  	if err == nil {
   580  		os.Remove(rbdFile)
   581  	}
   582  	return err
   583  }
   584  
   585  // CreateImage creates a RBD image.
   586  func (util *rbdUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDPersistentVolumeSource, size int, err error) {
   587  	var output []byte
   588  	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
   589  	// Convert to MB that rbd defaults on.
   590  	sz, err := volumehelpers.RoundUpToMiBInt(capacity)
   591  	if err != nil {
   592  		return nil, 0, err
   593  	}
   594  	volSz := fmt.Sprintf("%d", sz)
   595  	mon := util.kernelRBDMonitorsOpt(p.Mon)
   596  	if p.rbdMounter.imageFormat == rbdImageFormat2 {
   597  		klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key <masked>", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminID)
   598  	} else {
   599  		klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key <masked>", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminID)
   600  	}
   601  	args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminID, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
   602  	if p.rbdMounter.imageFormat == rbdImageFormat2 {
   603  		// If no image features is provided, it results in empty string
   604  		// which disable all RBD image format 2 features as expected.
   605  		features := strings.Join(p.rbdMounter.imageFeatures, ",")
   606  		args = append(args, "--image-feature", features)
   607  	}
   608  	output, err = p.exec.Command("rbd", args...).CombinedOutput()
   609  
   610  	if err != nil {
   611  		klog.Warningf("failed to create rbd image, output %v", string(output))
   612  		return nil, 0, fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output))
   613  	}
   614  
   615  	return &v1.RBDPersistentVolumeSource{
   616  		CephMonitors: p.rbdMounter.Mon,
   617  		RBDImage:     p.rbdMounter.Image,
   618  		RBDPool:      p.rbdMounter.Pool,
   619  	}, sz, nil
   620  }
   621  
   622  // DeleteImage deletes a RBD image.
   623  func (util *rbdUtil) DeleteImage(p *rbdVolumeDeleter) error {
   624  	var output []byte
   625  	found, rbdOutput, err := util.rbdStatus(p.rbdMounter)
   626  	if err != nil {
   627  		return fmt.Errorf("error %v, rbd output: %v", err, rbdOutput)
   628  	}
   629  	if found {
   630  		klog.Infof("rbd %s is still being used ", p.rbdMounter.Image)
   631  		return fmt.Errorf("rbd image %s/%s is still being used, rbd output: %v", p.rbdMounter.Pool, p.rbdMounter.Image, rbdOutput)
   632  	}
   633  	// rbd rm.
   634  	mon := util.kernelRBDMonitorsOpt(p.rbdMounter.Mon)
   635  	klog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key <masked>", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminID)
   636  	output, err = p.exec.Command("rbd",
   637  		"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminID, "-m", mon, "--key="+p.rbdMounter.adminSecret).CombinedOutput()
   638  	if err == nil {
   639  		return nil
   640  	}
   641  
   642  	klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
   643  	return fmt.Errorf("error %v, rbd output: %v", err, string(output))
   644  }
   645  
   646  // ExpandImage runs rbd resize command to resize the specified image.
   647  func (util *rbdUtil) ExpandImage(rbdExpander *rbdVolumeExpander, oldSize resource.Quantity, newSize resource.Quantity) (resource.Quantity, error) {
   648  	var output []byte
   649  	var err error
   650  
   651  	// Convert to MB that rbd defaults on.
   652  	sz, err := volumehelpers.RoundUpToMiBInt(newSize)
   653  	if err != nil {
   654  		return oldSize, err
   655  	}
   656  
   657  	newVolSz := fmt.Sprintf("%d", sz)
   658  	newSizeQuant := resource.MustParse(fmt.Sprintf("%dMi", sz))
   659  
   660  	// Check the current size of rbd image, if equals to or greater that the new request size, do nothing.
   661  	curSize, infoErr := util.rbdInfo(rbdExpander.rbdMounter)
   662  	if infoErr != nil {
   663  		return oldSize, fmt.Errorf("rbd info failed, error: %v", infoErr)
   664  	}
   665  	if curSize >= sz {
   666  		return newSizeQuant, nil
   667  	}
   668  
   669  	// rbd resize.
   670  	mon := util.kernelRBDMonitorsOpt(rbdExpander.rbdMounter.Mon)
   671  	klog.V(4).Infof("rbd: resize %s using mon %s, pool %s id %s key <masked>", rbdExpander.rbdMounter.Image, mon, rbdExpander.rbdMounter.Pool, rbdExpander.rbdMounter.adminID)
   672  	output, err = rbdExpander.exec.Command("rbd",
   673  		"resize", rbdExpander.rbdMounter.Image, "--size", newVolSz, "--pool", rbdExpander.rbdMounter.Pool, "--id", rbdExpander.rbdMounter.adminID, "-m", mon, "--key="+rbdExpander.rbdMounter.adminSecret).CombinedOutput()
   674  	if err == nil {
   675  		return newSizeQuant, nil
   676  	}
   677  
   678  	klog.Errorf("failed to resize rbd image: %v, command output: %s", err, string(output))
   679  	return oldSize, err
   680  }
   681  
   682  // rbdInfo runs `rbd info` command to get the current image size in MB.
   683  func (util *rbdUtil) rbdInfo(b *rbdMounter) (int, error) {
   684  	var err error
   685  	var output []byte
   686  
   687  	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
   688  	id := b.adminID
   689  	secret := b.adminSecret
   690  	if id == "" {
   691  		id = b.ID
   692  		secret = b.Secret
   693  	}
   694  
   695  	mon := util.kernelRBDMonitorsOpt(b.Mon)
   696  	// cmd "rbd info" get the image info with the following output:
   697  	//
   698  	// # image exists (exit=0)
   699  	// rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08
   700  	//    size 1024 MB in 256 objects
   701  	//    order 22 (4096 kB objects)
   702  	// 	  block_name_prefix: rbd_data.1253ac238e1f29
   703  	//    format: 2
   704  	//    ...
   705  	//
   706  	//  rbd info volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08 --format json
   707  	// {"name":"volume-4a5bcc8b-2b55-46da-ba04-0d3dc5227f08","size":1073741824,"objects":256,"order":22,"object_size":4194304,"block_name_prefix":"rbd_data.1253ac238e1f29","format":2,"features":["layering","exclusive-lock","object-map","fast-diff","deep-flatten"],"flags":[]}
   708  	//
   709  	//
   710  	// # image does not exist (exit=2)
   711  	// rbd: error opening image 1234: (2) No such file or directory
   712  	//
   713  	klog.V(4).Infof("rbd: info %s using mon %s, pool %s id %s key <masked>", b.Image, mon, b.Pool, id)
   714  	output, err = b.exec.Command("rbd",
   715  		"info", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret, "-k=/dev/null", "--format=json").Output()
   716  
   717  	if err, ok := err.(*exec.Error); ok {
   718  		if err.Err == exec.ErrNotFound {
   719  			klog.Errorf("rbd cmd not found")
   720  			// fail fast if rbd command is not found.
   721  			return 0, err
   722  		}
   723  	}
   724  
   725  	// If command never succeed, returns its last error.
   726  	if err != nil {
   727  		return 0, err
   728  	}
   729  
   730  	if len(output) == 0 {
   731  		return 0, fmt.Errorf("can not get image size info %s: %s", b.Image, string(output))
   732  	}
   733  
   734  	return getRbdImageSize(output)
   735  }
   736  
   737  func getRbdImageSize(output []byte) (int, error) {
   738  	info := struct {
   739  		Size int64 `json:"size"`
   740  	}{}
   741  	if err := json.Unmarshal(output, &info); err != nil {
   742  		return 0, fmt.Errorf("parse rbd info output failed: %s, %v", string(output), err)
   743  	}
   744  	return int(info.Size / rbdImageSizeUnitMiB), nil
   745  }
   746  
   747  // rbdStatus runs `rbd status` command to check if there is watcher on the image.
   748  func (util *rbdUtil) rbdStatus(b *rbdMounter) (bool, string, error) {
   749  	var err error
   750  	var output string
   751  	var cmd []byte
   752  
   753  	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
   754  	id := b.adminID
   755  	secret := b.adminSecret
   756  	if id == "" {
   757  		id = b.ID
   758  		secret = b.Secret
   759  	}
   760  
   761  	mon := util.kernelRBDMonitorsOpt(b.Mon)
   762  	// cmd "rbd status" list the rbd client watch with the following output:
   763  	//
   764  	// # there is a watcher (exit=0)
   765  	// Watchers:
   766  	//   watcher=10.16.153.105:0/710245699 client.14163 cookie=1
   767  	//
   768  	// # there is no watcher (exit=0)
   769  	// Watchers: none
   770  	//
   771  	// Otherwise, exit is non-zero, for example:
   772  	//
   773  	// # image does not exist (exit=2)
   774  	// rbd: error opening image kubernetes-dynamic-pvc-<UUID>: (2) No such file or directory
   775  	//
   776  	klog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key <masked>", b.Image, mon, b.Pool, id)
   777  	cmd, err = b.exec.Command("rbd",
   778  		"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", id, "--key="+secret).CombinedOutput()
   779  	output = string(cmd)
   780  
   781  	if err, ok := err.(*exec.Error); ok {
   782  		if err.Err == exec.ErrNotFound {
   783  			klog.Errorf("rbd cmd not found")
   784  			// fail fast if command not found
   785  			return false, output, err
   786  		}
   787  	}
   788  
   789  	// If command never succeed, returns its last error.
   790  	if err != nil {
   791  		return false, output, err
   792  	}
   793  
   794  	if strings.Contains(output, imageWatcherStr) {
   795  		klog.V(4).Infof("rbd: watchers on %s: %s", b.Image, output)
   796  		return true, output, nil
   797  	}
   798  	klog.Warningf("rbd: no watchers on %s", b.Image)
   799  	return false, output, nil
   800  }
   801  
   802  // getRbdImageInfo try to get rbdImageInfo from deviceMountPath.
   803  func getRbdImageInfo(deviceMountPath string) (*rbdImageInfo, error) {
   804  	deviceMountedPathSeps := strings.Split(filepath.Base(deviceMountPath), "-image-")
   805  	if len(deviceMountedPathSeps) != 2 {
   806  		return nil, fmt.Errorf("can't found devicePath for %s ", deviceMountPath)
   807  	}
   808  	return &rbdImageInfo{
   809  		pool: deviceMountedPathSeps[0],
   810  		name: deviceMountedPathSeps[1],
   811  	}, nil
   812  }
   813
View as plain text