scsi.go

Documentation: github.com/Microsoft/hcsshim/internal/guest/storage/scsi

     1  //go:build linux
     2  // +build linux
     3  
     4  package scsi
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"os"
    10  	"path"
    11  	"path/filepath"
    12  	"strconv"
    13  	"strings"
    14  	"time"
    15  
    16  	"github.com/pkg/errors"
    17  	"go.opencensus.io/trace"
    18  	"golang.org/x/sys/unix"
    19  
    20  	"github.com/Microsoft/hcsshim/internal/guest/storage"
    21  	"github.com/Microsoft/hcsshim/internal/guest/storage/crypt"
    22  	dm "github.com/Microsoft/hcsshim/internal/guest/storage/devicemapper"
    23  	"github.com/Microsoft/hcsshim/internal/log"
    24  	"github.com/Microsoft/hcsshim/internal/oc"
    25  	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
    26  	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
    27  )
    28  
    29  // Test dependencies.
    30  var (
    31  	osMkdirAll  = os.MkdirAll
    32  	osRemoveAll = os.RemoveAll
    33  	unixMount   = unix.Mount
    34  
    35  	// controllerLunToName is stubbed to make testing `Mount` easier.
    36  	controllerLunToName = ControllerLunToName
    37  	// createVerityTarget is stubbed for unit testing `Mount`.
    38  	createVerityTarget = dm.CreateVerityTarget
    39  	// removeDevice is stubbed for unit testing `Mount`.
    40  	removeDevice = dm.RemoveDevice
    41  	// encryptDevice is stubbed for unit testing `mount`
    42  	encryptDevice = crypt.EncryptDevice
    43  	// cleanupCryptDevice is stubbed for unit testing `mount`
    44  	cleanupCryptDevice = crypt.CleanupCryptDevice
    45  	// storageUnmountPath is stubbed for unit testing `unmount`
    46  	storageUnmountPath = storage.UnmountPath
    47  )
    48  
    49  const (
    50  	scsiDevicesPath  = "/sys/bus/scsi/devices"
    51  	vmbusDevicesPath = "/sys/bus/vmbus/devices"
    52  	verityDeviceFmt  = "dm-verity-scsi-contr%d-lun%d-%s"
    53  	cryptDeviceFmt   = "dm-crypt-scsi-contr%d-lun%d"
    54  )
    55  
    56  // ActualControllerNumber retrieves the actual controller number assigned to a SCSI controller
    57  // with number `passedController`.
    58  // When HCS creates the UVM it adds 4 SCSI controllers to the UVM but the 1st SCSI
    59  // controller according to HCS can actually show up as 2nd, 3rd or 4th controller inside
    60  // the UVM. So the i'th controller from HCS' perspective could actually be j'th controller
    61  // inside the UVM. However, we can refer to the SCSI controllers with their GUIDs (that
    62  // are hardcoded) and then using that GUID find out the SCSI controller number inside the
    63  // guest. This function does exactly that.
    64  func ActualControllerNumber(_ context.Context, passedController uint8) (uint8, error) {
    65  	// find the controller number by looking for a file named host<N> (e.g host1, host3 etc.)
    66  	// `N` is the controller number.
    67  	// Full file path would be /sys/bus/vmbus/devices/<controller-guid>/host<N>.
    68  	controllerDirPath := path.Join(vmbusDevicesPath, guestrequest.ScsiControllerGuids[passedController])
    69  	entries, err := os.ReadDir(controllerDirPath)
    70  	if err != nil {
    71  		return 0, err
    72  	}
    73  
    74  	for _, entry := range entries {
    75  		baseName := path.Base(entry.Name())
    76  		if !strings.HasPrefix(baseName, "host") {
    77  			continue
    78  		}
    79  		controllerStr := baseName[len("host"):]
    80  		controllerNum, err := strconv.ParseUint(controllerStr, 10, 8)
    81  		if err != nil {
    82  			return 0, fmt.Errorf("failed to parse controller number from %s: %w", baseName, err)
    83  		}
    84  		return uint8(controllerNum), nil
    85  	}
    86  	return 0, fmt.Errorf("host<N> directory not found inside %s", controllerDirPath)
    87  }
    88  
    89  // Mount creates a mount from the SCSI device on `controller` index `lun` to
    90  // `target`
    91  //
    92  // `target` will be created. On mount failure the created `target` will be
    93  // automatically cleaned up.
    94  //
    95  // If `encrypted` is set to true, the SCSI device will be encrypted using
    96  // dm-crypt.
    97  func Mount(
    98  	ctx context.Context,
    99  	controller,
   100  	lun uint8,
   101  	target string,
   102  	readonly bool,
   103  	encrypted bool,
   104  	options []string,
   105  	verityInfo *guestresource.DeviceVerityInfo) (err error) {
   106  	spnCtx, span := oc.StartSpan(ctx, "scsi::Mount")
   107  	defer span.End()
   108  	defer func() { oc.SetSpanStatus(span, err) }()
   109  
   110  	span.AddAttributes(
   111  		trace.Int64Attribute("controller", int64(controller)),
   112  		trace.Int64Attribute("lun", int64(lun)))
   113  
   114  	source, err := controllerLunToName(spnCtx, controller, lun)
   115  	if err != nil {
   116  		return err
   117  	}
   118  
   119  	if readonly {
   120  		var deviceHash string
   121  		if verityInfo != nil {
   122  			deviceHash = verityInfo.RootDigest
   123  		}
   124  
   125  		if verityInfo != nil {
   126  			dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, deviceHash)
   127  			if source, err = createVerityTarget(spnCtx, source, dmVerityName, verityInfo); err != nil {
   128  				return err
   129  			}
   130  			defer func() {
   131  				if err != nil {
   132  					if err := removeDevice(dmVerityName); err != nil {
   133  						log.G(spnCtx).WithError(err).WithField("verityTarget", dmVerityName).Debug("failed to cleanup verity target")
   134  					}
   135  				}
   136  			}()
   137  		}
   138  	}
   139  
   140  	if err := osMkdirAll(target, 0700); err != nil {
   141  		return err
   142  	}
   143  	defer func() {
   144  		if err != nil {
   145  			_ = osRemoveAll(target)
   146  		}
   147  	}()
   148  
   149  	// we only care about readonly mount option when mounting the device
   150  	var flags uintptr
   151  	data := ""
   152  	if readonly {
   153  		flags |= unix.MS_RDONLY
   154  		data = "noload"
   155  	}
   156  
   157  	mountType := "ext4"
   158  	if encrypted {
   159  		cryptDeviceName := fmt.Sprintf(cryptDeviceFmt, controller, lun)
   160  		encryptedSource, err := encryptDevice(spnCtx, source, cryptDeviceName)
   161  		if err != nil {
   162  			// todo (maksiman): add better retry logic, similar to how SCSI device mounts are
   163  			// retried on unix.ENOENT and unix.ENXIO. The retry should probably be on an
   164  			// error message rather than actual error, because we shell-out to cryptsetup.
   165  			time.Sleep(500 * time.Millisecond)
   166  			if encryptedSource, err = encryptDevice(spnCtx, source, cryptDeviceName); err != nil {
   167  				return fmt.Errorf("failed to mount encrypted device %s: %w", source, err)
   168  			}
   169  		}
   170  		source = encryptedSource
   171  		mountType = "xfs"
   172  	}
   173  
   174  	for {
   175  		if err := unixMount(source, target, mountType, flags, data); err != nil {
   176  			// The `source` found by controllerLunToName can take some time
   177  			// before its actually available under `/dev/sd*`. Retry while we
   178  			// wait for `source` to show up.
   179  			if errors.Is(err, unix.ENOENT) || errors.Is(err, unix.ENXIO) {
   180  				select {
   181  				case <-ctx.Done():
   182  					log.G(ctx).Warnf("mount system call failed with %s, context timed out while retrying", err)
   183  					return err
   184  				default:
   185  					time.Sleep(10 * time.Millisecond)
   186  					continue
   187  				}
   188  			}
   189  			return err
   190  		}
   191  		break
   192  	}
   193  
   194  	// remount the target to account for propagation flags
   195  	_, pgFlags, _ := storage.ParseMountOptions(options)
   196  	if len(pgFlags) != 0 {
   197  		for _, pg := range pgFlags {
   198  			if err := unixMount(target, target, "", pg, ""); err != nil {
   199  				return err
   200  			}
   201  		}
   202  	}
   203  
   204  	return nil
   205  }
   206  
   207  // Unmount SCSI device mounted at `target`. Cleanup associated dm-verity and
   208  // dm-crypt devices when necessary.
   209  func Unmount(
   210  	ctx context.Context,
   211  	controller,
   212  	lun uint8,
   213  	target string,
   214  	encrypted bool,
   215  	verityInfo *guestresource.DeviceVerityInfo,
   216  ) (err error) {
   217  	ctx, span := oc.StartSpan(ctx, "scsi::Unmount")
   218  	defer span.End()
   219  	defer func() { oc.SetSpanStatus(span, err) }()
   220  
   221  	span.AddAttributes(
   222  		trace.Int64Attribute("controller", int64(controller)),
   223  		trace.Int64Attribute("lun", int64(lun)),
   224  		trace.StringAttribute("target", target))
   225  
   226  	// unmount target
   227  	if err := storageUnmountPath(ctx, target, true); err != nil {
   228  		return errors.Wrapf(err, "unmount failed: %s", target)
   229  	}
   230  
   231  	if verityInfo != nil {
   232  		dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, verityInfo.RootDigest)
   233  		if err := removeDevice(dmVerityName); err != nil {
   234  			// Ignore failures, since the path has been unmounted at this point.
   235  			log.G(ctx).WithError(err).Debugf("failed to remove dm verity target: %s", dmVerityName)
   236  		}
   237  	}
   238  
   239  	if encrypted {
   240  		dmCryptName := fmt.Sprintf(cryptDeviceFmt, controller, lun)
   241  		if err := cleanupCryptDevice(dmCryptName); err != nil {
   242  			return fmt.Errorf("failed to cleanup dm-crypt target %s: %w", dmCryptName, err)
   243  		}
   244  	}
   245  
   246  	return nil
   247  }
   248  
   249  // ControllerLunToName finds the `/dev/sd*` path to the SCSI device on
   250  // `controller` index `lun`.
   251  func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string, err error) {
   252  	ctx, span := oc.StartSpan(ctx, "scsi::ControllerLunToName")
   253  	defer span.End()
   254  	defer func() { oc.SetSpanStatus(span, err) }()
   255  
   256  	span.AddAttributes(
   257  		trace.Int64Attribute("controller", int64(controller)),
   258  		trace.Int64Attribute("lun", int64(lun)))
   259  
   260  	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
   261  	// Devices matching the given SCSI code should each have a subdirectory
   262  	// under /sys/bus/scsi/devices/<scsiID>/block.
   263  	blockPath := filepath.Join(scsiDevicesPath, scsiID, "block")
   264  	var deviceNames []os.DirEntry
   265  	for {
   266  		deviceNames, err = os.ReadDir(blockPath)
   267  		if err != nil && !os.IsNotExist(err) {
   268  			return "", err
   269  		}
   270  		if len(deviceNames) == 0 {
   271  			select {
   272  			case <-ctx.Done():
   273  				return "", ctx.Err()
   274  			default:
   275  				time.Sleep(time.Millisecond * 10)
   276  				continue
   277  			}
   278  		}
   279  		break
   280  	}
   281  
   282  	if len(deviceNames) > 1 {
   283  		return "", errors.Errorf("more than one block device could match SCSI ID \"%s\"", scsiID)
   284  	}
   285  
   286  	devicePath := filepath.Join("/dev", deviceNames[0].Name())
   287  	log.G(ctx).WithField("devicePath", devicePath).Debug("found device path")
   288  	return devicePath, nil
   289  }
   290  
   291  // UnplugDevice finds the SCSI device on `controller` index `lun` and issues a
   292  // guest initiated unplug.
   293  //
   294  // If the device is not attached returns no error.
   295  func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
   296  	_, span := oc.StartSpan(ctx, "scsi::UnplugDevice")
   297  	defer span.End()
   298  	defer func() { oc.SetSpanStatus(span, err) }()
   299  
   300  	span.AddAttributes(
   301  		trace.Int64Attribute("controller", int64(controller)),
   302  		trace.Int64Attribute("lun", int64(lun)))
   303  
   304  	scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
   305  	f, err := os.OpenFile(filepath.Join(scsiDevicesPath, scsiID, "delete"), os.O_WRONLY, 0644)
   306  	if err != nil {
   307  		if os.IsNotExist(err) {
   308  			return nil
   309  		}
   310  		return err
   311  	}
   312  	defer f.Close()
   313  
   314  	if _, err := f.Write([]byte("1\n")); err != nil {
   315  		return err
   316  	}
   317  	return nil
   318  }
   319
View as plain text