scsi.go

Documentation: github.com/Microsoft/hcsshim/internal/uvm

     1  //go:build windows
     2  
     3  package uvm
     4  
     5  import (
     6  	"context"
     7  	"fmt"
     8  	"strings"
     9  
    10  	"github.com/pkg/errors"
    11  	"github.com/sirupsen/logrus"
    12  
    13  	"github.com/Microsoft/hcsshim/internal/hcs/resourcepaths"
    14  	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
    15  	"github.com/Microsoft/hcsshim/internal/log"
    16  	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
    17  	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
    18  	"github.com/Microsoft/hcsshim/internal/security"
    19  	"github.com/Microsoft/hcsshim/internal/wclayer"
    20  )
    21  
    22  // VMAccessType is used to determine the various types of access we can
    23  // grant for a given file.
    24  type VMAccessType int
    25  
    26  const (
    27  	// `VMAccessTypeNoop` indicates no additional access should be given. Note
    28  	// this should be used for layers and gpu vhd where we have given VM group
    29  	// access outside of the shim (containerd for layers, package installation
    30  	// for gpu vhd).
    31  	VMAccessTypeNoop VMAccessType = iota
    32  	// `VMAccessTypeGroup` indicates we should give access to a file for the VM group sid
    33  	VMAccessTypeGroup
    34  	// `VMAccessTypeIndividual` indicates we should give additional access to a file for
    35  	// the running VM only
    36  	VMAccessTypeIndividual
    37  )
    38  
    39  var (
    40  	ErrNoAvailableLocation      = fmt.Errorf("no available location")
    41  	ErrNotAttached              = fmt.Errorf("not attached")
    42  	ErrAlreadyAttached          = fmt.Errorf("already attached")
    43  	ErrNoSCSIControllers        = fmt.Errorf("no SCSI controllers configured for this utility VM")
    44  	ErrTooManyAttachments       = fmt.Errorf("too many SCSI attachments")
    45  	ErrSCSILayerWCOWUnsupported = fmt.Errorf("SCSI attached layers are not supported for WCOW")
    46  )
    47  
    48  // Release frees the resources of the corresponding Scsi Mount
    49  func (sm *SCSIMount) Release(ctx context.Context) error {
    50  	if err := sm.vm.RemoveSCSI(ctx, sm.HostPath); err != nil {
    51  		return fmt.Errorf("failed to remove SCSI device: %s", err)
    52  	}
    53  	return nil
    54  }
    55  
    56  // SCSIMount struct representing a SCSI mount point and the UVM
    57  // it belongs to.
    58  type SCSIMount struct {
    59  	// Utility VM the scsi mount belongs to
    60  	vm *UtilityVM
    61  	// path is the host path to the vhd that is mounted.
    62  	HostPath string
    63  	// path for the uvm
    64  	UVMPath string
    65  	// scsi controller
    66  	Controller int
    67  	// scsi logical unit number
    68  	LUN int32
    69  	// While most VHDs attached to SCSI are scratch spaces, in the case of LCOW
    70  	// when the size is over the size possible to attach to PMEM, we use SCSI for
    71  	// read-only layers. As RO layers are shared, we perform ref-counting.
    72  	isLayer  bool
    73  	refCount uint32
    74  	// specifies if this is an encrypted VHD
    75  	encrypted bool
    76  	// specifies if this is a readonly layer
    77  	readOnly bool
    78  	// "VirtualDisk" or "PassThru" or "ExtensibleVirtualDisk" disk attachment type.
    79  	attachmentType string
    80  	// If attachmentType is "ExtensibleVirtualDisk" then extensibleVirtualDiskType should
    81  	// specify the type of it (for e.g "space" for storage spaces). Otherwise this should be
    82  	// empty.
    83  	extensibleVirtualDiskType string
    84  
    85  	// A channel to wait on while mount of this SCSI disk is in progress.
    86  	waitCh chan struct{}
    87  	// The error field that is set if the mounting of this disk fails. Any other waiters on waitCh
    88  	// can use this waitErr after the channel is closed.
    89  	waitErr error
    90  }
    91  
    92  // addSCSIRequest is an internal struct used to hold all the parameters that are sent to
    93  // the addSCSIActual method.
    94  type addSCSIRequest struct {
    95  	// host path to the disk that should be added as a SCSI disk.
    96  	hostPath string
    97  	// the path inside the uvm at which this disk should show up. Can be empty.
    98  	uvmPath string
    99  	// attachmentType is required and `must` be `VirtualDisk` for vhd/vhdx
   100  	// attachments, `PassThru` for physical disk and `ExtensibleVirtualDisk` for
   101  	// Extensible virtual disks.
   102  	attachmentType string
   103  	// indicates if the VHD is encrypted
   104  	encrypted bool
   105  	// indicates if the attachment should be added read only.
   106  	readOnly bool
   107  	// guestOptions is a slice that contains optional information to pass to the guest
   108  	// service.
   109  	guestOptions []string
   110  	// indicates what access to grant the vm for the hostpath. Only required for
   111  	// `VirtualDisk` and `PassThru` disk types.
   112  	vmAccess VMAccessType
   113  	// `evdType` indicates the type of the extensible virtual disk if `attachmentType`
   114  	// is "ExtensibleVirtualDisk" should be empty otherwise.
   115  	evdType string
   116  }
   117  
   118  // RefCount returns the current refcount for the SCSI mount.
   119  func (sm *SCSIMount) RefCount() uint32 {
   120  	return sm.refCount
   121  }
   122  
   123  func (sm *SCSIMount) logFormat() logrus.Fields {
   124  	return logrus.Fields{
   125  		"HostPath":                  sm.HostPath,
   126  		"UVMPath":                   sm.UVMPath,
   127  		"isLayer":                   sm.isLayer,
   128  		"refCount":                  sm.refCount,
   129  		"Controller":                sm.Controller,
   130  		"LUN":                       sm.LUN,
   131  		"ExtensibleVirtualDiskType": sm.extensibleVirtualDiskType,
   132  	}
   133  }
   134  
   135  func newSCSIMount(
   136  	uvm *UtilityVM,
   137  	hostPath string,
   138  	uvmPath string,
   139  	attachmentType string,
   140  	evdType string,
   141  	refCount uint32,
   142  	controller int,
   143  	lun int32,
   144  	readOnly bool,
   145  	encrypted bool,
   146  ) *SCSIMount {
   147  	return &SCSIMount{
   148  		vm:                        uvm,
   149  		HostPath:                  hostPath,
   150  		UVMPath:                   uvmPath,
   151  		refCount:                  refCount,
   152  		Controller:                controller,
   153  		LUN:                       int32(lun),
   154  		encrypted:                 encrypted,
   155  		readOnly:                  readOnly,
   156  		attachmentType:            attachmentType,
   157  		extensibleVirtualDiskType: evdType,
   158  		waitCh:                    make(chan struct{}),
   159  	}
   160  }
   161  
   162  // allocateSCSISlot finds the next available slot on the
   163  // SCSI controllers associated with a utility VM to use.
   164  // Lock must be held when calling this function
   165  func (uvm *UtilityVM) allocateSCSISlot(ctx context.Context) (int, int, error) {
   166  	for controller := 0; controller < int(uvm.scsiControllerCount); controller++ {
   167  		for lun, sm := range uvm.scsiLocations[controller] {
   168  			// If sm is nil, we have found an open slot so we allocate a new SCSIMount
   169  			if sm == nil {
   170  				return controller, lun, nil
   171  			}
   172  		}
   173  	}
   174  	return -1, -1, ErrNoAvailableLocation
   175  }
   176  
   177  func (uvm *UtilityVM) deallocateSCSIMount(ctx context.Context, sm *SCSIMount) {
   178  	uvm.m.Lock()
   179  	defer uvm.m.Unlock()
   180  	if sm != nil {
   181  		log.G(ctx).WithFields(sm.logFormat()).Debug("removed SCSI location")
   182  		uvm.scsiLocations[sm.Controller][sm.LUN] = nil
   183  	}
   184  }
   185  
   186  // Lock must be held when calling this function.
   187  func (uvm *UtilityVM) findSCSIAttachment(ctx context.Context, findThisHostPath string) (*SCSIMount, error) {
   188  	for _, luns := range uvm.scsiLocations {
   189  		for _, sm := range luns {
   190  			if sm != nil && sm.HostPath == findThisHostPath {
   191  				log.G(ctx).WithFields(sm.logFormat()).Debug("found SCSI location")
   192  				return sm, nil
   193  			}
   194  		}
   195  	}
   196  	return nil, ErrNotAttached
   197  }
   198  
   199  // RemoveSCSI removes a SCSI disk from a utility VM.
   200  func (uvm *UtilityVM) RemoveSCSI(ctx context.Context, hostPath string) error {
   201  	uvm.m.Lock()
   202  	defer uvm.m.Unlock()
   203  
   204  	if uvm.scsiControllerCount == 0 {
   205  		return ErrNoSCSIControllers
   206  	}
   207  
   208  	// Make sure it is actually attached
   209  	sm, err := uvm.findSCSIAttachment(ctx, hostPath)
   210  	if err != nil {
   211  		return err
   212  	}
   213  
   214  	sm.refCount--
   215  	if sm.refCount > 0 {
   216  		return nil
   217  	}
   218  
   219  	scsiModification := &hcsschema.ModifySettingRequest{
   220  		RequestType:  guestrequest.RequestTypeRemove,
   221  		ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, guestrequest.ScsiControllerGuids[sm.Controller], sm.LUN),
   222  	}
   223  
   224  	var verity *guestresource.DeviceVerityInfo
   225  	if v, iErr := readVeritySuperBlock(ctx, hostPath); iErr != nil {
   226  		log.G(ctx).WithError(iErr).WithField("hostPath", sm.HostPath).Debug("unable to read dm-verity information from VHD")
   227  	} else {
   228  		if v != nil {
   229  			log.G(ctx).WithFields(logrus.Fields{
   230  				"hostPath":   hostPath,
   231  				"rootDigest": v.RootDigest,
   232  			}).Debug("removing SCSI with dm-verity")
   233  		}
   234  		verity = v
   235  	}
   236  
   237  	// Include the GuestRequest so that the GCS ejects the disk cleanly if the
   238  	// disk was attached/mounted
   239  	//
   240  	// Note: We always send a guest eject even if there is no UVM path in lcow
   241  	// so that we synchronize the guest state. This seems to always avoid SCSI
   242  	// related errors if this index quickly reused by another container.
   243  	if uvm.operatingSystem == "windows" && sm.UVMPath != "" {
   244  		scsiModification.GuestRequest = guestrequest.ModificationRequest{
   245  			ResourceType: guestresource.ResourceTypeMappedVirtualDisk,
   246  			RequestType:  guestrequest.RequestTypeRemove,
   247  			Settings: guestresource.WCOWMappedVirtualDisk{
   248  				ContainerPath: sm.UVMPath,
   249  				Lun:           sm.LUN,
   250  			},
   251  		}
   252  	} else {
   253  		scsiModification.GuestRequest = guestrequest.ModificationRequest{
   254  			ResourceType: guestresource.ResourceTypeMappedVirtualDisk,
   255  			RequestType:  guestrequest.RequestTypeRemove,
   256  			Settings: guestresource.LCOWMappedVirtualDisk{
   257  				MountPath:  sm.UVMPath, // May be blank in attach-only
   258  				Lun:        uint8(sm.LUN),
   259  				Controller: uint8(sm.Controller),
   260  				VerityInfo: verity,
   261  			},
   262  		}
   263  	}
   264  
   265  	if err := uvm.modify(ctx, scsiModification); err != nil {
   266  		return fmt.Errorf("failed to remove SCSI disk %s from container %s: %s", hostPath, uvm.id, err)
   267  	}
   268  	log.G(ctx).WithFields(sm.logFormat()).Debug("removed SCSI location")
   269  	uvm.scsiLocations[sm.Controller][sm.LUN] = nil
   270  	return nil
   271  }
   272  
   273  // AddSCSI adds a SCSI disk to a utility VM at the next available location. This
   274  // function should be called for adding a scratch layer, a read-only layer as an
   275  // alternative to VPMEM, or for other VHD mounts.
   276  //
   277  // `hostPath` is required and must point to a vhd/vhdx path.
   278  //
   279  // `uvmPath` is optional. If not provided, no guest request will be made
   280  //
   281  // `readOnly` set to `true` if the vhd/vhdx should be attached read only.
   282  //
   283  // `encrypted` set to `true` if the vhd/vhdx should be attached in encrypted mode.
   284  // The device will be formatted, so this option must be used only when creating
   285  // scratch vhd/vhdx.
   286  //
   287  // `guestOptions` is a slice that contains optional information to pass
   288  // to the guest service
   289  //
   290  // `vmAccess` indicates what access to grant the vm for the hostpath
   291  func (uvm *UtilityVM) AddSCSI(
   292  	ctx context.Context,
   293  	hostPath string,
   294  	uvmPath string,
   295  	readOnly bool,
   296  	encrypted bool,
   297  	guestOptions []string,
   298  	vmAccess VMAccessType,
   299  ) (*SCSIMount, error) {
   300  	addReq := &addSCSIRequest{
   301  		hostPath:       hostPath,
   302  		uvmPath:        uvmPath,
   303  		attachmentType: "VirtualDisk",
   304  		readOnly:       readOnly,
   305  		encrypted:      encrypted,
   306  		guestOptions:   guestOptions,
   307  		vmAccess:       vmAccess,
   308  	}
   309  	return uvm.addSCSIActual(ctx, addReq)
   310  }
   311  
   312  // AddSCSIPhysicalDisk attaches a physical disk from the host directly to the
   313  // Utility VM at the next available location.
   314  //
   315  // `hostPath` is required and `likely` start's with `\\.\PHYSICALDRIVE`.
   316  //
   317  // `uvmPath` is optional if a guest mount is not requested.
   318  //
   319  // `readOnly` set to `true` if the physical disk should be attached read only.
   320  //
   321  // `guestOptions` is a slice that contains optional information to pass
   322  // to the guest service
   323  func (uvm *UtilityVM) AddSCSIPhysicalDisk(ctx context.Context, hostPath, uvmPath string, readOnly bool, guestOptions []string) (*SCSIMount, error) {
   324  	addReq := &addSCSIRequest{
   325  		hostPath:       hostPath,
   326  		uvmPath:        uvmPath,
   327  		attachmentType: "PassThru",
   328  		readOnly:       readOnly,
   329  		guestOptions:   guestOptions,
   330  		vmAccess:       VMAccessTypeIndividual,
   331  	}
   332  	return uvm.addSCSIActual(ctx, addReq)
   333  }
   334  
   335  // AddSCSIExtensibleVirtualDisk adds an extensible virtual disk as a SCSI mount
   336  // to the utility VM at the next available location. All such disks which are not actual virtual disks
   337  // but provide the same SCSI interface are added to the UVM as Extensible Virtual disks.
   338  //
   339  // `hostPath` is required. Depending on the type of the extensible virtual disk the format of `hostPath` can
   340  // be different.
   341  // For example, in case of storage spaces the host path must be in the
   342  // `evd://space/{storage_pool_unique_ID}{virtual_disk_unique_ID}` format.
   343  //
   344  // `uvmPath` must be provided in order to be able to use this disk in a container.
   345  //
   346  // `readOnly` set to `true` if the virtual disk should be attached read only.
   347  //
   348  // `vmAccess` indicates what access to grant the vm for the hostpath
   349  func (uvm *UtilityVM) AddSCSIExtensibleVirtualDisk(ctx context.Context, hostPath, uvmPath string, readOnly bool) (*SCSIMount, error) {
   350  	if uvmPath == "" {
   351  		return nil, errors.New("uvmPath can not be empty for extensible virtual disk")
   352  	}
   353  	evdType, mountPath, err := ParseExtensibleVirtualDiskPath(hostPath)
   354  	if err != nil {
   355  		return nil, err
   356  	}
   357  	addReq := &addSCSIRequest{
   358  		hostPath:       mountPath,
   359  		uvmPath:        uvmPath,
   360  		attachmentType: "ExtensibleVirtualDisk",
   361  		readOnly:       readOnly,
   362  		guestOptions:   []string{},
   363  		vmAccess:       VMAccessTypeIndividual,
   364  		evdType:        evdType,
   365  	}
   366  	return uvm.addSCSIActual(ctx, addReq)
   367  }
   368  
   369  // addSCSIActual is the implementation behind the external functions AddSCSI,
   370  // AddSCSIPhysicalDisk, AddSCSIExtensibleVirtualDisk.
   371  //
   372  // We are in control of everything ourselves. Hence we have ref- counting and
   373  // so-on tracking what SCSI locations are available or used.
   374  //
   375  // Returns result from calling modify with the given scsi mount
   376  func (uvm *UtilityVM) addSCSIActual(ctx context.Context, addReq *addSCSIRequest) (_ *SCSIMount, err error) {
   377  	sm, existed, err := uvm.allocateSCSIMount(
   378  		ctx,
   379  		addReq.readOnly,
   380  		addReq.encrypted,
   381  		addReq.hostPath,
   382  		addReq.uvmPath,
   383  		addReq.attachmentType,
   384  		addReq.evdType,
   385  		addReq.vmAccess,
   386  	)
   387  	if err != nil {
   388  		return nil, err
   389  	}
   390  
   391  	if existed {
   392  		// another mount request might be in progress, wait for it to finish and if that operation
   393  		// fails return that error.
   394  		<-sm.waitCh
   395  		if sm.waitErr != nil {
   396  			return nil, sm.waitErr
   397  		}
   398  		return sm, nil
   399  	}
   400  
   401  	// This is the first goroutine to add this disk, close the waitCh after we are done.
   402  	defer func() {
   403  		if err != nil {
   404  			uvm.deallocateSCSIMount(ctx, sm)
   405  		}
   406  
   407  		// error must be set _before_ the channel is closed.
   408  		sm.waitErr = err
   409  		close(sm.waitCh)
   410  	}()
   411  
   412  	SCSIModification := &hcsschema.ModifySettingRequest{
   413  		RequestType: guestrequest.RequestTypeAdd,
   414  		Settings: hcsschema.Attachment{
   415  			Path:                      sm.HostPath,
   416  			Type_:                     addReq.attachmentType,
   417  			ReadOnly:                  addReq.readOnly,
   418  			ExtensibleVirtualDiskType: addReq.evdType,
   419  		},
   420  		ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, guestrequest.ScsiControllerGuids[sm.Controller], sm.LUN),
   421  	}
   422  
   423  	if sm.UVMPath != "" {
   424  		guestReq := guestrequest.ModificationRequest{
   425  			ResourceType: guestresource.ResourceTypeMappedVirtualDisk,
   426  			RequestType:  guestrequest.RequestTypeAdd,
   427  		}
   428  
   429  		if uvm.operatingSystem == "windows" {
   430  			guestReq.Settings = guestresource.WCOWMappedVirtualDisk{
   431  				ContainerPath: sm.UVMPath,
   432  				Lun:           sm.LUN,
   433  			}
   434  		} else {
   435  			var verity *guestresource.DeviceVerityInfo
   436  			if v, iErr := readVeritySuperBlock(ctx, sm.HostPath); iErr != nil {
   437  				log.G(ctx).WithError(iErr).WithField("hostPath", sm.HostPath).Debug("unable to read dm-verity information from VHD")
   438  			} else {
   439  				if v != nil {
   440  					log.G(ctx).WithFields(logrus.Fields{
   441  						"hostPath":   sm.HostPath,
   442  						"rootDigest": v.RootDigest,
   443  					}).Debug("adding SCSI with dm-verity")
   444  				}
   445  				verity = v
   446  			}
   447  
   448  			guestReq.Settings = guestresource.LCOWMappedVirtualDisk{
   449  				MountPath:  sm.UVMPath,
   450  				Lun:        uint8(sm.LUN),
   451  				Controller: uint8(sm.Controller),
   452  				ReadOnly:   addReq.readOnly,
   453  				Encrypted:  addReq.encrypted,
   454  				Options:    addReq.guestOptions,
   455  				VerityInfo: verity,
   456  			}
   457  		}
   458  		SCSIModification.GuestRequest = guestReq
   459  	}
   460  
   461  	if err := uvm.modify(ctx, SCSIModification); err != nil {
   462  		return nil, fmt.Errorf("failed to modify UVM with new SCSI mount: %s", err)
   463  	}
   464  	return sm, nil
   465  }
   466  
   467  // allocateSCSIMount grants vm access to hostpath and increments the ref count of an existing scsi
   468  // device or allocates a new one if not already present.
   469  // Returns the resulting *SCSIMount, a bool indicating if the scsi device was already present,
   470  // and error if any.
   471  func (uvm *UtilityVM) allocateSCSIMount(
   472  	ctx context.Context,
   473  	readOnly bool,
   474  	encrypted bool,
   475  	hostPath string,
   476  	uvmPath string,
   477  	attachmentType string,
   478  	evdType string,
   479  	vmAccess VMAccessType,
   480  ) (*SCSIMount, bool, error) {
   481  	if attachmentType != "ExtensibleVirtualDisk" {
   482  		// Ensure the utility VM has access
   483  		err := grantAccess(ctx, uvm.id, hostPath, vmAccess)
   484  		if err != nil {
   485  			return nil, false, errors.Wrapf(err, "failed to grant VM access for SCSI mount")
   486  		}
   487  	}
   488  	// We must hold the lock throughout the lookup (findSCSIAttachment) until
   489  	// after the possible allocation (allocateSCSISlot) has been completed to ensure
   490  	// there isn't a race condition for it being attached by another thread between
   491  	// these two operations.
   492  	uvm.m.Lock()
   493  	defer uvm.m.Unlock()
   494  	if sm, err := uvm.findSCSIAttachment(ctx, hostPath); err == nil {
   495  		sm.refCount++
   496  		return sm, true, nil
   497  	}
   498  
   499  	controller, lun, err := uvm.allocateSCSISlot(ctx)
   500  	if err != nil {
   501  		return nil, false, err
   502  	}
   503  
   504  	uvm.scsiLocations[controller][lun] = newSCSIMount(
   505  		uvm,
   506  		hostPath,
   507  		uvmPath,
   508  		attachmentType,
   509  		evdType,
   510  		1,
   511  		controller,
   512  		int32(lun),
   513  		readOnly,
   514  		encrypted,
   515  	)
   516  
   517  	log.G(ctx).WithFields(uvm.scsiLocations[controller][lun].logFormat()).Debug("allocated SCSI mount")
   518  
   519  	return uvm.scsiLocations[controller][lun], false, nil
   520  }
   521  
   522  // ScratchEncryptionEnabled is a getter for `uvm.encryptScratch`.
   523  //
   524  // Returns true if the scratch disks should be encrypted, false otherwise.
   525  func (uvm *UtilityVM) ScratchEncryptionEnabled() bool {
   526  	return uvm.encryptScratch
   527  }
   528  
   529  // grantAccess helper function to grant access to a file for the vm or vm group
   530  func grantAccess(ctx context.Context, uvmID string, hostPath string, vmAccess VMAccessType) error {
   531  	switch vmAccess {
   532  	case VMAccessTypeGroup:
   533  		log.G(ctx).WithField("path", hostPath).Debug("granting vm group access")
   534  		return security.GrantVmGroupAccess(hostPath)
   535  	case VMAccessTypeIndividual:
   536  		return wclayer.GrantVmAccess(ctx, uvmID, hostPath)
   537  	}
   538  	return nil
   539  }
   540  
   541  // ParseExtensibleVirtualDiskPath parses the evd path provided in the config.
   542  // extensible virtual disk path has format "evd://<evdType>/<evd-mount-path>"
   543  // this function parses that and returns the `evdType` and `evd-mount-path`.
   544  func ParseExtensibleVirtualDiskPath(hostPath string) (evdType, mountPath string, err error) {
   545  	trimmedPath := strings.TrimPrefix(hostPath, "evd://")
   546  	separatorIndex := strings.Index(trimmedPath, "/")
   547  	if separatorIndex <= 0 {
   548  		return "", "", errors.Errorf("invalid extensible vhd path: %s", hostPath)
   549  	}
   550  	return trimmedPath[:separatorIndex], trimmedPath[separatorIndex+1:], nil
   551  }
   552
View as plain text