devices.go

Documentation: github.com/Microsoft/hcsshim/internal/hcsoci

     1  //go:build windows
     2  
     3  package hcsoci
     4  
     5  import (
     6  	"context"
     7  	"encoding/json"
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  	"strconv"
    12  
    13  	specs "github.com/opencontainers/runtime-spec/specs-go"
    14  	"github.com/pkg/errors"
    15  
    16  	"github.com/Microsoft/hcsshim/internal/devices"
    17  	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
    18  	"github.com/Microsoft/hcsshim/internal/log"
    19  	"github.com/Microsoft/hcsshim/internal/oci"
    20  	"github.com/Microsoft/hcsshim/internal/resources"
    21  	"github.com/Microsoft/hcsshim/internal/uvm"
    22  	"github.com/Microsoft/hcsshim/osversion"
    23  	"github.com/Microsoft/hcsshim/pkg/annotations"
    24  )
    25  
    26  const deviceUtilExeName = "device-util.exe"
    27  
    28  // getSpecKernelDrivers gets any device drivers specified on the spec.
    29  // Drivers are optional, therefore do not return an error if none are on the spec.
    30  func getSpecKernelDrivers(annots map[string]string) ([]string, error) {
    31  	drivers := oci.ParseAnnotationCommaSeparated(annotations.VirtualMachineKernelDrivers, annots)
    32  	for _, driver := range drivers {
    33  		if _, err := os.Stat(driver); err != nil {
    34  			return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver)
    35  		}
    36  	}
    37  	return drivers, nil
    38  }
    39  
    40  // getDeviceExtensionPaths gets any device extensions paths specified on the spec.
    41  // device extensions are optional, therefore if none are on the spec, do not return an error.
    42  func getDeviceExtensionPaths(annots map[string]string) ([]string, error) {
    43  	extensions := oci.ParseAnnotationCommaSeparated(annotations.DeviceExtensions, annots)
    44  	for _, ext := range extensions {
    45  		if _, err := os.Stat(ext); err != nil {
    46  			return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext)
    47  		}
    48  	}
    49  	return extensions, nil
    50  }
    51  
    52  // getGPUVHDPath gets the gpu vhd path from the shim options or uses the default if no
    53  // shim option is set. Right now we only support Nvidia gpus, so this will default to
    54  // a gpu vhd with nvidia files
    55  func getGPUVHDPath(annot map[string]string) (string, error) {
    56  	gpuVHDPath, ok := annot[annotations.GPUVHDPath]
    57  	if !ok || gpuVHDPath == "" {
    58  		return "", errors.New("no gpu vhd specified")
    59  	}
    60  	if _, err := os.Stat(gpuVHDPath); err != nil {
    61  		return "", errors.Wrapf(err, "failed to find gpu support vhd %s", gpuVHDPath)
    62  	}
    63  	return gpuVHDPath, nil
    64  }
    65  
    66  // getDeviceUtilHostPath is a simple helper function to find the host path of the device-util tool
    67  func getDeviceUtilHostPath() string {
    68  	return filepath.Join(filepath.Dir(os.Args[0]), deviceUtilExeName)
    69  }
    70  
    71  func isDeviceExtensionsSupported() bool {
    72  	// device extensions support was added from LTSC2022 (20348) onwards.
    73  	return osversion.Build() >= osversion.LTSC2022
    74  }
    75  
    76  // getDeviceExtensions is a helper function to read the files at `extensionPaths` and unmarshal the contents
    77  // into a `hcsshema.DeviceExtension` to be added to a container's hcs create document.
    78  func getDeviceExtensions(annotations map[string]string) (*hcsschema.ContainerDefinitionDevice, error) {
    79  	extensionPaths, err := getDeviceExtensionPaths(annotations)
    80  	if err != nil {
    81  		return nil, err
    82  	}
    83  
    84  	if len(extensionPaths) == 0 {
    85  		return nil, nil
    86  	}
    87  
    88  	if !isDeviceExtensionsSupported() {
    89  		return nil, fmt.Errorf("device extensions are not supported on this build (%d)", osversion.Build())
    90  	}
    91  
    92  	results := &hcsschema.ContainerDefinitionDevice{
    93  		DeviceExtension: []hcsschema.DeviceExtension{},
    94  	}
    95  	for _, extensionPath := range extensionPaths {
    96  		data, err := os.ReadFile(extensionPath)
    97  		if err != nil {
    98  			return nil, errors.Wrapf(err, "failed to read extension file at %s", extensionPath)
    99  		}
   100  		extension := hcsschema.DeviceExtension{}
   101  		if err := json.Unmarshal(data, &extension); err != nil {
   102  			return nil, errors.Wrapf(err, "failed to unmarshal extension file at %s", extensionPath)
   103  		}
   104  		results.DeviceExtension = append(results.DeviceExtension, extension)
   105  	}
   106  	return results, nil
   107  }
   108  
   109  // handleAssignedDevicesWindows does all of the work to setup the hosting UVM, assign in devices
   110  // specified on the spec, and install any necessary, specified kernel drivers into the UVM.
   111  //
   112  // Drivers must be installed after the target devices are assigned into the UVM.
   113  // This ordering allows us to guarantee that driver installation on a device in the UVM is completed
   114  // before we attempt to create a container.
   115  func handleAssignedDevicesWindows(
   116  	ctx context.Context,
   117  	vm *uvm.UtilityVM,
   118  	annotations map[string]string,
   119  	specDevs []specs.WindowsDevice) (resultDevs []specs.WindowsDevice, closers []resources.ResourceCloser, err error) {
   120  	defer func() {
   121  		if err != nil {
   122  			// best effort clean up allocated resources on failure
   123  			for _, r := range closers {
   124  				if releaseErr := r.Release(ctx); releaseErr != nil {
   125  					log.G(ctx).WithError(releaseErr).Error("failed to release container resource")
   126  				}
   127  			}
   128  			closers = nil
   129  			resultDevs = nil
   130  		}
   131  	}()
   132  
   133  	// install the device util tool in the UVM
   134  	toolHostPath := getDeviceUtilHostPath()
   135  	options := vm.DefaultVSMBOptions(true)
   136  	toolsShare, err := vm.AddVSMB(ctx, toolHostPath, options)
   137  	if err != nil {
   138  		return nil, closers, fmt.Errorf("failed to add VSMB share to utility VM for path %+v: %s", toolHostPath, err)
   139  	}
   140  	closers = append(closers, toolsShare)
   141  	deviceUtilPath, err := vm.GetVSMBUvmPath(ctx, toolHostPath, true)
   142  	if err != nil {
   143  		return nil, closers, err
   144  	}
   145  
   146  	// assign device into UVM and create corresponding spec windows devices
   147  	for _, d := range specDevs {
   148  		pciID, index := getDeviceInfoFromPath(d.ID)
   149  		vpciCloser, locationPaths, err := devices.AddDevice(ctx, vm, d.IDType, pciID, index, deviceUtilPath)
   150  		if err != nil {
   151  			return nil, nil, err
   152  		}
   153  		closers = append(closers, vpciCloser)
   154  		for _, value := range locationPaths {
   155  			specDev := specs.WindowsDevice{
   156  				ID:     value,
   157  				IDType: uvm.VPCILocationPathIDType,
   158  			}
   159  			log.G(ctx).WithField("parsed devices", specDev).Info("added windows device to spec")
   160  			resultDevs = append(resultDevs, specDev)
   161  		}
   162  	}
   163  
   164  	return resultDevs, closers, nil
   165  }
   166  
   167  // handleAssignedDevicesLCOW does all of the work to setup the hosting UVM, assign in devices
   168  // specified on the spec
   169  //
   170  // For LCOW, drivers must be installed before the target devices are assigned into the UVM so they
   171  // can be linked on arrival.
   172  func handleAssignedDevicesLCOW(
   173  	ctx context.Context,
   174  	vm *uvm.UtilityVM,
   175  	annotations map[string]string,
   176  	specDevs []specs.WindowsDevice) (resultDevs []specs.WindowsDevice, closers []resources.ResourceCloser, err error) {
   177  	defer func() {
   178  		if err != nil {
   179  			// best effort clean up allocated resources on failure
   180  			for _, r := range closers {
   181  				if releaseErr := r.Release(ctx); releaseErr != nil {
   182  					log.G(ctx).WithError(releaseErr).Error("failed to release container resource")
   183  				}
   184  			}
   185  			closers = nil
   186  			resultDevs = nil
   187  		}
   188  	}()
   189  
   190  	gpuPresent := false
   191  
   192  	// assign device into UVM and create corresponding spec windows devices
   193  	for _, d := range specDevs {
   194  		switch d.IDType {
   195  		case uvm.VPCIDeviceIDType, uvm.VPCIDeviceIDTypeLegacy, uvm.GPUDeviceIDType:
   196  			gpuPresent = gpuPresent || d.IDType == uvm.GPUDeviceIDType
   197  			pciID, index := getDeviceInfoFromPath(d.ID)
   198  			vpci, err := vm.AssignDevice(ctx, pciID, index, "")
   199  			if err != nil {
   200  				return resultDevs, closers, errors.Wrapf(err, "failed to assign device %s, function %d to pod %s", pciID, index, vm.ID())
   201  			}
   202  			closers = append(closers, vpci)
   203  
   204  			// update device ID on the spec to the assigned device's resulting vmbus guid so gcs knows which devices to
   205  			// map into the container
   206  			d.ID = vpci.VMBusGUID
   207  			resultDevs = append(resultDevs, d)
   208  		default:
   209  			return resultDevs, closers, errors.Errorf("specified device %s has unsupported type %s", d.ID, d.IDType)
   210  		}
   211  	}
   212  
   213  	if gpuPresent {
   214  		gpuSupportVhdPath, err := getGPUVHDPath(annotations)
   215  		if err != nil {
   216  			return resultDevs, closers, errors.Wrapf(err, "failed to add gpu vhd to %v", vm.ID())
   217  		}
   218  		// gpuvhd must be granted VM Group access.
   219  		driverCloser, err := devices.InstallDrivers(ctx, vm, gpuSupportVhdPath, true)
   220  		if err != nil {
   221  			return resultDevs, closers, err
   222  		}
   223  		if driverCloser != nil {
   224  			closers = append(closers, driverCloser)
   225  		}
   226  	}
   227  
   228  	return resultDevs, closers, nil
   229  }
   230  
   231  // addSpecGuestDrivers is a helper function to install kernel drivers specified on a spec into the guest
   232  func addSpecGuestDrivers(ctx context.Context, vm *uvm.UtilityVM, annotations map[string]string) (closers []resources.ResourceCloser, err error) {
   233  	defer func() {
   234  		if err != nil {
   235  			// best effort clean up allocated resources on failure
   236  			for _, r := range closers {
   237  				if releaseErr := r.Release(ctx); releaseErr != nil {
   238  					log.G(ctx).WithError(releaseErr).Error("failed to release container resource")
   239  				}
   240  			}
   241  		}
   242  	}()
   243  
   244  	// get the spec specified kernel drivers and install them on the UVM
   245  	drivers, err := getSpecKernelDrivers(annotations)
   246  	if err != nil {
   247  		return closers, err
   248  	}
   249  	for _, d := range drivers {
   250  		driverCloser, err := devices.InstallDrivers(ctx, vm, d, false)
   251  		if err != nil {
   252  			return closers, err
   253  		}
   254  		if driverCloser != nil {
   255  			closers = append(closers, driverCloser)
   256  		}
   257  	}
   258  	return closers, err
   259  }
   260  
   261  func getDeviceInfoFromPath(rawDevicePath string) (string, uint16) {
   262  	indexString := filepath.Base(rawDevicePath)
   263  	index, err := strconv.ParseUint(indexString, 10, 16)
   264  	if err == nil {
   265  		// we have a vf index
   266  		return filepath.Dir(rawDevicePath), uint16(index)
   267  	}
   268  	// otherwise, just use default index and full device ID given
   269  	return rawDevicePath, 0
   270  }
   271
View as plain text