vpmem_mapped.go

Documentation: github.com/Microsoft/hcsshim/internal/uvm

     1  //go:build windows
     2  
     3  package uvm
     4  
     5  import (
     6  	"context"
     7  	"fmt"
     8  	"os"
     9  
    10  	"github.com/pkg/errors"
    11  	"github.com/sirupsen/logrus"
    12  
    13  	"github.com/Microsoft/hcsshim/internal/hcs/resourcepaths"
    14  	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
    15  	"github.com/Microsoft/hcsshim/internal/log"
    16  	"github.com/Microsoft/hcsshim/internal/memory"
    17  	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
    18  	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
    19  )
    20  
    21  const (
    22  	PageSize             = 0x1000
    23  	MaxMappedDeviceCount = 1024
    24  )
    25  
    26  const lcowPackedVPMemLayerFmt = "/run/layers/p%d-%d-%d"
    27  
    28  type mappedDeviceInfo struct {
    29  	vPMemInfoDefault
    30  	mappedRegion memory.MappedRegion
    31  	sizeInBytes  uint64
    32  }
    33  
    34  type vPMemInfoMulti struct {
    35  	memory.PoolAllocator
    36  	maxSize              uint64
    37  	maxMappedDeviceCount uint32
    38  	mappings             map[string]*mappedDeviceInfo
    39  }
    40  
    41  func newVPMemMappedDevice(hostPath, uvmPath string, sizeBytes uint64, memReg memory.MappedRegion) *mappedDeviceInfo {
    42  	return &mappedDeviceInfo{
    43  		vPMemInfoDefault: vPMemInfoDefault{
    44  			hostPath: hostPath,
    45  			uvmPath:  uvmPath,
    46  			refCount: 1,
    47  		},
    48  		mappedRegion: memReg,
    49  		sizeInBytes:  sizeBytes,
    50  	}
    51  }
    52  
    53  func newPackedVPMemDevice() *vPMemInfoMulti {
    54  	return &vPMemInfoMulti{
    55  		PoolAllocator:        memory.NewPoolMemoryAllocator(),
    56  		maxSize:              DefaultVPMemSizeBytes,
    57  		mappings:             make(map[string]*mappedDeviceInfo),
    58  		maxMappedDeviceCount: MaxMappedDeviceCount,
    59  	}
    60  }
    61  
    62  func pageAlign(t uint64) uint64 {
    63  	if t%PageSize == 0 {
    64  		return t
    65  	}
    66  	return (t/PageSize + 1) * PageSize
    67  }
    68  
    69  // newMappedVPMemModifyRequest creates an hcsschema.ModifySettingsRequest to modify VPMem devices/mappings
    70  // for the multi-mapping setup
    71  func newMappedVPMemModifyRequest(
    72  	ctx context.Context,
    73  	rType guestrequest.RequestType,
    74  	deviceNumber uint32,
    75  	md *mappedDeviceInfo,
    76  	uvm *UtilityVM,
    77  ) (*hcsschema.ModifySettingRequest, error) {
    78  	guestSettings := guestresource.LCOWMappedVPMemDevice{
    79  		DeviceNumber: deviceNumber,
    80  		MountPath:    md.uvmPath,
    81  		MappingInfo: &guestresource.LCOWVPMemMappingInfo{
    82  			DeviceOffsetInBytes: md.mappedRegion.Offset(),
    83  			DeviceSizeInBytes:   md.sizeInBytes,
    84  		},
    85  	}
    86  
    87  	if verity, err := readVeritySuperBlock(ctx, md.hostPath); err != nil {
    88  		log.G(ctx).WithError(err).WithField("hostPath", md.hostPath).Debug("unable to read dm-verity information from VHD")
    89  	} else {
    90  		log.G(ctx).WithFields(logrus.Fields{
    91  			"hostPath":   md.hostPath,
    92  			"rootDigest": verity.RootDigest,
    93  		}).Debug("adding multi-mapped VPMem with dm-verity")
    94  		guestSettings.VerityInfo = verity
    95  	}
    96  
    97  	request := &hcsschema.ModifySettingRequest{
    98  		RequestType: rType,
    99  		GuestRequest: guestrequest.ModificationRequest{
   100  			ResourceType: guestresource.ResourceTypeVPMemDevice,
   101  			RequestType:  rType,
   102  			Settings:     guestSettings,
   103  		},
   104  	}
   105  
   106  	pmem := uvm.vpmemDevicesMultiMapped[deviceNumber]
   107  	switch rType {
   108  	case guestrequest.RequestTypeAdd:
   109  		if pmem == nil {
   110  			request.Settings = hcsschema.VirtualPMemDevice{
   111  				ReadOnly:    true,
   112  				HostPath:    md.hostPath,
   113  				ImageFormat: "Vhd1",
   114  			}
   115  			request.ResourcePath = fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, deviceNumber)
   116  		} else {
   117  			request.Settings = hcsschema.VirtualPMemMapping{
   118  				HostPath:    md.hostPath,
   119  				ImageFormat: "Vhd1",
   120  			}
   121  			request.ResourcePath = fmt.Sprintf(resourcepaths.VPMemDeviceResourceFormat, deviceNumber, md.mappedRegion.Offset())
   122  		}
   123  	case guestrequest.RequestTypeRemove:
   124  		if pmem == nil {
   125  			return nil, errors.Errorf("no device found at location %d", deviceNumber)
   126  		}
   127  		request.ResourcePath = fmt.Sprintf(resourcepaths.VPMemDeviceResourceFormat, deviceNumber, md.mappedRegion.Offset())
   128  	default:
   129  		return nil, errors.New("unsupported request type")
   130  	}
   131  
   132  	log.G(ctx).WithFields(logrus.Fields{
   133  		"deviceNumber": deviceNumber,
   134  		"hostPath":     md.hostPath,
   135  		"uvmPath":      md.uvmPath,
   136  	}).Debugf("new mapped VPMem modify request: %v", request)
   137  	return request, nil
   138  }
   139  
   140  // mapVHDLayer adds `device` to mappings
   141  func (pmem *vPMemInfoMulti) mapVHDLayer(ctx context.Context, device *mappedDeviceInfo) (err error) {
   142  	if md, ok := pmem.mappings[device.hostPath]; ok {
   143  		md.refCount++
   144  		return nil
   145  	}
   146  
   147  	log.G(ctx).WithFields(logrus.Fields{
   148  		"hostPath":     device.hostPath,
   149  		"mountPath":    device.uvmPath,
   150  		"deviceOffset": device.mappedRegion.Offset(),
   151  		"deviceSize":   device.sizeInBytes,
   152  	}).Debug("mapped new device")
   153  
   154  	pmem.mappings[device.hostPath] = device
   155  	return nil
   156  }
   157  
   158  // unmapVHDLayer removes mapped device with `hostPath` from mappings and releases allocated memory
   159  func (pmem *vPMemInfoMulti) unmapVHDLayer(ctx context.Context, hostPath string) (err error) {
   160  	dev, ok := pmem.mappings[hostPath]
   161  	if !ok {
   162  		return ErrNotAttached
   163  	}
   164  
   165  	if dev.refCount > 1 {
   166  		dev.refCount--
   167  		return nil
   168  	}
   169  
   170  	if err := pmem.Release(dev.mappedRegion); err != nil {
   171  		return err
   172  	}
   173  	log.G(ctx).WithFields(logrus.Fields{
   174  		"hostPath": dev.hostPath,
   175  	}).Debugf("Done releasing resources: %s", dev.hostPath)
   176  	delete(pmem.mappings, hostPath)
   177  	return nil
   178  }
   179  
   180  // findVPMemMappedDevice finds a VHD device that's been mapped on VPMem surface
   181  func (uvm *UtilityVM) findVPMemMappedDevice(ctx context.Context, hostPath string) (uint32, *mappedDeviceInfo, error) {
   182  	for i := uint32(0); i < uvm.vpmemMaxCount; i++ {
   183  		vi := uvm.vpmemDevicesMultiMapped[i]
   184  		if vi != nil {
   185  			if vhd, ok := vi.mappings[hostPath]; ok {
   186  				log.G(ctx).WithFields(logrus.Fields{
   187  					"deviceNumber": i,
   188  					"hostPath":     hostPath,
   189  					"uvmPath":      vhd.uvmPath,
   190  					"refCount":     vhd.refCount,
   191  					"deviceSize":   vhd.sizeInBytes,
   192  					"deviceOffset": vhd.mappedRegion.Offset(),
   193  				}).Debug("found mapped VHD")
   194  				return i, vhd, nil
   195  			}
   196  		}
   197  	}
   198  	return 0, nil, ErrNotAttached
   199  }
   200  
   201  // allocateNextVPMemMappedDeviceLocation allocates a memory region with a minimum offset on the VPMem surface,
   202  // where the device with a given `devSize` can be mapped.
   203  func (uvm *UtilityVM) allocateNextVPMemMappedDeviceLocation(ctx context.Context, devSize uint64) (uint32, memory.MappedRegion, error) {
   204  	// device size has to be page aligned
   205  	devSize = pageAlign(devSize)
   206  
   207  	for i := uint32(0); i < uvm.vpmemMaxCount; i++ {
   208  		pmem := uvm.vpmemDevicesMultiMapped[i]
   209  		if pmem == nil {
   210  			pmem = newPackedVPMemDevice()
   211  			uvm.vpmemDevicesMultiMapped[i] = pmem
   212  		}
   213  
   214  		if len(pmem.mappings) >= int(pmem.maxMappedDeviceCount) {
   215  			continue
   216  		}
   217  
   218  		reg, err := pmem.Allocate(devSize)
   219  		if err != nil {
   220  			continue
   221  		}
   222  		log.G(ctx).WithFields(logrus.Fields{
   223  			"deviceNumber": i,
   224  			"deviceOffset": reg.Offset(),
   225  			"deviceSize":   devSize,
   226  		}).Debug("found offset for mapped VHD on an existing VPMem device")
   227  		return i, reg, nil
   228  	}
   229  	return 0, nil, ErrNoAvailableLocation
   230  }
   231  
   232  // addVPMemMappedDevice adds container layer as a mapped device, first mapped device is added as a regular
   233  // VPMem device, but subsequent additions will call into mapping APIs
   234  //
   235  // Lock MUST be held when calling this function
   236  func (uvm *UtilityVM) addVPMemMappedDevice(ctx context.Context, hostPath string) (_ string, err error) {
   237  	if _, dev, err := uvm.findVPMemMappedDevice(ctx, hostPath); err == nil {
   238  		dev.refCount++
   239  		return dev.uvmPath, nil
   240  	}
   241  
   242  	st, err := os.Stat(hostPath)
   243  	if err != nil {
   244  		return "", err
   245  	}
   246  	// NOTE: On the guest side devSize is used to create a device mapper linear target, which is then used to create
   247  	// device mapper verity target. Since the dm-verity hash device is appended after ext4 data, we need the full size
   248  	// on disk (minus VHD footer), otherwise the resulting linear target will have hash device truncated and verity
   249  	// target creation will fail as a result.
   250  	devSize := pageAlign(uint64(st.Size()))
   251  	deviceNumber, memReg, err := uvm.allocateNextVPMemMappedDeviceLocation(ctx, devSize)
   252  	if err != nil {
   253  		return "", err
   254  	}
   255  	defer func() {
   256  		if err != nil {
   257  			pmem := uvm.vpmemDevicesMultiMapped[deviceNumber]
   258  			if err := pmem.Release(memReg); err != nil {
   259  				log.G(ctx).WithError(err).Debugf("failed to reclaim pmem region: %s", err)
   260  			}
   261  		}
   262  	}()
   263  
   264  	uvmPath := fmt.Sprintf(lcowPackedVPMemLayerFmt, deviceNumber, memReg.Offset(), devSize)
   265  	md := newVPMemMappedDevice(hostPath, uvmPath, devSize, memReg)
   266  	modification, err := newMappedVPMemModifyRequest(ctx, guestrequest.RequestTypeAdd, deviceNumber, md, uvm)
   267  	if err := uvm.modify(ctx, modification); err != nil {
   268  		return "", errors.Errorf("uvm::addVPMemMappedDevice: failed to modify utility VM configuration: %s", err)
   269  	}
   270  	defer func() {
   271  		if err != nil {
   272  			rmRequest, _ := newMappedVPMemModifyRequest(ctx, guestrequest.RequestTypeRemove, deviceNumber, md, uvm)
   273  			if err := uvm.modify(ctx, rmRequest); err != nil {
   274  				log.G(ctx).WithError(err).Debugf("failed to rollback modification")
   275  			}
   276  		}
   277  	}()
   278  
   279  	pmem := uvm.vpmemDevicesMultiMapped[deviceNumber]
   280  	if err := pmem.mapVHDLayer(ctx, md); err != nil {
   281  		return "", errors.Wrapf(err, "failed to update internal state")
   282  	}
   283  	return uvmPath, nil
   284  }
   285  
   286  // removeVPMemMappedDevice removes a mapped container layer. The VPMem device itself
   287  // is never removed after being added.
   288  //
   289  // The bug occurs when we try to clean up a mapped VHD at non-zero offset.
   290  // What happens is, the device-mapper target that corresponds to the mapped
   291  // layer VHD is unmounted and removed inside the guest, however removal on
   292  // the host through HCS API fails with "not found", because HCS API doesn't
   293  // allow removal of a VPMem if it doesn't have a mapped device at offset 0, this
   294  // results in the reference not being decreased and hcsshim "thinking" that the
   295  // layer is still present. Next time this layer is used by a different
   296  // container, it appears as if the layer is still mounted inside the guest as
   297  // well, which results in overlayfs mount failures.
   298  //
   299  // Lock MUST be held when calling this function
   300  func (uvm *UtilityVM) removeVPMemMappedDevice(ctx context.Context, hostPath string) error {
   301  	devNum, md, err := uvm.findVPMemMappedDevice(ctx, hostPath)
   302  	if err != nil {
   303  		return err
   304  	}
   305  	if md.refCount > 1 {
   306  		md.refCount--
   307  		return nil
   308  	}
   309  
   310  	modification, err := newMappedVPMemModifyRequest(ctx, guestrequest.RequestTypeRemove, devNum, md, uvm)
   311  	if err != nil {
   312  		return err
   313  	}
   314  
   315  	if err := uvm.modify(ctx, modification); err != nil {
   316  		return errors.Errorf("failed to remove packed VPMem %s from UVM %s: %s", md.hostPath, uvm.id, err)
   317  	}
   318  
   319  	pmem := uvm.vpmemDevicesMultiMapped[devNum]
   320  	if err := pmem.unmapVHDLayer(ctx, hostPath); err != nil {
   321  		log.G(ctx).WithError(err).Debugf("failed unmapping VHD layer %s", hostPath)
   322  	}
   323  	if len(pmem.mappings) == 0 {
   324  		uvm.vpmemDevicesMultiMapped[devNum] = nil
   325  	}
   326  	return nil
   327  }
   328
View as plain text