cgroups.go

Documentation: edge-infra.dev/pkg/sds/devices/agent/cgroups

     1  //go:build linux
     2  
     3  package cgroups
     4  
     5  import (
     6  	"context"
     7  	"maps"
     8  	"path/filepath"
     9  	"slices"
    10  	"strconv"
    11  
    12  	"github.com/opencontainers/runc/libcontainer/devices"
    13  
    14  	edgedevices "edge-infra.dev/pkg/lib/kernel/devices"
    15  	"edge-infra.dev/pkg/sds/devices/logger"
    16  
    17  	"edge-infra.dev/pkg/lib/kernel/cgroup"
    18  )
    19  
    20  var (
    21  	sysFsCgroupPath = "/sys/fs/cgroup"
    22  	// kubernetesSlice is the kubernetes pod slice for cgroups
    23  	kubernetesSlice = "kubepods.slice"
    24  )
    25  
    26  // default virtual allowed devices (virtual tun, mem, block)
    27  var defaultDeviceRules = []*devices.Rule{
    28  	{Type: devices.CharDevice, Major: 136, Minor: -1, Permissions: "rwm", Allow: true},
    29  	{Type: devices.CharDevice, Major: -1, Minor: -1, Permissions: "m", Allow: true},
    30  	{Type: devices.BlockDevice, Major: -1, Minor: -1, Permissions: "m", Allow: true},
    31  }
    32  
    33  type CgroupRequest interface {
    34  	Apply(ctx context.Context)
    35  }
    36  
    37  type request struct {
    38  	// name of container
    39  	containerName string
    40  	// container id
    41  	containerID string
    42  	// namespace of containers pod
    43  	namespace string
    44  	// device classes that applies to request
    45  	devices map[string]edgedevices.Device
    46  	// list of container cgroups paths to update
    47  	cgroupPath string
    48  	// isContainerizedVM indicates if container is vm compute container
    49  	isContainerizedVM bool
    50  }
    51  
    52  func NewCgroupRequest(name, containerID, namespace, cgroupPath string, devices map[string]edgedevices.Device, isContainerizedVM bool) CgroupRequest {
    53  	return request{
    54  		containerName:     name,
    55  		containerID:       containerID,
    56  		namespace:         namespace,
    57  		cgroupPath:        cgroupPath,
    58  		devices:           devices,
    59  		isContainerizedVM: isContainerizedVM,
    60  	}
    61  }
    62  
    63  // Apply takes the cgroup request and attempts to update the cgroups
    64  // rules using containerds cgroups client
    65  func (req request) Apply(ctx context.Context) {
    66  	log := logger.FromContext(ctx)
    67  
    68  	rules := slices.Clone(defaultDeviceRules)
    69  	rules = append(rules, CgroupRules(ctx, req.devices)...)
    70  	if req.isContainerizedVM {
    71  		rules = addVMDeviceRules(rules)
    72  	}
    73  	log.Debug("applying rules to container", "isVirtualMachine", req.isContainerizedVM, "rules", rules, "devices", slices.Collect(maps.Keys(req.devices)))
    74  
    75  	path := filepath.Join(sysFsCgroupPath, kubernetesSlice, req.cgroupPath)
    76  	if err := cgroup.ApplyCgroups(path, rules); err != nil {
    77  		log.Error("failed to apply cgroups", "error", err)
    78  		return
    79  	}
    80  	log.Info("applied cgroups to container")
    81  }
    82  
    83  // CgroupRules converts list of devices to unix device cgroup rules
    84  func CgroupRules(ctx context.Context, deviceMap map[string]edgedevices.Device) []*devices.Rule {
    85  	log := logger.FromContext(ctx)
    86  	rules := []*devices.Rule{}
    87  	for _, dev := range deviceMap {
    88  		node, err := dev.Node()
    89  		if err != nil {
    90  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node", "sys path", dev.Path())
    91  			continue
    92  		}
    93  		devType, err := node.Type()
    94  		if err != nil || devType == "" {
    95  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node type", "node path", node.Path())
    96  			continue
    97  		}
    98  
    99  		major, exists, err := dev.Property("MAJOR")
   100  		if err != nil || !exists {
   101  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node major number", "node path", node.Path())
   102  			continue
   103  		}
   104  
   105  		minor, exists, err := dev.Property("MINOR")
   106  		if err != nil || !exists {
   107  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node minor number", "node path", node.Path())
   108  			continue
   109  		}
   110  
   111  		majorInt, err := strconv.ParseInt(major, 10, 64)
   112  		if err != nil {
   113  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node major number", "node path", node.Path(), "number", major)
   114  			continue
   115  		}
   116  
   117  		minorInt, err := strconv.ParseInt(minor, 10, 64)
   118  		if err != nil {
   119  			log.Log(ctx, logger.LevelTrace, "failed to fetch device node minor number", "node path", node.Path(), "number", major)
   120  			continue
   121  		}
   122  
   123  		rules = append(rules, &devices.Rule{
   124  			Type:        convertDeviceType(devType),
   125  			Major:       majorInt,
   126  			Minor:       minorInt,
   127  			Permissions: "rwm",
   128  			Allow:       true,
   129  		})
   130  	}
   131  	return rules
   132  }
   133  
   134  // converts device type i.e. "c" (char) to devices.Type
   135  func convertDeviceType(devType string) devices.Type {
   136  	switch devType {
   137  	case string(devices.BlockDevice):
   138  		return devices.BlockDevice
   139  	case string(devices.FifoDevice):
   140  		return devices.FifoDevice
   141  	default:
   142  		return devices.CharDevice
   143  	}
   144  }
   145  
   146  // addVMDeviceRules adds access to pty and vfio devices
   147  func addVMDeviceRules(rules []*devices.Rule) []*devices.Rule {
   148  	// https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/admin-guide/devices.txt?h=v5.14#n2084
   149  	const ptyFirstMajor int64 = 136
   150  	const ptyMajors int64 = 16
   151  
   152  	for i := int64(0); i < ptyMajors; i++ {
   153  		rules = append(rules,
   154  			&devices.Rule{
   155  				Type:        devices.CharDevice,
   156  				Major:       ptyFirstMajor + i,
   157  				Minor:       -1,
   158  				Permissions: "rwm",
   159  				Allow:       true,
   160  			})
   161  	}
   162  
   163  	// /dev/vfio/vfio device
   164  	rules = append(rules, &devices.Rule{
   165  		Type:        devices.CharDevice,
   166  		Major:       10,
   167  		Minor:       196,
   168  		Permissions: "rwm",
   169  		Allow:       true,
   170  	})
   171  	return rules
   172  }
   173
View as plain text