package agent import ( "context" "errors" "fmt" "path/filepath" "strings" "edge-infra.dev/pkg/lib/kernel/devices" cc "edge-infra.dev/pkg/sds/devices/agent/common" "edge-infra.dev/pkg/sds/devices/class" dsv1 "edge-infra.dev/pkg/sds/devices/k8s/apis/v1" "github.com/containerd/containerd/containers" typeurl "github.com/containerd/typeurl/v2" specs "github.com/opencontainers/runtime-spec/specs-go" "edge-infra.dev/pkg/sds/devices/agent/cgroups" "edge-infra.dev/pkg/sds/devices/agent/common" "edge-infra.dev/pkg/sds/devices/logger" ) var ( // errMissingContainerID is thrown when container has no ID attached ErrMissingContainerID = errors.New("missing container id") ) const ( annContainerName = "io.kubernetes.container.name" annPodName = "io.kubernetes.pod.name" annPodNamespace = "io.kubernetes.pod.namespace" burstableQos = "kubepods-burstable" bestEffortQos = "kubepods-besteffort" ) var cgroupRequestFn = cgroups.NewCgroupRequest // ApplyCgroupsToContainer accepts a container update and attempts to update any container that requires an update to its cgroups func ApplyCgroupsToContainer(ctx context.Context, requestID string, ctr *containers.Container, classes map[string]*dsv1.DeviceClass) { log := logger.FromContext(ctx) ctrName := ctr.Labels[annContainerName] podName := ctr.Labels[annPodName] podNamespace := ctr.Labels[annPodNamespace] ctrLog := log.With("requestId", requestID, "pod", podName, "namespace", podNamespace, "containerId", ctr.ID, "container", ctrName) spec := &specs.Spec{} if err := typeurl.UnmarshalTo(ctr.Spec, spec); err != nil { ctrLog.Error("error unmarshalling container spec", "error", err) return } cgroupPath, err := containerCGroupPath(spec.Linux.CgroupsPath) if err != nil { ctrLog.Error("invalid cgroup path", "error", err, "path", spec.Linux.CgroupsPath) return } // populate with devices from requested classes requestedDevices := map[string]devices.Device{} requestedClasses := []string{} for className := range ctr.Labels { if _, ok := classes[className]; !ok { continue } requestedClasses = append(requestedClasses, className) for _, dev := range classes[className].DeviceIter() { requestedDevices[dev.Path()] = dev } } isContainerVM := isContainerVirtualMachine(ctr) ctrCtx := withContainerLogger(ctx, requestID, ctr, requestedClasses, fullCgroupPath(cgroupPath)) cgroupRequestFn(ctrName, ctr.ID, podNamespace, cgroupPath, requestedDevices, isContainerVM).Apply(ctrCtx) } // isContainerVirtualMachine returns true if the container is a containerized vm func isContainerVirtualMachine(ctr *containers.Container) bool { isComputeContainer := ctr.Labels[cc.AnnContainerName] == "compute" requestsKVMResource := false for k := range ctr.Labels { if !class.IsDeviceClass(k) { continue } if k == class.FmtClassLabel("kvm") { requestsKVMResource = true } } return isComputeContainer && requestsKVMResource } // withContainerLogger given a container will return its logger set with the log level // from the annotation device-system.edge.ncr.com/log-level. func withContainerLogger(ctx context.Context, requestID string, ctr *containers.Container, requestedClasses []string, cgroupPath string) context.Context { logLevel := ctr.Labels[common.AnnDeviceLogLevel] ctrName := ctr.Labels[annContainerName] podName := ctr.Labels[annPodName] podNamespace := ctr.Labels[annPodNamespace] opts := []logger.Option{ logger.WithLevel(logger.ToLevel(logLevel)), } return logger.IntoContext(ctx, logger.New(opts...).WithGroup(ctrName).With("requestId", requestID, "container", ctrName, "containerId", ctr.ID, "pod", podName, "namespace", podNamespace, "requestedClasses", requestedClasses, "cgroupPath", cgroupPath)) } // containerCGroupPath will generate the full group path for the container func containerCGroupPath(cgroupPath string) (string, error) { splitPath := strings.Split(cgroupPath, ":") if len(splitPath) != 3 { return "", ErrMissingContainerID } qos := burstableQos if strings.Contains(cgroupPath, bestEffortQos) { qos = bestEffortQos } cgroupPathFmt := "%s.slice/%s/%s-%s.scope" return fmt.Sprintf(cgroupPathFmt, qos, splitPath[0], splitPath[1], splitPath[2]), nil } // fullyCgroupPath generates the full cgroup path which includes both the slice and group names. func fullCgroupPath(cgroupPath string) string { return filepath.Join("/sys/fs/cgroup/kubepods.slice/", cgroupPath) }