kuberuntime_sandbox.go

Documentation: k8s.io/kubernetes/pkg/kubelet/kuberuntime

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kuberuntime
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net/url"
    23  	"runtime"
    24  	"sort"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	kubetypes "k8s.io/apimachinery/pkg/types"
    28  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    29  	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
    30  	"k8s.io/klog/v2"
    31  	"k8s.io/kubelet/pkg/types"
    32  	"k8s.io/kubernetes/pkg/features"
    33  	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
    34  	runtimeutil "k8s.io/kubernetes/pkg/kubelet/kuberuntime/util"
    35  	"k8s.io/kubernetes/pkg/kubelet/util"
    36  	"k8s.io/kubernetes/pkg/kubelet/util/format"
    37  	netutils "k8s.io/utils/net"
    38  )
    39  
    40  // createPodSandbox creates a pod sandbox and returns (podSandBoxID, message, error).
    41  func (m *kubeGenericRuntimeManager) createPodSandbox(ctx context.Context, pod *v1.Pod, attempt uint32) (string, string, error) {
    42  	podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt)
    43  	if err != nil {
    44  		message := fmt.Sprintf("Failed to generate sandbox config for pod %q: %v", format.Pod(pod), err)
    45  		klog.ErrorS(err, "Failed to generate sandbox config for pod", "pod", klog.KObj(pod))
    46  		return "", message, err
    47  	}
    48  
    49  	// Create pod logs directory
    50  	err = m.osInterface.MkdirAll(podSandboxConfig.LogDirectory, 0755)
    51  	if err != nil {
    52  		message := fmt.Sprintf("Failed to create log directory for pod %q: %v", format.Pod(pod), err)
    53  		klog.ErrorS(err, "Failed to create log directory for pod", "pod", klog.KObj(pod))
    54  		return "", message, err
    55  	}
    56  
    57  	runtimeHandler := ""
    58  	if m.runtimeClassManager != nil {
    59  		runtimeHandler, err = m.runtimeClassManager.LookupRuntimeHandler(pod.Spec.RuntimeClassName)
    60  		if err != nil {
    61  			message := fmt.Sprintf("Failed to create sandbox for pod %q: %v", format.Pod(pod), err)
    62  			return "", message, err
    63  		}
    64  		if runtimeHandler != "" {
    65  			klog.V(2).InfoS("Running pod with runtime handler", "pod", klog.KObj(pod), "runtimeHandler", runtimeHandler)
    66  		}
    67  	}
    68  
    69  	podSandBoxID, err := m.runtimeService.RunPodSandbox(ctx, podSandboxConfig, runtimeHandler)
    70  	if err != nil {
    71  		message := fmt.Sprintf("Failed to create sandbox for pod %q: %v", format.Pod(pod), err)
    72  		klog.ErrorS(err, "Failed to create sandbox for pod", "pod", klog.KObj(pod))
    73  		return "", message, err
    74  	}
    75  
    76  	return podSandBoxID, "", nil
    77  }
    78  
    79  // generatePodSandboxConfig generates pod sandbox config from v1.Pod.
    80  func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attempt uint32) (*runtimeapi.PodSandboxConfig, error) {
    81  	// TODO: deprecating podsandbox resource requirements in favor of the pod level cgroup
    82  	// Refer https://github.com/kubernetes/kubernetes/issues/29871
    83  	podUID := string(pod.UID)
    84  	podSandboxConfig := &runtimeapi.PodSandboxConfig{
    85  		Metadata: &runtimeapi.PodSandboxMetadata{
    86  			Name:      pod.Name,
    87  			Namespace: pod.Namespace,
    88  			Uid:       podUID,
    89  			Attempt:   attempt,
    90  		},
    91  		Labels:      newPodLabels(pod),
    92  		Annotations: newPodAnnotations(pod),
    93  	}
    94  
    95  	dnsConfig, err := m.runtimeHelper.GetPodDNS(pod)
    96  	if err != nil {
    97  		return nil, err
    98  	}
    99  	podSandboxConfig.DnsConfig = dnsConfig
   100  
   101  	if !kubecontainer.IsHostNetworkPod(pod) {
   102  		// TODO: Add domain support in new runtime interface
   103  		podHostname, podDomain, err := m.runtimeHelper.GeneratePodHostNameAndDomain(pod)
   104  		if err != nil {
   105  			return nil, err
   106  		}
   107  		podHostname, err = util.GetNodenameForKernel(podHostname, podDomain, pod.Spec.SetHostnameAsFQDN)
   108  		if err != nil {
   109  			return nil, err
   110  		}
   111  		podSandboxConfig.Hostname = podHostname
   112  	}
   113  
   114  	logDir := BuildPodLogsDirectory(m.podLogsDirectory, pod.Namespace, pod.Name, pod.UID)
   115  	podSandboxConfig.LogDirectory = logDir
   116  
   117  	portMappings := []*runtimeapi.PortMapping{}
   118  	for _, c := range pod.Spec.Containers {
   119  		containerPortMappings := kubecontainer.MakePortMappings(&c)
   120  
   121  		for idx := range containerPortMappings {
   122  			port := containerPortMappings[idx]
   123  			hostPort := int32(port.HostPort)
   124  			containerPort := int32(port.ContainerPort)
   125  			protocol := toRuntimeProtocol(port.Protocol)
   126  			portMappings = append(portMappings, &runtimeapi.PortMapping{
   127  				HostIp:        port.HostIP,
   128  				HostPort:      hostPort,
   129  				ContainerPort: containerPort,
   130  				Protocol:      protocol,
   131  			})
   132  		}
   133  
   134  	}
   135  	if len(portMappings) > 0 {
   136  		podSandboxConfig.PortMappings = portMappings
   137  	}
   138  
   139  	lc, err := m.generatePodSandboxLinuxConfig(pod)
   140  	if err != nil {
   141  		return nil, err
   142  	}
   143  	podSandboxConfig.Linux = lc
   144  
   145  	if runtime.GOOS == "windows" {
   146  		wc, err := m.generatePodSandboxWindowsConfig(pod)
   147  		if err != nil {
   148  			return nil, err
   149  		}
   150  		podSandboxConfig.Windows = wc
   151  	}
   152  
   153  	// Update config to include overhead, sandbox level resources
   154  	if err := m.applySandboxResources(pod, podSandboxConfig); err != nil {
   155  		return nil, err
   156  	}
   157  	return podSandboxConfig, nil
   158  }
   159  
   160  // generatePodSandboxLinuxConfig generates LinuxPodSandboxConfig from v1.Pod.
   161  // We've to call PodSandboxLinuxConfig always irrespective of the underlying OS as securityContext is not part of
   162  // podSandboxConfig. It is currently part of LinuxPodSandboxConfig. In future, if we have securityContext pulled out
   163  // in podSandboxConfig we should be able to use it.
   164  func (m *kubeGenericRuntimeManager) generatePodSandboxLinuxConfig(pod *v1.Pod) (*runtimeapi.LinuxPodSandboxConfig, error) {
   165  	cgroupParent := m.runtimeHelper.GetPodCgroupParent(pod)
   166  	lc := &runtimeapi.LinuxPodSandboxConfig{
   167  		CgroupParent: cgroupParent,
   168  		SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
   169  			Privileged: kubecontainer.HasPrivilegedContainer(pod),
   170  
   171  			// Forcing sandbox to run as `runtime/default` allow users to
   172  			// use least privileged seccomp profiles at pod level. Issue #84623
   173  			Seccomp: &runtimeapi.SecurityProfile{
   174  				ProfileType: runtimeapi.SecurityProfile_RuntimeDefault,
   175  			},
   176  		},
   177  	}
   178  
   179  	sysctls := make(map[string]string)
   180  	if pod.Spec.SecurityContext != nil {
   181  		for _, c := range pod.Spec.SecurityContext.Sysctls {
   182  			sysctls[c.Name] = c.Value
   183  		}
   184  	}
   185  
   186  	lc.Sysctls = sysctls
   187  
   188  	if pod.Spec.SecurityContext != nil {
   189  		sc := pod.Spec.SecurityContext
   190  		if sc.RunAsUser != nil && runtime.GOOS != "windows" {
   191  			lc.SecurityContext.RunAsUser = &runtimeapi.Int64Value{Value: int64(*sc.RunAsUser)}
   192  		}
   193  		if sc.RunAsGroup != nil && runtime.GOOS != "windows" {
   194  			lc.SecurityContext.RunAsGroup = &runtimeapi.Int64Value{Value: int64(*sc.RunAsGroup)}
   195  		}
   196  		namespaceOptions, err := runtimeutil.NamespacesForPod(pod, m.runtimeHelper, m.runtimeClassManager)
   197  		if err != nil {
   198  			return nil, err
   199  		}
   200  		lc.SecurityContext.NamespaceOptions = namespaceOptions
   201  
   202  		if sc.FSGroup != nil && runtime.GOOS != "windows" {
   203  			lc.SecurityContext.SupplementalGroups = append(lc.SecurityContext.SupplementalGroups, int64(*sc.FSGroup))
   204  		}
   205  		if groups := m.runtimeHelper.GetExtraSupplementalGroupsForPod(pod); len(groups) > 0 {
   206  			lc.SecurityContext.SupplementalGroups = append(lc.SecurityContext.SupplementalGroups, groups...)
   207  		}
   208  		if sc.SupplementalGroups != nil {
   209  			for _, sg := range sc.SupplementalGroups {
   210  				lc.SecurityContext.SupplementalGroups = append(lc.SecurityContext.SupplementalGroups, int64(sg))
   211  			}
   212  		}
   213  		if sc.SELinuxOptions != nil && runtime.GOOS != "windows" {
   214  			lc.SecurityContext.SelinuxOptions = &runtimeapi.SELinuxOption{
   215  				User:  sc.SELinuxOptions.User,
   216  				Role:  sc.SELinuxOptions.Role,
   217  				Type:  sc.SELinuxOptions.Type,
   218  				Level: sc.SELinuxOptions.Level,
   219  			}
   220  		}
   221  	}
   222  
   223  	return lc, nil
   224  }
   225  
   226  // generatePodSandboxWindowsConfig generates WindowsPodSandboxConfig from v1.Pod.
   227  // On Windows this will get called in addition to LinuxPodSandboxConfig because not all relevant fields have been added to
   228  // WindowsPodSandboxConfig at this time.
   229  func (m *kubeGenericRuntimeManager) generatePodSandboxWindowsConfig(pod *v1.Pod) (*runtimeapi.WindowsPodSandboxConfig, error) {
   230  	wc := &runtimeapi.WindowsPodSandboxConfig{
   231  		SecurityContext: &runtimeapi.WindowsSandboxSecurityContext{},
   232  	}
   233  
   234  	if utilfeature.DefaultFeatureGate.Enabled(features.WindowsHostNetwork) {
   235  		wc.SecurityContext.NamespaceOptions = &runtimeapi.WindowsNamespaceOption{}
   236  		if kubecontainer.IsHostNetworkPod(pod) {
   237  			wc.SecurityContext.NamespaceOptions.Network = runtimeapi.NamespaceMode_NODE
   238  		} else {
   239  			wc.SecurityContext.NamespaceOptions.Network = runtimeapi.NamespaceMode_POD
   240  		}
   241  	}
   242  
   243  	// If all of the containers in a pod are HostProcess containers, set the pod's HostProcess field
   244  	// explicitly because the container runtime requires this information at sandbox creation time.
   245  	if kubecontainer.HasWindowsHostProcessContainer(pod) {
   246  		// At present Windows all containers in a Windows pod must be HostProcess containers
   247  		// and HostNetwork is required to be set.
   248  		if !kubecontainer.AllContainersAreWindowsHostProcess(pod) {
   249  			return nil, fmt.Errorf("pod must not contain both HostProcess and non-HostProcess containers")
   250  		}
   251  
   252  		if !kubecontainer.IsHostNetworkPod(pod) {
   253  			return nil, fmt.Errorf("hostNetwork is required if Pod contains HostProcess containers")
   254  		}
   255  
   256  		wc.SecurityContext.HostProcess = true
   257  	}
   258  
   259  	sc := pod.Spec.SecurityContext
   260  	if sc == nil || sc.WindowsOptions == nil {
   261  		return wc, nil
   262  	}
   263  
   264  	wo := sc.WindowsOptions
   265  	if wo.GMSACredentialSpec != nil {
   266  		wc.SecurityContext.CredentialSpec = *wo.GMSACredentialSpec
   267  	}
   268  
   269  	if wo.RunAsUserName != nil {
   270  		wc.SecurityContext.RunAsUsername = *wo.RunAsUserName
   271  	}
   272  
   273  	if kubecontainer.HasWindowsHostProcessContainer(pod) {
   274  
   275  		if wo.HostProcess != nil && !*wo.HostProcess {
   276  			return nil, fmt.Errorf("pod must not contain any HostProcess containers if Pod's WindowsOptions.HostProcess is set to false")
   277  		}
   278  	}
   279  
   280  	return wc, nil
   281  }
   282  
   283  // getKubeletSandboxes lists all (or just the running) sandboxes managed by kubelet.
   284  func (m *kubeGenericRuntimeManager) getKubeletSandboxes(ctx context.Context, all bool) ([]*runtimeapi.PodSandbox, error) {
   285  	var filter *runtimeapi.PodSandboxFilter
   286  	if !all {
   287  		readyState := runtimeapi.PodSandboxState_SANDBOX_READY
   288  		filter = &runtimeapi.PodSandboxFilter{
   289  			State: &runtimeapi.PodSandboxStateValue{
   290  				State: readyState,
   291  			},
   292  		}
   293  	}
   294  
   295  	resp, err := m.runtimeService.ListPodSandbox(ctx, filter)
   296  	if err != nil {
   297  		klog.ErrorS(err, "Failed to list pod sandboxes")
   298  		return nil, err
   299  	}
   300  
   301  	return resp, nil
   302  }
   303  
   304  // determinePodSandboxIP determines the IP addresses of the given pod sandbox.
   305  func (m *kubeGenericRuntimeManager) determinePodSandboxIPs(podNamespace, podName string, podSandbox *runtimeapi.PodSandboxStatus) []string {
   306  	podIPs := make([]string, 0)
   307  	if podSandbox.Network == nil {
   308  		klog.InfoS("Pod Sandbox status doesn't have network information, cannot report IPs", "pod", klog.KRef(podNamespace, podName))
   309  		return podIPs
   310  	}
   311  
   312  	// ip could be an empty string if runtime is not responsible for the
   313  	// IP (e.g., host networking).
   314  
   315  	// pick primary IP
   316  	if len(podSandbox.Network.Ip) != 0 {
   317  		if netutils.ParseIPSloppy(podSandbox.Network.Ip) == nil {
   318  			klog.InfoS("Pod Sandbox reported an unparseable primary IP", "pod", klog.KRef(podNamespace, podName), "IP", podSandbox.Network.Ip)
   319  			return nil
   320  		}
   321  		podIPs = append(podIPs, podSandbox.Network.Ip)
   322  	}
   323  
   324  	// pick additional ips, if cri reported them
   325  	for _, podIP := range podSandbox.Network.AdditionalIps {
   326  		if nil == netutils.ParseIPSloppy(podIP.Ip) {
   327  			klog.InfoS("Pod Sandbox reported an unparseable additional IP", "pod", klog.KRef(podNamespace, podName), "IP", podIP.Ip)
   328  			return nil
   329  		}
   330  		podIPs = append(podIPs, podIP.Ip)
   331  	}
   332  
   333  	return podIPs
   334  }
   335  
   336  // getPodSandboxID gets the sandbox id by podUID and returns ([]sandboxID, error).
   337  // Param state could be nil in order to get all sandboxes belonging to same pod.
   338  func (m *kubeGenericRuntimeManager) getSandboxIDByPodUID(ctx context.Context, podUID kubetypes.UID, state *runtimeapi.PodSandboxState) ([]string, error) {
   339  	filter := &runtimeapi.PodSandboxFilter{
   340  		LabelSelector: map[string]string{types.KubernetesPodUIDLabel: string(podUID)},
   341  	}
   342  	if state != nil {
   343  		filter.State = &runtimeapi.PodSandboxStateValue{
   344  			State: *state,
   345  		}
   346  	}
   347  	sandboxes, err := m.runtimeService.ListPodSandbox(ctx, filter)
   348  	if err != nil {
   349  		klog.ErrorS(err, "Failed to list sandboxes for pod", "podUID", podUID)
   350  		return nil, err
   351  	}
   352  
   353  	if len(sandboxes) == 0 {
   354  		return nil, nil
   355  	}
   356  
   357  	// Sort with newest first.
   358  	sandboxIDs := make([]string, len(sandboxes))
   359  	sort.Sort(podSandboxByCreated(sandboxes))
   360  	for i, s := range sandboxes {
   361  		sandboxIDs[i] = s.Id
   362  	}
   363  
   364  	return sandboxIDs, nil
   365  }
   366  
   367  // GetPortForward gets the endpoint the runtime will serve the port-forward request from.
   368  func (m *kubeGenericRuntimeManager) GetPortForward(ctx context.Context, podName, podNamespace string, podUID kubetypes.UID, ports []int32) (*url.URL, error) {
   369  	sandboxIDs, err := m.getSandboxIDByPodUID(ctx, podUID, nil)
   370  	if err != nil {
   371  		return nil, fmt.Errorf("failed to find sandboxID for pod %s: %v", format.PodDesc(podName, podNamespace, podUID), err)
   372  	}
   373  	if len(sandboxIDs) == 0 {
   374  		return nil, fmt.Errorf("failed to find sandboxID for pod %s", format.PodDesc(podName, podNamespace, podUID))
   375  	}
   376  	req := &runtimeapi.PortForwardRequest{
   377  		PodSandboxId: sandboxIDs[0],
   378  		Port:         ports,
   379  	}
   380  	resp, err := m.runtimeService.PortForward(ctx, req)
   381  	if err != nil {
   382  		return nil, err
   383  	}
   384  	return url.Parse(resp.Url)
   385  }
   386
View as plain text