swap_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"path/filepath"
    23  	"strconv"
    24  
    25  	"github.com/onsi/ginkgo/v2"
    26  	"github.com/onsi/gomega"
    27  	v1 "k8s.io/api/core/v1"
    28  	"k8s.io/apimachinery/pkg/api/resource"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/util/rand"
    31  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    32  	"k8s.io/kubernetes/pkg/features"
    33  	"k8s.io/kubernetes/pkg/kubelet/types"
    34  	"k8s.io/kubernetes/test/e2e/framework"
    35  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    36  	testutils "k8s.io/kubernetes/test/utils"
    37  	admissionapi "k8s.io/pod-security-admission/api"
    38  )
    39  
    40  const (
    41  	cgroupBasePath        = "/sys/fs/cgroup/"
    42  	cgroupV1SwapLimitFile = "/memory/memory.memsw.limit_in_bytes"
    43  	cgroupV2SwapLimitFile = "memory.swap.max"
    44  	cgroupV1MemLimitFile  = "/memory/memory.limit_in_bytes"
    45  )
    46  
    47  var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func() {
    48  	f := framework.NewDefaultFramework("swap-test")
    49  	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
    50  
    51  	ginkgo.DescribeTable("with configuration", func(qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) {
    52  		ginkgo.By(fmt.Sprintf("Creating a pod of QOS class %s. memoryRequestEqualLimit: %t", qosClass, memoryRequestEqualLimit))
    53  		pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit)
    54  		pod = runPodAndWaitUntilScheduled(f, pod)
    55  
    56  		isCgroupV2 := isPodCgroupV2(f, pod)
    57  		isLimitedSwap := isLimitedSwap(f, pod)
    58  		isNoSwap := isNoSwap(f, pod)
    59  
    60  		if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) {
    61  			ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable))
    62  			expectNoSwap(f, pod, isCgroupV2)
    63  			return
    64  		}
    65  
    66  		if !isLimitedSwap {
    67  			ginkgo.By("expecting no swap")
    68  			expectNoSwap(f, pod, isCgroupV2)
    69  			return
    70  		}
    71  
    72  		ginkgo.By("expecting limited swap")
    73  		expectedSwapLimit := calcSwapForBurstablePod(f, pod)
    74  		expectLimitedSwap(f, pod, expectedSwapLimit)
    75  	},
    76  		ginkgo.Entry("QOS Best-effort", v1.PodQOSBestEffort, false),
    77  		ginkgo.Entry("QOS Burstable", v1.PodQOSBurstable, false),
    78  		ginkgo.Entry("QOS Burstable with memory request equals to limit", v1.PodQOSBurstable, true),
    79  		ginkgo.Entry("QOS Guaranteed", v1.PodQOSGuaranteed, false),
    80  	)
    81  })
    82  
    83  // Note that memoryRequestEqualLimit is effective only when qosClass is PodQOSBestEffort.
    84  func getSwapTestPod(f *framework.Framework, qosClass v1.PodQOSClass, memoryRequestEqualLimit bool) *v1.Pod {
    85  	podMemoryAmount := resource.MustParse("128Mi")
    86  
    87  	var resources v1.ResourceRequirements
    88  	switch qosClass {
    89  	case v1.PodQOSBestEffort:
    90  		// nothing to do in this case
    91  	case v1.PodQOSBurstable:
    92  		resources = v1.ResourceRequirements{
    93  			Requests: v1.ResourceList{
    94  				v1.ResourceMemory: podMemoryAmount,
    95  			},
    96  		}
    97  
    98  		if memoryRequestEqualLimit {
    99  			resources.Limits = resources.Requests
   100  		}
   101  	case v1.PodQOSGuaranteed:
   102  		resources = v1.ResourceRequirements{
   103  			Limits: v1.ResourceList{
   104  				v1.ResourceCPU:    resource.MustParse("200m"),
   105  				v1.ResourceMemory: podMemoryAmount,
   106  			},
   107  		}
   108  		resources.Requests = resources.Limits
   109  	}
   110  
   111  	pod := &v1.Pod{
   112  		ObjectMeta: metav1.ObjectMeta{
   113  			Name:      "test-pod-swap-" + rand.String(5),
   114  			Namespace: f.Namespace.Name,
   115  		},
   116  		Spec: v1.PodSpec{
   117  			RestartPolicy: v1.RestartPolicyAlways,
   118  			Containers: []v1.Container{
   119  				{
   120  					Name:      "busybox-container",
   121  					Image:     busyboxImage,
   122  					Command:   []string{"sleep", "600"},
   123  					Resources: resources,
   124  				},
   125  			},
   126  		},
   127  	}
   128  
   129  	return pod
   130  }
   131  
   132  func runPodAndWaitUntilScheduled(f *framework.Framework, pod *v1.Pod) *v1.Pod {
   133  	ginkgo.By("running swap test pod")
   134  	podClient := e2epod.NewPodClient(f)
   135  
   136  	pod = podClient.CreateSync(context.Background(), pod)
   137  	pod, err := podClient.Get(context.Background(), pod.Name, metav1.GetOptions{})
   138  
   139  	framework.ExpectNoError(err)
   140  	isReady, err := testutils.PodRunningReady(pod)
   141  	framework.ExpectNoError(err)
   142  	gomega.ExpectWithOffset(1, isReady).To(gomega.BeTrue(), "pod should be ready")
   143  
   144  	return pod
   145  }
   146  
   147  func isSwapFeatureGateEnabled() bool {
   148  	ginkgo.By("figuring if NodeSwap feature gate is turned on")
   149  	return utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap)
   150  }
   151  
   152  func readCgroupFile(f *framework.Framework, pod *v1.Pod, filename string) string {
   153  	filePath := filepath.Join(cgroupBasePath, filename)
   154  
   155  	ginkgo.By("reading cgroup file " + filePath)
   156  	output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "cat "+filePath)
   157  
   158  	return output
   159  }
   160  
   161  func isPodCgroupV2(f *framework.Framework, pod *v1.Pod) bool {
   162  	ginkgo.By("figuring is test pod runs cgroup v2")
   163  	output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", `if test -f "/sys/fs/cgroup/cgroup.controllers"; then echo "true"; else echo "false"; fi`)
   164  
   165  	return output == "true"
   166  }
   167  
   168  func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) {
   169  	if isCgroupV2 {
   170  		swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
   171  		gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("0"), "max swap allowed should be zero")
   172  	} else {
   173  		swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile)
   174  		memLimit := readCgroupFile(f, pod, cgroupV1MemLimitFile)
   175  		gomega.ExpectWithOffset(1, swapPlusMemLimit).ToNot(gomega.BeEmpty())
   176  		gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal(memLimit))
   177  	}
   178  }
   179  
   180  // supports v2 only as v1 shouldn't support LimitedSwap
   181  func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) {
   182  	swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
   183  
   184  	swapLimit, err := strconv.Atoi(swapLimitStr)
   185  	framework.ExpectNoError(err, "cannot convert swap limit to int")
   186  
   187  	// cgroup values are always aligned w.r.t. the page size, which is usually 4Ki
   188  	const cgroupAlignment int64 = 4 * 1024 // 4Ki
   189  	const errMsg = "swap limitation is not as expected"
   190  
   191  	gomega.ExpectWithOffset(1, int64(swapLimit)).To(
   192  		gomega.Or(
   193  			gomega.BeNumerically(">=", expectedSwapLimit-cgroupAlignment),
   194  			gomega.BeNumerically("<=", expectedSwapLimit+cgroupAlignment),
   195  		),
   196  		errMsg,
   197  	)
   198  }
   199  
   200  func getSwapCapacity(f *framework.Framework, pod *v1.Pod) int64 {
   201  	output := e2epod.ExecCommandInContainer(f, pod.Name, pod.Spec.Containers[0].Name, "/bin/sh", "-ec", "free -b | grep Swap | xargs | cut -d\" \" -f2")
   202  
   203  	swapCapacity, err := strconv.Atoi(output)
   204  	framework.ExpectNoError(err, "cannot convert swap size to int")
   205  
   206  	ginkgo.By(fmt.Sprintf("providing swap capacity: %d", swapCapacity))
   207  
   208  	return int64(swapCapacity)
   209  }
   210  
   211  func getMemoryCapacity(f *framework.Framework, pod *v1.Pod) int64 {
   212  	nodes, err := f.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
   213  	framework.ExpectNoError(err, "failed listing nodes")
   214  
   215  	for _, node := range nodes.Items {
   216  		if node.Name != pod.Spec.NodeName {
   217  			continue
   218  		}
   219  
   220  		memCapacity := node.Status.Capacity[v1.ResourceMemory]
   221  		return memCapacity.Value()
   222  	}
   223  
   224  	framework.ExpectNoError(fmt.Errorf("node %s wasn't found", pod.Spec.NodeName))
   225  	return 0
   226  }
   227  
   228  func calcSwapForBurstablePod(f *framework.Framework, pod *v1.Pod) int64 {
   229  	nodeMemoryCapacity := getMemoryCapacity(f, pod)
   230  	nodeSwapCapacity := getSwapCapacity(f, pod)
   231  	containerMemoryRequest := pod.Spec.Containers[0].Resources.Requests.Memory().Value()
   232  
   233  	containerMemoryProportion := float64(containerMemoryRequest) / float64(nodeMemoryCapacity)
   234  	swapAllocation := containerMemoryProportion * float64(nodeSwapCapacity)
   235  	ginkgo.By(fmt.Sprintf("Calculating swap for burstable pods: nodeMemoryCapacity: %d, nodeSwapCapacity: %d, containerMemoryRequest: %d, swapAllocation: %d",
   236  		nodeMemoryCapacity, nodeSwapCapacity, containerMemoryRequest, int64(swapAllocation)))
   237  
   238  	return int64(swapAllocation)
   239  }
   240  
   241  func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool {
   242  	kubeletCfg, err := getCurrentKubeletConfig(context.Background())
   243  	framework.ExpectNoError(err, "cannot get kubelet config")
   244  
   245  	return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap
   246  }
   247  
   248  func isNoSwap(f *framework.Framework, pod *v1.Pod) bool {
   249  	kubeletCfg, err := getCurrentKubeletConfig(context.Background())
   250  	framework.ExpectNoError(err, "cannot get kubelet config")
   251  
   252  	return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == ""
   253  }
   254
View as plain text