1
2
3
4
19
20 package e2enode
21
22 import (
23 "context"
24 "fmt"
25 "os/exec"
26 "path"
27 "strconv"
28 "strings"
29 "time"
30
31 v1 "k8s.io/api/core/v1"
32 "k8s.io/apimachinery/pkg/api/resource"
33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34 "k8s.io/apimachinery/pkg/util/sets"
35 "k8s.io/apimachinery/pkg/util/uuid"
36 runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
37 "k8s.io/kubernetes/test/e2e/framework"
38 e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
39 "k8s.io/kubernetes/test/e2e/nodefeature"
40 imageutils "k8s.io/kubernetes/test/utils/image"
41 admissionapi "k8s.io/pod-security-admission/api"
42
43 "github.com/onsi/ginkgo/v2"
44 "github.com/onsi/gomega"
45 )
46
47 func getOOMScoreForPid(pid int) (int, error) {
48 procfsPath := path.Join("/proc", strconv.Itoa(pid), "oom_score_adj")
49 out, err := exec.Command("sudo", "cat", procfsPath).CombinedOutput()
50 if err != nil {
51 return 0, err
52 }
53 return strconv.Atoi(strings.TrimSpace(string(out)))
54 }
55
56 func validateOOMScoreAdjSetting(pid int, expectedOOMScoreAdj int) error {
57 oomScore, err := getOOMScoreForPid(pid)
58 if err != nil {
59 return fmt.Errorf("failed to get oom_score_adj for %d: %w", pid, err)
60 }
61 if expectedOOMScoreAdj != oomScore {
62 return fmt.Errorf("expected pid %d's oom_score_adj to be %d; found %d", pid, expectedOOMScoreAdj, oomScore)
63 }
64 return nil
65 }
66
67 func validateOOMScoreAdjSettingIsInRange(pid int, expectedMinOOMScoreAdj, expectedMaxOOMScoreAdj int) error {
68 oomScore, err := getOOMScoreForPid(pid)
69 if err != nil {
70 return fmt.Errorf("failed to get oom_score_adj for %d", pid)
71 }
72 if oomScore < expectedMinOOMScoreAdj {
73 return fmt.Errorf("expected pid %d's oom_score_adj to be >= %d; found %d", pid, expectedMinOOMScoreAdj, oomScore)
74 }
75 if oomScore >= expectedMaxOOMScoreAdj {
76 return fmt.Errorf("expected pid %d's oom_score_adj to be < %d; found %d", pid, expectedMaxOOMScoreAdj, oomScore)
77 }
78 return nil
79 }
80
81 var _ = SIGDescribe("Container Manager Misc", framework.WithSerial(), func() {
82 f := framework.NewDefaultFramework("kubelet-container-manager")
83 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
84 f.Describe("Validate OOM score adjustments", nodefeature.OOMScoreAdj, func() {
85 ginkgo.Context("once the node is setup", func() {
86 ginkgo.It("container runtime's oom-score-adj should be -999", func(ctx context.Context) {
87 runtimePids, err := getPidsForProcess(framework.TestContext.ContainerRuntimeProcessName, framework.TestContext.ContainerRuntimePidFile)
88 framework.ExpectNoError(err, "failed to get list of container runtime pids")
89 for _, pid := range runtimePids {
90 gomega.Eventually(ctx, func() error {
91 return validateOOMScoreAdjSetting(pid, -999)
92 }, 5*time.Minute, 30*time.Second).Should(gomega.BeNil())
93 }
94 })
95 ginkgo.It("Kubelet's oom-score-adj should be -999", func(ctx context.Context) {
96 kubeletPids, err := getPidsForProcess(kubeletProcessName, "")
97 framework.ExpectNoError(err, "failed to get list of kubelet pids")
98 gomega.Expect(kubeletPids).To(gomega.HaveLen(1), "expected only one kubelet process; found %d", len(kubeletPids))
99 gomega.Eventually(ctx, func() error {
100 return validateOOMScoreAdjSetting(kubeletPids[0], -999)
101 }, 5*time.Minute, 30*time.Second).Should(gomega.BeNil())
102 })
103 ginkgo.Context("", func() {
104 ginkgo.It("pod infra containers oom-score-adj should be -998 and best effort container's should be 1000", func(ctx context.Context) {
105
106
107
108 existingPausePIDs, err := getPidsForProcess("pause", "")
109 framework.ExpectNoError(err, "failed to list all pause processes on the node")
110 existingPausePIDSet := sets.NewInt(existingPausePIDs...)
111
112 podClient := e2epod.NewPodClient(f)
113 podName := "besteffort" + string(uuid.NewUUID())
114 podClient.Create(ctx, &v1.Pod{
115 ObjectMeta: metav1.ObjectMeta{
116 Name: podName,
117 },
118 Spec: v1.PodSpec{
119 Containers: []v1.Container{
120 {
121 Image: framework.ServeHostnameImage,
122 Name: podName,
123 },
124 },
125 },
126 })
127
128 var pausePids []int
129 ginkgo.By("checking infra container's oom-score-adj")
130 gomega.Eventually(ctx, func() error {
131 pausePids, err = getPidsForProcess("pause", "")
132 if err != nil {
133 return fmt.Errorf("failed to get list of pause pids: %w", err)
134 }
135 for _, pid := range pausePids {
136 if existingPausePIDSet.Has(pid) {
137
138 continue
139 }
140 if err := validateOOMScoreAdjSetting(pid, -998); err != nil {
141 return err
142 }
143 }
144 return nil
145 }, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
146 var shPids []int
147 ginkgo.By("checking besteffort container's oom-score-adj")
148 gomega.Eventually(ctx, func() error {
149 shPids, err = getPidsForProcess("agnhost", "")
150 if err != nil {
151 return fmt.Errorf("failed to get list of serve hostname process pids: %w", err)
152 }
153 if len(shPids) != 1 {
154 return fmt.Errorf("expected only one agnhost process; found %d", len(shPids))
155 }
156 return validateOOMScoreAdjSetting(shPids[0], 1000)
157 }, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
158 })
159
160 ginkgo.AfterEach(func() {
161 if ginkgo.CurrentSpecReport().Failed() {
162 ginkgo.By("Dump all running containers")
163 runtime, _, err := getCRIClient()
164 framework.ExpectNoError(err)
165 containers, err := runtime.ListContainers(context.Background(), &runtimeapi.ContainerFilter{
166 State: &runtimeapi.ContainerStateValue{
167 State: runtimeapi.ContainerState_CONTAINER_RUNNING,
168 },
169 })
170 framework.ExpectNoError(err)
171 framework.Logf("Running containers:")
172 for _, c := range containers {
173 framework.Logf("%+v", c)
174 }
175 }
176 })
177 })
178 ginkgo.It("guaranteed container's oom-score-adj should be -998", func(ctx context.Context) {
179 podClient := e2epod.NewPodClient(f)
180 podName := "guaranteed" + string(uuid.NewUUID())
181 podClient.Create(ctx, &v1.Pod{
182 ObjectMeta: metav1.ObjectMeta{
183 Name: podName,
184 },
185 Spec: v1.PodSpec{
186 Containers: []v1.Container{
187 {
188 Image: imageutils.GetE2EImage(imageutils.Nginx),
189 Name: podName,
190 Resources: v1.ResourceRequirements{
191 Limits: v1.ResourceList{
192 v1.ResourceCPU: resource.MustParse("100m"),
193 v1.ResourceMemory: resource.MustParse("50Mi"),
194 },
195 },
196 },
197 },
198 },
199 })
200 var (
201 ngPids []int
202 err error
203 )
204 gomega.Eventually(ctx, func() error {
205 ngPids, err = getPidsForProcess("nginx", "")
206 if err != nil {
207 return fmt.Errorf("failed to get list of nginx process pids: %w", err)
208 }
209 for _, pid := range ngPids {
210 if err := validateOOMScoreAdjSetting(pid, -998); err != nil {
211 return err
212 }
213 }
214
215 return nil
216 }, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
217
218 })
219 ginkgo.It("burstable container's oom-score-adj should be between [2, 1000)", func(ctx context.Context) {
220 podClient := e2epod.NewPodClient(f)
221 podName := "burstable" + string(uuid.NewUUID())
222 podClient.Create(ctx, &v1.Pod{
223 ObjectMeta: metav1.ObjectMeta{
224 Name: podName,
225 },
226 Spec: v1.PodSpec{
227 Containers: []v1.Container{
228 {
229 Image: imageutils.GetE2EImage(imageutils.Agnhost),
230 Args: []string{"test-webserver"},
231 Name: podName,
232 Resources: v1.ResourceRequirements{
233 Requests: v1.ResourceList{
234 v1.ResourceCPU: resource.MustParse("100m"),
235 v1.ResourceMemory: resource.MustParse("50Mi"),
236 },
237 },
238 },
239 },
240 },
241 })
242 var (
243 wsPids []int
244 err error
245 )
246 gomega.Eventually(ctx, func() error {
247 wsPids, err = getPidsForProcess("agnhost", "")
248 if err != nil {
249 return fmt.Errorf("failed to get list of test-webserver process pids: %w", err)
250 }
251 for _, pid := range wsPids {
252 if err := validateOOMScoreAdjSettingIsInRange(pid, 2, 1000); err != nil {
253 return err
254 }
255 }
256 return nil
257 }, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
258
259
260 })
261 })
262 })
263 })
264
View as plain text