1
16
17 package cpumanager
18
19 import (
20 "reflect"
21 "sort"
22 "testing"
23
24 cadvisorapi "github.com/google/cadvisor/info/v1"
25 v1 "k8s.io/api/core/v1"
26 "k8s.io/apimachinery/pkg/types"
27 utilfeature "k8s.io/apiserver/pkg/util/feature"
28 featuregatetesting "k8s.io/component-base/featuregate/testing"
29 pkgfeatures "k8s.io/kubernetes/pkg/features"
30 "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
31 "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
32 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
33 "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
34 "k8s.io/utils/cpuset"
35 )
36
37 type testCase struct {
38 name string
39 pod v1.Pod
40 container v1.Container
41 assignments state.ContainerCPUAssignments
42 defaultCPUSet cpuset.CPUSet
43 expectedHints []topologymanager.TopologyHint
44 }
45
46 func returnMachineInfo() cadvisorapi.MachineInfo {
47 return cadvisorapi.MachineInfo{
48 NumCores: 12,
49 Topology: []cadvisorapi.Node{
50 {Id: 0,
51 Cores: []cadvisorapi.Core{
52 {SocketID: 0, Id: 0, Threads: []int{0, 6}},
53 {SocketID: 0, Id: 1, Threads: []int{1, 7}},
54 {SocketID: 0, Id: 2, Threads: []int{2, 8}},
55 },
56 },
57 {Id: 1,
58 Cores: []cadvisorapi.Core{
59 {SocketID: 1, Id: 0, Threads: []int{3, 9}},
60 {SocketID: 1, Id: 1, Threads: []int{4, 10}},
61 {SocketID: 1, Id: 2, Threads: []int{5, 11}},
62 },
63 },
64 },
65 }
66 }
67
68 type containerOptions struct {
69 request string
70 limit string
71 restartPolicy v1.ContainerRestartPolicy
72 }
73
74 func TestPodGuaranteedCPUs(t *testing.T) {
75 options := [][]*containerOptions{
76 {
77 {request: "0", limit: "0"},
78 },
79 {
80 {request: "2", limit: "2"},
81 },
82 {
83 {request: "5", limit: "5"},
84 },
85 {
86 {request: "2", limit: "2"},
87 {request: "4", limit: "4"},
88 },
89 }
90
91 testPod1 := makeMultiContainerPodWithOptions(options[0], options[0])
92 testPod2 := makeMultiContainerPodWithOptions(options[0], options[1])
93 testPod3 := makeMultiContainerPodWithOptions(options[1], options[0])
94
95 testPod4 := makeMultiContainerPodWithOptions(options[1], options[1])
96 testPod5 := makeMultiContainerPodWithOptions(options[2], options[2])
97
98 testPod6 := makeMultiContainerPodWithOptions(options[1], options[2])
99 testPod7 := makeMultiContainerPodWithOptions(options[2], options[1])
100
101 testPod8 := makeMultiContainerPodWithOptions(options[3], options[3])
102
103 testPod9 := makeMultiContainerPodWithOptions([]*containerOptions{
104 {request: "1", limit: "1", restartPolicy: v1.ContainerRestartPolicyAlways},
105 }, []*containerOptions{
106 {request: "1", limit: "1"},
107 })
108 testPod10 := makeMultiContainerPodWithOptions([]*containerOptions{
109 {request: "5", limit: "5"},
110 {request: "1", limit: "1", restartPolicy: v1.ContainerRestartPolicyAlways},
111 {request: "2", limit: "2", restartPolicy: v1.ContainerRestartPolicyAlways},
112 {request: "3", limit: "3", restartPolicy: v1.ContainerRestartPolicyAlways},
113 }, []*containerOptions{
114 {request: "1", limit: "1"},
115 })
116 testPod11 := makeMultiContainerPodWithOptions([]*containerOptions{
117 {request: "5", limit: "5"},
118 {request: "1", limit: "1", restartPolicy: v1.ContainerRestartPolicyAlways},
119 {request: "2", limit: "2", restartPolicy: v1.ContainerRestartPolicyAlways},
120 {request: "5", limit: "5"},
121 {request: "3", limit: "3", restartPolicy: v1.ContainerRestartPolicyAlways},
122 }, []*containerOptions{
123 {request: "1", limit: "1"},
124 })
125 testPod12 := makeMultiContainerPodWithOptions([]*containerOptions{
126 {request: "10", limit: "10", restartPolicy: v1.ContainerRestartPolicyAlways},
127 {request: "200", limit: "200"},
128 }, []*containerOptions{
129 {request: "100", limit: "100"},
130 })
131
132 p := staticPolicy{}
133
134 tcases := []struct {
135 name string
136 pod *v1.Pod
137 expectedCPU int
138 }{
139 {
140 name: "TestCase01: if requestedCPU == 0, Pod is not Guaranteed Qos",
141 pod: testPod1,
142 expectedCPU: 0,
143 },
144 {
145 name: "TestCase02: if requestedCPU == 0, Pod is not Guaranteed Qos",
146 pod: testPod2,
147 expectedCPU: 0,
148 },
149 {
150 name: "TestCase03: if requestedCPU == 0, Pod is not Guaranteed Qos",
151 pod: testPod3,
152 expectedCPU: 0,
153 },
154 {
155 name: "TestCase04: Guaranteed Pod requests 2 CPUs",
156 pod: testPod4,
157 expectedCPU: 2,
158 },
159 {
160 name: "TestCase05: Guaranteed Pod requests 5 CPUs",
161 pod: testPod5,
162 expectedCPU: 5,
163 },
164 {
165 name: "TestCase06: The number of CPUs requested By app is bigger than the number of CPUs requested by init",
166 pod: testPod6,
167 expectedCPU: 5,
168 },
169 {
170 name: "TestCase07: The number of CPUs requested By init is bigger than the number of CPUs requested by app",
171 pod: testPod7,
172 expectedCPU: 5,
173 },
174 {
175 name: "TestCase08: Sum of CPUs requested by multiple containers",
176 pod: testPod8,
177 expectedCPU: 6,
178 },
179 {
180 name: "TestCase09: restartable init container + regular container",
181 pod: testPod9,
182 expectedCPU: 2,
183 },
184 {
185 name: "TestCase09: multiple restartable init containers",
186 pod: testPod10,
187 expectedCPU: 7,
188 },
189 {
190 name: "TestCase11: multiple restartable and regular init containers",
191 pod: testPod11,
192 expectedCPU: 8,
193 },
194 {
195 name: "TestCase12: restartable init, regular init and regular container",
196 pod: testPod12,
197 expectedCPU: 210,
198 },
199 }
200 for _, tc := range tcases {
201 t.Run(tc.name, func(t *testing.T) {
202 requestedCPU := p.podGuaranteedCPUs(tc.pod)
203
204 if requestedCPU != tc.expectedCPU {
205 t.Errorf("Expected in result to be %v , got %v", tc.expectedCPU, requestedCPU)
206 }
207 })
208 }
209 }
210
211 func TestGetTopologyHints(t *testing.T) {
212 machineInfo := returnMachineInfo()
213 tcases := returnTestCases()
214
215 for _, tc := range tcases {
216 topology, _ := topology.Discover(&machineInfo)
217
218 var activePods []*v1.Pod
219 for p := range tc.assignments {
220 pod := v1.Pod{}
221 pod.UID = types.UID(p)
222 for c := range tc.assignments[p] {
223 container := v1.Container{}
224 container.Name = c
225 pod.Spec.Containers = append(pod.Spec.Containers, container)
226 }
227 activePods = append(activePods, &pod)
228 }
229
230 m := manager{
231 policy: &staticPolicy{
232 topology: topology,
233 },
234 state: &mockState{
235 assignments: tc.assignments,
236 defaultCPUSet: tc.defaultCPUSet,
237 },
238 topology: topology,
239 activePods: func() []*v1.Pod { return activePods },
240 podStatusProvider: mockPodStatusProvider{},
241 sourcesReady: &sourcesReadyStub{},
242 }
243
244 hints := m.GetTopologyHints(&tc.pod, &tc.container)[string(v1.ResourceCPU)]
245 if len(tc.expectedHints) == 0 && len(hints) == 0 {
246 continue
247 }
248
249 if m.pendingAdmissionPod == nil {
250 t.Errorf("The pendingAdmissionPod should point to the current pod after the call to GetTopologyHints()")
251 }
252
253 sort.SliceStable(hints, func(i, j int) bool {
254 return hints[i].LessThan(hints[j])
255 })
256 sort.SliceStable(tc.expectedHints, func(i, j int) bool {
257 return tc.expectedHints[i].LessThan(tc.expectedHints[j])
258 })
259 if !reflect.DeepEqual(tc.expectedHints, hints) {
260 t.Errorf("Expected in result to be %v , got %v", tc.expectedHints, hints)
261 }
262 }
263 }
264
265 func TestGetPodTopologyHints(t *testing.T) {
266 machineInfo := returnMachineInfo()
267
268 for _, tc := range returnTestCases() {
269 topology, _ := topology.Discover(&machineInfo)
270
271 var activePods []*v1.Pod
272 for p := range tc.assignments {
273 pod := v1.Pod{}
274 pod.UID = types.UID(p)
275 for c := range tc.assignments[p] {
276 container := v1.Container{}
277 container.Name = c
278 pod.Spec.Containers = append(pod.Spec.Containers, container)
279 }
280 activePods = append(activePods, &pod)
281 }
282
283 m := manager{
284 policy: &staticPolicy{
285 topology: topology,
286 },
287 state: &mockState{
288 assignments: tc.assignments,
289 defaultCPUSet: tc.defaultCPUSet,
290 },
291 topology: topology,
292 activePods: func() []*v1.Pod { return activePods },
293 podStatusProvider: mockPodStatusProvider{},
294 sourcesReady: &sourcesReadyStub{},
295 }
296
297 podHints := m.GetPodTopologyHints(&tc.pod)[string(v1.ResourceCPU)]
298 if len(tc.expectedHints) == 0 && len(podHints) == 0 {
299 continue
300 }
301
302 sort.SliceStable(podHints, func(i, j int) bool {
303 return podHints[i].LessThan(podHints[j])
304 })
305 sort.SliceStable(tc.expectedHints, func(i, j int) bool {
306 return tc.expectedHints[i].LessThan(tc.expectedHints[j])
307 })
308 if !reflect.DeepEqual(tc.expectedHints, podHints) {
309 t.Errorf("Expected in result to be %v , got %v", tc.expectedHints, podHints)
310 }
311 }
312 }
313
314 func TestGetPodTopologyHintsWithPolicyOptions(t *testing.T) {
315 testPod1 := makePod("fakePod", "fakeContainer", "2", "2")
316 testContainer1 := &testPod1.Spec.Containers[0]
317
318 testPod2 := makePod("fakePod", "fakeContainer", "41", "41")
319 testContainer2 := &testPod1.Spec.Containers[0]
320
321 cpuSetAcrossSocket, _ := cpuset.Parse("0-28,40-57")
322
323 m0001, _ := bitmask.NewBitMask(0)
324 m0011, _ := bitmask.NewBitMask(0, 1)
325 m0101, _ := bitmask.NewBitMask(0, 2)
326 m1001, _ := bitmask.NewBitMask(0, 3)
327 m0111, _ := bitmask.NewBitMask(0, 1, 2)
328 m1011, _ := bitmask.NewBitMask(0, 1, 3)
329 m1101, _ := bitmask.NewBitMask(0, 2, 3)
330 m1111, _ := bitmask.NewBitMask(0, 1, 2, 3)
331
332 testCases := []struct {
333 description string
334 pod v1.Pod
335 container v1.Container
336 assignments state.ContainerCPUAssignments
337 defaultCPUSet cpuset.CPUSet
338 policyOptions map[string]string
339 topology *topology.CPUTopology
340 expectedHints []topologymanager.TopologyHint
341 }{
342 {
343
344 description: "AlignBySocket:false, Preferred hints does not contains socket aligned hints",
345 pod: *testPod1,
346 container: *testContainer1,
347 defaultCPUSet: cpuset.New(2, 3, 11),
348 topology: topoDualSocketMultiNumaPerSocketHT,
349 policyOptions: map[string]string{AlignBySocketOption: "false"},
350 expectedHints: []topologymanager.TopologyHint{
351 {
352 NUMANodeAffinity: m0001,
353 Preferred: true,
354 },
355 {
356 NUMANodeAffinity: m0011,
357 Preferred: false,
358 },
359 {
360 NUMANodeAffinity: m0101,
361 Preferred: false,
362 },
363 {
364 NUMANodeAffinity: m1001,
365 Preferred: false,
366 },
367 {
368 NUMANodeAffinity: m0111,
369 Preferred: false,
370 },
371 {
372 NUMANodeAffinity: m1011,
373 Preferred: false,
374 },
375 {
376 NUMANodeAffinity: m1101,
377 Preferred: false,
378 },
379 {
380 NUMANodeAffinity: m1111,
381 Preferred: false,
382 },
383 },
384 },
385 {
386
387 description: "AlignBySocket:true Preferred hints contains socket aligned hints",
388 pod: *testPod1,
389 container: *testContainer1,
390 defaultCPUSet: cpuset.New(2, 3, 11),
391 topology: topoDualSocketMultiNumaPerSocketHT,
392 policyOptions: map[string]string{AlignBySocketOption: "true"},
393 expectedHints: []topologymanager.TopologyHint{
394 {
395 NUMANodeAffinity: m0001,
396 Preferred: true,
397 },
398 {
399 NUMANodeAffinity: m0011,
400 Preferred: true,
401 },
402 {
403 NUMANodeAffinity: m0101,
404 Preferred: false,
405 },
406 {
407 NUMANodeAffinity: m1001,
408 Preferred: false,
409 },
410 {
411 NUMANodeAffinity: m0111,
412 Preferred: false,
413 },
414 {
415 NUMANodeAffinity: m1011,
416 Preferred: false,
417 },
418 {
419 NUMANodeAffinity: m1101,
420 Preferred: false,
421 },
422 {
423 NUMANodeAffinity: m1111,
424 Preferred: false,
425 },
426 },
427 },
428 {
429
430 description: "AlignBySocket:true Preferred hints are spread across socket since 2 sockets are required",
431 pod: *testPod2,
432 container: *testContainer2,
433 defaultCPUSet: cpuSetAcrossSocket,
434 topology: topoDualSocketMultiNumaPerSocketHT,
435 policyOptions: map[string]string{AlignBySocketOption: "true"},
436 expectedHints: []topologymanager.TopologyHint{
437 {
438 NUMANodeAffinity: m0111,
439 Preferred: true,
440 },
441 {
442 NUMANodeAffinity: m1111,
443 Preferred: true,
444 },
445 },
446 },
447 }
448
449 for _, testCase := range testCases {
450 t.Run(testCase.description, func(t *testing.T) {
451 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true)()
452
453 var activePods []*v1.Pod
454 for p := range testCase.assignments {
455 pod := v1.Pod{}
456 pod.UID = types.UID(p)
457 for c := range testCase.assignments[p] {
458 container := v1.Container{}
459 container.Name = c
460 pod.Spec.Containers = append(pod.Spec.Containers, container)
461 }
462 activePods = append(activePods, &pod)
463 }
464 policyOpt, _ := NewStaticPolicyOptions(testCase.policyOptions)
465 m := manager{
466 policy: &staticPolicy{
467 topology: testCase.topology,
468 options: policyOpt,
469 },
470 state: &mockState{
471 assignments: testCase.assignments,
472 defaultCPUSet: testCase.defaultCPUSet,
473 },
474 topology: testCase.topology,
475 activePods: func() []*v1.Pod { return activePods },
476 podStatusProvider: mockPodStatusProvider{},
477 sourcesReady: &sourcesReadyStub{},
478 }
479
480 podHints := m.GetPodTopologyHints(&testCase.pod)[string(v1.ResourceCPU)]
481 sort.SliceStable(podHints, func(i, j int) bool {
482 return podHints[i].LessThan(podHints[j])
483 })
484 sort.SliceStable(testCase.expectedHints, func(i, j int) bool {
485 return testCase.expectedHints[i].LessThan(testCase.expectedHints[j])
486 })
487 if !reflect.DeepEqual(testCase.expectedHints, podHints) {
488 t.Errorf("Expected in result to be %v , got %v", testCase.expectedHints, podHints)
489 }
490 })
491 }
492 }
493
494 func returnTestCases() []testCase {
495 testPod1 := makePod("fakePod", "fakeContainer", "2", "2")
496 testContainer1 := &testPod1.Spec.Containers[0]
497 testPod2 := makePod("fakePod", "fakeContainer", "5", "5")
498 testContainer2 := &testPod2.Spec.Containers[0]
499 testPod3 := makePod("fakePod", "fakeContainer", "7", "7")
500 testContainer3 := &testPod3.Spec.Containers[0]
501 testPod4 := makePod("fakePod", "fakeContainer", "11", "11")
502 testContainer4 := &testPod4.Spec.Containers[0]
503
504 firstSocketMask, _ := bitmask.NewBitMask(0)
505 secondSocketMask, _ := bitmask.NewBitMask(1)
506 crossSocketMask, _ := bitmask.NewBitMask(0, 1)
507
508 return []testCase{
509 {
510 name: "Request 2 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
511 pod: *testPod1,
512 container: *testContainer1,
513 defaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
514 expectedHints: []topologymanager.TopologyHint{
515 {
516 NUMANodeAffinity: firstSocketMask,
517 Preferred: true,
518 },
519 {
520 NUMANodeAffinity: secondSocketMask,
521 Preferred: true,
522 },
523 {
524 NUMANodeAffinity: crossSocketMask,
525 Preferred: false,
526 },
527 },
528 },
529 {
530 name: "Request 5 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
531 pod: *testPod2,
532 container: *testContainer2,
533 defaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
534 expectedHints: []topologymanager.TopologyHint{
535 {
536 NUMANodeAffinity: secondSocketMask,
537 Preferred: true,
538 },
539 {
540 NUMANodeAffinity: crossSocketMask,
541 Preferred: false,
542 },
543 },
544 },
545 {
546 name: "Request 7 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
547 pod: *testPod3,
548 container: *testContainer3,
549 defaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
550 expectedHints: []topologymanager.TopologyHint{
551 {
552 NUMANodeAffinity: crossSocketMask,
553 Preferred: true,
554 },
555 },
556 },
557 {
558 name: "Request 11 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
559 pod: *testPod4,
560 container: *testContainer4,
561 defaultCPUSet: cpuset.New(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
562 expectedHints: nil,
563 },
564 {
565 name: "Request 2 CPUs, 1 available on NUMA 0, 1 available on NUMA 1",
566 pod: *testPod1,
567 container: *testContainer1,
568 defaultCPUSet: cpuset.New(0, 3),
569 expectedHints: []topologymanager.TopologyHint{
570 {
571 NUMANodeAffinity: crossSocketMask,
572 Preferred: false,
573 },
574 },
575 },
576 {
577 name: "Request more CPUs than available",
578 pod: *testPod2,
579 container: *testContainer2,
580 defaultCPUSet: cpuset.New(0, 1, 2, 3),
581 expectedHints: nil,
582 },
583 {
584 name: "Regenerate Single-Node NUMA Hints if already allocated 1/2",
585 pod: *testPod1,
586 container: *testContainer1,
587 assignments: state.ContainerCPUAssignments{
588 string(testPod1.UID): map[string]cpuset.CPUSet{
589 testContainer1.Name: cpuset.New(0, 6),
590 },
591 },
592 defaultCPUSet: cpuset.New(),
593 expectedHints: []topologymanager.TopologyHint{
594 {
595 NUMANodeAffinity: firstSocketMask,
596 Preferred: true,
597 },
598 {
599 NUMANodeAffinity: crossSocketMask,
600 Preferred: false,
601 },
602 },
603 },
604 {
605 name: "Regenerate Single-Node NUMA Hints if already allocated 1/2",
606 pod: *testPod1,
607 container: *testContainer1,
608 assignments: state.ContainerCPUAssignments{
609 string(testPod1.UID): map[string]cpuset.CPUSet{
610 testContainer1.Name: cpuset.New(3, 9),
611 },
612 },
613 defaultCPUSet: cpuset.New(),
614 expectedHints: []topologymanager.TopologyHint{
615 {
616 NUMANodeAffinity: secondSocketMask,
617 Preferred: true,
618 },
619 {
620 NUMANodeAffinity: crossSocketMask,
621 Preferred: false,
622 },
623 },
624 },
625 {
626 name: "Regenerate Cross-NUMA Hints if already allocated",
627 pod: *testPod4,
628 container: *testContainer4,
629 assignments: state.ContainerCPUAssignments{
630 string(testPod4.UID): map[string]cpuset.CPUSet{
631 testContainer4.Name: cpuset.New(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10),
632 },
633 },
634 defaultCPUSet: cpuset.New(),
635 expectedHints: []topologymanager.TopologyHint{
636 {
637 NUMANodeAffinity: crossSocketMask,
638 Preferred: true,
639 },
640 },
641 },
642 {
643 name: "Requested less than already allocated",
644 pod: *testPod1,
645 container: *testContainer1,
646 assignments: state.ContainerCPUAssignments{
647 string(testPod1.UID): map[string]cpuset.CPUSet{
648 testContainer1.Name: cpuset.New(0, 6, 3, 9),
649 },
650 },
651 defaultCPUSet: cpuset.New(),
652 expectedHints: []topologymanager.TopologyHint{},
653 },
654 {
655 name: "Requested more than already allocated",
656 pod: *testPod4,
657 container: *testContainer4,
658 assignments: state.ContainerCPUAssignments{
659 string(testPod4.UID): map[string]cpuset.CPUSet{
660 testContainer4.Name: cpuset.New(0, 6, 3, 9),
661 },
662 },
663 defaultCPUSet: cpuset.New(),
664 expectedHints: []topologymanager.TopologyHint{},
665 },
666 }
667 }
668
View as plain text