1
16
17 package nodeaffinity
18
19 import (
20 "context"
21 "fmt"
22 "testing"
23
24 "github.com/google/go-cmp/cmp"
25 "github.com/stretchr/testify/require"
26 v1 "k8s.io/api/core/v1"
27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28 "k8s.io/apimachinery/pkg/util/sets"
29 "k8s.io/klog/v2/ktesting"
30 "k8s.io/kubernetes/pkg/scheduler/apis/config"
31 "k8s.io/kubernetes/pkg/scheduler/framework"
32 "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
33 "k8s.io/kubernetes/pkg/scheduler/internal/cache"
34 st "k8s.io/kubernetes/pkg/scheduler/testing"
35 tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
36 )
37
38
39 func TestNodeAffinity(t *testing.T) {
40 tests := []struct {
41 name string
42 pod *v1.Pod
43 labels map[string]string
44 nodeName string
45 wantStatus *framework.Status
46 wantPreFilterStatus *framework.Status
47 wantPreFilterResult *framework.PreFilterResult
48 args config.NodeAffinityArgs
49 runPreFilter bool
50 }{
51 {
52 name: "missing labels",
53 pod: st.MakePod().NodeSelector(map[string]string{
54 "foo": "bar",
55 }).Obj(),
56 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
57 runPreFilter: true,
58 },
59 {
60 name: "same labels",
61 pod: st.MakePod().NodeSelector(map[string]string{
62 "foo": "bar",
63 }).Obj(),
64 labels: map[string]string{
65 "foo": "bar",
66 },
67 runPreFilter: true,
68 },
69 {
70 name: "node labels are superset",
71 pod: st.MakePod().NodeSelector(map[string]string{
72 "foo": "bar",
73 }).Obj(),
74 labels: map[string]string{
75 "foo": "bar",
76 "baz": "blah",
77 },
78 runPreFilter: true,
79 },
80 {
81 name: "node labels are subset",
82 pod: st.MakePod().NodeSelector(map[string]string{
83 "foo": "bar",
84 "baz": "blah",
85 }).Obj(),
86 labels: map[string]string{
87 "foo": "bar",
88 },
89 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
90 runPreFilter: true,
91 },
92 {
93 name: "Pod with matchExpressions using In operator that matches the existing node",
94 pod: &v1.Pod{
95 Spec: v1.PodSpec{
96 Affinity: &v1.Affinity{
97 NodeAffinity: &v1.NodeAffinity{
98 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
99 NodeSelectorTerms: []v1.NodeSelectorTerm{
100 {
101 MatchExpressions: []v1.NodeSelectorRequirement{
102 {
103 Key: "foo",
104 Operator: v1.NodeSelectorOpIn,
105 Values: []string{"bar", "value2"},
106 },
107 },
108 },
109 },
110 },
111 },
112 },
113 },
114 },
115 labels: map[string]string{
116 "foo": "bar",
117 },
118 runPreFilter: true,
119 },
120 {
121 name: "Pod with matchExpressions using Gt operator that matches the existing node",
122 pod: &v1.Pod{
123 Spec: v1.PodSpec{
124 Affinity: &v1.Affinity{
125 NodeAffinity: &v1.NodeAffinity{
126 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
127 NodeSelectorTerms: []v1.NodeSelectorTerm{
128 {
129 MatchExpressions: []v1.NodeSelectorRequirement{
130 {
131 Key: "kernel-version",
132 Operator: v1.NodeSelectorOpGt,
133 Values: []string{"0204"},
134 },
135 },
136 },
137 },
138 },
139 },
140 },
141 },
142 },
143 labels: map[string]string{
144
145 "kernel-version": "0206",
146 },
147 runPreFilter: true,
148 },
149 {
150 name: "Pod with matchExpressions using NotIn operator that matches the existing node",
151 pod: &v1.Pod{
152 Spec: v1.PodSpec{
153 Affinity: &v1.Affinity{
154 NodeAffinity: &v1.NodeAffinity{
155 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
156 NodeSelectorTerms: []v1.NodeSelectorTerm{
157 {
158 MatchExpressions: []v1.NodeSelectorRequirement{
159 {
160 Key: "mem-type",
161 Operator: v1.NodeSelectorOpNotIn,
162 Values: []string{"DDR", "DDR2"},
163 },
164 },
165 },
166 },
167 },
168 },
169 },
170 },
171 },
172 labels: map[string]string{
173 "mem-type": "DDR3",
174 },
175 runPreFilter: true,
176 },
177 {
178 name: "Pod with matchExpressions using Exists operator that matches the existing node",
179 pod: &v1.Pod{
180 Spec: v1.PodSpec{
181 Affinity: &v1.Affinity{
182 NodeAffinity: &v1.NodeAffinity{
183 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
184 NodeSelectorTerms: []v1.NodeSelectorTerm{
185 {
186 MatchExpressions: []v1.NodeSelectorRequirement{
187 {
188 Key: "GPU",
189 Operator: v1.NodeSelectorOpExists,
190 },
191 },
192 },
193 },
194 },
195 },
196 },
197 },
198 },
199 labels: map[string]string{
200 "GPU": "NVIDIA-GRID-K1",
201 },
202 runPreFilter: true,
203 },
204 {
205 name: "Pod with affinity that don't match node's labels won't schedule onto the node",
206 pod: &v1.Pod{
207 Spec: v1.PodSpec{
208 Affinity: &v1.Affinity{
209 NodeAffinity: &v1.NodeAffinity{
210 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
211 NodeSelectorTerms: []v1.NodeSelectorTerm{
212 {
213 MatchExpressions: []v1.NodeSelectorRequirement{
214 {
215 Key: "foo",
216 Operator: v1.NodeSelectorOpIn,
217 Values: []string{"value1", "value2"},
218 },
219 },
220 },
221 },
222 },
223 },
224 },
225 },
226 },
227 labels: map[string]string{
228 "foo": "bar",
229 },
230 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
231 runPreFilter: true,
232 },
233 {
234 name: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node",
235 pod: &v1.Pod{
236 Spec: v1.PodSpec{
237 Affinity: &v1.Affinity{
238 NodeAffinity: &v1.NodeAffinity{
239 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
240 NodeSelectorTerms: []v1.NodeSelectorTerm{
241 {
242 MatchExpressions: []v1.NodeSelectorRequirement{},
243 },
244 },
245 },
246 },
247 },
248 },
249 },
250 labels: map[string]string{
251 "foo": "bar",
252 },
253 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
254 runPreFilter: true,
255 },
256 {
257 name: "Pod with no Affinity will schedule onto a node",
258 pod: &v1.Pod{},
259 labels: map[string]string{
260 "foo": "bar",
261 },
262 wantPreFilterStatus: framework.NewStatus(framework.Skip),
263 runPreFilter: true,
264 },
265 {
266 name: "Pod with Affinity but nil NodeSelector will schedule onto a node",
267 pod: &v1.Pod{
268 Spec: v1.PodSpec{
269 Affinity: &v1.Affinity{
270 NodeAffinity: &v1.NodeAffinity{
271 RequiredDuringSchedulingIgnoredDuringExecution: nil,
272 },
273 },
274 },
275 },
276 labels: map[string]string{
277 "foo": "bar",
278 },
279 wantPreFilterStatus: framework.NewStatus(framework.Skip),
280 runPreFilter: true,
281 },
282 {
283 name: "Pod with multiple matchExpressions ANDed that matches the existing node",
284 pod: &v1.Pod{
285 Spec: v1.PodSpec{
286 Affinity: &v1.Affinity{
287 NodeAffinity: &v1.NodeAffinity{
288 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
289 NodeSelectorTerms: []v1.NodeSelectorTerm{
290 {
291 MatchExpressions: []v1.NodeSelectorRequirement{
292 {
293 Key: "GPU",
294 Operator: v1.NodeSelectorOpExists,
295 }, {
296 Key: "GPU",
297 Operator: v1.NodeSelectorOpNotIn,
298 Values: []string{"AMD", "INTER"},
299 },
300 },
301 },
302 },
303 },
304 },
305 },
306 },
307 },
308 labels: map[string]string{
309 "GPU": "NVIDIA-GRID-K1",
310 },
311 runPreFilter: true,
312 },
313 {
314 name: "Pod with multiple matchExpressions ANDed that doesn't match the existing node",
315 pod: &v1.Pod{
316 Spec: v1.PodSpec{
317 Affinity: &v1.Affinity{
318 NodeAffinity: &v1.NodeAffinity{
319 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
320 NodeSelectorTerms: []v1.NodeSelectorTerm{
321 {
322 MatchExpressions: []v1.NodeSelectorRequirement{
323 {
324 Key: "GPU",
325 Operator: v1.NodeSelectorOpExists,
326 }, {
327 Key: "GPU",
328 Operator: v1.NodeSelectorOpIn,
329 Values: []string{"AMD", "INTER"},
330 },
331 },
332 },
333 },
334 },
335 },
336 },
337 },
338 },
339 labels: map[string]string{
340 "GPU": "NVIDIA-GRID-K1",
341 },
342 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
343 runPreFilter: true,
344 },
345 {
346 name: "Pod with multiple NodeSelectorTerms ORed in affinity, matches the node's labels and will schedule onto the node",
347 pod: &v1.Pod{
348 Spec: v1.PodSpec{
349 Affinity: &v1.Affinity{
350 NodeAffinity: &v1.NodeAffinity{
351 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
352 NodeSelectorTerms: []v1.NodeSelectorTerm{
353 {
354 MatchExpressions: []v1.NodeSelectorRequirement{
355 {
356 Key: "foo",
357 Operator: v1.NodeSelectorOpIn,
358 Values: []string{"bar", "value2"},
359 },
360 },
361 },
362 {
363 MatchExpressions: []v1.NodeSelectorRequirement{
364 {
365 Key: "diffkey",
366 Operator: v1.NodeSelectorOpIn,
367 Values: []string{"wrong", "value2"},
368 },
369 },
370 },
371 },
372 },
373 },
374 },
375 },
376 },
377 labels: map[string]string{
378 "foo": "bar",
379 },
380 runPreFilter: true,
381 },
382 {
383 name: "Pod with an Affinity and a PodSpec.NodeSelector(the old thing that we are deprecating) " +
384 "both are satisfied, will schedule onto the node",
385 pod: &v1.Pod{
386 Spec: v1.PodSpec{
387 NodeSelector: map[string]string{
388 "foo": "bar",
389 },
390 Affinity: &v1.Affinity{
391 NodeAffinity: &v1.NodeAffinity{
392 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
393 NodeSelectorTerms: []v1.NodeSelectorTerm{
394 {
395 MatchExpressions: []v1.NodeSelectorRequirement{
396 {
397 Key: "foo",
398 Operator: v1.NodeSelectorOpExists,
399 },
400 },
401 },
402 },
403 },
404 },
405 },
406 },
407 },
408 labels: map[string]string{
409 "foo": "bar",
410 },
411 runPreFilter: true,
412 },
413 {
414 name: "Pod with an Affinity matches node's labels but the PodSpec.NodeSelector(the old thing that we are deprecating) " +
415 "is not satisfied, won't schedule onto the node",
416 pod: &v1.Pod{
417 Spec: v1.PodSpec{
418 NodeSelector: map[string]string{
419 "foo": "bar",
420 },
421 Affinity: &v1.Affinity{
422 NodeAffinity: &v1.NodeAffinity{
423 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
424 NodeSelectorTerms: []v1.NodeSelectorTerm{
425 {
426 MatchExpressions: []v1.NodeSelectorRequirement{
427 {
428 Key: "foo",
429 Operator: v1.NodeSelectorOpExists,
430 },
431 },
432 },
433 },
434 },
435 },
436 },
437 },
438 },
439 labels: map[string]string{
440 "foo": "barrrrrr",
441 },
442 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
443 runPreFilter: true,
444 },
445 {
446 name: "Pod with an invalid value in Affinity term won't be scheduled onto the node",
447 pod: &v1.Pod{
448 Spec: v1.PodSpec{
449 Affinity: &v1.Affinity{
450 NodeAffinity: &v1.NodeAffinity{
451 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
452 NodeSelectorTerms: []v1.NodeSelectorTerm{
453 {
454 MatchExpressions: []v1.NodeSelectorRequirement{
455 {
456 Key: "foo",
457 Operator: v1.NodeSelectorOpNotIn,
458 Values: []string{"invalid value: ___@#$%^"},
459 },
460 },
461 },
462 },
463 },
464 },
465 },
466 },
467 },
468 labels: map[string]string{
469 "foo": "bar",
470 },
471 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
472 runPreFilter: true,
473 },
474 {
475 name: "Pod with matchFields using In operator that matches the existing node",
476 pod: &v1.Pod{
477 Spec: v1.PodSpec{
478 Affinity: &v1.Affinity{
479 NodeAffinity: &v1.NodeAffinity{
480 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
481 NodeSelectorTerms: []v1.NodeSelectorTerm{
482 {
483 MatchFields: []v1.NodeSelectorRequirement{
484 {
485 Key: metav1.ObjectNameField,
486 Operator: v1.NodeSelectorOpIn,
487 Values: []string{"node1"},
488 },
489 },
490 },
491 },
492 },
493 },
494 },
495 },
496 },
497 nodeName: "node1",
498 wantPreFilterResult: &framework.PreFilterResult{NodeNames: sets.New("node1")},
499 runPreFilter: true,
500 },
501 {
502 name: "Pod with matchFields using In operator that does not match the existing node",
503 pod: &v1.Pod{
504 Spec: v1.PodSpec{
505 Affinity: &v1.Affinity{
506 NodeAffinity: &v1.NodeAffinity{
507 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
508 NodeSelectorTerms: []v1.NodeSelectorTerm{
509 {
510 MatchFields: []v1.NodeSelectorRequirement{
511 {
512 Key: metav1.ObjectNameField,
513 Operator: v1.NodeSelectorOpIn,
514 Values: []string{"node1"},
515 },
516 },
517 },
518 },
519 },
520 },
521 },
522 },
523 },
524 nodeName: "node2",
525 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
526 wantPreFilterResult: &framework.PreFilterResult{NodeNames: sets.New("node1")},
527 runPreFilter: true,
528 },
529 {
530 name: "Pod with two terms: matchFields does not match, but matchExpressions matches",
531 pod: &v1.Pod{
532 Spec: v1.PodSpec{
533 Affinity: &v1.Affinity{
534 NodeAffinity: &v1.NodeAffinity{
535 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
536 NodeSelectorTerms: []v1.NodeSelectorTerm{
537 {
538 MatchFields: []v1.NodeSelectorRequirement{
539 {
540 Key: metav1.ObjectNameField,
541 Operator: v1.NodeSelectorOpIn,
542 Values: []string{"node1"},
543 },
544 {
545 Key: metav1.ObjectNameField,
546 Operator: v1.NodeSelectorOpIn,
547 Values: []string{"node2"},
548 },
549 },
550 },
551 {
552 MatchExpressions: []v1.NodeSelectorRequirement{
553 {
554 Key: "foo",
555 Operator: v1.NodeSelectorOpIn,
556 Values: []string{"bar"},
557 },
558 },
559 },
560 },
561 },
562 },
563 },
564 },
565 },
566 nodeName: "node2",
567 labels: map[string]string{"foo": "bar"},
568 runPreFilter: true,
569 },
570 {
571 name: "Pod with one term: matchFields does not match, but matchExpressions matches",
572 pod: &v1.Pod{
573 Spec: v1.PodSpec{
574 Affinity: &v1.Affinity{
575 NodeAffinity: &v1.NodeAffinity{
576 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
577 NodeSelectorTerms: []v1.NodeSelectorTerm{
578 {
579 MatchFields: []v1.NodeSelectorRequirement{
580 {
581 Key: metav1.ObjectNameField,
582 Operator: v1.NodeSelectorOpIn,
583 Values: []string{"node1"},
584 },
585 },
586 MatchExpressions: []v1.NodeSelectorRequirement{
587 {
588 Key: "foo",
589 Operator: v1.NodeSelectorOpIn,
590 Values: []string{"bar"},
591 },
592 },
593 },
594 },
595 },
596 },
597 },
598 },
599 },
600 nodeName: "node2",
601 labels: map[string]string{"foo": "bar"},
602 wantPreFilterResult: &framework.PreFilterResult{NodeNames: sets.New("node1")},
603 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
604 runPreFilter: true,
605 },
606 {
607 name: "Pod with one term: both matchFields and matchExpressions match",
608 pod: &v1.Pod{
609 Spec: v1.PodSpec{
610 Affinity: &v1.Affinity{
611 NodeAffinity: &v1.NodeAffinity{
612 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
613 NodeSelectorTerms: []v1.NodeSelectorTerm{
614 {
615 MatchFields: []v1.NodeSelectorRequirement{
616 {
617 Key: metav1.ObjectNameField,
618 Operator: v1.NodeSelectorOpIn,
619 Values: []string{"node1"},
620 },
621 },
622 MatchExpressions: []v1.NodeSelectorRequirement{
623 {
624 Key: "foo",
625 Operator: v1.NodeSelectorOpIn,
626 Values: []string{"bar"},
627 },
628 },
629 },
630 },
631 },
632 },
633 },
634 },
635 },
636 nodeName: "node1",
637 labels: map[string]string{"foo": "bar"},
638 wantPreFilterResult: &framework.PreFilterResult{NodeNames: sets.New("node1")},
639 runPreFilter: true,
640 },
641 {
642 name: "Pod with two terms: both matchFields and matchExpressions do not match",
643 pod: &v1.Pod{
644 Spec: v1.PodSpec{
645 Affinity: &v1.Affinity{
646 NodeAffinity: &v1.NodeAffinity{
647 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
648 NodeSelectorTerms: []v1.NodeSelectorTerm{
649 {
650 MatchFields: []v1.NodeSelectorRequirement{
651 {
652 Key: metav1.ObjectNameField,
653 Operator: v1.NodeSelectorOpIn,
654 Values: []string{"node1"},
655 },
656 },
657 },
658 {
659 MatchExpressions: []v1.NodeSelectorRequirement{
660 {
661 Key: "foo",
662 Operator: v1.NodeSelectorOpIn,
663 Values: []string{"not-match-to-bar"},
664 },
665 },
666 },
667 },
668 },
669 },
670 },
671 },
672 },
673 nodeName: "node2",
674 labels: map[string]string{"foo": "bar"},
675 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
676 runPreFilter: true,
677 },
678 {
679 name: "Pod with two terms of node.Name affinity",
680 pod: &v1.Pod{
681 Spec: v1.PodSpec{
682 Affinity: &v1.Affinity{
683 NodeAffinity: &v1.NodeAffinity{
684 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
685 NodeSelectorTerms: []v1.NodeSelectorTerm{
686 {
687 MatchFields: []v1.NodeSelectorRequirement{
688 {
689 Key: metav1.ObjectNameField,
690 Operator: v1.NodeSelectorOpIn,
691 Values: []string{"node1"},
692 },
693 },
694 },
695 {
696 MatchFields: []v1.NodeSelectorRequirement{
697 {
698 Key: metav1.ObjectNameField,
699 Operator: v1.NodeSelectorOpIn,
700 Values: []string{"node2"},
701 },
702 },
703 },
704 },
705 },
706 },
707 },
708 },
709 },
710 nodeName: "node2",
711 wantPreFilterResult: &framework.PreFilterResult{NodeNames: sets.New("node1", "node2")},
712 runPreFilter: true,
713 },
714 {
715 name: "Pod with two conflicting mach field requirements",
716 pod: &v1.Pod{
717 Spec: v1.PodSpec{
718 Affinity: &v1.Affinity{
719 NodeAffinity: &v1.NodeAffinity{
720 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
721 NodeSelectorTerms: []v1.NodeSelectorTerm{
722 {
723 MatchFields: []v1.NodeSelectorRequirement{
724 {
725 Key: metav1.ObjectNameField,
726 Operator: v1.NodeSelectorOpIn,
727 Values: []string{"node1"},
728 },
729 {
730 Key: metav1.ObjectNameField,
731 Operator: v1.NodeSelectorOpIn,
732 Values: []string{"node2"},
733 },
734 },
735 },
736 },
737 },
738 },
739 },
740 },
741 },
742 nodeName: "node2",
743 labels: map[string]string{"foo": "bar"},
744 wantPreFilterStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, errReasonConflict),
745 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
746 runPreFilter: true,
747 },
748 {
749 name: "Matches added affinity and Pod's node affinity",
750 pod: &v1.Pod{
751 Spec: v1.PodSpec{
752 Affinity: &v1.Affinity{
753 NodeAffinity: &v1.NodeAffinity{
754 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
755 NodeSelectorTerms: []v1.NodeSelectorTerm{
756 {
757 MatchExpressions: []v1.NodeSelectorRequirement{
758 {
759 Key: "zone",
760 Operator: v1.NodeSelectorOpIn,
761 Values: []string{"foo"},
762 },
763 },
764 },
765 },
766 },
767 },
768 },
769 },
770 },
771 nodeName: "node2",
772 labels: map[string]string{"zone": "foo"},
773 args: config.NodeAffinityArgs{
774 AddedAffinity: &v1.NodeAffinity{
775 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
776 NodeSelectorTerms: []v1.NodeSelectorTerm{{
777 MatchFields: []v1.NodeSelectorRequirement{{
778 Key: metav1.ObjectNameField,
779 Operator: v1.NodeSelectorOpIn,
780 Values: []string{"node2"},
781 }},
782 }},
783 },
784 },
785 },
786 runPreFilter: true,
787 },
788 {
789 name: "Matches added affinity but not Pod's node affinity",
790 pod: &v1.Pod{
791 Spec: v1.PodSpec{
792 Affinity: &v1.Affinity{
793 NodeAffinity: &v1.NodeAffinity{
794 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
795 NodeSelectorTerms: []v1.NodeSelectorTerm{
796 {
797 MatchExpressions: []v1.NodeSelectorRequirement{
798 {
799 Key: "zone",
800 Operator: v1.NodeSelectorOpIn,
801 Values: []string{"bar"},
802 },
803 },
804 },
805 },
806 },
807 },
808 },
809 },
810 },
811 nodeName: "node2",
812 labels: map[string]string{"zone": "foo"},
813 args: config.NodeAffinityArgs{
814 AddedAffinity: &v1.NodeAffinity{
815 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
816 NodeSelectorTerms: []v1.NodeSelectorTerm{{
817 MatchFields: []v1.NodeSelectorRequirement{{
818 Key: metav1.ObjectNameField,
819 Operator: v1.NodeSelectorOpIn,
820 Values: []string{"node2"},
821 }},
822 }},
823 },
824 },
825 },
826 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonPod),
827 runPreFilter: true,
828 },
829 {
830 name: "Doesn't match added affinity",
831 pod: &v1.Pod{},
832 nodeName: "node2",
833 labels: map[string]string{"zone": "foo"},
834 args: config.NodeAffinityArgs{
835 AddedAffinity: &v1.NodeAffinity{
836 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
837 NodeSelectorTerms: []v1.NodeSelectorTerm{{
838 MatchExpressions: []v1.NodeSelectorRequirement{
839 {
840 Key: "zone",
841 Operator: v1.NodeSelectorOpIn,
842 Values: []string{"bar"},
843 },
844 },
845 }},
846 },
847 },
848 },
849 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, errReasonEnforced),
850 runPreFilter: true,
851 },
852 {
853 name: "Matches node selector correctly even if PreFilter is not called",
854 pod: &v1.Pod{
855 Spec: v1.PodSpec{
856 NodeSelector: map[string]string{
857 "foo": "bar",
858 },
859 },
860 },
861 labels: map[string]string{
862 "foo": "bar",
863 "baz": "blah",
864 },
865 runPreFilter: false,
866 },
867 {
868 name: "Matches node affinity correctly even if PreFilter is not called",
869 pod: &v1.Pod{
870 Spec: v1.PodSpec{
871 Affinity: &v1.Affinity{
872 NodeAffinity: &v1.NodeAffinity{
873 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
874 NodeSelectorTerms: []v1.NodeSelectorTerm{
875 {
876 MatchExpressions: []v1.NodeSelectorRequirement{
877 {
878 Key: "GPU",
879 Operator: v1.NodeSelectorOpExists,
880 }, {
881 Key: "GPU",
882 Operator: v1.NodeSelectorOpNotIn,
883 Values: []string{"AMD", "INTER"},
884 },
885 },
886 },
887 },
888 },
889 },
890 },
891 },
892 },
893 labels: map[string]string{
894 "GPU": "NVIDIA-GRID-K1",
895 },
896 runPreFilter: false,
897 },
898 }
899
900 for _, test := range tests {
901 t.Run(test.name, func(t *testing.T) {
902 _, ctx := ktesting.NewTestContext(t)
903 node := v1.Node{ObjectMeta: metav1.ObjectMeta{
904 Name: test.nodeName,
905 Labels: test.labels,
906 }}
907 nodeInfo := framework.NewNodeInfo()
908 nodeInfo.SetNode(&node)
909
910 p, err := New(ctx, &test.args, nil)
911 if err != nil {
912 t.Fatalf("Creating plugin: %v", err)
913 }
914
915 state := framework.NewCycleState()
916 var gotStatus *framework.Status
917 if test.runPreFilter {
918 gotPreFilterResult, gotStatus := p.(framework.PreFilterPlugin).PreFilter(context.Background(), state, test.pod)
919 if diff := cmp.Diff(test.wantPreFilterStatus, gotStatus); diff != "" {
920 t.Errorf("unexpected PreFilter Status (-want,+got):\n%s", diff)
921 }
922 if diff := cmp.Diff(test.wantPreFilterResult, gotPreFilterResult); diff != "" {
923 t.Errorf("unexpected PreFilterResult (-want,+got):\n%s", diff)
924 }
925 }
926 gotStatus = p.(framework.FilterPlugin).Filter(context.Background(), state, test.pod, nodeInfo)
927 if diff := cmp.Diff(test.wantStatus, gotStatus); diff != "" {
928 t.Errorf("unexpected Filter Status (-want,+got):\n%s", diff)
929 }
930 })
931 }
932 }
933
934 func TestNodeAffinityPriority(t *testing.T) {
935 label1 := map[string]string{"foo": "bar"}
936 label2 := map[string]string{"key": "value"}
937 label3 := map[string]string{"az": "az1"}
938 label4 := map[string]string{"abc": "az11", "def": "az22"}
939 label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"}
940
941 affinity1 := &v1.Affinity{
942 NodeAffinity: &v1.NodeAffinity{
943 PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{
944 Weight: 2,
945 Preference: v1.NodeSelectorTerm{
946 MatchExpressions: []v1.NodeSelectorRequirement{{
947 Key: "foo",
948 Operator: v1.NodeSelectorOpIn,
949 Values: []string{"bar"},
950 }},
951 },
952 }},
953 },
954 }
955
956 affinity2 := &v1.Affinity{
957 NodeAffinity: &v1.NodeAffinity{
958 PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
959 {
960 Weight: 2,
961 Preference: v1.NodeSelectorTerm{
962 MatchExpressions: []v1.NodeSelectorRequirement{
963 {
964 Key: "foo",
965 Operator: v1.NodeSelectorOpIn,
966 Values: []string{"bar"},
967 },
968 },
969 },
970 },
971 {
972 Weight: 4,
973 Preference: v1.NodeSelectorTerm{
974 MatchExpressions: []v1.NodeSelectorRequirement{
975 {
976 Key: "key",
977 Operator: v1.NodeSelectorOpIn,
978 Values: []string{"value"},
979 },
980 },
981 },
982 },
983 {
984 Weight: 5,
985 Preference: v1.NodeSelectorTerm{
986 MatchExpressions: []v1.NodeSelectorRequirement{
987 {
988 Key: "foo",
989 Operator: v1.NodeSelectorOpIn,
990 Values: []string{"bar"},
991 },
992 {
993 Key: "key",
994 Operator: v1.NodeSelectorOpIn,
995 Values: []string{"value"},
996 },
997 {
998 Key: "az",
999 Operator: v1.NodeSelectorOpIn,
1000 Values: []string{"az1"},
1001 },
1002 },
1003 },
1004 },
1005 },
1006 },
1007 }
1008
1009 tests := []struct {
1010 name string
1011 pod *v1.Pod
1012 nodes []*v1.Node
1013 expectedList framework.NodeScoreList
1014 args config.NodeAffinityArgs
1015 runPreScore bool
1016 wantPreScoreStatus *framework.Status
1017 }{
1018 {
1019 name: "all nodes are same priority as NodeAffinity is nil",
1020 pod: &v1.Pod{
1021 ObjectMeta: metav1.ObjectMeta{
1022 Annotations: map[string]string{},
1023 },
1024 },
1025 nodes: []*v1.Node{
1026 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1027 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1028 {ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: label3}},
1029 },
1030 expectedList: []framework.NodeScore{{Name: "node1", Score: 0}, {Name: "node2", Score: 0}, {Name: "node3", Score: 0}},
1031 },
1032 {
1033
1034 name: "Skip is returned in PreScore when NodeAffinity is nil",
1035 pod: &v1.Pod{
1036 ObjectMeta: metav1.ObjectMeta{
1037 Annotations: map[string]string{},
1038 },
1039 },
1040 nodes: []*v1.Node{
1041 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1042 },
1043 runPreScore: true,
1044 wantPreScoreStatus: framework.NewStatus(framework.Skip),
1045 },
1046 {
1047 name: "PreScore returns error when an incoming Pod has a broken affinity",
1048 pod: &v1.Pod{
1049 ObjectMeta: metav1.ObjectMeta{
1050 Annotations: map[string]string{},
1051 },
1052 Spec: v1.PodSpec{
1053 Affinity: &v1.Affinity{
1054 NodeAffinity: &v1.NodeAffinity{
1055 PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
1056 {
1057 Weight: 2,
1058 Preference: v1.NodeSelectorTerm{
1059 MatchExpressions: []v1.NodeSelectorRequirement{
1060 {
1061 Key: "invalid key",
1062 Operator: v1.NodeSelectorOpIn,
1063 Values: []string{"bar"},
1064 },
1065 },
1066 },
1067 },
1068 },
1069 },
1070 },
1071 },
1072 },
1073 nodes: []*v1.Node{
1074 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1075 },
1076 runPreScore: true,
1077 wantPreScoreStatus: framework.AsStatus(fmt.Errorf(`[0].matchExpressions[0].key: Invalid value: "invalid key": name part must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]')`)),
1078 },
1079 {
1080 name: "no node matches preferred scheduling requirements in NodeAffinity of pod so all nodes' priority is zero",
1081 pod: &v1.Pod{
1082 Spec: v1.PodSpec{
1083 Affinity: affinity1,
1084 },
1085 },
1086 nodes: []*v1.Node{
1087 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label4}},
1088 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1089 {ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: label3}},
1090 },
1091 expectedList: []framework.NodeScore{{Name: "node1", Score: 0}, {Name: "node2", Score: 0}, {Name: "node3", Score: 0}},
1092 runPreScore: true,
1093 },
1094 {
1095 name: "only node1 matches the preferred scheduling requirements of pod",
1096 pod: &v1.Pod{
1097 Spec: v1.PodSpec{
1098 Affinity: affinity1,
1099 },
1100 },
1101 nodes: []*v1.Node{
1102 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1103 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1104 {ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: label3}},
1105 },
1106 expectedList: []framework.NodeScore{{Name: "node1", Score: framework.MaxNodeScore}, {Name: "node2", Score: 0}, {Name: "node3", Score: 0}},
1107 runPreScore: true,
1108 },
1109 {
1110 name: "all nodes matches the preferred scheduling requirements of pod but with different priorities ",
1111 pod: &v1.Pod{
1112 Spec: v1.PodSpec{
1113 Affinity: affinity2,
1114 },
1115 },
1116 nodes: []*v1.Node{
1117 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1118 {ObjectMeta: metav1.ObjectMeta{Name: "node5", Labels: label5}},
1119 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1120 },
1121 expectedList: []framework.NodeScore{{Name: "node1", Score: 18}, {Name: "node5", Score: framework.MaxNodeScore}, {Name: "node2", Score: 36}},
1122 runPreScore: true,
1123 },
1124 {
1125 name: "added affinity",
1126 pod: &v1.Pod{},
1127 nodes: []*v1.Node{
1128 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1129 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1130 },
1131 expectedList: []framework.NodeScore{{Name: "node1", Score: framework.MaxNodeScore}, {Name: "node2", Score: 0}},
1132 args: config.NodeAffinityArgs{
1133 AddedAffinity: affinity1.NodeAffinity,
1134 },
1135 runPreScore: true,
1136 },
1137 {
1138 name: "added affinity and pod has default affinity",
1139 pod: &v1.Pod{
1140 Spec: v1.PodSpec{
1141 Affinity: affinity1,
1142 },
1143 },
1144 nodes: []*v1.Node{
1145 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1146 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1147 {ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: label5}},
1148 },
1149 expectedList: []framework.NodeScore{{Name: "node1", Score: 40}, {Name: "node2", Score: 60}, {Name: "node3", Score: framework.MaxNodeScore}},
1150 args: config.NodeAffinityArgs{
1151 AddedAffinity: &v1.NodeAffinity{
1152 PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
1153 {
1154 Weight: 3,
1155 Preference: v1.NodeSelectorTerm{
1156 MatchExpressions: []v1.NodeSelectorRequirement{
1157 {
1158 Key: "key",
1159 Operator: v1.NodeSelectorOpIn,
1160 Values: []string{"value"},
1161 },
1162 },
1163 },
1164 },
1165 },
1166 },
1167 },
1168 runPreScore: true,
1169 },
1170 {
1171 name: "calculate the priorities correctly even if PreScore is not called",
1172 pod: &v1.Pod{
1173 Spec: v1.PodSpec{
1174 Affinity: affinity2,
1175 },
1176 },
1177 nodes: []*v1.Node{
1178 {ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: label1}},
1179 {ObjectMeta: metav1.ObjectMeta{Name: "node5", Labels: label5}},
1180 {ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: label2}},
1181 },
1182 expectedList: []framework.NodeScore{{Name: "node1", Score: 18}, {Name: "node5", Score: framework.MaxNodeScore}, {Name: "node2", Score: 36}},
1183 runPreScore: true,
1184 },
1185 }
1186
1187 for _, test := range tests {
1188 t.Run(test.name, func(t *testing.T) {
1189 _, ctx := ktesting.NewTestContext(t)
1190 ctx, cancel := context.WithCancel(ctx)
1191 defer cancel()
1192
1193 state := framework.NewCycleState()
1194 fh, _ := runtime.NewFramework(ctx, nil, nil, runtime.WithSnapshotSharedLister(cache.NewSnapshot(nil, test.nodes)))
1195 p, err := New(ctx, &test.args, fh)
1196 if err != nil {
1197 t.Fatalf("Creating plugin: %v", err)
1198 }
1199 var status *framework.Status
1200 if test.runPreScore {
1201 status = p.(framework.PreScorePlugin).PreScore(ctx, state, test.pod, tf.BuildNodeInfos(test.nodes))
1202 if status.Code() != test.wantPreScoreStatus.Code() {
1203 t.Errorf("unexpected status code from PreScore: want: %v got: %v", test.wantPreScoreStatus.Code().String(), status.Code().String())
1204 }
1205 if status.Message() != test.wantPreScoreStatus.Message() {
1206 t.Errorf("unexpected status message from PreScore: want: %v got: %v", test.wantPreScoreStatus.Message(), status.Message())
1207 }
1208 if !status.IsSuccess() {
1209
1210 return
1211 }
1212 }
1213 var gotList framework.NodeScoreList
1214 for _, n := range test.nodes {
1215 nodeName := n.ObjectMeta.Name
1216 score, status := p.(framework.ScorePlugin).Score(ctx, state, test.pod, nodeName)
1217 if !status.IsSuccess() {
1218 t.Errorf("unexpected error: %v", status)
1219 }
1220 gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
1221 }
1222
1223 status = p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(ctx, state, test.pod, gotList)
1224 if !status.IsSuccess() {
1225 t.Errorf("unexpected error: %v", status)
1226 }
1227
1228 if diff := cmp.Diff(test.expectedList, gotList); diff != "" {
1229 t.Errorf("obtained scores (-want,+got):\n%s", diff)
1230 }
1231 })
1232 }
1233 }
1234
1235 func Test_isSchedulableAfterNodeChange(t *testing.T) {
1236 podWithNodeAffinity := st.MakePod().NodeAffinityIn("foo", []string{"bar"})
1237 testcases := map[string]struct {
1238 args *config.NodeAffinityArgs
1239 pod *v1.Pod
1240 oldObj, newObj interface{}
1241 expectedHint framework.QueueingHint
1242 expectedErr bool
1243 }{
1244 "backoff-wrong-new-object": {
1245 args: &config.NodeAffinityArgs{},
1246 pod: podWithNodeAffinity.Obj(),
1247 newObj: "not-a-node",
1248 expectedHint: framework.Queue,
1249 expectedErr: true,
1250 },
1251 "backoff-wrong-old-object": {
1252 args: &config.NodeAffinityArgs{},
1253 pod: podWithNodeAffinity.Obj(),
1254 oldObj: "not-a-node",
1255 newObj: st.MakeNode().Obj(),
1256 expectedHint: framework.Queue,
1257 expectedErr: true,
1258 },
1259 "skip-queue-on-add": {
1260 args: &config.NodeAffinityArgs{},
1261 pod: podWithNodeAffinity.Obj(),
1262 newObj: st.MakeNode().Obj(),
1263 expectedHint: framework.QueueSkip,
1264 },
1265 "queue-on-add": {
1266 args: &config.NodeAffinityArgs{},
1267 pod: podWithNodeAffinity.Obj(),
1268 newObj: st.MakeNode().Label("foo", "bar").Obj(),
1269 expectedHint: framework.Queue,
1270 },
1271 "skip-unrelated-changes": {
1272 args: &config.NodeAffinityArgs{},
1273 pod: podWithNodeAffinity.Obj(),
1274 oldObj: st.MakeNode().Obj(),
1275 newObj: st.MakeNode().Capacity(nil).Obj(),
1276 expectedHint: framework.QueueSkip,
1277 },
1278 "skip-unrelated-changes-on-labels": {
1279 args: &config.NodeAffinityArgs{},
1280 pod: podWithNodeAffinity.DeepCopy(),
1281 oldObj: st.MakeNode().Obj(),
1282 newObj: st.MakeNode().Label("k", "v").Obj(),
1283 expectedHint: framework.QueueSkip,
1284 },
1285 "skip-labels-changes-on-node-from-suitable-to-unsuitable": {
1286 args: &config.NodeAffinityArgs{},
1287 pod: podWithNodeAffinity.DeepCopy(),
1288 oldObj: st.MakeNode().Label("foo", "bar").Obj(),
1289 newObj: st.MakeNode().Label("k", "v").Obj(),
1290 expectedHint: framework.QueueSkip,
1291 },
1292 "queue-on-labels-change-makes-pod-schedulable": {
1293 args: &config.NodeAffinityArgs{},
1294 pod: podWithNodeAffinity.Obj(),
1295 oldObj: st.MakeNode().Obj(),
1296 newObj: st.MakeNode().Label("foo", "bar").Obj(),
1297 expectedHint: framework.Queue,
1298 },
1299 "skip-queue-on-add-scheduler-enforced-node-affinity": {
1300 args: &config.NodeAffinityArgs{
1301 AddedAffinity: &v1.NodeAffinity{
1302 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
1303 NodeSelectorTerms: []v1.NodeSelectorTerm{
1304 {
1305 MatchExpressions: []v1.NodeSelectorRequirement{
1306 {
1307 Key: "foo",
1308 Operator: v1.NodeSelectorOpIn,
1309 Values: []string{"bar"},
1310 },
1311 },
1312 },
1313 },
1314 },
1315 },
1316 },
1317 pod: podWithNodeAffinity.Obj(),
1318 newObj: st.MakeNode().Obj(),
1319 expectedHint: framework.QueueSkip,
1320 },
1321 "queue-on-add-scheduler-enforced-node-affinity": {
1322 args: &config.NodeAffinityArgs{
1323 AddedAffinity: &v1.NodeAffinity{
1324 RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
1325 NodeSelectorTerms: []v1.NodeSelectorTerm{
1326 {
1327 MatchExpressions: []v1.NodeSelectorRequirement{
1328 {
1329 Key: "foo",
1330 Operator: v1.NodeSelectorOpIn,
1331 Values: []string{"bar"},
1332 },
1333 },
1334 },
1335 },
1336 },
1337 },
1338 },
1339 pod: podWithNodeAffinity.Obj(),
1340 newObj: st.MakeNode().Label("foo", "bar").Obj(),
1341 expectedHint: framework.Queue,
1342 },
1343 }
1344
1345 for name, tc := range testcases {
1346 t.Run(name, func(t *testing.T) {
1347 logger, ctx := ktesting.NewTestContext(t)
1348 p, err := New(ctx, tc.args, nil)
1349 if err != nil {
1350 t.Fatalf("Creating plugin: %v", err)
1351 }
1352
1353 actualHint, err := p.(*NodeAffinity).isSchedulableAfterNodeChange(logger, tc.pod, tc.oldObj, tc.newObj)
1354 if tc.expectedErr {
1355 require.Error(t, err)
1356 return
1357 }
1358 require.NoError(t, err)
1359 require.Equal(t, tc.expectedHint, actualHint)
1360 })
1361 }
1362 }
1363
View as plain text