1
16
17 package scheduler
18
19 import (
20 "context"
21 "reflect"
22 "testing"
23 "time"
24
25 v1 "k8s.io/api/core/v1"
26 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27 "k8s.io/apimachinery/pkg/util/sets"
28 "k8s.io/client-go/informers"
29 clientsetfake "k8s.io/client-go/kubernetes/fake"
30 "k8s.io/klog/v2/ktesting"
31 extenderv1 "k8s.io/kube-scheduler/extender/v1"
32 schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
33 "k8s.io/kubernetes/pkg/scheduler/framework"
34 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
35 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
36 "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
37 internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
38 internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
39 st "k8s.io/kubernetes/pkg/scheduler/testing"
40 tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
41 )
42
43 func TestSchedulerWithExtenders(t *testing.T) {
44 tests := []struct {
45 name string
46 registerPlugins []tf.RegisterPluginFunc
47 extenders []tf.FakeExtender
48 nodes []string
49 expectedResult ScheduleResult
50 expectsErr bool
51 }{
52 {
53 registerPlugins: []tf.RegisterPluginFunc{
54 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
55 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
56 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
57 },
58 extenders: []tf.FakeExtender{
59 {
60 ExtenderName: "FakeExtender1",
61 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
62 },
63 {
64 ExtenderName: "FakeExtender2",
65 Predicates: []tf.FitPredicate{tf.ErrorPredicateExtender},
66 },
67 },
68 nodes: []string{"node1", "node2"},
69 expectsErr: true,
70 name: "test 1",
71 },
72 {
73 registerPlugins: []tf.RegisterPluginFunc{
74 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
75 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
76 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
77 },
78 extenders: []tf.FakeExtender{
79 {
80 ExtenderName: "FakeExtender1",
81 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
82 },
83 {
84 ExtenderName: "FakeExtender2",
85 Predicates: []tf.FitPredicate{tf.FalsePredicateExtender},
86 },
87 },
88 nodes: []string{"node1", "node2"},
89 expectsErr: true,
90 name: "test 2",
91 },
92 {
93 registerPlugins: []tf.RegisterPluginFunc{
94 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
95 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
96 tf.RegisterScorePlugin("EqualPrioritizerPlugin", tf.NewEqualPrioritizerPlugin(), 1),
97 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
98 },
99 extenders: []tf.FakeExtender{
100 {
101 ExtenderName: "FakeExtender1",
102 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
103 },
104 {
105 ExtenderName: "FakeExtender2",
106 Predicates: []tf.FitPredicate{tf.Node1PredicateExtender},
107 },
108 },
109 nodes: []string{"node1", "node2"},
110 expectedResult: ScheduleResult{
111 SuggestedHost: "node1",
112 EvaluatedNodes: 2,
113 FeasibleNodes: 1,
114 },
115 name: "test 3",
116 },
117 {
118 registerPlugins: []tf.RegisterPluginFunc{
119 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
120 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
121 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
122 },
123 extenders: []tf.FakeExtender{
124 {
125 ExtenderName: "FakeExtender1",
126 Predicates: []tf.FitPredicate{tf.Node2PredicateExtender},
127 },
128 {
129 ExtenderName: "FakeExtender2",
130 Predicates: []tf.FitPredicate{tf.Node1PredicateExtender},
131 },
132 },
133 nodes: []string{"node1", "node2"},
134 expectsErr: true,
135 name: "test 4",
136 },
137 {
138 registerPlugins: []tf.RegisterPluginFunc{
139 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
140 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
141 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
142 },
143 extenders: []tf.FakeExtender{
144 {
145 ExtenderName: "FakeExtender1",
146 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
147 Prioritizers: []tf.PriorityConfig{{Function: tf.ErrorPrioritizerExtender, Weight: 10}},
148 Weight: 1,
149 },
150 },
151 nodes: []string{"node1"},
152 expectedResult: ScheduleResult{
153 SuggestedHost: "node1",
154 EvaluatedNodes: 1,
155 FeasibleNodes: 1,
156 },
157 name: "test 5",
158 },
159 {
160 registerPlugins: []tf.RegisterPluginFunc{
161 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
162 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
163 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
164 },
165 extenders: []tf.FakeExtender{
166 {
167 ExtenderName: "FakeExtender1",
168 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
169 Prioritizers: []tf.PriorityConfig{{Function: tf.Node1PrioritizerExtender, Weight: 10}},
170 Weight: 1,
171 },
172 {
173 ExtenderName: "FakeExtender2",
174 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
175 Prioritizers: []tf.PriorityConfig{{Function: tf.Node2PrioritizerExtender, Weight: 10}},
176 Weight: 5,
177 },
178 },
179 nodes: []string{"node1", "node2"},
180 expectedResult: ScheduleResult{
181 SuggestedHost: "node2",
182 EvaluatedNodes: 2,
183 FeasibleNodes: 2,
184 },
185 name: "test 6",
186 },
187 {
188 registerPlugins: []tf.RegisterPluginFunc{
189 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
190 tf.RegisterScorePlugin("Node2Prioritizer", tf.NewNode2PrioritizerPlugin(), 20),
191 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
192 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
193 },
194 extenders: []tf.FakeExtender{
195 {
196 ExtenderName: "FakeExtender1",
197 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
198 Prioritizers: []tf.PriorityConfig{{Function: tf.Node1PrioritizerExtender, Weight: 10}},
199 Weight: 1,
200 },
201 },
202 nodes: []string{"node1", "node2"},
203 expectedResult: ScheduleResult{
204 SuggestedHost: "node2",
205 EvaluatedNodes: 2,
206 FeasibleNodes: 2,
207 },
208 name: "test 7",
209 },
210 {
211
212
213
214
215
216
217
218 registerPlugins: []tf.RegisterPluginFunc{
219 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
220 tf.RegisterScorePlugin("Node2Prioritizer", tf.NewNode2PrioritizerPlugin(), 1),
221 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
222 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
223 },
224 extenders: []tf.FakeExtender{
225 {
226 ExtenderName: "FakeExtender1",
227 Predicates: []tf.FitPredicate{tf.ErrorPredicateExtender},
228 Prioritizers: []tf.PriorityConfig{{Function: tf.ErrorPrioritizerExtender, Weight: 10}},
229 UnInterested: true,
230 },
231 },
232 nodes: []string{"node1", "node2"},
233 expectsErr: false,
234 expectedResult: ScheduleResult{
235 SuggestedHost: "node2",
236 EvaluatedNodes: 2,
237 FeasibleNodes: 2,
238 },
239 name: "test 8",
240 },
241 {
242
243
244
245
246
247 registerPlugins: []tf.RegisterPluginFunc{
248 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
249 tf.RegisterScorePlugin("EqualPrioritizerPlugin", tf.NewEqualPrioritizerPlugin(), 1),
250 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
251 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
252 },
253 extenders: []tf.FakeExtender{
254 {
255 ExtenderName: "FakeExtender1",
256 Predicates: []tf.FitPredicate{tf.ErrorPredicateExtender},
257 Ignorable: true,
258 },
259 {
260 ExtenderName: "FakeExtender2",
261 Predicates: []tf.FitPredicate{tf.Node1PredicateExtender},
262 },
263 },
264 nodes: []string{"node1", "node2"},
265 expectsErr: false,
266 expectedResult: ScheduleResult{
267 SuggestedHost: "node1",
268 EvaluatedNodes: 2,
269 FeasibleNodes: 1,
270 },
271 name: "test 9",
272 },
273 {
274 registerPlugins: []tf.RegisterPluginFunc{
275 tf.RegisterFilterPlugin("TrueFilter", tf.NewTrueFilterPlugin),
276 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
277 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
278 },
279 extenders: []tf.FakeExtender{
280 {
281 ExtenderName: "FakeExtender1",
282 Predicates: []tf.FitPredicate{tf.TruePredicateExtender},
283 },
284 {
285 ExtenderName: "FakeExtender2",
286 Predicates: []tf.FitPredicate{tf.Node1PredicateExtender},
287 },
288 },
289 nodes: []string{"node1", "node2"},
290 expectedResult: ScheduleResult{
291 SuggestedHost: "node1",
292 EvaluatedNodes: 2,
293 FeasibleNodes: 1,
294 },
295 name: "test 10 - no scoring, extender filters configured, multiple feasible nodes are evaluated",
296 },
297 {
298 registerPlugins: []tf.RegisterPluginFunc{
299 tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
300 tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
301 },
302 extenders: []tf.FakeExtender{
303 {
304 ExtenderName: "FakeExtender1",
305 Binder: func() error { return nil },
306 },
307 },
308 nodes: []string{"node1", "node2"},
309 expectedResult: ScheduleResult{
310 SuggestedHost: "node1",
311 EvaluatedNodes: 1,
312 FeasibleNodes: 1,
313 },
314 name: "test 11 - no scoring, no prefilters or extender filters configured, a single feasible node is evaluated",
315 },
316 }
317
318 for _, test := range tests {
319 t.Run(test.name, func(t *testing.T) {
320 client := clientsetfake.NewSimpleClientset()
321 informerFactory := informers.NewSharedInformerFactory(client, 0)
322
323 var extenders []framework.Extender
324 for ii := range test.extenders {
325 extenders = append(extenders, &test.extenders[ii])
326 }
327 logger, ctx := ktesting.NewTestContext(t)
328 ctx, cancel := context.WithCancel(ctx)
329 defer cancel()
330
331 cache := internalcache.New(ctx, time.Duration(0))
332 for _, name := range test.nodes {
333 cache.AddNode(logger, createNode(name))
334 }
335 fwk, err := tf.NewFramework(
336 ctx,
337 test.registerPlugins, "",
338 runtime.WithClientSet(client),
339 runtime.WithInformerFactory(informerFactory),
340 runtime.WithPodNominator(internalqueue.NewPodNominator(informerFactory.Core().V1().Pods().Lister())),
341 runtime.WithLogger(logger),
342 )
343 if err != nil {
344 t.Fatal(err)
345 }
346
347 sched := &Scheduler{
348 Cache: cache,
349 nodeInfoSnapshot: emptySnapshot,
350 percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
351 Extenders: extenders,
352 logger: logger,
353 }
354 sched.applyDefaultHandlers()
355
356 podIgnored := &v1.Pod{}
357 result, err := sched.SchedulePod(ctx, fwk, framework.NewCycleState(), podIgnored)
358 if test.expectsErr {
359 if err == nil {
360 t.Errorf("Unexpected non-error, result %+v", result)
361 }
362 } else {
363 if err != nil {
364 t.Errorf("Unexpected error: %v", err)
365 return
366 }
367
368 if !reflect.DeepEqual(result, test.expectedResult) {
369 t.Errorf("Expected: %+v, Saw: %+v", test.expectedResult, result)
370 }
371 }
372 })
373 }
374 }
375
376 func createNode(name string) *v1.Node {
377 return &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}}
378 }
379
380 func TestIsInterested(t *testing.T) {
381 mem := &HTTPExtender{
382 managedResources: sets.New[string](),
383 }
384 mem.managedResources.Insert("memory")
385
386 for _, tc := range []struct {
387 label string
388 extender *HTTPExtender
389 pod *v1.Pod
390 want bool
391 }{
392 {
393 label: "Empty managed resources",
394 extender: &HTTPExtender{
395 managedResources: sets.New[string](),
396 },
397 pod: &v1.Pod{},
398 want: true,
399 },
400 {
401 label: "Managed memory, empty resources",
402 extender: mem,
403 pod: st.MakePod().Container("app").Obj(),
404 want: false,
405 },
406 {
407 label: "Managed memory, container memory with Requests",
408 extender: mem,
409 pod: st.MakePod().Req(map[v1.ResourceName]string{
410 "memory": "0",
411 }).Obj(),
412 want: true,
413 },
414 {
415 label: "Managed memory, container memory with Limits",
416 extender: mem,
417 pod: st.MakePod().Lim(map[v1.ResourceName]string{
418 "memory": "0",
419 }).Obj(),
420 want: true,
421 },
422 {
423 label: "Managed memory, init container memory",
424 extender: mem,
425 pod: st.MakePod().Container("app").InitReq(map[v1.ResourceName]string{
426 "memory": "0",
427 }).Obj(),
428 want: true,
429 },
430 } {
431 t.Run(tc.label, func(t *testing.T) {
432 if got := tc.extender.IsInterested(tc.pod); got != tc.want {
433 t.Fatalf("IsInterested(%v) = %v, wanted %v", tc.pod, got, tc.want)
434 }
435 })
436 }
437 }
438
439 func TestConvertToMetaVictims(t *testing.T) {
440 tests := []struct {
441 name string
442 nodeNameToVictims map[string]*extenderv1.Victims
443 want map[string]*extenderv1.MetaVictims
444 }{
445 {
446 name: "test NumPDBViolations is transferred from nodeNameToVictims to nodeNameToMetaVictims",
447 nodeNameToVictims: map[string]*extenderv1.Victims{
448 "node1": {
449 Pods: []*v1.Pod{
450 st.MakePod().Name("pod1").UID("uid1").Obj(),
451 st.MakePod().Name("pod3").UID("uid3").Obj(),
452 },
453 NumPDBViolations: 1,
454 },
455 "node2": {
456 Pods: []*v1.Pod{
457 st.MakePod().Name("pod2").UID("uid2").Obj(),
458 st.MakePod().Name("pod4").UID("uid4").Obj(),
459 },
460 NumPDBViolations: 2,
461 },
462 },
463 want: map[string]*extenderv1.MetaVictims{
464 "node1": {
465 Pods: []*extenderv1.MetaPod{
466 {UID: "uid1"},
467 {UID: "uid3"},
468 },
469 NumPDBViolations: 1,
470 },
471 "node2": {
472 Pods: []*extenderv1.MetaPod{
473 {UID: "uid2"},
474 {UID: "uid4"},
475 },
476 NumPDBViolations: 2,
477 },
478 },
479 },
480 }
481 for _, tt := range tests {
482 t.Run(tt.name, func(t *testing.T) {
483 if got := convertToMetaVictims(tt.nodeNameToVictims); !reflect.DeepEqual(got, tt.want) {
484 t.Errorf("convertToMetaVictims() = %v, want %v", got, tt.want)
485 }
486 })
487 }
488 }
489
490 func TestConvertToVictims(t *testing.T) {
491 tests := []struct {
492 name string
493 httpExtender *HTTPExtender
494 nodeNameToMetaVictims map[string]*extenderv1.MetaVictims
495 nodeNames []string
496 podsInNodeList []*v1.Pod
497 nodeInfos framework.NodeInfoLister
498 want map[string]*extenderv1.Victims
499 wantErr bool
500 }{
501 {
502 name: "test NumPDBViolations is transferred from NodeNameToMetaVictims to newNodeNameToVictims",
503 httpExtender: &HTTPExtender{},
504 nodeNameToMetaVictims: map[string]*extenderv1.MetaVictims{
505 "node1": {
506 Pods: []*extenderv1.MetaPod{
507 {UID: "uid1"},
508 {UID: "uid3"},
509 },
510 NumPDBViolations: 1,
511 },
512 "node2": {
513 Pods: []*extenderv1.MetaPod{
514 {UID: "uid2"},
515 {UID: "uid4"},
516 },
517 NumPDBViolations: 2,
518 },
519 },
520 nodeNames: []string{"node1", "node2"},
521 podsInNodeList: []*v1.Pod{
522 st.MakePod().Name("pod1").UID("uid1").Obj(),
523 st.MakePod().Name("pod2").UID("uid2").Obj(),
524 st.MakePod().Name("pod3").UID("uid3").Obj(),
525 st.MakePod().Name("pod4").UID("uid4").Obj(),
526 },
527 nodeInfos: nil,
528 want: map[string]*extenderv1.Victims{
529 "node1": {
530 Pods: []*v1.Pod{
531 st.MakePod().Name("pod1").UID("uid1").Obj(),
532 st.MakePod().Name("pod3").UID("uid3").Obj(),
533 },
534 NumPDBViolations: 1,
535 },
536 "node2": {
537 Pods: []*v1.Pod{
538 st.MakePod().Name("pod2").UID("uid2").Obj(),
539 st.MakePod().Name("pod4").UID("uid4").Obj(),
540 },
541 NumPDBViolations: 2,
542 },
543 },
544 },
545 }
546 for _, tt := range tests {
547 t.Run(tt.name, func(t *testing.T) {
548
549 nodeInfoList := make([]*framework.NodeInfo, 0, len(tt.nodeNames))
550 for i, nm := range tt.nodeNames {
551 nodeInfo := framework.NewNodeInfo()
552 node := createNode(nm)
553 nodeInfo.SetNode(node)
554 nodeInfo.AddPod(tt.podsInNodeList[i])
555 nodeInfo.AddPod(tt.podsInNodeList[i+2])
556 nodeInfoList = append(nodeInfoList, nodeInfo)
557 }
558 tt.nodeInfos = tf.NodeInfoLister(nodeInfoList)
559
560 got, err := tt.httpExtender.convertToVictims(tt.nodeNameToMetaVictims, tt.nodeInfos)
561 if (err != nil) != tt.wantErr {
562 t.Errorf("convertToVictims() error = %v, wantErr %v", err, tt.wantErr)
563 return
564 }
565 if !reflect.DeepEqual(got, tt.want) {
566 t.Errorf("convertToVictims() got = %v, want %v", got, tt.want)
567 }
568 })
569 }
570 }
571
View as plain text