1
16
17 package tainttoleration
18
19 import (
20 "context"
21 "reflect"
22 "testing"
23
24 v1 "k8s.io/api/core/v1"
25 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26 "k8s.io/klog/v2/ktesting"
27 "k8s.io/kubernetes/pkg/scheduler/framework"
28 "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
29 "k8s.io/kubernetes/pkg/scheduler/internal/cache"
30 tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
31 )
32
33 func nodeWithTaints(nodeName string, taints []v1.Taint) *v1.Node {
34 return &v1.Node{
35 ObjectMeta: metav1.ObjectMeta{
36 Name: nodeName,
37 },
38 Spec: v1.NodeSpec{
39 Taints: taints,
40 },
41 }
42 }
43
44 func podWithTolerations(podName string, tolerations []v1.Toleration) *v1.Pod {
45 return &v1.Pod{
46 ObjectMeta: metav1.ObjectMeta{
47 Name: podName,
48 },
49 Spec: v1.PodSpec{
50 Tolerations: tolerations,
51 },
52 }
53 }
54
55 func TestTaintTolerationScore(t *testing.T) {
56 tests := []struct {
57 name string
58 pod *v1.Pod
59 nodes []*v1.Node
60 expectedList framework.NodeScoreList
61 }{
62
63 {
64 name: "node with taints tolerated by the pod, gets a higher score than those node with intolerable taints",
65 pod: podWithTolerations("pod1", []v1.Toleration{{
66 Key: "foo",
67 Operator: v1.TolerationOpEqual,
68 Value: "bar",
69 Effect: v1.TaintEffectPreferNoSchedule,
70 }}),
71 nodes: []*v1.Node{
72 nodeWithTaints("nodeA", []v1.Taint{{
73 Key: "foo",
74 Value: "bar",
75 Effect: v1.TaintEffectPreferNoSchedule,
76 }}),
77 nodeWithTaints("nodeB", []v1.Taint{{
78 Key: "foo",
79 Value: "blah",
80 Effect: v1.TaintEffectPreferNoSchedule,
81 }}),
82 },
83 expectedList: []framework.NodeScore{
84 {Name: "nodeA", Score: framework.MaxNodeScore},
85 {Name: "nodeB", Score: 0},
86 },
87 },
88
89 {
90 name: "the nodes that all of their taints are tolerated by the pod, get the same score, no matter how many tolerable taints a node has",
91 pod: podWithTolerations("pod1", []v1.Toleration{
92 {
93 Key: "cpu-type",
94 Operator: v1.TolerationOpEqual,
95 Value: "arm64",
96 Effect: v1.TaintEffectPreferNoSchedule,
97 }, {
98 Key: "disk-type",
99 Operator: v1.TolerationOpEqual,
100 Value: "ssd",
101 Effect: v1.TaintEffectPreferNoSchedule,
102 },
103 }),
104 nodes: []*v1.Node{
105 nodeWithTaints("nodeA", []v1.Taint{}),
106 nodeWithTaints("nodeB", []v1.Taint{
107 {
108 Key: "cpu-type",
109 Value: "arm64",
110 Effect: v1.TaintEffectPreferNoSchedule,
111 },
112 }),
113 nodeWithTaints("nodeC", []v1.Taint{
114 {
115 Key: "cpu-type",
116 Value: "arm64",
117 Effect: v1.TaintEffectPreferNoSchedule,
118 }, {
119 Key: "disk-type",
120 Value: "ssd",
121 Effect: v1.TaintEffectPreferNoSchedule,
122 },
123 }),
124 },
125 expectedList: []framework.NodeScore{
126 {Name: "nodeA", Score: framework.MaxNodeScore},
127 {Name: "nodeB", Score: framework.MaxNodeScore},
128 {Name: "nodeC", Score: framework.MaxNodeScore},
129 },
130 },
131
132 {
133 name: "the more intolerable taints a node has, the lower score it gets.",
134 pod: podWithTolerations("pod1", []v1.Toleration{{
135 Key: "foo",
136 Operator: v1.TolerationOpEqual,
137 Value: "bar",
138 Effect: v1.TaintEffectPreferNoSchedule,
139 }}),
140 nodes: []*v1.Node{
141 nodeWithTaints("nodeA", []v1.Taint{}),
142 nodeWithTaints("nodeB", []v1.Taint{
143 {
144 Key: "cpu-type",
145 Value: "arm64",
146 Effect: v1.TaintEffectPreferNoSchedule,
147 },
148 }),
149 nodeWithTaints("nodeC", []v1.Taint{
150 {
151 Key: "cpu-type",
152 Value: "arm64",
153 Effect: v1.TaintEffectPreferNoSchedule,
154 }, {
155 Key: "disk-type",
156 Value: "ssd",
157 Effect: v1.TaintEffectPreferNoSchedule,
158 },
159 }),
160 },
161 expectedList: []framework.NodeScore{
162 {Name: "nodeA", Score: framework.MaxNodeScore},
163 {Name: "nodeB", Score: 50},
164 {Name: "nodeC", Score: 0},
165 },
166 },
167
168 {
169 name: "only taints and tolerations that have effect PreferNoSchedule are checked by taints-tolerations priority function",
170 pod: podWithTolerations("pod1", []v1.Toleration{
171 {
172 Key: "cpu-type",
173 Operator: v1.TolerationOpEqual,
174 Value: "arm64",
175 Effect: v1.TaintEffectNoSchedule,
176 }, {
177 Key: "disk-type",
178 Operator: v1.TolerationOpEqual,
179 Value: "ssd",
180 Effect: v1.TaintEffectNoSchedule,
181 },
182 }),
183 nodes: []*v1.Node{
184 nodeWithTaints("nodeA", []v1.Taint{}),
185 nodeWithTaints("nodeB", []v1.Taint{
186 {
187 Key: "cpu-type",
188 Value: "arm64",
189 Effect: v1.TaintEffectNoSchedule,
190 },
191 }),
192 nodeWithTaints("nodeC", []v1.Taint{
193 {
194 Key: "cpu-type",
195 Value: "arm64",
196 Effect: v1.TaintEffectPreferNoSchedule,
197 }, {
198 Key: "disk-type",
199 Value: "ssd",
200 Effect: v1.TaintEffectPreferNoSchedule,
201 },
202 }),
203 },
204 expectedList: []framework.NodeScore{
205 {Name: "nodeA", Score: framework.MaxNodeScore},
206 {Name: "nodeB", Score: framework.MaxNodeScore},
207 {Name: "nodeC", Score: 0},
208 },
209 },
210 {
211 name: "Default behaviour No taints and tolerations, lands on node with no taints",
212
213 pod: podWithTolerations("pod1", []v1.Toleration{}),
214 nodes: []*v1.Node{
215
216 nodeWithTaints("nodeA", []v1.Taint{}),
217 nodeWithTaints("nodeB", []v1.Taint{
218 {
219 Key: "cpu-type",
220 Value: "arm64",
221 Effect: v1.TaintEffectPreferNoSchedule,
222 },
223 }),
224 },
225 expectedList: []framework.NodeScore{
226 {Name: "nodeA", Score: framework.MaxNodeScore},
227 {Name: "nodeB", Score: 0},
228 },
229 },
230 }
231 for _, test := range tests {
232 t.Run(test.name, func(t *testing.T) {
233 _, ctx := ktesting.NewTestContext(t)
234 ctx, cancel := context.WithCancel(ctx)
235 defer cancel()
236
237 state := framework.NewCycleState()
238 snapshot := cache.NewSnapshot(nil, test.nodes)
239 fh, _ := runtime.NewFramework(ctx, nil, nil, runtime.WithSnapshotSharedLister(snapshot))
240
241 p, err := New(ctx, nil, fh)
242 if err != nil {
243 t.Fatalf("creating plugin: %v", err)
244 }
245 status := p.(framework.PreScorePlugin).PreScore(ctx, state, test.pod, tf.BuildNodeInfos(test.nodes))
246 if !status.IsSuccess() {
247 t.Errorf("unexpected error: %v", status)
248 }
249 var gotList framework.NodeScoreList
250 for _, n := range test.nodes {
251 nodeName := n.ObjectMeta.Name
252 score, status := p.(framework.ScorePlugin).Score(ctx, state, test.pod, nodeName)
253 if !status.IsSuccess() {
254 t.Errorf("unexpected error: %v", status)
255 }
256 gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
257 }
258
259 status = p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(ctx, state, test.pod, gotList)
260 if !status.IsSuccess() {
261 t.Errorf("unexpected error: %v", status)
262 }
263
264 if !reflect.DeepEqual(test.expectedList, gotList) {
265 t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList)
266 }
267 })
268 }
269 }
270
271 func TestTaintTolerationFilter(t *testing.T) {
272 tests := []struct {
273 name string
274 pod *v1.Pod
275 node *v1.Node
276 wantStatus *framework.Status
277 }{
278 {
279 name: "A pod having no tolerations can't be scheduled onto a node with nonempty taints",
280 pod: podWithTolerations("pod1", []v1.Toleration{}),
281 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
282 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
283 "node(s) had untolerated taint {dedicated: user1}"),
284 },
285 {
286 name: "A pod which can be scheduled on a dedicated node assigned to user1 with effect NoSchedule",
287 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
288 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
289 },
290 {
291 name: "A pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule",
292 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}),
293 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}),
294 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
295 "node(s) had untolerated taint {dedicated: user1}"),
296 },
297 {
298 name: "A pod can be scheduled onto the node, with a toleration uses operator Exists that tolerates the taints on the node",
299 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Exists", Effect: "NoSchedule"}}),
300 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
301 },
302 {
303 name: "A pod has multiple tolerations, node has multiple taints, all the taints are tolerated, pod can be scheduled onto the node",
304 pod: podWithTolerations("pod1", []v1.Toleration{
305 {Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"},
306 {Key: "foo", Operator: "Exists", Effect: "NoSchedule"},
307 }),
308 node: nodeWithTaints("nodeA", []v1.Taint{
309 {Key: "dedicated", Value: "user2", Effect: "NoSchedule"},
310 {Key: "foo", Value: "bar", Effect: "NoSchedule"},
311 }),
312 },
313 {
314 name: "A pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
315 "can't be scheduled onto the node",
316 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}}),
317 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
318 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable,
319 "node(s) had untolerated taint {foo: bar}"),
320 },
321 {
322 name: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
323 "and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
324 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar"}}),
325 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}),
326 },
327 {
328 name: "The pod has a toleration that key and value don't match the taint on the node, " +
329 "but the effect of taint on node is PreferNoSchedule. Pod can be scheduled onto the node",
330 pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}),
331 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}),
332 },
333 {
334 name: "The pod has no toleration, " +
335 "but the effect of taint on node is PreferNoSchedule. Pod can be scheduled onto the node",
336 pod: podWithTolerations("pod1", []v1.Toleration{}),
337 node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "PreferNoSchedule"}}),
338 },
339 }
340 for _, test := range tests {
341 t.Run(test.name, func(t *testing.T) {
342 _, ctx := ktesting.NewTestContext(t)
343 nodeInfo := framework.NewNodeInfo()
344 nodeInfo.SetNode(test.node)
345 p, err := New(ctx, nil, nil)
346 if err != nil {
347 t.Fatalf("creating plugin: %v", err)
348 }
349 gotStatus := p.(framework.FilterPlugin).Filter(ctx, nil, test.pod, nodeInfo)
350 if !reflect.DeepEqual(gotStatus, test.wantStatus) {
351 t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
352 }
353 })
354 }
355 }
356
View as plain text