1
16
17 package storage
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "reflect"
24 "strings"
25 "testing"
26
27 policyv1 "k8s.io/api/policy/v1"
28 apierrors "k8s.io/apimachinery/pkg/api/errors"
29 apimeta "k8s.io/apimachinery/pkg/api/meta"
30 metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32 "k8s.io/apimachinery/pkg/runtime"
33 "k8s.io/apimachinery/pkg/runtime/schema"
34 "k8s.io/apimachinery/pkg/watch"
35 genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
36 "k8s.io/apiserver/pkg/registry/rest"
37 utilfeature "k8s.io/apiserver/pkg/util/feature"
38 "k8s.io/client-go/kubernetes/fake"
39 featuregatetesting "k8s.io/component-base/featuregate/testing"
40 podapi "k8s.io/kubernetes/pkg/api/pod"
41 api "k8s.io/kubernetes/pkg/apis/core"
42 "k8s.io/kubernetes/pkg/apis/policy"
43 "k8s.io/kubernetes/pkg/features"
44 )
45
46 func TestEviction(t *testing.T) {
47 testcases := []struct {
48 name string
49 pdbs []runtime.Object
50 policies []*policyv1.UnhealthyPodEvictionPolicyType
51 eviction *policy.Eviction
52
53 badNameInURL bool
54
55 expectError string
56 expectDeleted bool
57 podPhase api.PodPhase
58 podName string
59 }{
60 {
61 name: "matching pdbs with no disruptions allowed, pod running",
62 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
63 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
64 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
65 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
66 }},
67 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t1", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
68 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
69 podPhase: api.PodRunning,
70 podName: "t1",
71 policies: []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)},
72 },
73 {
74 name: "matching pdbs with no disruptions allowed, pod pending",
75 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
76 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
77 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
78 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
79 }},
80 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
81 expectError: "",
82 podPhase: api.PodPending,
83 expectDeleted: true,
84 podName: "t2",
85 },
86 {
87 name: "matching pdbs with no disruptions allowed, pod succeeded",
88 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
89 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
90 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
91 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
92 }},
93 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t3", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
94 expectError: "",
95 podPhase: api.PodSucceeded,
96 expectDeleted: true,
97 podName: "t3",
98 },
99 {
100 name: "matching pdbs with no disruptions allowed, pod failed",
101 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
102 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
103 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
104 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
105 }},
106 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t4", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
107 expectError: "",
108 podPhase: api.PodFailed,
109 expectDeleted: true,
110 podName: "t4",
111 },
112 {
113 name: "matching pdbs with disruptions allowed",
114 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
115 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
116 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
117 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
118 }},
119 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t5", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
120 expectDeleted: true,
121 podName: "t5",
122 },
123 {
124 name: "non-matching pdbs",
125 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
126 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
127 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"b": "true"}}},
128 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
129 }},
130 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t6", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
131 expectDeleted: true,
132 podName: "t6",
133 },
134 {
135 name: "matching pdbs with disruptions allowed but bad name in Url",
136 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
137 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
138 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
139 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
140 }},
141 badNameInURL: true,
142 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t7", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
143 expectError: "name in URL does not match name in Eviction object: BadRequest",
144 podName: "t7",
145 },
146 {
147 name: "matching pdbs with no disruptions allowed, pod running, empty selector",
148 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
149 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
150 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{}},
151 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
152 }},
153 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
154 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
155 podPhase: api.PodRunning,
156 podName: "t8",
157 policies: []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)},
158 },
159 }
160
161 for _, unhealthyPodEvictionPolicy := range []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget), unhealthyPolicyPtr(policyv1.AlwaysAllow)} {
162 for _, tc := range testcases {
163 if len(tc.policies) > 0 && !hasUnhealthyPolicy(tc.policies, unhealthyPodEvictionPolicy) {
164
165 continue
166 }
167 t.Run(fmt.Sprintf("%v with %v policy", tc.name, unhealthyPolicyStr(unhealthyPodEvictionPolicy)), func(t *testing.T) {
168 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, true)()
169
170
171 evictionCopy := tc.eviction.DeepCopy()
172 var pdbsCopy []runtime.Object
173 for _, pdb := range tc.pdbs {
174 pdbCopy := pdb.DeepCopyObject()
175 pdbCopy.(*policyv1.PodDisruptionBudget).Spec.UnhealthyPodEvictionPolicy = unhealthyPodEvictionPolicy
176 pdbsCopy = append(pdbsCopy, pdbCopy)
177 }
178
179 testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
180 storage, _, statusStorage, server := newStorage(t)
181 defer server.Terminate(t)
182 defer storage.Store.DestroyFunc()
183
184 pod := validNewPod()
185 pod.Name = tc.podName
186 pod.Labels = map[string]string{"a": "true"}
187 pod.Spec.NodeName = "foo"
188 if _, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{}); err != nil {
189 t.Error(err)
190 }
191
192 if tc.podPhase != "" {
193 pod.Status.Phase = tc.podPhase
194 _, _, err := statusStorage.Update(testContext, pod.Name, rest.DefaultUpdatedObjectInfo(pod), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
195 if err != nil {
196 t.Errorf("Unexpected error: %v", err)
197 }
198 }
199
200 client := fake.NewSimpleClientset(pdbsCopy...)
201 evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
202
203 name := pod.Name
204 if tc.badNameInURL {
205 name += "bad-name"
206 }
207
208 _, err := evictionRest.Create(testContext, name, evictionCopy, nil, &metav1.CreateOptions{})
209 gotErr := errToString(err)
210 if gotErr != tc.expectError {
211 t.Errorf("error mismatch: expected %v, got %v; name %v", tc.expectError, gotErr, pod.Name)
212 return
213 }
214 if tc.badNameInURL {
215 if err == nil {
216 t.Error("expected error here, but got nil")
217 return
218 }
219 if err.Error() != "name in URL does not match name in Eviction object" {
220 t.Errorf("got unexpected error: %v", err)
221 }
222 }
223 if tc.expectError != "" {
224 return
225 }
226
227 existingPod, err := storage.Get(testContext, pod.Name, &metav1.GetOptions{})
228 if tc.expectDeleted {
229 if !apierrors.IsNotFound(err) {
230 t.Errorf("expected to be deleted, lookup returned %#v", existingPod)
231 }
232 return
233 } else if apierrors.IsNotFound(err) {
234 t.Errorf("expected graceful deletion, got %v", err)
235 return
236 }
237
238 if err != nil {
239 t.Errorf("%#v", err)
240 return
241 }
242
243 if existingPod.(*api.Pod).DeletionTimestamp == nil {
244 t.Errorf("expected gracefully deleted pod with deletionTimestamp set, got %#v", existingPod)
245 }
246 })
247 }
248 }
249 }
250
251 func TestEvictionIgnorePDB(t *testing.T) {
252 testcases := []struct {
253 name string
254 pdbs []runtime.Object
255 policies []*policyv1.UnhealthyPodEvictionPolicyType
256 eviction *policy.Eviction
257
258 expectError string
259 podPhase api.PodPhase
260 podName string
261 expectedDeleteCount int
262 podTerminating bool
263 prc *api.PodCondition
264 }{
265 {
266 name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod still pending, pod deleted successfully",
267 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
268 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
269 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
270 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
271 }},
272 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t1", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
273 expectError: "",
274 podPhase: api.PodPending,
275 podName: "t1",
276 expectedDeleteCount: 3,
277 },
278
279
280
281 {
282 name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod becomes running, continueToPDBs",
283 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
284 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
285 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
286 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
287 }},
288 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
289 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
290 podPhase: api.PodPending,
291 podName: "t2",
292 expectedDeleteCount: 1,
293 policies: []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)},
294 },
295 {
296 name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod becomes running, skip PDB check, conflict",
297 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
298 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
299 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
300 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
301 }},
302 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
303 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo is still being processed by the server.",
304 podPhase: api.PodPending,
305 podName: "t2",
306 podTerminating: false,
307 expectedDeleteCount: 2,
308 prc: &api.PodCondition{
309 Type: api.PodReady,
310 Status: api.ConditionFalse,
311 },
312 policies: []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow)},
313 },
314 {
315 name: "pdbs disruptions allowed, pod pending, first delete conflict, pod becomes running, continueToPDBs",
316 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
317 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
318 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
319 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
320 }},
321 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t3", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
322 expectError: "",
323 podPhase: api.PodPending,
324 podName: "t3",
325 expectedDeleteCount: 2,
326 policies: []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)},
327 },
328 {
329 name: "pod pending, always conflict on delete",
330 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
331 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
332 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
333 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
334 }},
335 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t4", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
336 expectError: `Operation cannot be fulfilled on tests "2": message: Conflict`,
337 podPhase: api.PodPending,
338 podName: "t4",
339 expectedDeleteCount: EvictionsRetry.Steps,
340 },
341 {
342 name: "pod pending, always conflict on delete, user provided ResourceVersion constraint",
343 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
344 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
345 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
346 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
347 }},
348 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t5", Namespace: "default"}, DeleteOptions: metav1.NewRVDeletionPrecondition("userProvided")},
349 expectError: `Operation cannot be fulfilled on tests "2": message: Conflict`,
350 podPhase: api.PodPending,
351 podName: "t5",
352 expectedDeleteCount: 1,
353 },
354 {
355 name: "matching pdbs with no disruptions allowed, pod terminating",
356 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
357 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
358 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
359 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
360 }},
361 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t6", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(300)},
362 expectError: "",
363 podName: "t6",
364 expectedDeleteCount: 1,
365 podTerminating: true,
366 },
367 {
368 name: "matching pdbs with no disruptions allowed, pod running, pod healthy, unhealthy pod not ours",
369 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
370 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
371 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
372 Status: policyv1.PodDisruptionBudgetStatus{
373
374 DisruptionsAllowed: 0,
375 CurrentHealthy: 2,
376 DesiredHealthy: 2,
377 },
378 }},
379 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t7", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
380 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 2 healthy pods and has 2 currently",
381 podName: "t7",
382 expectedDeleteCount: 0,
383 podTerminating: false,
384 podPhase: api.PodRunning,
385 prc: &api.PodCondition{
386 Type: api.PodReady,
387 Status: api.ConditionTrue,
388 },
389 },
390 {
391 name: "matching pdbs with disruptions allowed, pod running, pod healthy, healthy pod ours, deletes pod by honoring the PDB",
392 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
393 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
394 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
395 Status: policyv1.PodDisruptionBudgetStatus{
396 DisruptionsAllowed: 1,
397 CurrentHealthy: 3,
398 DesiredHealthy: 3,
399 },
400 }},
401 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
402 expectError: "",
403 podName: "t8",
404 expectedDeleteCount: 1,
405 podTerminating: false,
406 podPhase: api.PodRunning,
407 prc: &api.PodCondition{
408 Type: api.PodReady,
409 Status: api.ConditionTrue,
410 },
411 },
412 {
413 name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours",
414 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
415 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
416 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
417 Status: policyv1.PodDisruptionBudgetStatus{
418
419 DisruptionsAllowed: 0,
420 CurrentHealthy: 2,
421 DesiredHealthy: 2,
422 },
423 }},
424 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
425 expectError: "",
426 podName: "t8",
427 expectedDeleteCount: 1,
428 podTerminating: false,
429 podPhase: api.PodRunning,
430 prc: &api.PodCondition{
431 Type: api.PodReady,
432 Status: api.ConditionFalse,
433 },
434 },
435 {
436 name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, skips the PDB check and deletes",
437 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
438 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
439 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
440 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
441 }},
442 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
443 expectError: "",
444 podName: "t8",
445 expectedDeleteCount: 1,
446 podTerminating: false,
447 podPhase: api.PodRunning,
448 prc: &api.PodCondition{
449 Type: api.PodReady,
450 Status: api.ConditionFalse,
451 },
452 policies: []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow)},
453 },
454 {
455
456 name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, resource version conflict",
457 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
458 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
459 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
460 Status: policyv1.PodDisruptionBudgetStatus{
461
462 DisruptionsAllowed: 0,
463 CurrentHealthy: 2,
464 DesiredHealthy: 2,
465 },
466 }},
467 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t9", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
468 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo is still being processed by the server.",
469 podName: "t9",
470 expectedDeleteCount: 1,
471 podTerminating: false,
472 podPhase: api.PodRunning,
473 prc: &api.PodCondition{
474 Type: api.PodReady,
475 Status: api.ConditionFalse,
476 },
477 },
478 {
479
480 name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, other error on delete",
481 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
482 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
483 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
484 Status: policyv1.PodDisruptionBudgetStatus{
485
486 DisruptionsAllowed: 0,
487 CurrentHealthy: 2,
488 DesiredHealthy: 2,
489 },
490 }},
491 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t10", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
492 expectError: "test designed to error: BadRequest",
493 podName: "t10",
494 expectedDeleteCount: 1,
495 podTerminating: false,
496 podPhase: api.PodRunning,
497 prc: &api.PodCondition{
498 Type: api.PodReady,
499 Status: api.ConditionFalse,
500 },
501 },
502 {
503 name: "matching pdbs with no disruptions allowed, pod running, pod healthy, empty selector, pod not deleted by honoring the PDB",
504 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
505 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
506 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{}},
507 Status: policyv1.PodDisruptionBudgetStatus{
508 DisruptionsAllowed: 0,
509 CurrentHealthy: 3,
510 DesiredHealthy: 3,
511 },
512 }},
513 eviction: &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t11", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
514 expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 3 healthy pods and has 3 currently",
515 podName: "t11",
516 expectedDeleteCount: 0,
517 podTerminating: false,
518 podPhase: api.PodRunning,
519 prc: &api.PodCondition{
520 Type: api.PodReady,
521 Status: api.ConditionTrue,
522 },
523 },
524 }
525
526 for _, unhealthyPodEvictionPolicy := range []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow), nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)} {
527 for _, tc := range testcases {
528 if len(tc.policies) > 0 && !hasUnhealthyPolicy(tc.policies, unhealthyPodEvictionPolicy) {
529
530 continue
531 }
532 t.Run(fmt.Sprintf("%v with %v policy", tc.name, unhealthyPolicyStr(unhealthyPodEvictionPolicy)), func(t *testing.T) {
533 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, true)()
534
535
536 evictionCopy := tc.eviction.DeepCopy()
537 prcCopy := tc.prc.DeepCopy()
538 var pdbsCopy []runtime.Object
539 for _, pdb := range tc.pdbs {
540 pdbCopy := pdb.DeepCopyObject()
541 pdbCopy.(*policyv1.PodDisruptionBudget).Spec.UnhealthyPodEvictionPolicy = unhealthyPodEvictionPolicy
542 pdbsCopy = append(pdbsCopy, pdbCopy)
543 }
544
545 testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
546 ms := &mockStore{
547 deleteCount: 0,
548 }
549
550 pod := validNewPod()
551 pod.Name = tc.podName
552 pod.Labels = map[string]string{"a": "true"}
553 pod.Spec.NodeName = "foo"
554 if tc.podPhase != "" {
555 pod.Status.Phase = tc.podPhase
556 }
557
558 if tc.podTerminating {
559 currentTime := metav1.Now()
560 pod.ObjectMeta.DeletionTimestamp = ¤tTime
561 }
562
563
564 if tc.prc != nil {
565 if !podapi.UpdatePodCondition(&pod.Status, prcCopy) {
566 t.Fatalf("Unable to update pod ready condition")
567 }
568 }
569
570 client := fake.NewSimpleClientset(pdbsCopy...)
571 evictionRest := newEvictionStorage(ms, client.PolicyV1())
572
573 name := pod.Name
574 ms.pod = pod
575
576 _, err := evictionRest.Create(testContext, name, evictionCopy, nil, &metav1.CreateOptions{})
577 gotErr := errToString(err)
578 if gotErr != tc.expectError {
579 t.Errorf("error mismatch: expected %v, got %v; name %v", tc.expectError, gotErr, pod.Name)
580 return
581 }
582
583 if tc.expectedDeleteCount != ms.deleteCount {
584 t.Errorf("expected delete count=%v, got %v; name %v", tc.expectedDeleteCount, ms.deleteCount, pod.Name)
585 }
586 })
587 }
588 }
589 }
590
591 func TestEvictionDryRun(t *testing.T) {
592 testcases := []struct {
593 name string
594 evictionOptions *metav1.DeleteOptions
595 requestOptions *metav1.CreateOptions
596 pdbs []runtime.Object
597 }{
598 {
599 name: "just request-options",
600 requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}},
601 evictionOptions: &metav1.DeleteOptions{},
602 },
603 {
604 name: "just eviction-options",
605 requestOptions: &metav1.CreateOptions{},
606 evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
607 },
608 {
609 name: "both options",
610 evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
611 requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}},
612 },
613 {
614 name: "with pdbs",
615 evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
616 requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}},
617 pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
618 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
619 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
620 Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
621 }},
622 },
623 }
624
625 for _, tc := range testcases {
626 t.Run(tc.name, func(t *testing.T) {
627 testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
628 storage, _, _, server := newStorage(t)
629 defer server.Terminate(t)
630 defer storage.Store.DestroyFunc()
631
632 pod := validNewPod()
633 pod.Labels = map[string]string{"a": "true"}
634 pod.Spec.NodeName = "foo"
635 if _, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{}); err != nil {
636 t.Error(err)
637 }
638
639 client := fake.NewSimpleClientset(tc.pdbs...)
640 evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
641 eviction := &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: tc.evictionOptions}
642 _, err := evictionRest.Create(testContext, pod.Name, eviction, nil, tc.requestOptions)
643 if err != nil {
644 t.Fatalf("Failed to run eviction: %v", err)
645 }
646 })
647 }
648 }
649
650 func TestEvictionPDBStatus(t *testing.T) {
651 testcases := []struct {
652 name string
653 pdb *policyv1.PodDisruptionBudget
654 expectedDisruptionsAllowed int32
655 expectedReason string
656 }{
657 {
658 name: "pdb status is updated after eviction",
659 pdb: &policyv1.PodDisruptionBudget{
660 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
661 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
662 Status: policyv1.PodDisruptionBudgetStatus{
663 DisruptionsAllowed: 1,
664 Conditions: []metav1.Condition{
665 {
666 Type: policyv1.DisruptionAllowedCondition,
667 Reason: policyv1.SufficientPodsReason,
668 Status: metav1.ConditionTrue,
669 },
670 },
671 },
672 },
673 expectedDisruptionsAllowed: 0,
674 expectedReason: policyv1.InsufficientPodsReason,
675 },
676 {
677 name: "condition reason is only updated if AllowedDisruptions becomes 0",
678 pdb: &policyv1.PodDisruptionBudget{
679 ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
680 Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
681 Status: policyv1.PodDisruptionBudgetStatus{
682 DisruptionsAllowed: 3,
683 Conditions: []metav1.Condition{
684 {
685 Type: policyv1.DisruptionAllowedCondition,
686 Reason: policyv1.SufficientPodsReason,
687 Status: metav1.ConditionTrue,
688 },
689 },
690 },
691 },
692 expectedDisruptionsAllowed: 2,
693 expectedReason: policyv1.SufficientPodsReason,
694 },
695 }
696
697 for _, tc := range testcases {
698 t.Run(tc.name, func(t *testing.T) {
699 testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
700 storage, _, statusStorage, server := newStorage(t)
701 defer server.Terminate(t)
702 defer storage.Store.DestroyFunc()
703
704 client := fake.NewSimpleClientset(tc.pdb)
705 for _, podName := range []string{"foo-1", "foo-2"} {
706 pod := validNewPod()
707 pod.Labels = map[string]string{"a": "true"}
708 pod.ObjectMeta.Name = podName
709 pod.Spec.NodeName = "foo"
710 newPod, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{})
711 if err != nil {
712 t.Error(err)
713 }
714 (newPod.(*api.Pod)).Status.Phase = api.PodRunning
715 _, _, err = statusStorage.Update(testContext, pod.Name, rest.DefaultUpdatedObjectInfo(newPod),
716 nil, nil, false, &metav1.UpdateOptions{})
717 if err != nil {
718 t.Error(err)
719 }
720 }
721
722 evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
723 eviction := &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo-1", Namespace: "default"}, DeleteOptions: &metav1.DeleteOptions{}}
724 _, err := evictionRest.Create(testContext, "foo-1", eviction, nil, &metav1.CreateOptions{})
725 if err != nil {
726 t.Fatalf("Failed to run eviction: %v", err)
727 }
728
729 existingPDB, err := client.PolicyV1().PodDisruptionBudgets(metav1.NamespaceDefault).Get(context.TODO(), tc.pdb.Name, metav1.GetOptions{})
730 if err != nil {
731 t.Errorf("%#v", err)
732 return
733 }
734
735 if want, got := tc.expectedDisruptionsAllowed, existingPDB.Status.DisruptionsAllowed; got != want {
736 t.Errorf("expected DisruptionsAllowed to be %d, but got %d", want, got)
737 }
738
739 cond := apimeta.FindStatusCondition(existingPDB.Status.Conditions, policyv1.DisruptionAllowedCondition)
740 if want, got := tc.expectedReason, cond.Reason; want != got {
741 t.Errorf("expected Reason to be %q, but got %q", want, got)
742 }
743 })
744 }
745 }
746
747 func TestAddConditionAndDelete(t *testing.T) {
748 cases := []struct {
749 name string
750 initialPod bool
751 makeDeleteOptions func(*api.Pod) *metav1.DeleteOptions
752 expectErr string
753 }{
754 {
755 name: "simple",
756 initialPod: true,
757 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions { return &metav1.DeleteOptions{} },
758 },
759 {
760 name: "missing",
761 initialPod: false,
762 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions { return &metav1.DeleteOptions{} },
763 expectErr: "not found",
764 },
765 {
766 name: "valid uid",
767 initialPod: true,
768 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
769 return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &pod.UID}}
770 },
771 },
772 {
773 name: "invalid uid",
774 initialPod: true,
775 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
776 badUID := pod.UID + "1"
777 return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &badUID}}
778 },
779 expectErr: "The object might have been deleted and then recreated",
780 },
781 {
782 name: "valid resourceVersion",
783 initialPod: true,
784 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
785 return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &pod.ResourceVersion}}
786 },
787 },
788 {
789 name: "invalid resourceVersion",
790 initialPod: true,
791 makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
792 badRV := pod.ResourceVersion + "1"
793 return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &badRV}}
794 },
795 expectErr: "The object might have been modified",
796 },
797 }
798
799 testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
800
801 storage, _, _, server := newStorage(t)
802 defer server.Terminate(t)
803 defer storage.Store.DestroyFunc()
804
805 client := fake.NewSimpleClientset()
806 evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
807
808 for _, tc := range cases {
809 for _, conditionsEnabled := range []bool{true, false} {
810 name := fmt.Sprintf("%s_conditions=%v", tc.name, conditionsEnabled)
811 t.Run(name, func(t *testing.T) {
812 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDisruptionConditions, conditionsEnabled)()
813 var deleteOptions *metav1.DeleteOptions
814 if tc.initialPod {
815 newPod := validNewPod()
816 createdObj, err := storage.Create(testContext, newPod, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
817 if err != nil {
818 t.Fatal(err)
819 }
820 t.Cleanup(func() {
821 zero := int64(0)
822 storage.Delete(testContext, newPod.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{GracePeriodSeconds: &zero})
823 })
824 deleteOptions = tc.makeDeleteOptions(createdObj.(*api.Pod))
825 } else {
826 deleteOptions = tc.makeDeleteOptions(nil)
827 }
828 if deleteOptions == nil {
829 deleteOptions = &metav1.DeleteOptions{}
830 }
831
832 err := addConditionAndDeletePod(evictionRest, testContext, "foo", rest.ValidateAllObjectFunc, deleteOptions)
833 if err == nil {
834 if tc.expectErr != "" {
835 t.Fatalf("expected err containing %q, got none", tc.expectErr)
836 }
837 return
838 }
839 if tc.expectErr == "" {
840 t.Fatalf("unexpected err: %v", err)
841 }
842 if !strings.Contains(err.Error(), tc.expectErr) {
843 t.Fatalf("expected err containing %q, got %v", tc.expectErr, err)
844 }
845 })
846 }
847 }
848 }
849
850 func resource(resource string) schema.GroupResource {
851 return schema.GroupResource{Group: "", Resource: resource}
852 }
853
854 type mockStore struct {
855 deleteCount int
856 pod *api.Pod
857 }
858
859 func (ms *mockStore) mutatorDeleteFunc(count int, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
860 if ms.pod.Name == "t4" {
861
862 return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
863 }
864 if ms.pod.Name == "t6" || ms.pod.Name == "t8" {
865
866
867 return nil, true, nil
868 }
869 if ms.pod.Name == "t10" {
870 return nil, false, apierrors.NewBadRequest("test designed to error")
871 }
872 if count == 1 {
873
874
875 if ms.pod.Name != "t1" && ms.pod.Name != "t5" {
876 ms.pod.Status.Phase = api.PodRunning
877 }
878 ms.pod.ResourceVersion = "999"
879
880 return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
881 }
882
883 if options == nil || options.Preconditions == nil || options.Preconditions.ResourceVersion == nil {
884 return nil, true, nil
885 } else if *options.Preconditions.ResourceVersion != "1000" {
886
887
888
889
890 ms.pod.ResourceVersion = "1000"
891 return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
892 }
893 return nil, true, nil
894 }
895
896 func (ms *mockStore) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
897 ms.deleteCount++
898 return ms.mutatorDeleteFunc(ms.deleteCount, options)
899 }
900
901 func (ms *mockStore) Watch(ctx context.Context, options *metainternalversion.ListOptions) (watch.Interface, error) {
902 return nil, nil
903 }
904
905 func (ms *mockStore) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
906 return ms.pod, false, nil
907 }
908
909 func (ms *mockStore) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
910 return ms.pod, nil
911 }
912
913 func (ms *mockStore) New() runtime.Object {
914 return nil
915 }
916
917 func (ms *mockStore) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
918 return nil, nil
919 }
920
921 func (ms *mockStore) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
922 return nil, nil
923 }
924
925 func (ms *mockStore) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
926 return nil, nil
927 }
928
929 func (ms *mockStore) NewList() runtime.Object {
930 return nil
931 }
932
933 func (ms *mockStore) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
934 return nil, nil
935 }
936
937 func (ms *mockStore) Destroy() {
938 }
939
940 func unhealthyPolicyPtr(unhealthyPodEvictionPolicy policyv1.UnhealthyPodEvictionPolicyType) *policyv1.UnhealthyPodEvictionPolicyType {
941 return &unhealthyPodEvictionPolicy
942 }
943
944 func unhealthyPolicyStr(unhealthyPodEvictionPolicy *policyv1.UnhealthyPodEvictionPolicyType) string {
945 if unhealthyPodEvictionPolicy == nil {
946 return "nil"
947 }
948 return string(*unhealthyPodEvictionPolicy)
949 }
950
951 func hasUnhealthyPolicy(unhealthyPolicies []*policyv1.UnhealthyPodEvictionPolicyType, unhealthyPolicy *policyv1.UnhealthyPodEvictionPolicyType) bool {
952 for _, p := range unhealthyPolicies {
953 if reflect.DeepEqual(unhealthyPolicy, p) {
954 return true
955 }
956 }
957 return false
958 }
959
960 func errToString(err error) string {
961 result := ""
962 if err != nil {
963 result = err.Error()
964 if statusErr, ok := err.(*apierrors.StatusError); ok {
965 result += fmt.Sprintf(": %v", statusErr.ErrStatus.Reason)
966 if statusErr.ErrStatus.Details != nil {
967 for _, cause := range statusErr.ErrStatus.Details.Causes {
968 if !strings.HasSuffix(err.Error(), cause.Message) {
969 result += fmt.Sprintf(": %v", cause.Message)
970 }
971 }
972 }
973 }
974 }
975 return result
976 }
977
View as plain text