1
16
17 package drain
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "math"
24 "os"
25 "reflect"
26 "sort"
27 "strconv"
28 "testing"
29 "time"
30
31 corev1 "k8s.io/api/core/v1"
32 policyv1 "k8s.io/api/policy/v1"
33 policyv1beta1 "k8s.io/api/policy/v1beta1"
34 apierrors "k8s.io/apimachinery/pkg/api/errors"
35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
36 "k8s.io/apimachinery/pkg/runtime"
37 "k8s.io/apimachinery/pkg/runtime/schema"
38 "k8s.io/apimachinery/pkg/types"
39 "k8s.io/apimachinery/pkg/util/wait"
40 "k8s.io/client-go/kubernetes/fake"
41 ktest "k8s.io/client-go/testing"
42 )
43
44 func TestDeletePods(t *testing.T) {
45 ifHasBeenCalled := map[string]bool{}
46 tests := []struct {
47 description string
48 interval time.Duration
49 timeout time.Duration
50 ctxTimeoutEarly bool
51 expectPendingPods bool
52 expectError bool
53 expectedError *error
54 getPodFn func(namespace, name string) (*corev1.Pod, error)
55 }{
56 {
57 description: "Wait for deleting to complete",
58 interval: 100 * time.Millisecond,
59 timeout: 10 * time.Second,
60 expectPendingPods: false,
61 expectError: false,
62 expectedError: nil,
63 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
64 oldPodMap, _ := createPods(false)
65 newPodMap, _ := createPods(true)
66 if oldPod, found := oldPodMap[name]; found {
67 if _, ok := ifHasBeenCalled[name]; !ok {
68 ifHasBeenCalled[name] = true
69 return &oldPod, nil
70 }
71 if oldPod.ObjectMeta.Generation < 4 {
72 newPod := newPodMap[name]
73 return &newPod, nil
74 }
75 return &corev1.Pod{}, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, name)
76 }
77 return &corev1.Pod{}, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, name)
78 },
79 },
80 {
81 description: "Pod found with same name but different UID",
82 interval: 100 * time.Millisecond,
83 timeout: 10 * time.Second,
84 expectPendingPods: false,
85 expectError: false,
86 expectedError: nil,
87 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
88
89 return &corev1.Pod{
90 ObjectMeta: metav1.ObjectMeta{
91 Namespace: namespace,
92 Name: name,
93 UID: "SOME_OTHER_UID",
94 },
95 }, nil
96 },
97 },
98 {
99 description: "Deleting could timeout",
100 interval: 200 * time.Millisecond,
101 timeout: 3 * time.Second,
102 expectPendingPods: true,
103 expectError: true,
104 expectedError: &wait.ErrWaitTimeout,
105 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
106 oldPodMap, _ := createPods(false)
107 if oldPod, found := oldPodMap[name]; found {
108 return &oldPod, nil
109 }
110 return &corev1.Pod{}, fmt.Errorf("%q: not found", name)
111 },
112 },
113 {
114 description: "Context Canceled",
115 interval: 1000 * time.Millisecond,
116 timeout: 5 * time.Second,
117 ctxTimeoutEarly: true,
118 expectPendingPods: true,
119 expectError: true,
120 expectedError: &wait.ErrWaitTimeout,
121 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
122 oldPodMap, _ := createPods(false)
123 if oldPod, found := oldPodMap[name]; found {
124 return &oldPod, nil
125 }
126 return &corev1.Pod{}, fmt.Errorf("%q: not found", name)
127 },
128 },
129 {
130 description: "Skip Deleted Pod",
131 interval: 200 * time.Millisecond,
132 timeout: 3 * time.Second,
133 expectPendingPods: false,
134 expectError: false,
135 expectedError: nil,
136 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
137 oldPodMap, _ := createPods(false)
138 if oldPod, found := oldPodMap[name]; found {
139 dTime := &metav1.Time{Time: time.Now().Add(time.Duration(100) * time.Second * -1)}
140 oldPod.ObjectMeta.SetDeletionTimestamp(dTime)
141 return &oldPod, nil
142 }
143 return &corev1.Pod{}, fmt.Errorf("%q: not found", name)
144 },
145 },
146 {
147 description: "Client error could be passed out",
148 interval: 200 * time.Millisecond,
149 timeout: 5 * time.Second,
150 expectPendingPods: true,
151 expectError: true,
152 expectedError: nil,
153 getPodFn: func(namespace, name string) (*corev1.Pod, error) {
154 return &corev1.Pod{}, errors.New("This is a random error for testing")
155 },
156 },
157 }
158
159 for _, test := range tests {
160 t.Run(test.description, func(t *testing.T) {
161 _, pods := createPods(false)
162 var ctx context.Context
163 var cancel context.CancelFunc
164 ctx = context.Background()
165 if test.ctxTimeoutEarly {
166 ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond)
167 defer cancel()
168 }
169 params := waitForDeleteParams{
170 ctx: ctx,
171 pods: pods,
172 interval: test.interval,
173 timeout: test.timeout,
174 usingEviction: false,
175 getPodFn: test.getPodFn,
176 onDoneFn: nil,
177 globalTimeout: time.Duration(math.MaxInt64),
178 out: os.Stdout,
179 skipWaitForDeleteTimeoutSeconds: 10,
180 }
181 start := time.Now()
182 pendingPods, err := waitForDelete(params)
183 elapsed := time.Since(start)
184
185 if test.expectError {
186 if err == nil {
187 t.Fatalf("%s: unexpected non-error", test.description)
188 } else if test.expectedError != nil {
189 if test.ctxTimeoutEarly {
190 if elapsed >= test.timeout {
191 t.Fatalf("%s: the supplied context did not effectively cancel the waitForDelete", test.description)
192 }
193 } else if *test.expectedError != err {
194 t.Fatalf("%s: the error does not match expected error", test.description)
195 }
196 }
197 }
198 if !test.expectError && err != nil {
199 t.Fatalf("%s: unexpected error", test.description)
200 }
201 if test.expectPendingPods && len(pendingPods) == 0 {
202 t.Fatalf("%s: unexpected empty pods", test.description)
203 }
204 if !test.expectPendingPods && len(pendingPods) > 0 {
205 t.Fatalf("%s: unexpected pending pods", test.description)
206 }
207 })
208 }
209 }
210
211 func createPods(ifCreateNewPods bool) (map[string]corev1.Pod, []corev1.Pod) {
212 podMap := make(map[string]corev1.Pod)
213 podSlice := []corev1.Pod{}
214 for i := 0; i < 8; i++ {
215 var uid types.UID
216 if ifCreateNewPods {
217 uid = types.UID(strconv.Itoa(i))
218 } else {
219 uid = types.UID(strconv.Itoa(i) + strconv.Itoa(i))
220 }
221 pod := corev1.Pod{
222 ObjectMeta: metav1.ObjectMeta{
223 Name: "pod" + strconv.Itoa(i),
224 Namespace: "default",
225 UID: uid,
226 Generation: int64(i),
227 },
228 }
229 podMap[pod.Name] = pod
230 podSlice = append(podSlice, pod)
231 }
232 return podMap, podSlice
233 }
234
235 func addCoreNonEvictionSupport(t *testing.T, k *fake.Clientset) {
236 coreResources := &metav1.APIResourceList{
237 GroupVersion: "v1",
238 }
239 k.Resources = append(k.Resources, coreResources)
240 }
241
242
243 func addEvictionSupport(t *testing.T, k *fake.Clientset, version string) {
244 podsEviction := metav1.APIResource{
245 Name: "pods/eviction",
246 Kind: "Eviction",
247 Group: "policy",
248 Version: version,
249 }
250 coreResources := &metav1.APIResourceList{
251 GroupVersion: "v1",
252 APIResources: []metav1.APIResource{podsEviction},
253 }
254
255 policyResources := &metav1.APIResourceList{
256 GroupVersion: "policy/v1",
257 }
258 k.Resources = append(k.Resources, coreResources, policyResources)
259
260
261 k.PrependReactor("create", "pods", func(action ktest.Action) (bool, runtime.Object, error) {
262 if action.GetSubresource() != "eviction" {
263 return false, nil, nil
264 }
265
266 namespace := ""
267 name := ""
268 switch version {
269 case "v1":
270 eviction := *action.(ktest.CreateAction).GetObject().(*policyv1.Eviction)
271 namespace = eviction.Namespace
272 name = eviction.Name
273 case "v1beta1":
274 eviction := *action.(ktest.CreateAction).GetObject().(*policyv1beta1.Eviction)
275 namespace = eviction.Namespace
276 name = eviction.Name
277 default:
278 t.Errorf("unknown version %s", version)
279 }
280
281 go func() {
282 err := k.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
283 if err != nil {
284
285 t.Errorf("failed to delete pod: %s/%s", namespace, name)
286 }
287 }()
288
289 return true, nil, nil
290 })
291 }
292
293 func TestCheckEvictionSupport(t *testing.T) {
294 for _, evictionVersion := range []string{"", "v1", "v1beta1"} {
295 t.Run(fmt.Sprintf("evictionVersion=%v", evictionVersion),
296 func(t *testing.T) {
297 k := fake.NewSimpleClientset()
298 if len(evictionVersion) > 0 {
299 addEvictionSupport(t, k, evictionVersion)
300 } else {
301 addCoreNonEvictionSupport(t, k)
302 }
303
304 apiGroup, err := CheckEvictionSupport(k)
305 if err != nil {
306 t.Fatalf("unexpected error: %v", err)
307 }
308 expectedAPIGroup := schema.GroupVersion{}
309 if len(evictionVersion) > 0 {
310 expectedAPIGroup = schema.GroupVersion{Group: "policy", Version: evictionVersion}
311 }
312 if apiGroup != expectedAPIGroup {
313 t.Fatalf("expected apigroup %q, actual=%q", expectedAPIGroup, apiGroup)
314 }
315 })
316 }
317 }
318
319 func TestDeleteOrEvict(t *testing.T) {
320 tests := []struct {
321 description string
322 evictionSupported bool
323 disableEviction bool
324 }{
325 {
326 description: "eviction supported/enabled",
327 evictionSupported: true,
328 disableEviction: false,
329 },
330 {
331 description: "eviction unsupported/disabled",
332 evictionSupported: false,
333 disableEviction: false,
334 },
335 {
336 description: "eviction supported/disabled",
337 evictionSupported: true,
338 disableEviction: true,
339 },
340 {
341 description: "eviction unsupported/disabled",
342 evictionSupported: false,
343 disableEviction: false,
344 },
345 }
346 for _, tc := range tests {
347 t.Run(tc.description, func(t *testing.T) {
348 h := &Helper{
349 Out: os.Stdout,
350 GracePeriodSeconds: 10,
351 OnPodDeletionOrEvictionStarted: func(pod *corev1.Pod, usingEviction bool) {
352 if tc.evictionSupported && !tc.disableEviction {
353 if !usingEviction {
354 t.Errorf("%s: OnPodDeletionOrEvictionStarted callback failed while evicting; actual\n\t%v\nexpected\n\t%v", tc.description, usingEviction, !usingEviction)
355 }
356 } else if tc.evictionSupported && tc.disableEviction {
357 if usingEviction {
358 t.Errorf("%s: OnPodDeletionOrEvictionStarted callback failed while deleting; actual\n\t%v\nexpected\n\t%v", tc.description, !usingEviction, usingEviction)
359 }
360 }
361 },
362 OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) {
363 if tc.evictionSupported && !tc.disableEviction {
364 if !usingEviction {
365 t.Errorf("%s: OnPodDeletedOrEvicted callback failed while evicting; actual\n\t%v\nexpected\n\t%v", tc.description, usingEviction, !usingEviction)
366 }
367 } else if tc.evictionSupported && tc.disableEviction {
368 if usingEviction {
369 t.Errorf("%s: OnPodDeletedOrEvicted callback failed while deleting; actual\n\t%v\nexpected\n\t%v", tc.description, !usingEviction, usingEviction)
370 }
371 }
372 },
373 OnPodDeletionOrEvictionFinished: func(pod *corev1.Pod, usingEviction bool, err error) {
374 if tc.evictionSupported && !tc.disableEviction {
375 if !usingEviction {
376 t.Errorf("%s: OnPodDeletionOrEvictionFinished callback failed while evicting; actual\n\t%v\nexpected\n\t%v", tc.description, usingEviction, !usingEviction)
377 }
378 } else if tc.evictionSupported && tc.disableEviction {
379 if usingEviction {
380 t.Errorf("%s: OnPodDeletionOrEvictionFinished callback failed while deleting; actual\n\t%v\nexpected\n\t%v", tc.description, !usingEviction, usingEviction)
381 }
382 }
383 },
384 }
385
386
387 var expectedEvictions []policyv1.Eviction
388 var create []runtime.Object
389 deletePods := []corev1.Pod{}
390 for i := 1; i <= 4; i++ {
391 pod := &corev1.Pod{}
392 pod.Name = fmt.Sprintf("mypod-%d", i)
393 pod.Namespace = "default"
394
395 create = append(create, pod)
396 if i <= 2 {
397 deletePods = append(deletePods, *pod)
398
399 if tc.evictionSupported && !tc.disableEviction {
400 eviction := policyv1.Eviction{}
401 eviction.Namespace = pod.Namespace
402 eviction.Name = pod.Name
403
404 gracePeriodSeconds := int64(h.GracePeriodSeconds)
405 eviction.DeleteOptions = &metav1.DeleteOptions{
406 GracePeriodSeconds: &gracePeriodSeconds,
407 }
408
409 expectedEvictions = append(expectedEvictions, eviction)
410 }
411 }
412 }
413
414
415 k := fake.NewSimpleClientset(create...)
416 if tc.evictionSupported {
417 addEvictionSupport(t, k, "v1")
418 } else {
419 addCoreNonEvictionSupport(t, k)
420 }
421 h.Client = k
422 h.DisableEviction = tc.disableEviction
423
424 if err := h.DeleteOrEvictPods(deletePods); err != nil {
425 t.Fatalf("error from DeleteOrEvictPods: %v", err)
426 }
427
428
429 var remainingPods []string
430 {
431 podList, err := k.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
432 if err != nil {
433 t.Fatalf("error listing pods: %v", err)
434 }
435
436 for _, pod := range podList.Items {
437 remainingPods = append(remainingPods, pod.Namespace+"/"+pod.Name)
438 }
439 sort.Strings(remainingPods)
440 }
441 expected := []string{"default/mypod-3", "default/mypod-4"}
442 if !reflect.DeepEqual(remainingPods, expected) {
443 t.Errorf("%s: unexpected remaining pods after DeleteOrEvictPods; actual %v; expected %v", tc.description, remainingPods, expected)
444 }
445
446
447 var actualEvictions []policyv1.Eviction
448 for _, action := range k.Actions() {
449 if action.GetVerb() != "create" || action.GetResource().Resource != "pods" || action.GetSubresource() != "eviction" {
450 continue
451 }
452 eviction := *action.(ktest.CreateAction).GetObject().(*policyv1.Eviction)
453 actualEvictions = append(actualEvictions, eviction)
454 }
455 sort.Slice(actualEvictions, func(i, j int) bool {
456 return actualEvictions[i].Name < actualEvictions[j].Name
457 })
458 if !reflect.DeepEqual(actualEvictions, expectedEvictions) {
459 t.Errorf("%s: unexpected evictions; actual\n\t%v\nexpected\n\t%v", tc.description, actualEvictions, expectedEvictions)
460 }
461 })
462 }
463 }
464
465 func mockFilterSkip(_ corev1.Pod) PodDeleteStatus {
466 return MakePodDeleteStatusSkip()
467 }
468
469 func mockFilterOkay(_ corev1.Pod) PodDeleteStatus {
470 return MakePodDeleteStatusOkay()
471 }
472
473 func TestFilterPods(t *testing.T) {
474 tCases := []struct {
475 description string
476 expectedPodListLen int
477 additionalFilters []PodFilter
478 }{
479 {
480 description: "AdditionalFilter skip all",
481 expectedPodListLen: 0,
482 additionalFilters: []PodFilter{
483 mockFilterSkip,
484 mockFilterOkay,
485 },
486 },
487 {
488 description: "AdditionalFilter okay all",
489 expectedPodListLen: 1,
490 additionalFilters: []PodFilter{
491 mockFilterOkay,
492 },
493 },
494 {
495 description: "AdditionalFilter Skip after Okay all skip",
496 expectedPodListLen: 0,
497 additionalFilters: []PodFilter{
498 mockFilterOkay,
499 mockFilterSkip,
500 },
501 },
502 {
503 description: "No additionalFilters okay all",
504 expectedPodListLen: 1,
505 },
506 }
507 for _, tc := range tCases {
508 t.Run(tc.description, func(t *testing.T) {
509 h := &Helper{
510 Force: true,
511 AdditionalFilters: tc.additionalFilters,
512 }
513 pod := corev1.Pod{
514 ObjectMeta: metav1.ObjectMeta{
515 Name: "pod",
516 Namespace: "default",
517 },
518 }
519 podList := corev1.PodList{
520 Items: []corev1.Pod{
521 pod,
522 },
523 }
524
525 list := filterPods(&podList, h.makeFilters())
526 podsLen := len(list.Pods())
527 if podsLen != tc.expectedPodListLen {
528 t.Errorf("%s: unexpected evictions; actual %v; expected %v", tc.description, podsLen, tc.expectedPodListLen)
529 }
530 })
531 }
532 }
533
View as plain text