...

Source file src/k8s.io/kubernetes/pkg/registry/core/pod/storage/eviction_test.go

Documentation: k8s.io/kubernetes/pkg/registry/core/pod/storage

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storage
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"reflect"
    24  	"strings"
    25  	"testing"
    26  
    27  	policyv1 "k8s.io/api/policy/v1"
    28  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    29  	apimeta "k8s.io/apimachinery/pkg/api/meta"
    30  	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/runtime"
    33  	"k8s.io/apimachinery/pkg/runtime/schema"
    34  	"k8s.io/apimachinery/pkg/watch"
    35  	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
    36  	"k8s.io/apiserver/pkg/registry/rest"
    37  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    38  	"k8s.io/client-go/kubernetes/fake"
    39  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    40  	podapi "k8s.io/kubernetes/pkg/api/pod"
    41  	api "k8s.io/kubernetes/pkg/apis/core"
    42  	"k8s.io/kubernetes/pkg/apis/policy"
    43  	"k8s.io/kubernetes/pkg/features"
    44  )
    45  
    46  func TestEviction(t *testing.T) {
    47  	testcases := []struct {
    48  		name     string
    49  		pdbs     []runtime.Object
    50  		policies []*policyv1.UnhealthyPodEvictionPolicyType
    51  		eviction *policy.Eviction
    52  
    53  		badNameInURL bool
    54  
    55  		expectError   string
    56  		expectDeleted bool
    57  		podPhase      api.PodPhase
    58  		podName       string
    59  	}{
    60  		{
    61  			name: "matching pdbs with no disruptions allowed, pod running",
    62  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
    63  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
    64  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
    65  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
    66  			}},
    67  			eviction:    &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t1", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
    68  			expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
    69  			podPhase:    api.PodRunning,
    70  			podName:     "t1",
    71  			policies:    []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)}, // AlwaysAllow would terminate the pod since Running pods are not guarded by this policy
    72  		},
    73  		{
    74  			name: "matching pdbs with no disruptions allowed, pod pending",
    75  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
    76  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
    77  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
    78  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
    79  			}},
    80  			eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
    81  			expectError:   "",
    82  			podPhase:      api.PodPending,
    83  			expectDeleted: true,
    84  			podName:       "t2",
    85  		},
    86  		{
    87  			name: "matching pdbs with no disruptions allowed, pod succeeded",
    88  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
    89  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
    90  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
    91  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
    92  			}},
    93  			eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t3", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
    94  			expectError:   "",
    95  			podPhase:      api.PodSucceeded,
    96  			expectDeleted: true,
    97  			podName:       "t3",
    98  		},
    99  		{
   100  			name: "matching pdbs with no disruptions allowed, pod failed",
   101  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   102  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   103  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   104  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   105  			}},
   106  			eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t4", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   107  			expectError:   "",
   108  			podPhase:      api.PodFailed,
   109  			expectDeleted: true,
   110  			podName:       "t4",
   111  		},
   112  		{
   113  			name: "matching pdbs with disruptions allowed",
   114  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   115  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   116  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   117  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
   118  			}},
   119  			eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t5", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   120  			expectDeleted: true,
   121  			podName:       "t5",
   122  		},
   123  		{
   124  			name: "non-matching pdbs",
   125  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   126  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   127  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"b": "true"}}},
   128  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   129  			}},
   130  			eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t6", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   131  			expectDeleted: true,
   132  			podName:       "t6",
   133  		},
   134  		{
   135  			name: "matching pdbs with disruptions allowed but bad name in Url",
   136  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   137  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   138  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   139  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
   140  			}},
   141  			badNameInURL: true,
   142  			eviction:     &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t7", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   143  			expectError:  "name in URL does not match name in Eviction object: BadRequest",
   144  			podName:      "t7",
   145  		},
   146  		{
   147  			name: "matching pdbs with no disruptions allowed, pod running, empty selector",
   148  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   149  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   150  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{}},
   151  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   152  			}},
   153  			eviction:    &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   154  			expectError: "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
   155  			podPhase:    api.PodRunning,
   156  			podName:     "t8",
   157  			policies:    []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)}, // AlwaysAllow would terminate the pod since Running pods are not guarded by this policy
   158  		},
   159  	}
   160  
   161  	for _, unhealthyPodEvictionPolicy := range []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget), unhealthyPolicyPtr(policyv1.AlwaysAllow)} {
   162  		for _, tc := range testcases {
   163  			if len(tc.policies) > 0 && !hasUnhealthyPolicy(tc.policies, unhealthyPodEvictionPolicy) {
   164  				// unhealthyPodEvictionPolicy is not covered by this test
   165  				continue
   166  			}
   167  			t.Run(fmt.Sprintf("%v with %v policy", tc.name, unhealthyPolicyStr(unhealthyPodEvictionPolicy)), func(t *testing.T) {
   168  				defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, true)()
   169  
   170  				// same test runs multiple times, make copy of objects to have unique ones
   171  				evictionCopy := tc.eviction.DeepCopy()
   172  				var pdbsCopy []runtime.Object
   173  				for _, pdb := range tc.pdbs {
   174  					pdbCopy := pdb.DeepCopyObject()
   175  					pdbCopy.(*policyv1.PodDisruptionBudget).Spec.UnhealthyPodEvictionPolicy = unhealthyPodEvictionPolicy
   176  					pdbsCopy = append(pdbsCopy, pdbCopy)
   177  				}
   178  
   179  				testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
   180  				storage, _, statusStorage, server := newStorage(t)
   181  				defer server.Terminate(t)
   182  				defer storage.Store.DestroyFunc()
   183  
   184  				pod := validNewPod()
   185  				pod.Name = tc.podName
   186  				pod.Labels = map[string]string{"a": "true"}
   187  				pod.Spec.NodeName = "foo"
   188  				if _, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{}); err != nil {
   189  					t.Error(err)
   190  				}
   191  
   192  				if tc.podPhase != "" {
   193  					pod.Status.Phase = tc.podPhase
   194  					_, _, err := statusStorage.Update(testContext, pod.Name, rest.DefaultUpdatedObjectInfo(pod), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{})
   195  					if err != nil {
   196  						t.Errorf("Unexpected error: %v", err)
   197  					}
   198  				}
   199  
   200  				client := fake.NewSimpleClientset(pdbsCopy...)
   201  				evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
   202  
   203  				name := pod.Name
   204  				if tc.badNameInURL {
   205  					name += "bad-name"
   206  				}
   207  
   208  				_, err := evictionRest.Create(testContext, name, evictionCopy, nil, &metav1.CreateOptions{})
   209  				gotErr := errToString(err)
   210  				if gotErr != tc.expectError {
   211  					t.Errorf("error mismatch: expected %v, got %v; name %v", tc.expectError, gotErr, pod.Name)
   212  					return
   213  				}
   214  				if tc.badNameInURL {
   215  					if err == nil {
   216  						t.Error("expected error here, but got nil")
   217  						return
   218  					}
   219  					if err.Error() != "name in URL does not match name in Eviction object" {
   220  						t.Errorf("got unexpected error: %v", err)
   221  					}
   222  				}
   223  				if tc.expectError != "" {
   224  					return
   225  				}
   226  
   227  				existingPod, err := storage.Get(testContext, pod.Name, &metav1.GetOptions{})
   228  				if tc.expectDeleted {
   229  					if !apierrors.IsNotFound(err) {
   230  						t.Errorf("expected to be deleted, lookup returned %#v", existingPod)
   231  					}
   232  					return
   233  				} else if apierrors.IsNotFound(err) {
   234  					t.Errorf("expected graceful deletion, got %v", err)
   235  					return
   236  				}
   237  
   238  				if err != nil {
   239  					t.Errorf("%#v", err)
   240  					return
   241  				}
   242  
   243  				if existingPod.(*api.Pod).DeletionTimestamp == nil {
   244  					t.Errorf("expected gracefully deleted pod with deletionTimestamp set, got %#v", existingPod)
   245  				}
   246  			})
   247  		}
   248  	}
   249  }
   250  
   251  func TestEvictionIgnorePDB(t *testing.T) {
   252  	testcases := []struct {
   253  		name     string
   254  		pdbs     []runtime.Object
   255  		policies []*policyv1.UnhealthyPodEvictionPolicyType
   256  		eviction *policy.Eviction
   257  
   258  		expectError         string
   259  		podPhase            api.PodPhase
   260  		podName             string
   261  		expectedDeleteCount int
   262  		podTerminating      bool
   263  		prc                 *api.PodCondition
   264  	}{
   265  		{
   266  			name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod still pending, pod deleted successfully",
   267  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   268  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   269  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   270  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   271  			}},
   272  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t1", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   273  			expectError:         "",
   274  			podPhase:            api.PodPending,
   275  			podName:             "t1",
   276  			expectedDeleteCount: 3,
   277  		},
   278  		// This test case is critical. If it is removed or broken we may
   279  		// regress and allow a pod to be deleted without checking PDBs when the
   280  		// pod should not be deleted.
   281  		{
   282  			name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod becomes running, continueToPDBs",
   283  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   284  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   285  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   286  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   287  			}},
   288  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   289  			expectError:         "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 0 healthy pods and has 0 currently",
   290  			podPhase:            api.PodPending,
   291  			podName:             "t2",
   292  			expectedDeleteCount: 1,
   293  			policies:            []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)}, // AlwaysAllow does not continueToPDBs, but straight to deletion
   294  		},
   295  		{
   296  			name: "pdbs No disruptions allowed, pod pending, first delete conflict, pod becomes running, skip PDB check, conflict",
   297  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   298  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   299  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   300  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   301  			}},
   302  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t2", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   303  			expectError:         "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo is still being processed by the server.",
   304  			podPhase:            api.PodPending,
   305  			podName:             "t2",
   306  			podTerminating:      false,
   307  			expectedDeleteCount: 2,
   308  			prc: &api.PodCondition{
   309  				Type:   api.PodReady,
   310  				Status: api.ConditionFalse,
   311  			},
   312  			policies: []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow)}, // nil, IfHealthyBudget continueToPDBs
   313  		},
   314  		{
   315  			name: "pdbs disruptions allowed, pod pending, first delete conflict, pod becomes running, continueToPDBs",
   316  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   317  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   318  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   319  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
   320  			}},
   321  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t3", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   322  			expectError:         "",
   323  			podPhase:            api.PodPending,
   324  			podName:             "t3",
   325  			expectedDeleteCount: 2,
   326  			policies:            []*policyv1.UnhealthyPodEvictionPolicyType{nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)}, // AlwaysAllow does not continueToPDBs, but straight to deletion if pod not healthy (ready)
   327  		},
   328  		{
   329  			name: "pod pending, always conflict on delete",
   330  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   331  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   332  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   333  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   334  			}},
   335  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t4", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   336  			expectError:         `Operation cannot be fulfilled on tests "2": message: Conflict`,
   337  			podPhase:            api.PodPending,
   338  			podName:             "t4",
   339  			expectedDeleteCount: EvictionsRetry.Steps,
   340  		},
   341  		{
   342  			name: "pod pending, always conflict on delete, user provided ResourceVersion constraint",
   343  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   344  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   345  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   346  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   347  			}},
   348  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t5", Namespace: "default"}, DeleteOptions: metav1.NewRVDeletionPrecondition("userProvided")},
   349  			expectError:         `Operation cannot be fulfilled on tests "2": message: Conflict`,
   350  			podPhase:            api.PodPending,
   351  			podName:             "t5",
   352  			expectedDeleteCount: 1,
   353  		},
   354  		{
   355  			name: "matching pdbs with no disruptions allowed, pod terminating",
   356  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   357  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   358  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   359  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   360  			}},
   361  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t6", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(300)},
   362  			expectError:         "",
   363  			podName:             "t6",
   364  			expectedDeleteCount: 1,
   365  			podTerminating:      true,
   366  		},
   367  		{
   368  			name: "matching pdbs with no disruptions allowed, pod running, pod healthy, unhealthy pod not ours",
   369  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   370  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   371  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   372  				Status: policyv1.PodDisruptionBudgetStatus{
   373  					// This simulates 3 pods expected, our pod healthy, unhealthy pod is not ours.
   374  					DisruptionsAllowed: 0,
   375  					CurrentHealthy:     2,
   376  					DesiredHealthy:     2,
   377  				},
   378  			}},
   379  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t7", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   380  			expectError:         "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 2 healthy pods and has 2 currently",
   381  			podName:             "t7",
   382  			expectedDeleteCount: 0,
   383  			podTerminating:      false,
   384  			podPhase:            api.PodRunning,
   385  			prc: &api.PodCondition{
   386  				Type:   api.PodReady,
   387  				Status: api.ConditionTrue,
   388  			},
   389  		},
   390  		{
   391  			name: "matching pdbs with disruptions allowed, pod running, pod healthy, healthy pod ours, deletes pod by honoring the PDB",
   392  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   393  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   394  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   395  				Status: policyv1.PodDisruptionBudgetStatus{
   396  					DisruptionsAllowed: 1,
   397  					CurrentHealthy:     3,
   398  					DesiredHealthy:     3,
   399  				},
   400  			}},
   401  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   402  			expectError:         "",
   403  			podName:             "t8",
   404  			expectedDeleteCount: 1,
   405  			podTerminating:      false,
   406  			podPhase:            api.PodRunning,
   407  			prc: &api.PodCondition{
   408  				Type:   api.PodReady,
   409  				Status: api.ConditionTrue,
   410  			},
   411  		},
   412  		{
   413  			name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours",
   414  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   415  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   416  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   417  				Status: policyv1.PodDisruptionBudgetStatus{
   418  					// This simulates 3 pods expected, our pod unhealthy
   419  					DisruptionsAllowed: 0,
   420  					CurrentHealthy:     2,
   421  					DesiredHealthy:     2,
   422  				},
   423  			}},
   424  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   425  			expectError:         "",
   426  			podName:             "t8",
   427  			expectedDeleteCount: 1,
   428  			podTerminating:      false,
   429  			podPhase:            api.PodRunning,
   430  			prc: &api.PodCondition{
   431  				Type:   api.PodReady,
   432  				Status: api.ConditionFalse,
   433  			},
   434  		},
   435  		{
   436  			name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, skips the PDB check and deletes",
   437  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   438  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   439  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   440  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
   441  			}},
   442  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t8", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   443  			expectError:         "",
   444  			podName:             "t8",
   445  			expectedDeleteCount: 1,
   446  			podTerminating:      false,
   447  			podPhase:            api.PodRunning,
   448  			prc: &api.PodCondition{
   449  				Type:   api.PodReady,
   450  				Status: api.ConditionFalse,
   451  			},
   452  			policies: []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow)}, // nil, IfHealthyBudget would not skip the PDB check
   453  		},
   454  		{
   455  			// This case should return the 529 retry error.
   456  			name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, resource version conflict",
   457  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   458  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   459  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   460  				Status: policyv1.PodDisruptionBudgetStatus{
   461  					// This simulates 3 pods expected, our pod unhealthy
   462  					DisruptionsAllowed: 0,
   463  					CurrentHealthy:     2,
   464  					DesiredHealthy:     2,
   465  				},
   466  			}},
   467  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t9", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   468  			expectError:         "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo is still being processed by the server.",
   469  			podName:             "t9",
   470  			expectedDeleteCount: 1,
   471  			podTerminating:      false,
   472  			podPhase:            api.PodRunning,
   473  			prc: &api.PodCondition{
   474  				Type:   api.PodReady,
   475  				Status: api.ConditionFalse,
   476  			},
   477  		},
   478  		{
   479  			// This case should return the 529 retry error.
   480  			name: "matching pdbs with no disruptions allowed, pod running, pod unhealthy, unhealthy pod ours, other error on delete",
   481  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   482  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   483  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   484  				Status: policyv1.PodDisruptionBudgetStatus{
   485  					// This simulates 3 pods expected, our pod unhealthy
   486  					DisruptionsAllowed: 0,
   487  					CurrentHealthy:     2,
   488  					DesiredHealthy:     2,
   489  				},
   490  			}},
   491  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t10", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   492  			expectError:         "test designed to error: BadRequest",
   493  			podName:             "t10",
   494  			expectedDeleteCount: 1,
   495  			podTerminating:      false,
   496  			podPhase:            api.PodRunning,
   497  			prc: &api.PodCondition{
   498  				Type:   api.PodReady,
   499  				Status: api.ConditionFalse,
   500  			},
   501  		},
   502  		{
   503  			name: "matching pdbs with no disruptions allowed, pod running, pod healthy, empty selector, pod not deleted by honoring the PDB",
   504  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   505  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   506  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{}},
   507  				Status: policyv1.PodDisruptionBudgetStatus{
   508  					DisruptionsAllowed: 0,
   509  					CurrentHealthy:     3,
   510  					DesiredHealthy:     3,
   511  				},
   512  			}},
   513  			eviction:            &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "t11", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)},
   514  			expectError:         "Cannot evict pod as it would violate the pod's disruption budget.: TooManyRequests: The disruption budget foo needs 3 healthy pods and has 3 currently",
   515  			podName:             "t11",
   516  			expectedDeleteCount: 0,
   517  			podTerminating:      false,
   518  			podPhase:            api.PodRunning,
   519  			prc: &api.PodCondition{
   520  				Type:   api.PodReady,
   521  				Status: api.ConditionTrue,
   522  			},
   523  		},
   524  	}
   525  
   526  	for _, unhealthyPodEvictionPolicy := range []*policyv1.UnhealthyPodEvictionPolicyType{unhealthyPolicyPtr(policyv1.AlwaysAllow), nil, unhealthyPolicyPtr(policyv1.IfHealthyBudget)} {
   527  		for _, tc := range testcases {
   528  			if len(tc.policies) > 0 && !hasUnhealthyPolicy(tc.policies, unhealthyPodEvictionPolicy) {
   529  				// unhealthyPodEvictionPolicy is not covered by this test
   530  				continue
   531  			}
   532  			t.Run(fmt.Sprintf("%v with %v policy", tc.name, unhealthyPolicyStr(unhealthyPodEvictionPolicy)), func(t *testing.T) {
   533  				defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, true)()
   534  
   535  				// same test runs 3 times, make copy of objects to have unique ones
   536  				evictionCopy := tc.eviction.DeepCopy()
   537  				prcCopy := tc.prc.DeepCopy()
   538  				var pdbsCopy []runtime.Object
   539  				for _, pdb := range tc.pdbs {
   540  					pdbCopy := pdb.DeepCopyObject()
   541  					pdbCopy.(*policyv1.PodDisruptionBudget).Spec.UnhealthyPodEvictionPolicy = unhealthyPodEvictionPolicy
   542  					pdbsCopy = append(pdbsCopy, pdbCopy)
   543  				}
   544  
   545  				testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
   546  				ms := &mockStore{
   547  					deleteCount: 0,
   548  				}
   549  
   550  				pod := validNewPod()
   551  				pod.Name = tc.podName
   552  				pod.Labels = map[string]string{"a": "true"}
   553  				pod.Spec.NodeName = "foo"
   554  				if tc.podPhase != "" {
   555  					pod.Status.Phase = tc.podPhase
   556  				}
   557  
   558  				if tc.podTerminating {
   559  					currentTime := metav1.Now()
   560  					pod.ObjectMeta.DeletionTimestamp = &currentTime
   561  				}
   562  
   563  				// Setup pod condition
   564  				if tc.prc != nil {
   565  					if !podapi.UpdatePodCondition(&pod.Status, prcCopy) {
   566  						t.Fatalf("Unable to update pod ready condition")
   567  					}
   568  				}
   569  
   570  				client := fake.NewSimpleClientset(pdbsCopy...)
   571  				evictionRest := newEvictionStorage(ms, client.PolicyV1())
   572  
   573  				name := pod.Name
   574  				ms.pod = pod
   575  
   576  				_, err := evictionRest.Create(testContext, name, evictionCopy, nil, &metav1.CreateOptions{})
   577  				gotErr := errToString(err)
   578  				if gotErr != tc.expectError {
   579  					t.Errorf("error mismatch: expected %v, got %v; name %v", tc.expectError, gotErr, pod.Name)
   580  					return
   581  				}
   582  
   583  				if tc.expectedDeleteCount != ms.deleteCount {
   584  					t.Errorf("expected delete count=%v, got %v; name %v", tc.expectedDeleteCount, ms.deleteCount, pod.Name)
   585  				}
   586  			})
   587  		}
   588  	}
   589  }
   590  
   591  func TestEvictionDryRun(t *testing.T) {
   592  	testcases := []struct {
   593  		name            string
   594  		evictionOptions *metav1.DeleteOptions
   595  		requestOptions  *metav1.CreateOptions
   596  		pdbs            []runtime.Object
   597  	}{
   598  		{
   599  			name:            "just request-options",
   600  			requestOptions:  &metav1.CreateOptions{DryRun: []string{"All"}},
   601  			evictionOptions: &metav1.DeleteOptions{},
   602  		},
   603  		{
   604  			name:            "just eviction-options",
   605  			requestOptions:  &metav1.CreateOptions{},
   606  			evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
   607  		},
   608  		{
   609  			name:            "both options",
   610  			evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
   611  			requestOptions:  &metav1.CreateOptions{DryRun: []string{"All"}},
   612  		},
   613  		{
   614  			name:            "with pdbs",
   615  			evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}},
   616  			requestOptions:  &metav1.CreateOptions{DryRun: []string{"All"}},
   617  			pdbs: []runtime.Object{&policyv1.PodDisruptionBudget{
   618  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   619  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   620  				Status:     policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 1},
   621  			}},
   622  		},
   623  	}
   624  
   625  	for _, tc := range testcases {
   626  		t.Run(tc.name, func(t *testing.T) {
   627  			testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
   628  			storage, _, _, server := newStorage(t)
   629  			defer server.Terminate(t)
   630  			defer storage.Store.DestroyFunc()
   631  
   632  			pod := validNewPod()
   633  			pod.Labels = map[string]string{"a": "true"}
   634  			pod.Spec.NodeName = "foo"
   635  			if _, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{}); err != nil {
   636  				t.Error(err)
   637  			}
   638  
   639  			client := fake.NewSimpleClientset(tc.pdbs...)
   640  			evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
   641  			eviction := &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: tc.evictionOptions}
   642  			_, err := evictionRest.Create(testContext, pod.Name, eviction, nil, tc.requestOptions)
   643  			if err != nil {
   644  				t.Fatalf("Failed to run eviction: %v", err)
   645  			}
   646  		})
   647  	}
   648  }
   649  
   650  func TestEvictionPDBStatus(t *testing.T) {
   651  	testcases := []struct {
   652  		name                       string
   653  		pdb                        *policyv1.PodDisruptionBudget
   654  		expectedDisruptionsAllowed int32
   655  		expectedReason             string
   656  	}{
   657  		{
   658  			name: "pdb status is updated after eviction",
   659  			pdb: &policyv1.PodDisruptionBudget{
   660  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   661  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   662  				Status: policyv1.PodDisruptionBudgetStatus{
   663  					DisruptionsAllowed: 1,
   664  					Conditions: []metav1.Condition{
   665  						{
   666  							Type:   policyv1.DisruptionAllowedCondition,
   667  							Reason: policyv1.SufficientPodsReason,
   668  							Status: metav1.ConditionTrue,
   669  						},
   670  					},
   671  				},
   672  			},
   673  			expectedDisruptionsAllowed: 0,
   674  			expectedReason:             policyv1.InsufficientPodsReason,
   675  		},
   676  		{
   677  			name: "condition reason is only updated if AllowedDisruptions becomes 0",
   678  			pdb: &policyv1.PodDisruptionBudget{
   679  				ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
   680  				Spec:       policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}},
   681  				Status: policyv1.PodDisruptionBudgetStatus{
   682  					DisruptionsAllowed: 3,
   683  					Conditions: []metav1.Condition{
   684  						{
   685  							Type:   policyv1.DisruptionAllowedCondition,
   686  							Reason: policyv1.SufficientPodsReason,
   687  							Status: metav1.ConditionTrue,
   688  						},
   689  					},
   690  				},
   691  			},
   692  			expectedDisruptionsAllowed: 2,
   693  			expectedReason:             policyv1.SufficientPodsReason,
   694  		},
   695  	}
   696  
   697  	for _, tc := range testcases {
   698  		t.Run(tc.name, func(t *testing.T) {
   699  			testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
   700  			storage, _, statusStorage, server := newStorage(t)
   701  			defer server.Terminate(t)
   702  			defer storage.Store.DestroyFunc()
   703  
   704  			client := fake.NewSimpleClientset(tc.pdb)
   705  			for _, podName := range []string{"foo-1", "foo-2"} {
   706  				pod := validNewPod()
   707  				pod.Labels = map[string]string{"a": "true"}
   708  				pod.ObjectMeta.Name = podName
   709  				pod.Spec.NodeName = "foo"
   710  				newPod, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{})
   711  				if err != nil {
   712  					t.Error(err)
   713  				}
   714  				(newPod.(*api.Pod)).Status.Phase = api.PodRunning
   715  				_, _, err = statusStorage.Update(testContext, pod.Name, rest.DefaultUpdatedObjectInfo(newPod),
   716  					nil, nil, false, &metav1.UpdateOptions{})
   717  				if err != nil {
   718  					t.Error(err)
   719  				}
   720  			}
   721  
   722  			evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
   723  			eviction := &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo-1", Namespace: "default"}, DeleteOptions: &metav1.DeleteOptions{}}
   724  			_, err := evictionRest.Create(testContext, "foo-1", eviction, nil, &metav1.CreateOptions{})
   725  			if err != nil {
   726  				t.Fatalf("Failed to run eviction: %v", err)
   727  			}
   728  
   729  			existingPDB, err := client.PolicyV1().PodDisruptionBudgets(metav1.NamespaceDefault).Get(context.TODO(), tc.pdb.Name, metav1.GetOptions{})
   730  			if err != nil {
   731  				t.Errorf("%#v", err)
   732  				return
   733  			}
   734  
   735  			if want, got := tc.expectedDisruptionsAllowed, existingPDB.Status.DisruptionsAllowed; got != want {
   736  				t.Errorf("expected DisruptionsAllowed to be %d, but got %d", want, got)
   737  			}
   738  
   739  			cond := apimeta.FindStatusCondition(existingPDB.Status.Conditions, policyv1.DisruptionAllowedCondition)
   740  			if want, got := tc.expectedReason, cond.Reason; want != got {
   741  				t.Errorf("expected Reason to be %q, but got %q", want, got)
   742  			}
   743  		})
   744  	}
   745  }
   746  
   747  func TestAddConditionAndDelete(t *testing.T) {
   748  	cases := []struct {
   749  		name              string
   750  		initialPod        bool
   751  		makeDeleteOptions func(*api.Pod) *metav1.DeleteOptions
   752  		expectErr         string
   753  	}{
   754  		{
   755  			name:              "simple",
   756  			initialPod:        true,
   757  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions { return &metav1.DeleteOptions{} },
   758  		},
   759  		{
   760  			name:              "missing",
   761  			initialPod:        false,
   762  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions { return &metav1.DeleteOptions{} },
   763  			expectErr:         "not found",
   764  		},
   765  		{
   766  			name:       "valid uid",
   767  			initialPod: true,
   768  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
   769  				return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &pod.UID}}
   770  			},
   771  		},
   772  		{
   773  			name:       "invalid uid",
   774  			initialPod: true,
   775  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
   776  				badUID := pod.UID + "1"
   777  				return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &badUID}}
   778  			},
   779  			expectErr: "The object might have been deleted and then recreated",
   780  		},
   781  		{
   782  			name:       "valid resourceVersion",
   783  			initialPod: true,
   784  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
   785  				return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &pod.ResourceVersion}}
   786  			},
   787  		},
   788  		{
   789  			name:       "invalid resourceVersion",
   790  			initialPod: true,
   791  			makeDeleteOptions: func(pod *api.Pod) *metav1.DeleteOptions {
   792  				badRV := pod.ResourceVersion + "1"
   793  				return &metav1.DeleteOptions{Preconditions: &metav1.Preconditions{ResourceVersion: &badRV}}
   794  			},
   795  			expectErr: "The object might have been modified",
   796  		},
   797  	}
   798  
   799  	testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault)
   800  
   801  	storage, _, _, server := newStorage(t)
   802  	defer server.Terminate(t)
   803  	defer storage.Store.DestroyFunc()
   804  
   805  	client := fake.NewSimpleClientset()
   806  	evictionRest := newEvictionStorage(storage.Store, client.PolicyV1())
   807  
   808  	for _, tc := range cases {
   809  		for _, conditionsEnabled := range []bool{true, false} {
   810  			name := fmt.Sprintf("%s_conditions=%v", tc.name, conditionsEnabled)
   811  			t.Run(name, func(t *testing.T) {
   812  				defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDisruptionConditions, conditionsEnabled)()
   813  				var deleteOptions *metav1.DeleteOptions
   814  				if tc.initialPod {
   815  					newPod := validNewPod()
   816  					createdObj, err := storage.Create(testContext, newPod, rest.ValidateAllObjectFunc, &metav1.CreateOptions{})
   817  					if err != nil {
   818  						t.Fatal(err)
   819  					}
   820  					t.Cleanup(func() {
   821  						zero := int64(0)
   822  						storage.Delete(testContext, newPod.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{GracePeriodSeconds: &zero})
   823  					})
   824  					deleteOptions = tc.makeDeleteOptions(createdObj.(*api.Pod))
   825  				} else {
   826  					deleteOptions = tc.makeDeleteOptions(nil)
   827  				}
   828  				if deleteOptions == nil {
   829  					deleteOptions = &metav1.DeleteOptions{}
   830  				}
   831  
   832  				err := addConditionAndDeletePod(evictionRest, testContext, "foo", rest.ValidateAllObjectFunc, deleteOptions)
   833  				if err == nil {
   834  					if tc.expectErr != "" {
   835  						t.Fatalf("expected err containing %q, got none", tc.expectErr)
   836  					}
   837  					return
   838  				}
   839  				if tc.expectErr == "" {
   840  					t.Fatalf("unexpected err: %v", err)
   841  				}
   842  				if !strings.Contains(err.Error(), tc.expectErr) {
   843  					t.Fatalf("expected err containing %q, got %v", tc.expectErr, err)
   844  				}
   845  			})
   846  		}
   847  	}
   848  }
   849  
   850  func resource(resource string) schema.GroupResource {
   851  	return schema.GroupResource{Group: "", Resource: resource}
   852  }
   853  
   854  type mockStore struct {
   855  	deleteCount int
   856  	pod         *api.Pod
   857  }
   858  
   859  func (ms *mockStore) mutatorDeleteFunc(count int, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
   860  	if ms.pod.Name == "t4" {
   861  		// Always return error for this pod
   862  		return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
   863  	}
   864  	if ms.pod.Name == "t6" || ms.pod.Name == "t8" {
   865  		// t6: This pod has a deletionTimestamp and should not raise conflict on delete
   866  		// t8: This pod should not have a resource conflict.
   867  		return nil, true, nil
   868  	}
   869  	if ms.pod.Name == "t10" {
   870  		return nil, false, apierrors.NewBadRequest("test designed to error")
   871  	}
   872  	if count == 1 {
   873  		// This is a hack to ensure that some test pods don't change phase
   874  		// but do change resource version
   875  		if ms.pod.Name != "t1" && ms.pod.Name != "t5" {
   876  			ms.pod.Status.Phase = api.PodRunning
   877  		}
   878  		ms.pod.ResourceVersion = "999"
   879  		// Always return conflict on the first attempt
   880  		return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
   881  	}
   882  	// Compare enforce deletionOptions
   883  	if options == nil || options.Preconditions == nil || options.Preconditions.ResourceVersion == nil {
   884  		return nil, true, nil
   885  	} else if *options.Preconditions.ResourceVersion != "1000" {
   886  		// Here we're simulating that the pod has changed resource version again
   887  		// pod "t4" should make it here, this validates we're getting the latest
   888  		// resourceVersion of the pod and successfully delete on the next deletion
   889  		// attempt after this one.
   890  		ms.pod.ResourceVersion = "1000"
   891  		return nil, false, apierrors.NewConflict(resource("tests"), "2", errors.New("message"))
   892  	}
   893  	return nil, true, nil
   894  }
   895  
   896  func (ms *mockStore) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
   897  	ms.deleteCount++
   898  	return ms.mutatorDeleteFunc(ms.deleteCount, options)
   899  }
   900  
   901  func (ms *mockStore) Watch(ctx context.Context, options *metainternalversion.ListOptions) (watch.Interface, error) {
   902  	return nil, nil
   903  }
   904  
   905  func (ms *mockStore) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
   906  	return ms.pod, false, nil
   907  }
   908  
   909  func (ms *mockStore) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
   910  	return ms.pod, nil
   911  }
   912  
   913  func (ms *mockStore) New() runtime.Object {
   914  	return nil
   915  }
   916  
   917  func (ms *mockStore) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
   918  	return nil, nil
   919  }
   920  
   921  func (ms *mockStore) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
   922  	return nil, nil
   923  }
   924  
   925  func (ms *mockStore) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
   926  	return nil, nil
   927  }
   928  
   929  func (ms *mockStore) NewList() runtime.Object {
   930  	return nil
   931  }
   932  
   933  func (ms *mockStore) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
   934  	return nil, nil
   935  }
   936  
   937  func (ms *mockStore) Destroy() {
   938  }
   939  
   940  func unhealthyPolicyPtr(unhealthyPodEvictionPolicy policyv1.UnhealthyPodEvictionPolicyType) *policyv1.UnhealthyPodEvictionPolicyType {
   941  	return &unhealthyPodEvictionPolicy
   942  }
   943  
   944  func unhealthyPolicyStr(unhealthyPodEvictionPolicy *policyv1.UnhealthyPodEvictionPolicyType) string {
   945  	if unhealthyPodEvictionPolicy == nil {
   946  		return "nil"
   947  	}
   948  	return string(*unhealthyPodEvictionPolicy)
   949  }
   950  
   951  func hasUnhealthyPolicy(unhealthyPolicies []*policyv1.UnhealthyPodEvictionPolicyType, unhealthyPolicy *policyv1.UnhealthyPodEvictionPolicyType) bool {
   952  	for _, p := range unhealthyPolicies {
   953  		if reflect.DeepEqual(unhealthyPolicy, p) {
   954  			return true
   955  		}
   956  	}
   957  	return false
   958  }
   959  
   960  func errToString(err error) string {
   961  	result := ""
   962  	if err != nil {
   963  		result = err.Error()
   964  		if statusErr, ok := err.(*apierrors.StatusError); ok {
   965  			result += fmt.Sprintf(": %v", statusErr.ErrStatus.Reason)
   966  			if statusErr.ErrStatus.Details != nil {
   967  				for _, cause := range statusErr.ErrStatus.Details.Causes {
   968  					if !strings.HasSuffix(err.Error(), cause.Message) {
   969  						result += fmt.Sprintf(": %v", cause.Message)
   970  					}
   971  				}
   972  			}
   973  		}
   974  	}
   975  	return result
   976  }
   977  

View as plain text