Source file src/k8s.io/kubernetes/test/integration/evictions/evictions_test.go

Documentation: k8s.io/kubernetes/test/integration/evictions

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package evictions
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"reflect"
    24  	"strings"
    25  	"sync"
    26  	"sync/atomic"
    27  	"testing"
    28  	"time"
    29  
    30  	v1 "k8s.io/api/core/v1"
    31  	policyv1 "k8s.io/api/policy/v1"
    32  	policyv1beta1 "k8s.io/api/policy/v1beta1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    36  	"k8s.io/apimachinery/pkg/runtime"
    37  	"k8s.io/apimachinery/pkg/runtime/schema"
    38  	"k8s.io/apimachinery/pkg/types"
    39  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    40  	"k8s.io/apimachinery/pkg/util/intstr"
    41  	"k8s.io/apimachinery/pkg/util/uuid"
    42  	"k8s.io/apimachinery/pkg/util/wait"
    43  	"k8s.io/apiserver/pkg/util/feature"
    44  	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
    45  	"k8s.io/client-go/dynamic"
    46  	"k8s.io/client-go/informers"
    47  	clientset "k8s.io/client-go/kubernetes"
    48  	policyv1client "k8s.io/client-go/kubernetes/typed/policy/v1"
    49  	restclient "k8s.io/client-go/rest"
    50  	"k8s.io/client-go/restmapper"
    51  	"k8s.io/client-go/scale"
    52  	"k8s.io/client-go/tools/cache"
    53  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    54  	"k8s.io/klog/v2"
    55  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    56  	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
    57  	"k8s.io/kubernetes/pkg/controller/disruption"
    58  	"k8s.io/kubernetes/pkg/features"
    59  	"k8s.io/kubernetes/test/integration/framework"
    60  	"k8s.io/kubernetes/test/utils/ktesting"
    61  )
    62  
    63  const (
    64  	numOfEvictions = 10
    65  )
    66  
    67  // TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to
    68  // handle concurrent eviction requests. Original issue:#37605
    69  func TestConcurrentEvictionRequests(t *testing.T) {
    70  	podNameFormat := "test-pod-%d"
    71  
    72  	tCtx := ktesting.Init(t)
    73  	closeFn, rm, informers, _, clientSet := rmSetup(tCtx, t)
    74  	defer closeFn()
    75  
    76  	ns := framework.CreateNamespaceOrDie(clientSet, "concurrent-eviction-requests", t)
    77  	defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
    78  	defer tCtx.Cancel("test has completed")
    79  
    80  	informers.Start(tCtx.Done())
    81  	go rm.Run(tCtx)
    82  
    83  	var gracePeriodSeconds int64 = 30
    84  	deleteOption := metav1.DeleteOptions{
    85  		GracePeriodSeconds: &gracePeriodSeconds,
    86  	}
    87  
    88  	// Generate numOfEvictions pods to evict
    89  	for i := 0; i < numOfEvictions; i++ {
    90  		podName := fmt.Sprintf(podNameFormat, i)
    91  		pod := newPod(podName)
    92  
    93  		if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil {
    94  			t.Errorf("Failed to create pod: %v", err)
    95  		}
    96  		pod.Status.Phase = v1.PodRunning
    97  		addPodConditionReady(pod)
    98  		if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil {
    99  			t.Fatal(err)
   100  		}
   101  	}
   102  
   103  	waitToObservePods(t, informers.Core().V1().Pods().Informer(), numOfEvictions, v1.PodRunning)
   104  
   105  	pdb := newPDB()
   106  	if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil {
   107  		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
   108  	}
   109  
   110  	waitPDBStable(t, clientSet, ns.Name, pdb.Name, numOfEvictions)
   111  
   112  	var numberPodsEvicted uint32
   113  	errCh := make(chan error, 3*numOfEvictions)
   114  	var wg sync.WaitGroup
   115  	// spawn numOfEvictions goroutines to concurrently evict the pods
   116  	for i := 0; i < numOfEvictions; i++ {
   117  		wg.Add(1)
   118  		go func(id int, errCh chan error) {
   119  			defer wg.Done()
   120  			podName := fmt.Sprintf(podNameFormat, id)
   121  			eviction := newV1Eviction(ns.Name, podName, deleteOption)
   122  
   123  			err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
   124  				e := clientSet.PolicyV1().Evictions(ns.Name).Evict(context.TODO(), eviction)
   125  				switch {
   126  				case apierrors.IsTooManyRequests(e):
   127  					return false, nil
   128  				case apierrors.IsConflict(e):
   129  					return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
   130  				case e == nil:
   131  					return true, nil
   132  				default:
   133  					return false, e
   134  				}
   135  			})
   136  
   137  			if err != nil {
   138  				errCh <- err
   139  				// should not return here otherwise we would leak the pod
   140  			}
   141  
   142  			_, err = clientSet.CoreV1().Pods(ns.Name).Get(context.TODO(), podName, metav1.GetOptions{})
   143  			switch {
   144  			case apierrors.IsNotFound(err):
   145  				atomic.AddUint32(&numberPodsEvicted, 1)
   146  				// pod was evicted and deleted so return from goroutine immediately
   147  				return
   148  			case err == nil:
   149  				// this shouldn't happen if the pod was evicted successfully
   150  				errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName)
   151  			default:
   152  				errCh <- err
   153  			}
   154  
   155  			// delete pod which still exists due to error
   156  			e := clientSet.CoreV1().Pods(ns.Name).Delete(context.TODO(), podName, deleteOption)
   157  			if e != nil {
   158  				errCh <- e
   159  			}
   160  
   161  		}(i, errCh)
   162  	}
   163  
   164  	wg.Wait()
   165  
   166  	close(errCh)
   167  	var errList []error
   168  	if err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Delete(context.TODO(), pdb.Name, deleteOption); err != nil {
   169  		errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err))
   170  	}
   171  	for err := range errCh {
   172  		errList = append(errList, err)
   173  	}
   174  	if len(errList) > 0 {
   175  		t.Fatal(utilerrors.NewAggregate(errList))
   176  	}
   177  
   178  	if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions {
   179  		t.Fatalf("fewer number of successful evictions than expected : %d", numberPodsEvicted)
   180  	}
   181  }
   182  
   183  // TestTerminalPodEviction ensures that PDB is not checked for terminal pods.
   184  func TestTerminalPodEviction(t *testing.T) {
   185  	tCtx := ktesting.Init(t)
   186  	closeFn, rm, informers, _, clientSet := rmSetup(tCtx, t)
   187  	defer closeFn()
   188  
   189  	ns := framework.CreateNamespaceOrDie(clientSet, "terminalpod-eviction", t)
   190  	defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
   191  	defer tCtx.Cancel("test has completed")
   192  
   193  	informers.Start(tCtx.Done())
   194  	go rm.Run(tCtx)
   195  
   196  	var gracePeriodSeconds int64 = 30
   197  	deleteOption := metav1.DeleteOptions{
   198  		GracePeriodSeconds: &gracePeriodSeconds,
   199  	}
   200  	pod := newPod("test-terminal-pod1")
   201  	if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil {
   202  		t.Errorf("Failed to create pod: %v", err)
   203  	}
   204  
   205  	pod.Status.Phase = v1.PodSucceeded
   206  	addPodConditionReady(pod)
   207  	if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil {
   208  		t.Fatal(err)
   209  	}
   210  
   211  	waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodSucceeded)
   212  
   213  	pdb := newPDB()
   214  	if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil {
   215  		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
   216  	}
   217  
   218  	waitPDBStable(t, clientSet, ns.Name, pdb.Name, 1)
   219  
   220  	pdbList, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).List(context.TODO(), metav1.ListOptions{})
   221  	if err != nil {
   222  		t.Fatalf("Error while listing pod disruption budget")
   223  	}
   224  	oldPdb := pdbList.Items[0]
   225  	eviction := newV1Eviction(ns.Name, pod.Name, deleteOption)
   226  	err = wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
   227  		e := clientSet.PolicyV1().Evictions(ns.Name).Evict(context.TODO(), eviction)
   228  		switch {
   229  		case apierrors.IsTooManyRequests(e):
   230  			return false, nil
   231  		case apierrors.IsConflict(e):
   232  			return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
   233  		case e == nil:
   234  			return true, nil
   235  		default:
   236  			return false, e
   237  		}
   238  	})
   239  	if err != nil {
   240  		t.Fatalf("Eviction of pod failed %v", err)
   241  	}
   242  	pdbList, err = clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).List(context.TODO(), metav1.ListOptions{})
   243  	if err != nil {
   244  		t.Fatalf("Error while listing pod disruption budget")
   245  	}
   246  	newPdb := pdbList.Items[0]
   247  	// We shouldn't see an update in pod disruption budget status' generation number as we are evicting terminal pods without checking for pod disruption.
   248  	if !reflect.DeepEqual(newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration) {
   249  		t.Fatalf("Expected the pdb generation to be of same value %v but got %v", newPdb.Status.ObservedGeneration, oldPdb.Status.ObservedGeneration)
   250  	}
   251  
   252  	if err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Delete(context.TODO(), pdb.Name, deleteOption); err != nil {
   253  		t.Fatalf("Failed to delete pod disruption budget")
   254  	}
   255  }
   256  
   257  // TestEvictionVersions ensures the eviction endpoint accepts and returns the correct API versions
   258  func TestEvictionVersions(t *testing.T) {
   259  	tCtx := ktesting.Init(t)
   260  	closeFn, rm, informers, config, clientSet := rmSetup(tCtx, t)
   261  	defer closeFn()
   262  	defer tCtx.Cancel("test has completed")
   263  
   264  	informers.Start(tCtx.Done())
   265  	go rm.Run(tCtx)
   266  
   267  	ns := "default"
   268  	subresource := "eviction"
   269  	pod := newPod("test")
   270  	if _, err := clientSet.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil {
   271  		t.Errorf("Failed to create pod: %v", err)
   272  	}
   273  
   274  	dynamicClient, err := dynamic.NewForConfig(config)
   275  	if err != nil {
   276  		t.Fatalf("Failed to create clientset: %v", err)
   277  	}
   278  
   279  	podClient := dynamicClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}).Namespace(ns)
   280  
   281  	// get should not be supported
   282  	if _, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}, subresource); !apierrors.IsMethodNotSupported(err) {
   283  		t.Fatalf("expected MethodNotSupported for GET, got %v", err)
   284  	}
   285  
   286  	// patch should not be supported
   287  	for _, patchType := range []types.PatchType{types.JSONPatchType, types.MergePatchType, types.StrategicMergePatchType, types.ApplyPatchType} {
   288  		if _, err := podClient.Patch(context.TODO(), pod.Name, patchType, []byte{}, metav1.PatchOptions{}, subresource); !apierrors.IsMethodNotSupported(err) {
   289  			t.Fatalf("expected MethodNotSupported for GET, got %v", err)
   290  		}
   291  	}
   292  
   293  	allowedEvictions := []runtime.Object{
   294  		// v1beta1, no apiVersion/kind
   295  		&policyv1beta1.Eviction{
   296  			TypeMeta:      metav1.TypeMeta{},
   297  			ObjectMeta:    metav1.ObjectMeta{Name: pod.Name},
   298  			DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}},
   299  		},
   300  		// v1beta1, apiVersion/kind
   301  		&policyv1beta1.Eviction{
   302  			TypeMeta:      metav1.TypeMeta{APIVersion: "policy/v1beta1", Kind: "Eviction"},
   303  			ObjectMeta:    metav1.ObjectMeta{Name: pod.Name},
   304  			DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}},
   305  		},
   306  		// v1, no apiVersion/kind
   307  		&policyv1.Eviction{
   308  			TypeMeta:      metav1.TypeMeta{},
   309  			ObjectMeta:    metav1.ObjectMeta{Name: pod.Name},
   310  			DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}},
   311  		},
   312  		// v1, apiVersion/kind
   313  		&policyv1.Eviction{
   314  			TypeMeta:      metav1.TypeMeta{APIVersion: "policy/v1", Kind: "Eviction"},
   315  			ObjectMeta:    metav1.ObjectMeta{Name: pod.Name},
   316  			DeleteOptions: &metav1.DeleteOptions{DryRun: []string{metav1.DryRunAll}},
   317  		},
   318  	}
   319  	v1Status := schema.GroupVersionKind{Version: "v1", Kind: "Status"}
   320  	for _, allowedEviction := range allowedEvictions {
   321  		data, _ := json.Marshal(allowedEviction)
   322  		u := &unstructured.Unstructured{}
   323  		json.Unmarshal(data, u)
   324  		result, err := podClient.Create(context.TODO(), u, metav1.CreateOptions{}, subresource)
   325  		if err != nil {
   326  			t.Fatalf("error posting %s: %v", string(data), err)
   327  		}
   328  		if result.GroupVersionKind() != v1Status {
   329  			t.Fatalf("expected v1 Status, got %#v", result)
   330  		}
   331  	}
   332  
   333  	// create unknown eviction version with apiVersion/kind should fail
   334  	u := &unstructured.Unstructured{Object: map[string]interface{}{
   335  		"metadata":   map[string]interface{}{"name": pod.Name},
   336  		"apiVersion": "policy/v2",
   337  		"kind":       "Eviction",
   338  	}}
   339  	if _, err := podClient.Create(context.TODO(), u, metav1.CreateOptions{}, subresource); err == nil {
   340  		t.Fatal("expected error posting unknown Eviction version, got none")
   341  	} else if !strings.Contains(err.Error(), "policy/v2") {
   342  		t.Fatalf("expected error about policy/v2, got %#v", err)
   343  	}
   344  }
   345  
   346  // TestEvictionWithFinalizers tests eviction with the use of finalizers
   347  func TestEvictionWithFinalizers(t *testing.T) {
   348  	cases := map[string]struct {
   349  		enablePodDisruptionConditions bool
   350  		phase                         v1.PodPhase
   351  		dryRun                        bool
   352  		wantDisruptionTargetCond      bool
   353  	}{
   354  		"terminal pod with PodDisruptionConditions enabled": {
   355  			enablePodDisruptionConditions: true,
   356  			phase:                         v1.PodSucceeded,
   357  			wantDisruptionTargetCond:      true,
   358  		},
   359  		"terminal pod with PodDisruptionConditions disabled": {
   360  			enablePodDisruptionConditions: false,
   361  			phase:                         v1.PodSucceeded,
   362  			wantDisruptionTargetCond:      false,
   363  		},
   364  		"running pod with PodDisruptionConditions enabled": {
   365  			enablePodDisruptionConditions: true,
   366  			phase:                         v1.PodRunning,
   367  			wantDisruptionTargetCond:      true,
   368  		},
   369  		"running pod with PodDisruptionConditions disabled": {
   370  			enablePodDisruptionConditions: false,
   371  			phase:                         v1.PodRunning,
   372  			wantDisruptionTargetCond:      false,
   373  		},
   374  		"running pod with PodDisruptionConditions enabled should not update conditions in dry-run mode": {
   375  			enablePodDisruptionConditions: true,
   376  			phase:                         v1.PodRunning,
   377  			dryRun:                        true,
   378  			wantDisruptionTargetCond:      false,
   379  		},
   380  	}
   381  	for name, tc := range cases {
   382  		t.Run(name, func(t *testing.T) {
   383  			tCtx := ktesting.Init(t)
   384  			closeFn, rm, informers, _, clientSet := rmSetup(tCtx, t)
   385  			defer closeFn()
   386  
   387  			ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-finalizers", t)
   388  			defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
   389  			defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.PodDisruptionConditions, tc.enablePodDisruptionConditions)()
   390  			defer tCtx.Cancel("test has completed")
   391  
   392  			informers.Start(tCtx.Done())
   393  			go rm.Run(tCtx)
   394  
   395  			pod := newPod("pod")
   396  			pod.ObjectMeta.Finalizers = []string{"test.k8s.io/finalizer"}
   397  			if _, err := clientSet.CoreV1().Pods(ns.Name).Create(tCtx, pod, metav1.CreateOptions{}); err != nil {
   398  				t.Errorf("Failed to create pod: %v", err)
   399  			}
   400  
   401  			pod.Status.Phase = tc.phase
   402  			addPodConditionReady(pod)
   403  			if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, pod, metav1.UpdateOptions{}); err != nil {
   404  				t.Fatal(err)
   405  			}
   406  
   407  			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, tc.phase)
   408  			deleteOption := metav1.DeleteOptions{}
   409  			if tc.dryRun {
   410  				deleteOption.DryRun = []string{metav1.DryRunAll}
   411  			}
   412  
   413  			eviction := newV1Eviction(ns.Name, pod.Name, deleteOption)
   414  
   415  			err := clientSet.PolicyV1().Evictions(ns.Name).Evict(tCtx, eviction)
   416  			if err != nil {
   417  				t.Fatalf("Eviction of pod failed %v", err)
   418  			}
   419  
   420  			updatedPod, e := clientSet.CoreV1().Pods(ns.Name).Get(tCtx, pod.Name, metav1.GetOptions{})
   421  			if e != nil {
   422  				t.Fatalf("Failed to get the pod %q with error: %q", klog.KObj(pod), e)
   423  			}
   424  			_, cond := podutil.GetPodCondition(&updatedPod.Status, v1.PodConditionType(v1.DisruptionTarget))
   425  			if tc.wantDisruptionTargetCond && cond == nil {
   426  				t.Errorf("Pod %q does not have the expected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget)
   427  			} else if !tc.wantDisruptionTargetCond && cond != nil {
   428  				t.Errorf("Pod %q has an unexpected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget)
   429  			}
   430  		})
   431  	}
   432  }
   433  
   434  // TestEvictionWithUnhealthyPodEvictionPolicy tests eviction with a PDB that has a UnhealthyPodEvictionPolicy
   435  func TestEvictionWithUnhealthyPodEvictionPolicy(t *testing.T) {
   436  	cases := map[string]struct {
   437  		enableUnhealthyPodEvictionPolicy bool
   438  		unhealthyPodEvictionPolicy       *policyv1.UnhealthyPodEvictionPolicyType
   439  		isPodReady                       bool
   440  	}{
   441  		"UnhealthyPodEvictionPolicy disabled and policy not set": {
   442  			enableUnhealthyPodEvictionPolicy: false,
   443  			unhealthyPodEvictionPolicy:       nil,
   444  			isPodReady:                       true,
   445  		},
   446  		"UnhealthyPodEvictionPolicy enabled but policy not set": {
   447  			enableUnhealthyPodEvictionPolicy: true,
   448  			unhealthyPodEvictionPolicy:       nil,
   449  			isPodReady:                       true,
   450  		},
   451  		"UnhealthyPodEvictionPolicy enabled but policy set to IfHealthyBudget with ready pod": {
   452  			enableUnhealthyPodEvictionPolicy: true,
   453  			unhealthyPodEvictionPolicy:       unhealthyPolicyPtr(policyv1.IfHealthyBudget),
   454  			isPodReady:                       true,
   455  		},
   456  		"UnhealthyPodEvictionPolicy enabled but policy set to AlwaysAllow with ready pod": {
   457  			enableUnhealthyPodEvictionPolicy: true,
   458  			unhealthyPodEvictionPolicy:       unhealthyPolicyPtr(policyv1.AlwaysAllow),
   459  			isPodReady:                       true,
   460  		},
   461  		"UnhealthyPodEvictionPolicy enabled but policy set to AlwaysAllow with unready pod": {
   462  			enableUnhealthyPodEvictionPolicy: true,
   463  			unhealthyPodEvictionPolicy:       unhealthyPolicyPtr(policyv1.AlwaysAllow),
   464  			isPodReady:                       false,
   465  		},
   466  	}
   467  	for name, tc := range cases {
   468  		t.Run(name, func(t *testing.T) {
   469  			tCtx := ktesting.Init(t)
   470  			defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.PDBUnhealthyPodEvictionPolicy, tc.enableUnhealthyPodEvictionPolicy)()
   471  			closeFn, rm, informers, _, clientSet := rmSetup(tCtx, t)
   472  			defer closeFn()
   473  
   474  			ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-pdb-pod-healthy-policy", t)
   475  			defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
   476  			defer tCtx.Cancel("test has completed")
   477  
   478  			informers.Start(tCtx.Done())
   479  			go rm.Run(tCtx)
   480  
   481  			pod := newPod("pod")
   482  			if _, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil {
   483  				t.Errorf("Failed to create pod: %v", err)
   484  			}
   485  
   486  			pod.Status.Phase = v1.PodRunning
   487  			if tc.isPodReady {
   488  				addPodConditionReady(pod)
   489  			}
   490  
   491  			if _, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, pod, metav1.UpdateOptions{}); err != nil {
   492  				t.Fatal(err)
   493  			}
   494  
   495  			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodRunning)
   496  
   497  			pdb := newPDB()
   498  			pdb.Spec.UnhealthyPodEvictionPolicy = tc.unhealthyPodEvictionPolicy
   499  			if _, err := clientSet.PolicyV1().PodDisruptionBudgets(ns.Name).Create(context.TODO(), pdb, metav1.CreateOptions{}); err != nil {
   500  				t.Errorf("Failed to create PodDisruptionBudget: %v", err)
   501  			}
   502  
   503  			if tc.isPodReady {
   504  				waitPDBStable(t, clientSet, ns.Name, pdb.Name, 1)
   505  			} else {
   506  				waitPDB(t, clientSet, ns.Name, pdb.Name, func(pdb *policyv1.PodDisruptionBudget) bool {
   507  					return pdb.Status.ExpectedPods == 1
   508  				})
   509  			}
   510  			// Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s
   511  			// Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s
   512  			policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()}
   513  			policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient)
   514  
   515  			deleteOption := metav1.DeleteOptions{}
   516  			eviction := newV1Eviction(ns.Name, pod.Name, deleteOption)
   517  			err := policyV1NoRetriesClient.Evictions(ns.Name).Evict(tCtx, eviction)
   518  			if err != nil {
   519  				t.Fatalf("Eviction of pod failed %v", err)
   520  			}
   521  			if policyV1NoRetriesRESTClient.postCalls != 1 {
   522  				t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls)
   523  			}
   524  
   525  			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 0, v1.PodRunning)
   526  			waitPDBStable(t, clientSet, ns.Name, pdb.Name, 0)
   527  		})
   528  	}
   529  }
   530  
   531  // TestEvictionWithPrecondition tests eviction with delete preconditions
   532  func TestEvictionWithPrecondition(t *testing.T) {
   533  	cases := map[string]struct {
   534  		enforceResourceVersion     bool
   535  		injectWrongResourceVersion bool
   536  		enforceUID                 bool
   537  		injectWrongUID             bool
   538  		shouldErr                  bool
   539  	}{
   540  		"eviction enforcing resource version": {
   541  			enforceResourceVersion: true,
   542  		},
   543  		"eviction enforcing UID": {
   544  			enforceUID: true,
   545  		},
   546  		"eviction enforcing resource version and UID": {
   547  			enforceUID:             true,
   548  			enforceResourceVersion: true,
   549  		},
   550  		"eviction enforcing wrong resource version should fail": {
   551  			enforceResourceVersion:     true,
   552  			injectWrongResourceVersion: true,
   553  			shouldErr:                  true,
   554  		},
   555  		"eviction enforcing wrong UID should fail": {
   556  			enforceUID:     true,
   557  			injectWrongUID: true,
   558  			shouldErr:      true,
   559  		},
   560  	}
   561  	for name, tc := range cases {
   562  		t.Run(name, func(t *testing.T) {
   563  			tCtx := ktesting.Init(t)
   564  			closeFn, rm, informers, _, clientSet := rmSetup(tCtx, t)
   565  			defer closeFn()
   566  
   567  			ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-preconditions", t)
   568  			defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
   569  
   570  			defer tCtx.Cancel("test has completed")
   571  			informers.Start(tCtx.Done())
   572  			go rm.Run(tCtx)
   573  
   574  			pod := newPod("pod")
   575  			pod, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{})
   576  			if err != nil {
   577  				t.Errorf("Failed to create pod: %q", err)
   578  			}
   579  
   580  			pod.Status.Phase = v1.PodRunning
   581  			addPodConditionReady(pod)
   582  
   583  			// generate a new resource version
   584  			updatedPod, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, pod, metav1.UpdateOptions{})
   585  			if err != nil {
   586  				t.Fatal(err)
   587  			}
   588  
   589  			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodRunning)
   590  
   591  			deleteOption := metav1.DeleteOptions{}
   592  
   593  			if tc.enforceResourceVersion || tc.enforceUID {
   594  				deleteOption.Preconditions = &metav1.Preconditions{}
   595  			}
   596  
   597  			if tc.enforceResourceVersion {
   598  				if tc.injectWrongResourceVersion {
   599  					deleteOption.Preconditions.ResourceVersion = &pod.ResourceVersion
   600  				} else {
   601  					deleteOption.Preconditions.ResourceVersion = &updatedPod.ResourceVersion
   602  				}
   603  
   604  			}
   605  			if tc.enforceUID {
   606  				if tc.injectWrongUID {
   607  					newUID := uuid.NewUUID()
   608  					deleteOption.Preconditions.UID = &newUID
   609  				} else {
   610  					deleteOption.Preconditions.UID = &updatedPod.UID
   611  				}
   612  			}
   613  
   614  			// Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s
   615  			// Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s
   616  			policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()}
   617  			policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient)
   618  
   619  			eviction := newV1Eviction(ns.Name, updatedPod.Name, deleteOption)
   620  			err = policyV1NoRetriesClient.Evictions(ns.Name).Evict(tCtx, eviction)
   621  			if err != nil && !tc.shouldErr {
   622  				t.Fatalf("Eviction of pod failed %q", err)
   623  			}
   624  			if err == nil && tc.shouldErr {
   625  				t.Fatal("Eviction of pod should fail")
   626  			}
   627  			if policyV1NoRetriesRESTClient.postCalls != 1 {
   628  				t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls)
   629  			}
   630  		})
   631  	}
   632  }
   633  
   634  func newPod(podName string) *v1.Pod {
   635  	return &v1.Pod{
   636  		ObjectMeta: metav1.ObjectMeta{
   637  			Name:   podName,
   638  			Labels: map[string]string{"app": "test-evictions"},
   639  		},
   640  		Spec: v1.PodSpec{
   641  			Containers: []v1.Container{
   642  				{
   643  					Name:  "fake-name",
   644  					Image: "fakeimage",
   645  				},
   646  			},
   647  		},
   648  	}
   649  }
   650  
   651  func addPodConditionReady(pod *v1.Pod) {
   652  	pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{
   653  		Type:   v1.PodReady,
   654  		Status: v1.ConditionTrue,
   655  	})
   656  }
   657  
   658  func newPDB() *policyv1.PodDisruptionBudget {
   659  	return &policyv1.PodDisruptionBudget{
   660  		ObjectMeta: metav1.ObjectMeta{
   661  			Name: "test-pdb",
   662  		},
   663  		Spec: policyv1.PodDisruptionBudgetSpec{
   664  			MinAvailable: &intstr.IntOrString{
   665  				Type:   intstr.Int,
   666  				IntVal: 0,
   667  			},
   668  			Selector: &metav1.LabelSelector{
   669  				MatchLabels: map[string]string{"app": "test-evictions"},
   670  			},
   671  		},
   672  	}
   673  }
   674  
   675  func newV1Eviction(ns, evictionName string, deleteOption metav1.DeleteOptions) *policyv1.Eviction {
   676  	return &policyv1.Eviction{
   677  		TypeMeta: metav1.TypeMeta{
   678  			APIVersion: "policy/v1",
   679  			Kind:       "Eviction",
   680  		},
   681  		ObjectMeta: metav1.ObjectMeta{
   682  			Name:      evictionName,
   683  			Namespace: ns,
   684  		},
   685  		DeleteOptions: &deleteOption,
   686  	}
   687  }
   688  
   689  func rmSetup(ctx context.Context, t *testing.T) (kubeapiservertesting.TearDownFunc, *disruption.DisruptionController, informers.SharedInformerFactory, *restclient.Config, clientset.Interface) {
   690  	// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
   691  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd())
   692  
   693  	config := restclient.CopyConfig(server.ClientConfig)
   694  	clientSet, err := clientset.NewForConfig(config)
   695  	if err != nil {
   696  		t.Fatalf("Error in create clientset: %v", err)
   697  	}
   698  	resyncPeriod := 12 * time.Hour
   699  	informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "pdb-informers")), resyncPeriod)
   700  
   701  	client := clientset.NewForConfigOrDie(restclient.AddUserAgent(config, "disruption-controller"))
   702  
   703  	discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery())
   704  	mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
   705  
   706  	scaleKindResolver := scale.NewDiscoveryScaleKindResolver(client.Discovery())
   707  	scaleClient, err := scale.NewForConfig(config, mapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver)
   708  	if err != nil {
   709  		t.Fatalf("Error in create scaleClient: %v", err)
   710  	}
   711  
   712  	rm := disruption.NewDisruptionController(
   713  		ctx,
   714  		informers.Core().V1().Pods(),
   715  		informers.Policy().V1().PodDisruptionBudgets(),
   716  		informers.Core().V1().ReplicationControllers(),
   717  		informers.Apps().V1().ReplicaSets(),
   718  		informers.Apps().V1().Deployments(),
   719  		informers.Apps().V1().StatefulSets(),
   720  		client,
   721  		mapper,
   722  		scaleClient,
   723  		client.Discovery(),
   724  	)
   725  	return server.TearDownFn, rm, informers, config, clientSet
   726  }
   727  
   728  // wait for the podInformer to observe the pods. Call this function before
   729  // running the RS controller to prevent the rc manager from creating new pods
   730  // rather than adopting the existing ones.
   731  func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int, phase v1.PodPhase) {
   732  	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
   733  		objects := podInformer.GetIndexer().List()
   734  		if len(objects) != podNum {
   735  			return false, nil
   736  		}
   737  		for _, obj := range objects {
   738  			pod := obj.(*v1.Pod)
   739  			if pod.Status.Phase != phase {
   740  				return false, nil
   741  			}
   742  		}
   743  		return true, nil
   744  	}); err != nil {
   745  		t.Fatal(err)
   746  	}
   747  }
   748  
   749  func waitPDBStable(t *testing.T, clientSet clientset.Interface, ns, pdbName string, podNum int32) {
   750  	waitPDB(t, clientSet, ns, pdbName, func(pdb *policyv1.PodDisruptionBudget) bool {
   751  		return pdb.Status.CurrentHealthy == podNum
   752  	})
   753  }
   754  
   755  func waitPDB(t *testing.T, clientSet clientset.Interface, ns, pdbName string, condition func(budget *policyv1.PodDisruptionBudget) bool) {
   756  	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
   757  		pdb, err := clientSet.PolicyV1().PodDisruptionBudgets(ns).Get(context.TODO(), pdbName, metav1.GetOptions{})
   758  		if err != nil {
   759  			return false, err
   760  		}
   761  		return condition(pdb), nil
   762  	}); err != nil {
   763  		t.Fatal(err)
   764  	}
   765  }
   766  
   767  func unhealthyPolicyPtr(unhealthyPodEvictionPolicy policyv1.UnhealthyPodEvictionPolicyType) *policyv1.UnhealthyPodEvictionPolicyType {
   768  	return &unhealthyPodEvictionPolicy
   769  }
   770  
   771  type noRetriesRESTClient struct {
   772  	mu        sync.Mutex
   773  	postCalls int
   774  	restclient.Interface
   775  }
   776  
   777  func (n *noRetriesRESTClient) Post() *restclient.Request {
   778  	n.mu.Lock()
   779  	defer n.mu.Unlock()
   780  	n.postCalls++
   781  	return n.Interface.Post().MaxRetries(0)
   782  }
   783  

View as plain text