Source file src/k8s.io/kubernetes/test/e2e/apimachinery/garbage_collector.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"sync/atomic"
    24  	"time"
    25  
    26  	appsv1 "k8s.io/api/apps/v1"
    27  	batchv1 "k8s.io/api/batch/v1"
    28  	v1 "k8s.io/api/core/v1"
    29  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    30  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    31  	apiextensionstestserver "k8s.io/apiextensions-apiserver/test/integration/fixtures"
    32  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    35  	"k8s.io/apimachinery/pkg/runtime/schema"
    36  	"k8s.io/apimachinery/pkg/types"
    37  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/apiserver/pkg/storage/names"
    40  	clientset "k8s.io/client-go/kubernetes"
    41  	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
    42  	"k8s.io/kubernetes/test/e2e/framework"
    43  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    44  	e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
    45  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    46  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    47  	imageutils "k8s.io/kubernetes/test/utils/image"
    48  	admissionapi "k8s.io/pod-security-admission/api"
    49  
    50  	"github.com/onsi/ginkgo/v2"
    51  	"github.com/onsi/gomega"
    52  )
    53  
    54  // estimateMaximumPods estimates how many pods the cluster can handle
    55  // with some wiggle room, to prevent pods being unable to schedule due
    56  // to max pod constraints.
    57  func estimateMaximumPods(ctx context.Context, c clientset.Interface, min, max int32) int32 {
    58  	nodes, err := e2enode.GetReadySchedulableNodes(ctx, c)
    59  	framework.ExpectNoError(err)
    60  
    61  	availablePods := int32(0)
    62  	for _, node := range nodes.Items {
    63  		if q, ok := node.Status.Allocatable["pods"]; ok {
    64  			if num, ok := q.AsInt64(); ok {
    65  				availablePods += int32(num)
    66  				continue
    67  			}
    68  		}
    69  		// best guess per node, since default maxPerCore is 10 and most nodes have at least
    70  		// one core.
    71  		availablePods += 10
    72  	}
    73  	//avoid creating exactly max pods
    74  	availablePods = int32(float32(availablePods) * 0.5)
    75  	// bound the top and bottom
    76  	if availablePods > max {
    77  		availablePods = max
    78  	}
    79  	if availablePods < min {
    80  		availablePods = min
    81  	}
    82  	return availablePods
    83  }
    84  
    85  func getForegroundOptions() metav1.DeleteOptions {
    86  	policy := metav1.DeletePropagationForeground
    87  	return metav1.DeleteOptions{PropagationPolicy: &policy}
    88  }
    89  
    90  func getBackgroundOptions() metav1.DeleteOptions {
    91  	policy := metav1.DeletePropagationBackground
    92  	return metav1.DeleteOptions{PropagationPolicy: &policy}
    93  }
    94  
    95  func getOrphanOptions() metav1.DeleteOptions {
    96  	policy := metav1.DeletePropagationOrphan
    97  	return metav1.DeleteOptions{PropagationPolicy: &policy}
    98  }
    99  
   100  var (
   101  	zero       = int64(0)
   102  	lablecount = int64(0)
   103  )
   104  
   105  const (
   106  	// The GC controller periodically rediscovers available APIs and syncs running informers for those resources.
   107  	// If previously available APIs are removed during that resync process, the sync process can fail and need to be retried.
   108  	//
   109  	// During e2e runs, parallel tests add/remove API resources (by creating/deleting CRDs and aggregated APIs),
   110  	// which makes it likely GC will need to retry informer resync at some point during an e2e run.
   111  	//
   112  	// This timeout covers two resync/retry periods, and should be added to wait timeouts to account for delays
   113  	// to the GC controller caused by API changes in other tests.
   114  	gcInformerResyncRetryTimeout = time.Minute
   115  
   116  	// Many operations in these tests are per-replica and may require 100 mutating requests. The
   117  	// default client QPS of a controller is 5. If the qps is saturated, it will take 20s complete
   118  	// 100 requests. The e2e tests are running in parallel, so a controller might be stuck
   119  	// processing other tests.
   120  	replicaSyncTimeout = 2 * time.Minute
   121  )
   122  
   123  func getPodTemplateSpec(labels map[string]string) v1.PodTemplateSpec {
   124  	return v1.PodTemplateSpec{
   125  		ObjectMeta: metav1.ObjectMeta{
   126  			Labels: labels,
   127  		},
   128  		Spec: v1.PodSpec{
   129  			TerminationGracePeriodSeconds: &zero,
   130  			Containers: []v1.Container{
   131  				{
   132  					Name:  "nginx",
   133  					Image: imageutils.GetE2EImage(imageutils.Nginx),
   134  				},
   135  			},
   136  		},
   137  	}
   138  }
   139  
   140  func newOwnerDeployment(f *framework.Framework, deploymentName string, labels map[string]string) *appsv1.Deployment {
   141  	return e2edeployment.NewDeployment(deploymentName, 2, labels, "nginx", imageutils.GetE2EImage(imageutils.Nginx), appsv1.RollingUpdateDeploymentStrategyType)
   142  }
   143  
   144  func newOwnerRC(f *framework.Framework, name string, replicas int32, labels map[string]string) *v1.ReplicationController {
   145  	template := getPodTemplateSpec(labels)
   146  	return &v1.ReplicationController{
   147  		TypeMeta: metav1.TypeMeta{
   148  			Kind:       "ReplicationController",
   149  			APIVersion: "v1",
   150  		},
   151  		ObjectMeta: metav1.ObjectMeta{
   152  			Namespace: f.Namespace.Name,
   153  			Name:      name,
   154  		},
   155  		Spec: v1.ReplicationControllerSpec{
   156  			Replicas: &replicas,
   157  			Selector: labels,
   158  			Template: &template,
   159  		},
   160  	}
   161  }
   162  
   163  func newGCPod(name string) *v1.Pod {
   164  	return &v1.Pod{
   165  		TypeMeta: metav1.TypeMeta{
   166  			Kind:       "Pod",
   167  			APIVersion: "v1",
   168  		},
   169  		ObjectMeta: metav1.ObjectMeta{
   170  			Name: name,
   171  		},
   172  		Spec: v1.PodSpec{
   173  			TerminationGracePeriodSeconds: new(int64),
   174  			Containers: []v1.Container{
   175  				{
   176  					Name:  "nginx",
   177  					Image: imageutils.GetE2EImage(imageutils.Nginx),
   178  				},
   179  			},
   180  		},
   181  	}
   182  }
   183  
   184  // verifyRemainingObjects verifies if the number of remaining objects.
   185  // It returns error if the communication with the API server fails.
   186  func verifyRemainingObjects(ctx context.Context, f *framework.Framework, objects map[string]int) (bool, error) {
   187  	var ret = true
   188  
   189  	for object, num := range objects {
   190  		switch object {
   191  		case "Pods":
   192  			pods, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   193  			if err != nil {
   194  				return false, fmt.Errorf("failed to list pods: %w", err)
   195  			}
   196  			if len(pods.Items) != num {
   197  				ret = false
   198  				ginkgo.By(fmt.Sprintf("expected %d pods, got %d pods", num, len(pods.Items)))
   199  			}
   200  		case "Deployments":
   201  			deployments, err := f.ClientSet.AppsV1().Deployments(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   202  			if err != nil {
   203  				return false, fmt.Errorf("failed to list deployments: %w", err)
   204  			}
   205  			if len(deployments.Items) != num {
   206  				ret = false
   207  				ginkgo.By(fmt.Sprintf("expected %d Deployments, got %d Deployments", num, len(deployments.Items)))
   208  			}
   209  		case "ReplicaSets":
   210  			rs, err := f.ClientSet.AppsV1().ReplicaSets(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   211  			if err != nil {
   212  				return false, fmt.Errorf("failed to list rs: %w", err)
   213  			}
   214  			if len(rs.Items) != num {
   215  				ret = false
   216  				ginkgo.By(fmt.Sprintf("expected %d rs, got %d rs", num, len(rs.Items)))
   217  			}
   218  		case "ReplicationControllers":
   219  			rcs, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   220  			if err != nil {
   221  				return false, fmt.Errorf("failed to list replication controllers: %w", err)
   222  			}
   223  			if len(rcs.Items) != num {
   224  				ret = false
   225  				ginkgo.By(fmt.Sprintf("expected %d RCs, got %d RCs", num, len(rcs.Items)))
   226  			}
   227  		case "CronJobs":
   228  			cronJobs, err := f.ClientSet.BatchV1().CronJobs(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   229  			if err != nil {
   230  				return false, fmt.Errorf("failed to list cronjobs: %w", err)
   231  			}
   232  			if len(cronJobs.Items) != num {
   233  				ret = false
   234  				ginkgo.By(fmt.Sprintf("expected %d cronjobs, got %d cronjobs", num, len(cronJobs.Items)))
   235  			}
   236  		case "Jobs":
   237  			jobs, err := f.ClientSet.BatchV1().Jobs(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   238  			if err != nil {
   239  				return false, fmt.Errorf("failed to list jobs: %w", err)
   240  			}
   241  			if len(jobs.Items) != num {
   242  				ret = false
   243  				ginkgo.By(fmt.Sprintf("expected %d jobs, got %d jobs", num, len(jobs.Items)))
   244  			}
   245  		default:
   246  			return false, fmt.Errorf("object %s is not supported", object)
   247  		}
   248  	}
   249  
   250  	return ret, nil
   251  }
   252  
   253  func gatherMetrics(ctx context.Context, f *framework.Framework) {
   254  	ginkgo.By("Gathering metrics")
   255  	var summary framework.TestDataSummary
   256  	grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), false, false, true, false, false, false)
   257  	if err != nil {
   258  		framework.Logf("Failed to create MetricsGrabber. Skipping metrics gathering.")
   259  	} else {
   260  		received, err := grabber.Grab(ctx)
   261  		if err != nil {
   262  			framework.Logf("MetricsGrabber failed grab metrics. Skipping metrics gathering.")
   263  		} else {
   264  			summary = (*e2emetrics.ComponentCollection)(&received)
   265  			framework.Logf(summary.PrintHumanReadable())
   266  		}
   267  	}
   268  }
   269  
   270  func newCronJob(name, schedule string) *batchv1.CronJob {
   271  	parallelism := int32(1)
   272  	completions := int32(1)
   273  	return &batchv1.CronJob{
   274  		ObjectMeta: metav1.ObjectMeta{
   275  			Name: name,
   276  		},
   277  		TypeMeta: metav1.TypeMeta{
   278  			Kind: "CronJob",
   279  		},
   280  		Spec: batchv1.CronJobSpec{
   281  			Schedule: schedule,
   282  			JobTemplate: batchv1.JobTemplateSpec{
   283  				Spec: batchv1.JobSpec{
   284  					Parallelism: &parallelism,
   285  					Completions: &completions,
   286  					Template: v1.PodTemplateSpec{
   287  						Spec: v1.PodSpec{
   288  							RestartPolicy:                 v1.RestartPolicyOnFailure,
   289  							TerminationGracePeriodSeconds: &zero,
   290  							Containers: []v1.Container{
   291  								{
   292  									Name:    "c",
   293  									Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   294  									Command: []string{"sleep", "300"},
   295  								},
   296  							},
   297  						},
   298  					},
   299  				},
   300  			},
   301  		},
   302  	}
   303  }
   304  
   305  // getUniqLabel returns a UniqLabel based on labeLkey and labelvalue.
   306  func getUniqLabel(labelkey, labelvalue string) map[string]string {
   307  	count := atomic.AddInt64(&lablecount, 1)
   308  	uniqlabelkey := fmt.Sprintf("%s-%05d", labelkey, count)
   309  	uniqlabelvalue := fmt.Sprintf("%s-%05d", labelvalue, count)
   310  	return map[string]string{uniqlabelkey: uniqlabelvalue}
   311  }
   312  
   313  var _ = SIGDescribe("Garbage collector", func() {
   314  	f := framework.NewDefaultFramework("gc")
   315  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
   316  
   317  	/*
   318  		Release: v1.9
   319  		Testname: Garbage Collector, delete replication controller, propagation policy background
   320  		Description: Create a replication controller with 2 Pods. Once RC is created and the first Pod is created, delete RC with deleteOptions.PropagationPolicy set to Background. Deleting the Replication Controller MUST cause pods created by that RC to be deleted.
   321  	*/
   322  	framework.ConformanceIt("should delete pods created by rc when not orphaning", func(ctx context.Context) {
   323  		clientSet := f.ClientSet
   324  		rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
   325  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   326  		rcName := "simpletest.rc"
   327  		uniqLabels := getUniqLabel("gctest", "delete_pods")
   328  		rc := newOwnerRC(f, rcName, 2, uniqLabels)
   329  		ginkgo.By("create the rc")
   330  		rc, err := rcClient.Create(ctx, rc, metav1.CreateOptions{})
   331  		if err != nil {
   332  			framework.Failf("Failed to create replication controller: %v", err)
   333  		}
   334  		// wait for rc to create some pods
   335  		if err := wait.Poll(5*time.Second, 30*time.Second, func() (bool, error) {
   336  			pods, err := podClient.List(ctx, metav1.ListOptions{})
   337  			if err != nil {
   338  				return false, fmt.Errorf("failed to list pods: %w", err)
   339  			}
   340  			// We intentionally don't wait the number of pods to reach
   341  			// rc.Spec.Replicas. We want to see if the garbage collector and the
   342  			// rc manager work properly if the rc is deleted before it reaches
   343  			// stasis.
   344  			if len(pods.Items) > 0 {
   345  				return true, nil
   346  			}
   347  			return false, nil
   348  
   349  		}); err != nil {
   350  			framework.Failf("failed to wait for the rc to create some pods: %v", err)
   351  		}
   352  		ginkgo.By("delete the rc")
   353  		deleteOptions := getBackgroundOptions()
   354  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(rc.UID))
   355  		if err := rcClient.Delete(ctx, rc.ObjectMeta.Name, deleteOptions); err != nil {
   356  			framework.Failf("failed to delete the rc: %v", err)
   357  		}
   358  		ginkgo.By("wait for all pods to be garbage collected")
   359  		// wait for the RCs and Pods to reach the expected numbers.
   360  		if err := wait.PollWithContext(ctx, 5*time.Second, (60*time.Second)+gcInformerResyncRetryTimeout, func(ctx context.Context) (bool, error) {
   361  			objects := map[string]int{"ReplicationControllers": 0, "Pods": 0}
   362  			return verifyRemainingObjects(ctx, f, objects)
   363  		}); err != nil {
   364  			framework.Failf("failed to wait for all pods to be deleted: %v", err)
   365  			remainingPods, err := podClient.List(ctx, metav1.ListOptions{})
   366  			if err != nil {
   367  				framework.Failf("failed to list pods post mortem: %v", err)
   368  			} else {
   369  				framework.Failf("remaining pods are: %#v", remainingPods)
   370  			}
   371  		}
   372  		gatherMetrics(ctx, f)
   373  	})
   374  
   375  	/*
   376  		Release: v1.9
   377  		Testname: Garbage Collector, delete replication controller, propagation policy orphan
   378  		Description: Create a replication controller with maximum allocatable Pods between 10 and 100 replicas. Once RC is created and the all Pods are created, delete RC with deleteOptions.PropagationPolicy set to Orphan. Deleting the Replication Controller MUST cause pods created by that RC to be orphaned.
   379  	*/
   380  	framework.ConformanceIt("should orphan pods created by rc if delete options say so", func(ctx context.Context) {
   381  		clientSet := f.ClientSet
   382  		rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
   383  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   384  		rcName := "simpletest.rc"
   385  		uniqLabels := getUniqLabel("gctest", "orphan_pods")
   386  		rc := newOwnerRC(f, rcName, estimateMaximumPods(ctx, clientSet, 10, 100), uniqLabels)
   387  		ginkgo.By("create the rc")
   388  		rc, err := rcClient.Create(ctx, rc, metav1.CreateOptions{})
   389  		if err != nil {
   390  			framework.Failf("Failed to create replication controller: %v", err)
   391  		}
   392  		// wait for rc to create pods
   393  		waitForReplicas(ctx, rc, rcClient)
   394  
   395  		ginkgo.By("delete the rc")
   396  		deleteOptions := getOrphanOptions()
   397  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(rc.UID))
   398  		if err := rcClient.Delete(ctx, rc.ObjectMeta.Name, deleteOptions); err != nil {
   399  			framework.Failf("failed to delete the rc: %v", err)
   400  		}
   401  		ginkgo.By("wait for the rc to be deleted")
   402  		// Orphaning the 100 pods takes 100 PATCH operations. The default qps of
   403  		// a client is 5. If the qps is saturated, it will take 20s to orphan
   404  		// the pods. However, apiserver takes hundreds of ms to finish one
   405  		// PATCH, and the gc sends the patching in a single thread, so the
   406  		// actual qps is less than 5. Also, the e2e tests are running in
   407  		// parallel, the GC controller might get distracted by other tests.
   408  		// According to the test logs, 120s is enough time.
   409  		if err := wait.PollWithContext(ctx, 5*time.Second, 120*time.Second+gcInformerResyncRetryTimeout, func(ctx context.Context) (bool, error) {
   410  			rcs, err := rcClient.List(ctx, metav1.ListOptions{})
   411  			if err != nil {
   412  				return false, fmt.Errorf("failed to list rcs: %w", err)
   413  			}
   414  			if len(rcs.Items) != 0 {
   415  				return false, nil
   416  			}
   417  			return true, nil
   418  		}); err != nil {
   419  			framework.Failf("%v", err)
   420  		}
   421  		ginkgo.By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods")
   422  		time.Sleep(30 * time.Second)
   423  		pods, err := podClient.List(ctx, metav1.ListOptions{})
   424  		if err != nil {
   425  			framework.Failf("Failed to list pods: %v", err)
   426  		}
   427  		if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a {
   428  			framework.Failf("expect %d pods, got %d pods", e, a)
   429  		}
   430  		gatherMetrics(ctx, f)
   431  		if err = e2epod.DeletePodsWithGracePeriod(ctx, clientSet, pods.Items, 0); err != nil {
   432  			framework.Logf("WARNING: failed to delete pods: %v", err)
   433  		}
   434  	})
   435  
   436  	// deleteOptions.OrphanDependents is deprecated in 1.7 and preferred to use the PropagationPolicy.
   437  	// Discussion is tracked under https://github.com/kubernetes/kubernetes/issues/65427 to promote for conformance in future.
   438  	ginkgo.It("should orphan pods created by rc if deleteOptions.OrphanDependents is nil", func(ctx context.Context) {
   439  		clientSet := f.ClientSet
   440  		rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
   441  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   442  		rcName := "simpletest.rc"
   443  		uniqLabels := getUniqLabel("gctest", "orphan_pods_nil_option")
   444  		rc := newOwnerRC(f, rcName, 2, uniqLabels)
   445  		ginkgo.By("create the rc")
   446  		rc, err := rcClient.Create(ctx, rc, metav1.CreateOptions{})
   447  		if err != nil {
   448  			framework.Failf("Failed to create replication controller: %v", err)
   449  		}
   450  		// wait for rc to create some pods
   451  		waitForReplicas(ctx, rc, rcClient)
   452  
   453  		ginkgo.By("delete the rc")
   454  		deleteOptions := metav1.DeleteOptions{
   455  			Preconditions: metav1.NewUIDPreconditions(string(rc.UID)),
   456  		}
   457  		if err := rcClient.Delete(ctx, rc.ObjectMeta.Name, deleteOptions); err != nil {
   458  			framework.Failf("failed to delete the rc: %v", err)
   459  		}
   460  		ginkgo.By("wait for 30 seconds to see if the garbage collector mistakenly deletes the pods")
   461  		time.Sleep(30 * time.Second)
   462  		pods, err := podClient.List(ctx, metav1.ListOptions{})
   463  		if err != nil {
   464  			framework.Failf("Failed to list pods: %v", err)
   465  		}
   466  		if e, a := int(*(rc.Spec.Replicas)), len(pods.Items); e != a {
   467  			framework.Failf("expect %d pods, got %d pods", e, a)
   468  		}
   469  		gatherMetrics(ctx, f)
   470  		if err = e2epod.DeletePodsWithGracePeriod(ctx, clientSet, pods.Items, 0); err != nil {
   471  			framework.Logf("WARNING: failed to delete pods: %v", err)
   472  		}
   473  	})
   474  
   475  	/*
   476  		Release: v1.9
   477  		Testname: Garbage Collector, delete deployment,  propagation policy background
   478  		Description: Create a deployment with a replicaset. Once replicaset is created , delete the deployment  with deleteOptions.PropagationPolicy set to Background. Deleting the deployment MUST delete the replicaset created by the deployment and also the Pods that belong to the deployments MUST be deleted.
   479  	*/
   480  	framework.ConformanceIt("should delete RS created by deployment when not orphaning", func(ctx context.Context) {
   481  		clientSet := f.ClientSet
   482  		deployClient := clientSet.AppsV1().Deployments(f.Namespace.Name)
   483  		rsClient := clientSet.AppsV1().ReplicaSets(f.Namespace.Name)
   484  		deploymentName := "simpletest.deployment"
   485  		uniqLabels := getUniqLabel("gctest", "delete_rs")
   486  		deployment := newOwnerDeployment(f, deploymentName, uniqLabels)
   487  		ginkgo.By("create the deployment")
   488  		createdDeployment, err := deployClient.Create(ctx, deployment, metav1.CreateOptions{})
   489  		if err != nil {
   490  			framework.Failf("Failed to create deployment: %v", err)
   491  		}
   492  		// wait for deployment to create some rs
   493  		ginkgo.By("Wait for the Deployment to create new ReplicaSet")
   494  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
   495  			rsList, err := rsClient.List(ctx, metav1.ListOptions{})
   496  			if err != nil {
   497  				return false, fmt.Errorf("failed to list rs: %w", err)
   498  			}
   499  			return len(rsList.Items) > 0, nil
   500  
   501  		})
   502  		if err != nil {
   503  			framework.Failf("Failed to wait for the Deployment to create some ReplicaSet: %v", err)
   504  		}
   505  
   506  		ginkgo.By("delete the deployment")
   507  		deleteOptions := getBackgroundOptions()
   508  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(createdDeployment.UID))
   509  		if err := deployClient.Delete(ctx, deployment.ObjectMeta.Name, deleteOptions); err != nil {
   510  			framework.Failf("failed to delete the deployment: %v", err)
   511  		}
   512  		ginkgo.By("wait for all rs to be garbage collected")
   513  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute+gcInformerResyncRetryTimeout, true, func(ctx context.Context) (bool, error) {
   514  			objects := map[string]int{"Deployments": 0, "ReplicaSets": 0, "Pods": 0}
   515  			return verifyRemainingObjects(ctx, f, objects)
   516  		})
   517  		if err != nil {
   518  			errList := make([]error, 0)
   519  			errList = append(errList, err)
   520  			remainingRSs, err := rsClient.List(ctx, metav1.ListOptions{})
   521  			if err != nil {
   522  				errList = append(errList, fmt.Errorf("failed to list RSs post mortem: %w", err))
   523  			} else {
   524  				errList = append(errList, fmt.Errorf("remaining rs are: %#v", remainingRSs))
   525  			}
   526  			aggregatedError := utilerrors.NewAggregate(errList)
   527  			framework.Failf("Failed to wait for all rs to be garbage collected: %v", aggregatedError)
   528  
   529  		}
   530  
   531  		gatherMetrics(ctx, f)
   532  	})
   533  
   534  	/*
   535  		Release: v1.9
   536  		Testname: Garbage Collector, delete deployment, propagation policy orphan
   537  		Description: Create a deployment with a replicaset. Once replicaset is created , delete the deployment  with deleteOptions.PropagationPolicy set to Orphan. Deleting the deployment MUST cause the replicaset created by the deployment to be orphaned, also the Pods created by the deployments MUST be orphaned.
   538  	*/
   539  	framework.ConformanceIt("should orphan RS created by deployment when deleteOptions.PropagationPolicy is Orphan", func(ctx context.Context) {
   540  		clientSet := f.ClientSet
   541  		deployClient := clientSet.AppsV1().Deployments(f.Namespace.Name)
   542  		rsClient := clientSet.AppsV1().ReplicaSets(f.Namespace.Name)
   543  		deploymentName := "simpletest.deployment"
   544  		uniqLabels := getUniqLabel("gctest", "orphan_rs")
   545  		deployment := newOwnerDeployment(f, deploymentName, uniqLabels)
   546  		ginkgo.By("create the deployment")
   547  		createdDeployment, err := deployClient.Create(ctx, deployment, metav1.CreateOptions{})
   548  		if err != nil {
   549  			framework.Failf("Failed to create deployment: %v", err)
   550  		}
   551  		// wait for deployment to create some rs
   552  		ginkgo.By("Wait for the Deployment to create new ReplicaSet")
   553  		var replicaset appsv1.ReplicaSet
   554  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
   555  			rsList, err := rsClient.List(ctx, metav1.ListOptions{})
   556  			if err != nil {
   557  				return false, fmt.Errorf("failed to list rs: %w", err)
   558  			}
   559  			if len(rsList.Items) > 0 {
   560  				replicaset = rsList.Items[0]
   561  				return true, nil
   562  			}
   563  			return false, nil
   564  
   565  		})
   566  		if err != nil {
   567  			framework.Failf("Failed to wait for the Deployment to create some ReplicaSet: %v", err)
   568  		}
   569  
   570  		desiredGeneration := replicaset.Generation
   571  		if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 60*time.Second, true, func(ctx context.Context) (bool, error) {
   572  			newRS, err := clientSet.AppsV1().ReplicaSets(replicaset.Namespace).Get(ctx, replicaset.Name, metav1.GetOptions{})
   573  			if err != nil {
   574  				return false, err
   575  			}
   576  			return newRS.Status.ObservedGeneration >= desiredGeneration && newRS.Status.Replicas == *replicaset.Spec.Replicas, nil
   577  		}); err != nil {
   578  			framework.Failf("failed to verify .Status.Replicas is equal to .Spec.Replicas for replicaset %q: %v", replicaset.Name, err)
   579  		}
   580  
   581  		ginkgo.By("delete the deployment")
   582  		deleteOptions := getOrphanOptions()
   583  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(createdDeployment.UID))
   584  		if err := deployClient.Delete(ctx, deployment.ObjectMeta.Name, deleteOptions); err != nil {
   585  			framework.Failf("failed to delete the deployment: %v", err)
   586  		}
   587  		ginkgo.By("wait for deployment deletion to see if the garbage collector mistakenly deletes the rs")
   588  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute+gcInformerResyncRetryTimeout, true, func(ctx context.Context) (bool, error) {
   589  			dList, err := deployClient.List(ctx, metav1.ListOptions{})
   590  			if err != nil {
   591  				return false, fmt.Errorf("failed to list deployments: %w", err)
   592  			}
   593  			return len(dList.Items) == 0, nil
   594  		})
   595  		if err != nil {
   596  			framework.Failf("Failed to wait for the Deployment to be deleted: %v", err)
   597  		}
   598  		// Once the deployment object is gone, we'll know the GC has finished performing any relevant actions.
   599  		objects := map[string]int{"Deployments": 0, "ReplicaSets": 1, "Pods": 2}
   600  		ok, err := verifyRemainingObjects(ctx, f, objects)
   601  		if err != nil {
   602  			framework.Failf("Unexpected error while verifying remaining deployments, rs, and pods: %v", err)
   603  		}
   604  		if !ok {
   605  			errList := make([]error, 0)
   606  			remainingRSs, err := rsClient.List(ctx, metav1.ListOptions{})
   607  			if err != nil {
   608  				errList = append(errList, fmt.Errorf("failed to list RSs post mortem: %w", err))
   609  			} else {
   610  				errList = append(errList, fmt.Errorf("remaining rs post mortem: %#v", remainingRSs))
   611  			}
   612  			remainingDSs, err := deployClient.List(ctx, metav1.ListOptions{})
   613  			if err != nil {
   614  				errList = append(errList, fmt.Errorf("failed to list Deployments post mortem: %w", err))
   615  			} else {
   616  				errList = append(errList, fmt.Errorf("remaining deployment's post mortem: %#v", remainingDSs))
   617  			}
   618  			aggregatedError := utilerrors.NewAggregate(errList)
   619  			framework.Failf("Failed to verify remaining deployments, rs, and pods: %v", aggregatedError)
   620  		}
   621  		rs, err := clientSet.AppsV1().ReplicaSets(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   622  		if err != nil {
   623  			framework.Failf("Failed to list ReplicaSet %v", err)
   624  		}
   625  		for _, replicaSet := range rs.Items {
   626  			if metav1.GetControllerOf(&replicaSet.ObjectMeta) != nil {
   627  				framework.Failf("Found ReplicaSet with non nil ownerRef %v", replicaSet)
   628  			}
   629  		}
   630  
   631  		gatherMetrics(ctx, f)
   632  	})
   633  
   634  	/*
   635  		Release: v1.9
   636  		Testname: Garbage Collector, delete replication controller, after owned pods
   637  		Description: Create a replication controller with maximum allocatable Pods between 10 and 100 replicas. Once RC is created and the all Pods are created, delete RC with deleteOptions.PropagationPolicy set to Foreground. Deleting the Replication Controller MUST cause pods created by that RC to be deleted before the RC is deleted.
   638  	*/
   639  	framework.ConformanceIt("should keep the rc around until all its pods are deleted if the deleteOptions says so", func(ctx context.Context) {
   640  		clientSet := f.ClientSet
   641  		rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
   642  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   643  		rcName := "simpletest.rc"
   644  		uniqLabels := getUniqLabel("gctest", "delete_pods_foreground")
   645  		rc := newOwnerRC(f, rcName, estimateMaximumPods(ctx, clientSet, 10, 100), uniqLabels)
   646  		ginkgo.By("create the rc")
   647  		rc, err := rcClient.Create(ctx, rc, metav1.CreateOptions{})
   648  		if err != nil {
   649  			framework.Failf("Failed to create replication controller: %v", err)
   650  		}
   651  		// wait for rc to create pods
   652  		waitForReplicas(ctx, rc, rcClient)
   653  
   654  		ginkgo.By("delete the rc")
   655  		deleteOptions := getForegroundOptions()
   656  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(rc.UID))
   657  		if err := rcClient.Delete(ctx, rc.ObjectMeta.Name, deleteOptions); err != nil {
   658  			framework.Failf("failed to delete the rc: %v", err)
   659  		}
   660  		ginkgo.By("wait for the rc to be deleted")
   661  		// default client QPS is 20, deleting each pod requires 2 requests, so 30s should be enough
   662  		// TODO: 30s is enough assuming immediate processing of dependents following
   663  		// owner deletion, but in practice there can be a long delay between owner
   664  		// deletion and dependent deletion processing. For now, increase the timeout
   665  		// and investigate the processing delay.
   666  		if err := wait.PollWithContext(ctx, 1*time.Second, 30*time.Second+gcInformerResyncRetryTimeout, func(ctx context.Context) (bool, error) {
   667  			_, err := rcClient.Get(ctx, rc.Name, metav1.GetOptions{})
   668  			if err == nil {
   669  				pods, _ := podClient.List(ctx, metav1.ListOptions{})
   670  				framework.Logf("%d pods remaining", len(pods.Items))
   671  				count := 0
   672  				for _, pod := range pods.Items {
   673  					if pod.ObjectMeta.DeletionTimestamp == nil {
   674  						count++
   675  					}
   676  				}
   677  				framework.Logf("%d pods has nil DeletionTimestamp", count)
   678  				framework.Logf("")
   679  				return false, nil
   680  			}
   681  			if apierrors.IsNotFound(err) {
   682  				return true, nil
   683  			}
   684  			return false, err
   685  		}); err != nil {
   686  			pods, err2 := podClient.List(ctx, metav1.ListOptions{})
   687  			if err2 != nil {
   688  				framework.Failf("%v", err2)
   689  			}
   690  			framework.Logf("%d remaining pods are:", len(pods.Items))
   691  			framework.Logf("The ObjectMeta of the remaining pods are:")
   692  			for _, pod := range pods.Items {
   693  				framework.Logf("%#v", pod.ObjectMeta)
   694  			}
   695  			framework.Failf("failed to delete the rc: %v", err)
   696  		}
   697  		// There shouldn't be any pods
   698  		pods, err := podClient.List(ctx, metav1.ListOptions{})
   699  		if err != nil {
   700  			framework.Failf("%v", err)
   701  		}
   702  		if len(pods.Items) != 0 {
   703  			framework.Failf("expected no pods, got %#v", pods)
   704  		}
   705  		gatherMetrics(ctx, f)
   706  	})
   707  
   708  	// TODO: this should be an integration test
   709  	/*
   710  		Release: v1.9
   711  		Testname: Garbage Collector, multiple owners
   712  		Description: Create a replication controller RC1, with maximum allocatable Pods between 10 and 100 replicas. Create second replication controller RC2 and set RC2 as owner for half of those replicas. Once RC1 is created and the all Pods are created, delete RC1 with deleteOptions.PropagationPolicy set to Foreground. Half of the Pods that has RC2 as owner MUST not be deleted or have a deletion timestamp. Deleting the Replication Controller MUST not delete Pods that are owned by multiple replication controllers.
   713  	*/
   714  	framework.ConformanceIt("should not delete dependents that have both valid owner and owner that's waiting for dependents to be deleted", func(ctx context.Context) {
   715  		clientSet := f.ClientSet
   716  		rcClient := clientSet.CoreV1().ReplicationControllers(f.Namespace.Name)
   717  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   718  		rc1Name := "simpletest-rc-to-be-deleted"
   719  		replicas := estimateMaximumPods(ctx, clientSet, 10, 100)
   720  		halfReplicas := int(replicas / 2)
   721  		uniqLabelsDeleted := getUniqLabel("gctest_d", "valid_and_pending_owners_d")
   722  		rc1 := newOwnerRC(f, rc1Name, replicas, uniqLabelsDeleted)
   723  		ginkgo.By("create the rc1")
   724  		rc1, err := rcClient.Create(ctx, rc1, metav1.CreateOptions{})
   725  		if err != nil {
   726  			framework.Failf("Failed to create replication controller: %v", err)
   727  		}
   728  		rc2Name := "simpletest-rc-to-stay"
   729  		uniqLabelsStay := getUniqLabel("gctest_s", "valid_and_pending_owners_s")
   730  		rc2 := newOwnerRC(f, rc2Name, 0, uniqLabelsStay)
   731  		ginkgo.By("create the rc2")
   732  		rc2, err = rcClient.Create(ctx, rc2, metav1.CreateOptions{})
   733  		if err != nil {
   734  			framework.Failf("Failed to create replication controller: %v", err)
   735  		}
   736  		// wait for rc1 to be stable
   737  		waitForReplicas(ctx, rc1, rcClient)
   738  
   739  		ginkgo.By(fmt.Sprintf("set half of pods created by rc %s to have rc %s as owner as well", rc1Name, rc2Name))
   740  		pods, err := podClient.List(ctx, metav1.ListOptions{})
   741  		framework.ExpectNoError(err, "failed to list pods in namespace: %s", f.Namespace.Name)
   742  		patch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"apiVersion":"v1","kind":"ReplicationController","name":"%s","uid":"%s"}]}}`, rc2.ObjectMeta.Name, rc2.ObjectMeta.UID)
   743  		for i := 0; i < halfReplicas; i++ {
   744  			pod := pods.Items[i]
   745  			_, err := podClient.Patch(ctx, pod.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
   746  			framework.ExpectNoError(err, "failed to apply to pod %s in namespace %s, a strategic merge patch: %s", pod.Name, f.Namespace.Name, patch)
   747  		}
   748  
   749  		ginkgo.By(fmt.Sprintf("delete the rc %s", rc1Name))
   750  		deleteOptions := getForegroundOptions()
   751  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(rc1.UID))
   752  		if err := rcClient.Delete(ctx, rc1.ObjectMeta.Name, deleteOptions); err != nil {
   753  			framework.Failf("failed to delete the rc: %v", err)
   754  		}
   755  		ginkgo.By("wait for the rc to be deleted")
   756  		// TODO: shorten the timeout when we make GC's periodic API rediscovery more efficient.
   757  		// Tracked at https://github.com/kubernetes/kubernetes/issues/50046.
   758  		if err := wait.PollWithContext(ctx, 5*time.Second, 90*time.Second, func(ctx context.Context) (bool, error) {
   759  			_, err := rcClient.Get(ctx, rc1.Name, metav1.GetOptions{})
   760  			if err == nil {
   761  				pods, _ := podClient.List(ctx, metav1.ListOptions{})
   762  				framework.Logf("%d pods remaining", len(pods.Items))
   763  				count := 0
   764  				for _, pod := range pods.Items {
   765  					if pod.ObjectMeta.DeletionTimestamp == nil {
   766  						count++
   767  					}
   768  				}
   769  				framework.Logf("%d pods has nil DeletionTimestamp", count)
   770  				framework.Logf("")
   771  				return false, nil
   772  			}
   773  			if apierrors.IsNotFound(err) {
   774  				return true, nil
   775  			}
   776  			return false, err
   777  		}); err != nil {
   778  			pods, err2 := podClient.List(ctx, metav1.ListOptions{})
   779  			if err2 != nil {
   780  				framework.Failf("%v", err2)
   781  			}
   782  			framework.Logf("%d remaining pods are:", len(pods.Items))
   783  			framework.Logf("ObjectMeta of remaining pods are:")
   784  			for _, pod := range pods.Items {
   785  				framework.Logf("%#v", pod.ObjectMeta)
   786  			}
   787  			framework.Failf("failed to delete rc %s, err: %v", rc1Name, err)
   788  		}
   789  		// half of the pods should still exist,
   790  		pods, err = podClient.List(ctx, metav1.ListOptions{})
   791  		if err != nil {
   792  			framework.Failf("%v", err)
   793  		}
   794  		if len(pods.Items) != halfReplicas {
   795  			framework.Failf("expected %d pods, got %d", halfReplicas, len(pods.Items))
   796  		}
   797  		for _, pod := range pods.Items {
   798  			if pod.ObjectMeta.DeletionTimestamp != nil {
   799  				framework.Failf("expected pod DeletionTimestamp to be nil, got %#v", pod.ObjectMeta)
   800  			}
   801  			// they should only have 1 ownerReference left
   802  			if len(pod.ObjectMeta.OwnerReferences) != 1 {
   803  				framework.Failf("expected pod to only have 1 owner, got %#v", pod.ObjectMeta.OwnerReferences)
   804  			}
   805  		}
   806  		gatherMetrics(ctx, f)
   807  		if err = e2epod.DeletePodsWithGracePeriod(ctx, clientSet, pods.Items, 0); err != nil {
   808  			framework.Logf("WARNING: failed to delete pods: %v", err)
   809  		}
   810  	})
   811  
   812  	// TODO: should be an integration test
   813  	/*
   814  		Release: v1.9
   815  		Testname: Garbage Collector, dependency cycle
   816  		Description: Create three pods, patch them with Owner references such that pod1 has pod3, pod2 has pod1 and pod3 has pod2 as owner references respectively. Delete pod1 MUST delete all pods. The dependency cycle MUST not block the garbage collection.
   817  	*/
   818  	framework.ConformanceIt("should not be blocked by dependency circle", func(ctx context.Context) {
   819  		clientSet := f.ClientSet
   820  		podClient := clientSet.CoreV1().Pods(f.Namespace.Name)
   821  		pod1Name := "pod1"
   822  		pod1 := newGCPod(pod1Name)
   823  		pod1, err := podClient.Create(ctx, pod1, metav1.CreateOptions{})
   824  		framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", pod1Name, f.Namespace.Name)
   825  		pod2Name := "pod2"
   826  		pod2 := newGCPod(pod2Name)
   827  		pod2, err = podClient.Create(ctx, pod2, metav1.CreateOptions{})
   828  		framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", pod2Name, f.Namespace.Name)
   829  		pod3Name := "pod3"
   830  		pod3 := newGCPod(pod3Name)
   831  		pod3, err = podClient.Create(ctx, pod3, metav1.CreateOptions{})
   832  		framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", pod3Name, f.Namespace.Name)
   833  		// create circular dependency
   834  		addRefPatch := func(name string, uid types.UID) []byte {
   835  			return []byte(fmt.Sprintf(`{"metadata":{"ownerReferences":[{"apiVersion":"v1","kind":"Pod","name":"%s","uid":"%s","controller":true,"blockOwnerDeletion":true}]}}`, name, uid))
   836  		}
   837  		patch1 := addRefPatch(pod3.Name, pod3.UID)
   838  		pod1, err = podClient.Patch(ctx, pod1.Name, types.StrategicMergePatchType, patch1, metav1.PatchOptions{})
   839  		framework.ExpectNoError(err, "failed to apply to pod %s in namespace %s, a strategic merge patch: %s", pod1.Name, f.Namespace.Name, patch1)
   840  		framework.Logf("pod1.ObjectMeta.OwnerReferences=%#v", pod1.ObjectMeta.OwnerReferences)
   841  		patch2 := addRefPatch(pod1.Name, pod1.UID)
   842  		pod2, err = podClient.Patch(ctx, pod2.Name, types.StrategicMergePatchType, patch2, metav1.PatchOptions{})
   843  		framework.ExpectNoError(err, "failed to apply to pod %s in namespace %s, a strategic merge patch: %s", pod2.Name, f.Namespace.Name, patch2)
   844  		framework.Logf("pod2.ObjectMeta.OwnerReferences=%#v", pod2.ObjectMeta.OwnerReferences)
   845  		patch3 := addRefPatch(pod2.Name, pod2.UID)
   846  		pod3, err = podClient.Patch(ctx, pod3.Name, types.StrategicMergePatchType, patch3, metav1.PatchOptions{})
   847  		framework.ExpectNoError(err, "failed to apply to pod %s in namespace %s, a strategic merge patch: %s", pod3.Name, f.Namespace.Name, patch3)
   848  		framework.Logf("pod3.ObjectMeta.OwnerReferences=%#v", pod3.ObjectMeta.OwnerReferences)
   849  		// delete one pod, should result in the deletion of all pods
   850  		deleteOptions := getForegroundOptions()
   851  		deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(pod1.UID))
   852  		err = podClient.Delete(ctx, pod1.ObjectMeta.Name, deleteOptions)
   853  		framework.ExpectNoError(err, "failed to delete pod %s in namespace: %s", pod1.Name, f.Namespace.Name)
   854  		var pods *v1.PodList
   855  		var err2 error
   856  		// TODO: shorten the timeout when we make GC's periodic API rediscovery more efficient.
   857  		// Tracked at https://github.com/kubernetes/kubernetes/issues/50046.
   858  		if err := wait.PollWithContext(ctx, 5*time.Second, 90*time.Second+gcInformerResyncRetryTimeout, func(ctx context.Context) (bool, error) {
   859  			pods, err2 = podClient.List(ctx, metav1.ListOptions{})
   860  			if err2 != nil {
   861  				return false, fmt.Errorf("failed to list pods: %w", err)
   862  			}
   863  			if len(pods.Items) == 0 {
   864  				return true, nil
   865  			}
   866  			return false, nil
   867  		}); err != nil {
   868  			data, _ := json.Marshal(pods.Items)
   869  			framework.Logf("pods are %s", string(data))
   870  			framework.Failf("failed to wait for all pods to be deleted: %v", err)
   871  		}
   872  	})
   873  
   874  	ginkgo.It("should support cascading deletion of custom resources", func(ctx context.Context) {
   875  		config, err := framework.LoadConfig()
   876  		if err != nil {
   877  			framework.Failf("failed to load config: %v", err)
   878  		}
   879  
   880  		apiExtensionClient, err := apiextensionsclientset.NewForConfig(config)
   881  		if err != nil {
   882  			framework.Failf("failed to initialize apiExtensionClient: %v", err)
   883  		}
   884  
   885  		// Create a random custom resource definition and ensure it's available for
   886  		// use.
   887  		definition := apiextensionstestserver.NewRandomNameV1CustomResourceDefinition(apiextensionsv1.ClusterScoped)
   888  		defer func() {
   889  			err = apiextensionstestserver.DeleteV1CustomResourceDefinition(definition, apiExtensionClient)
   890  			if err != nil && !apierrors.IsNotFound(err) {
   891  				framework.Failf("failed to delete CustomResourceDefinition: %v", err)
   892  			}
   893  		}()
   894  		definition, err = apiextensionstestserver.CreateNewV1CustomResourceDefinition(definition, apiExtensionClient, f.DynamicClient)
   895  		if err != nil {
   896  			framework.Failf("failed to create CustomResourceDefinition: %v", err)
   897  		}
   898  		gomega.Expect(definition.Spec.Versions).To(gomega.HaveLen(1), "custom resource definition should have one version")
   899  		version := definition.Spec.Versions[0]
   900  
   901  		// Get a client for the custom resource.
   902  		gvr := schema.GroupVersionResource{Group: definition.Spec.Group, Version: version.Name, Resource: definition.Spec.Names.Plural}
   903  		resourceClient := f.DynamicClient.Resource(gvr)
   904  
   905  		apiVersion := definition.Spec.Group + "/" + version.Name
   906  
   907  		// Create a custom owner resource.
   908  		ownerName := names.SimpleNameGenerator.GenerateName("owner")
   909  		owner := &unstructured.Unstructured{
   910  			Object: map[string]interface{}{
   911  				"apiVersion": apiVersion,
   912  				"kind":       definition.Spec.Names.Kind,
   913  				"metadata": map[string]interface{}{
   914  					"name": ownerName,
   915  				},
   916  			},
   917  		}
   918  		persistedOwner, err := resourceClient.Create(ctx, owner, metav1.CreateOptions{})
   919  		if err != nil {
   920  			framework.Failf("failed to create owner resource %q: %v", ownerName, err)
   921  		}
   922  		framework.Logf("created owner resource %q", ownerName)
   923  
   924  		// Create a custom dependent resource.
   925  		dependentName := names.SimpleNameGenerator.GenerateName("dependent")
   926  		dependent := &unstructured.Unstructured{
   927  			Object: map[string]interface{}{
   928  				"apiVersion": apiVersion,
   929  				"kind":       definition.Spec.Names.Kind,
   930  				"metadata": map[string]interface{}{
   931  					"name": dependentName,
   932  					"ownerReferences": []interface{}{
   933  						map[string]interface{}{
   934  							"uid":        string(persistedOwner.GetUID()),
   935  							"apiVersion": apiVersion,
   936  							"kind":       definition.Spec.Names.Kind,
   937  							"name":       ownerName,
   938  						},
   939  					},
   940  				},
   941  			},
   942  		}
   943  		persistedDependent, err := resourceClient.Create(ctx, dependent, metav1.CreateOptions{})
   944  		if err != nil {
   945  			framework.Failf("failed to create dependent resource %q: %v", dependentName, err)
   946  		}
   947  		framework.Logf("created dependent resource %q", dependentName)
   948  
   949  		// Delete the owner.
   950  		background := metav1.DeletePropagationBackground
   951  		err = resourceClient.Delete(ctx, ownerName, metav1.DeleteOptions{PropagationPolicy: &background})
   952  		if err != nil {
   953  			framework.Failf("failed to delete owner resource %q: %v", ownerName, err)
   954  		}
   955  
   956  		// Create and delete an unrelated instance of the custom resource in foreground deletion mode,
   957  		// so we have a signal when GC is aware of this custom resource type
   958  		canaryName := names.SimpleNameGenerator.GenerateName("canary")
   959  		canary := &unstructured.Unstructured{
   960  			Object: map[string]interface{}{
   961  				"apiVersion": apiVersion,
   962  				"kind":       definition.Spec.Names.Kind,
   963  				"metadata":   map[string]interface{}{"name": canaryName}},
   964  		}
   965  		_, err = resourceClient.Create(ctx, canary, metav1.CreateOptions{})
   966  		if err != nil {
   967  			framework.Failf("failed to create canary resource %q: %v", canaryName, err)
   968  		}
   969  		framework.Logf("created canary resource %q", canaryName)
   970  		foreground := metav1.DeletePropagationForeground
   971  		err = resourceClient.Delete(ctx, canaryName, metav1.DeleteOptions{PropagationPolicy: &foreground})
   972  		if err != nil {
   973  			framework.Failf("failed to delete canary resource %q: %v", canaryName, err)
   974  		}
   975  		// Wait for the canary foreground finalization to complete, which means GC is aware of our new custom resource type
   976  		var lastCanary *unstructured.Unstructured
   977  		if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) {
   978  			lastCanary, err = resourceClient.Get(ctx, dependentName, metav1.GetOptions{})
   979  			return apierrors.IsNotFound(err), nil
   980  		}); err != nil {
   981  			framework.Logf("canary last state: %#v", lastCanary)
   982  			framework.Failf("failed waiting for canary resource %q to be deleted", canaryName)
   983  		}
   984  
   985  		// Ensure the dependent is deleted.
   986  		var lastDependent *unstructured.Unstructured
   987  		var err2 error
   988  		if err := wait.PollWithContext(ctx, 5*time.Second, 60*time.Second, func(ctx context.Context) (bool, error) {
   989  			lastDependent, err2 = resourceClient.Get(ctx, dependentName, metav1.GetOptions{})
   990  			return apierrors.IsNotFound(err2), nil
   991  		}); err != nil {
   992  			framework.Logf("owner: %#v", persistedOwner)
   993  			framework.Logf("dependent: %#v", persistedDependent)
   994  			framework.Logf("dependent last state: %#v", lastDependent)
   995  			framework.Failf("failed waiting for dependent resource %q to be deleted", dependentName)
   996  		}
   997  
   998  		// Ensure the owner is deleted.
   999  		_, err = resourceClient.Get(ctx, ownerName, metav1.GetOptions{})
  1000  		if err == nil {
  1001  			framework.Failf("expected owner resource %q to be deleted", ownerName)
  1002  		} else {
  1003  			if !apierrors.IsNotFound(err) {
  1004  				framework.Failf("unexpected error getting owner resource %q: %v", ownerName, err)
  1005  			}
  1006  		}
  1007  	})
  1008  
  1009  	ginkgo.It("should support orphan deletion of custom resources", func(ctx context.Context) {
  1010  		config, err := framework.LoadConfig()
  1011  		if err != nil {
  1012  			framework.Failf("failed to load config: %v", err)
  1013  		}
  1014  
  1015  		apiExtensionClient, err := apiextensionsclientset.NewForConfig(config)
  1016  		if err != nil {
  1017  			framework.Failf("failed to initialize apiExtensionClient: %v", err)
  1018  		}
  1019  
  1020  		// Create a random custom resource definition and ensure it's available for
  1021  		// use.
  1022  		definition := apiextensionstestserver.NewRandomNameV1CustomResourceDefinition(apiextensionsv1.ClusterScoped)
  1023  		defer func() {
  1024  			err = apiextensionstestserver.DeleteV1CustomResourceDefinition(definition, apiExtensionClient)
  1025  			if err != nil && !apierrors.IsNotFound(err) {
  1026  				framework.Failf("failed to delete CustomResourceDefinition: %v", err)
  1027  			}
  1028  		}()
  1029  		definition, err = apiextensionstestserver.CreateNewV1CustomResourceDefinition(definition, apiExtensionClient, f.DynamicClient)
  1030  		if err != nil {
  1031  			framework.Failf("failed to create CustomResourceDefinition: %v", err)
  1032  		}
  1033  		gomega.Expect(definition.Spec.Versions).To(gomega.HaveLen(1), "custom resource definition should have one version")
  1034  		version := definition.Spec.Versions[0]
  1035  
  1036  		// Get a client for the custom resource.
  1037  		gvr := schema.GroupVersionResource{Group: definition.Spec.Group, Version: version.Name, Resource: definition.Spec.Names.Plural}
  1038  		resourceClient := f.DynamicClient.Resource(gvr)
  1039  
  1040  		apiVersion := definition.Spec.Group + "/" + version.Name
  1041  
  1042  		// Create a custom owner resource.
  1043  		ownerName := names.SimpleNameGenerator.GenerateName("owner")
  1044  		owner := &unstructured.Unstructured{
  1045  			Object: map[string]interface{}{
  1046  				"apiVersion": apiVersion,
  1047  				"kind":       definition.Spec.Names.Kind,
  1048  				"metadata": map[string]interface{}{
  1049  					"name": ownerName,
  1050  				},
  1051  			},
  1052  		}
  1053  		persistedOwner, err := resourceClient.Create(ctx, owner, metav1.CreateOptions{})
  1054  		if err != nil {
  1055  			framework.Failf("failed to create owner resource %q: %v", ownerName, err)
  1056  		}
  1057  		framework.Logf("created owner resource %q", ownerName)
  1058  
  1059  		// Create a custom dependent resource.
  1060  		dependentName := names.SimpleNameGenerator.GenerateName("dependent")
  1061  		dependent := &unstructured.Unstructured{
  1062  			Object: map[string]interface{}{
  1063  				"apiVersion": apiVersion,
  1064  				"kind":       definition.Spec.Names.Kind,
  1065  				"metadata": map[string]interface{}{
  1066  					"name": dependentName,
  1067  					"ownerReferences": []map[string]string{
  1068  						{
  1069  							"uid":        string(persistedOwner.GetUID()),
  1070  							"apiVersion": apiVersion,
  1071  							"kind":       definition.Spec.Names.Kind,
  1072  							"name":       ownerName,
  1073  						},
  1074  					},
  1075  				},
  1076  			},
  1077  		}
  1078  		_, err = resourceClient.Create(ctx, dependent, metav1.CreateOptions{})
  1079  		if err != nil {
  1080  			framework.Failf("failed to create dependent resource %q: %v", dependentName, err)
  1081  		}
  1082  		framework.Logf("created dependent resource %q", dependentName)
  1083  
  1084  		// Delete the owner and orphan the dependent.
  1085  		err = resourceClient.Delete(ctx, ownerName, getOrphanOptions())
  1086  		if err != nil {
  1087  			framework.Failf("failed to delete owner resource %q: %v", ownerName, err)
  1088  		}
  1089  
  1090  		ginkgo.By("wait for the owner to be deleted")
  1091  		if err := wait.PollWithContext(ctx, 5*time.Second, 120*time.Second, func(ctx context.Context) (bool, error) {
  1092  			_, err = resourceClient.Get(ctx, ownerName, metav1.GetOptions{})
  1093  			if err == nil {
  1094  				return false, nil
  1095  			}
  1096  			if err != nil && !apierrors.IsNotFound(err) {
  1097  				return false, fmt.Errorf("failed to get owner: %w", err)
  1098  			}
  1099  			return true, nil
  1100  		}); err != nil {
  1101  			framework.Failf("timeout in waiting for the owner to be deleted: %v", err)
  1102  		}
  1103  
  1104  		timeout := 30*time.Second + gcInformerResyncRetryTimeout
  1105  		ginkgo.By(fmt.Sprintf("wait for %s to see if the garbage collector mistakenly deletes the dependent crd\n", timeout))
  1106  		gomega.Consistently(ctx, framework.HandleRetry(func(ctx context.Context) (*unstructured.Unstructured, error) {
  1107  			return resourceClient.Get(ctx, dependentName, metav1.GetOptions{})
  1108  		})).WithTimeout(timeout).WithPolling(5 * time.Second).ShouldNot(gomega.BeNil())
  1109  	})
  1110  
  1111  	ginkgo.It("should delete jobs and pods created by cronjob", func(ctx context.Context) {
  1112  
  1113  		ginkgo.By("Create the cronjob")
  1114  		cronJob := newCronJob("simple", "*/1 * * * ?")
  1115  		cronJob, err := f.ClientSet.BatchV1().CronJobs(f.Namespace.Name).Create(ctx, cronJob, metav1.CreateOptions{})
  1116  		framework.ExpectNoError(err, "failed to create cronjob: %+v, in namespace: %s", cronJob, f.Namespace.Name)
  1117  
  1118  		ginkgo.By("Wait for the CronJob to create new Job")
  1119  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
  1120  			jobs, err := f.ClientSet.BatchV1().Jobs(f.Namespace.Name).List(ctx, metav1.ListOptions{})
  1121  			if err != nil {
  1122  				return false, fmt.Errorf("failed to list jobs: %w", err)
  1123  			}
  1124  			return len(jobs.Items) > 0, nil
  1125  		})
  1126  		if err != nil {
  1127  			framework.Failf("Failed to wait for the CronJob to create some Jobs: %v", err)
  1128  		}
  1129  
  1130  		ginkgo.By("Delete the cronjob")
  1131  		if err := f.ClientSet.BatchV1().CronJobs(f.Namespace.Name).Delete(ctx, cronJob.Name, getBackgroundOptions()); err != nil {
  1132  			framework.Failf("Failed to delete the CronJob: %v", err)
  1133  		}
  1134  		ginkgo.By("Verify if cronjob does not leave jobs nor pods behind")
  1135  		err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, true, func(ctx context.Context) (bool, error) {
  1136  			objects := map[string]int{"CronJobs": 0, "Jobs": 0, "Pods": 0}
  1137  			return verifyRemainingObjects(ctx, f, objects)
  1138  		})
  1139  		if err != nil {
  1140  			framework.Failf("Failed to wait for all jobs and pods to be deleted: %v", err)
  1141  		}
  1142  
  1143  		gatherMetrics(ctx, f)
  1144  	})
  1145  })
  1146  
  1147  // TODO(106575): Migrate away from generic polling function.
  1148  func waitForReplicas(ctx context.Context, rc *v1.ReplicationController, rcClient clientv1.ReplicationControllerInterface) {
  1149  	var (
  1150  		lastObservedRC *v1.ReplicationController
  1151  		err            error
  1152  	)
  1153  	if err := wait.PollWithContext(ctx, framework.Poll, replicaSyncTimeout, func(ctx context.Context) (bool, error) {
  1154  		lastObservedRC, err = rcClient.Get(ctx, rc.Name, metav1.GetOptions{})
  1155  		if err != nil {
  1156  			return false, err
  1157  		}
  1158  		if lastObservedRC.Status.Replicas == *rc.Spec.Replicas {
  1159  			return true, nil
  1160  		}
  1161  		return false, nil
  1162  	}); err != nil {
  1163  		if lastObservedRC == nil {
  1164  			framework.Failf("Failed to get ReplicationController %q: %v", rc.Name, err)
  1165  		} else {
  1166  			framework.Failf("failed to wait for the rc.Status.Replicas (%d) to reach rc.Spec.Replicas (%d): %v",
  1167  				lastObservedRC.Status.Replicas, *lastObservedRC.Spec.Replicas, err)
  1168  		}
  1169  	}
  1170  }
  1171  

View as plain text