Source file src/k8s.io/kubernetes/test/integration/garbagecollector/garbage_collector_test.go

Documentation: k8s.io/kubernetes/test/integration/garbagecollector

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package garbagecollector
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strconv"
    23  	"strings"
    24  	"sync"
    25  	"testing"
    26  	"time"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    30  	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    31  	apiextensionstestserver "k8s.io/apiextensions-apiserver/test/integration/fixtures"
    32  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    33  	"k8s.io/apimachinery/pkg/api/meta"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    36  	"k8s.io/apimachinery/pkg/runtime/schema"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/apiserver/pkg/storage/names"
    40  	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
    41  	"k8s.io/client-go/dynamic"
    42  	"k8s.io/client-go/informers"
    43  	clientset "k8s.io/client-go/kubernetes"
    44  	"k8s.io/client-go/metadata"
    45  	"k8s.io/client-go/metadata/metadatainformer"
    46  	"k8s.io/client-go/restmapper"
    47  	"k8s.io/client-go/tools/cache"
    48  	"k8s.io/controller-manager/pkg/informerfactory"
    49  	"k8s.io/klog/v2"
    50  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    51  	"k8s.io/kubernetes/pkg/controller/garbagecollector"
    52  	"k8s.io/kubernetes/test/integration"
    53  	"k8s.io/kubernetes/test/integration/framework"
    54  	"k8s.io/kubernetes/test/utils/ktesting"
    55  	"k8s.io/utils/ptr"
    56  )
    57  
    58  func getForegroundOptions() metav1.DeleteOptions {
    59  	policy := metav1.DeletePropagationForeground
    60  	return metav1.DeleteOptions{PropagationPolicy: &policy}
    61  }
    62  
    63  func getOrphanOptions() metav1.DeleteOptions {
    64  	var trueVar = true
    65  	return metav1.DeleteOptions{OrphanDependents: &trueVar}
    66  }
    67  
    68  func getPropagateOrphanOptions() metav1.DeleteOptions {
    69  	policy := metav1.DeletePropagationOrphan
    70  	return metav1.DeleteOptions{PropagationPolicy: &policy}
    71  }
    72  
    73  func getNonOrphanOptions() metav1.DeleteOptions {
    74  	var falseVar = false
    75  	return metav1.DeleteOptions{OrphanDependents: &falseVar}
    76  }
    77  
    78  const garbageCollectedPodName = "test.pod.1"
    79  const independentPodName = "test.pod.2"
    80  const oneValidOwnerPodName = "test.pod.3"
    81  const toBeDeletedRCName = "test.rc.1"
    82  const remainingRCName = "test.rc.2"
    83  
    84  func newPod(podName, podNamespace string, ownerReferences []metav1.OwnerReference) *v1.Pod {
    85  	for i := 0; i < len(ownerReferences); i++ {
    86  		if len(ownerReferences[i].Kind) == 0 {
    87  			ownerReferences[i].Kind = "ReplicationController"
    88  		}
    89  		ownerReferences[i].APIVersion = "v1"
    90  	}
    91  	return &v1.Pod{
    92  		TypeMeta: metav1.TypeMeta{
    93  			Kind:       "Pod",
    94  			APIVersion: "v1",
    95  		},
    96  		ObjectMeta: metav1.ObjectMeta{
    97  			Name:            podName,
    98  			Namespace:       podNamespace,
    99  			OwnerReferences: ownerReferences,
   100  		},
   101  		Spec: v1.PodSpec{
   102  			Containers: []v1.Container{
   103  				{
   104  					Name:  "fake-name",
   105  					Image: "fakeimage",
   106  				},
   107  			},
   108  		},
   109  	}
   110  }
   111  
   112  func newOwnerRC(name, namespace string) *v1.ReplicationController {
   113  	return &v1.ReplicationController{
   114  		TypeMeta: metav1.TypeMeta{
   115  			Kind:       "ReplicationController",
   116  			APIVersion: "v1",
   117  		},
   118  		ObjectMeta: metav1.ObjectMeta{
   119  			Namespace: namespace,
   120  			Name:      name,
   121  		},
   122  		Spec: v1.ReplicationControllerSpec{
   123  			Selector: map[string]string{"name": "test"},
   124  			Template: &v1.PodTemplateSpec{
   125  				ObjectMeta: metav1.ObjectMeta{
   126  					Labels: map[string]string{"name": "test"},
   127  				},
   128  				Spec: v1.PodSpec{
   129  					Containers: []v1.Container{
   130  						{
   131  							Name:  "fake-name",
   132  							Image: "fakeimage",
   133  						},
   134  					},
   135  				},
   136  			},
   137  		},
   138  	}
   139  }
   140  
   141  func newCRDInstance(definition *apiextensionsv1.CustomResourceDefinition, namespace, name string) *unstructured.Unstructured {
   142  	return &unstructured.Unstructured{
   143  		Object: map[string]interface{}{
   144  			"kind":       definition.Spec.Names.Kind,
   145  			"apiVersion": definition.Spec.Group + "/" + definition.Spec.Versions[0].Name,
   146  			"metadata": map[string]interface{}{
   147  				"name":      name,
   148  				"namespace": namespace,
   149  			},
   150  		},
   151  	}
   152  }
   153  
   154  func newConfigMap(namespace, name string) *v1.ConfigMap {
   155  	return &v1.ConfigMap{
   156  		TypeMeta: metav1.TypeMeta{
   157  			Kind:       "ConfigMap",
   158  			APIVersion: "v1",
   159  		},
   160  		ObjectMeta: metav1.ObjectMeta{
   161  			Namespace: namespace,
   162  			Name:      name,
   163  		},
   164  	}
   165  }
   166  
   167  func link(t *testing.T, owner, dependent metav1.Object) {
   168  	ownerType, err := meta.TypeAccessor(owner)
   169  	if err != nil {
   170  		t.Fatalf("failed to get type info for %#v: %v", owner, err)
   171  	}
   172  	ref := metav1.OwnerReference{
   173  		Kind:       ownerType.GetKind(),
   174  		APIVersion: ownerType.GetAPIVersion(),
   175  		Name:       owner.GetName(),
   176  		UID:        owner.GetUID(),
   177  	}
   178  	dependent.SetOwnerReferences(append(dependent.GetOwnerReferences(), ref))
   179  }
   180  
   181  func createRandomCustomResourceDefinition(
   182  	t *testing.T, apiExtensionClient apiextensionsclientset.Interface,
   183  	dynamicClient dynamic.Interface,
   184  	namespace string,
   185  ) (*apiextensionsv1.CustomResourceDefinition, dynamic.ResourceInterface) {
   186  	// Create a random custom resource definition and ensure it's available for
   187  	// use.
   188  	definition := apiextensionstestserver.NewRandomNameV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
   189  
   190  	definition, err := apiextensionstestserver.CreateNewV1CustomResourceDefinition(definition, apiExtensionClient, dynamicClient)
   191  	if err != nil {
   192  		t.Fatalf("failed to create CustomResourceDefinition: %v", err)
   193  	}
   194  
   195  	// Get a client for the custom resource.
   196  	gvr := schema.GroupVersionResource{Group: definition.Spec.Group, Version: definition.Spec.Versions[0].Name, Resource: definition.Spec.Names.Plural}
   197  
   198  	resourceClient := dynamicClient.Resource(gvr).Namespace(namespace)
   199  
   200  	return definition, resourceClient
   201  }
   202  
   203  type testContext struct {
   204  	logger             klog.Logger
   205  	tearDown           func()
   206  	gc                 *garbagecollector.GarbageCollector
   207  	clientSet          clientset.Interface
   208  	apiExtensionClient apiextensionsclientset.Interface
   209  	dynamicClient      dynamic.Interface
   210  	metadataClient     metadata.Interface
   211  	startGC            func(workers int)
   212  	// syncPeriod is how often the GC started with startGC will be resynced.
   213  	syncPeriod time.Duration
   214  }
   215  
   216  // if workerCount > 0, will start the GC, otherwise it's up to the caller to Run() the GC.
   217  func setup(t *testing.T, workerCount int) *testContext {
   218  	return setupWithServer(t, kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()), workerCount)
   219  }
   220  
   221  func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, workerCount int) *testContext {
   222  	clientSet, err := clientset.NewForConfig(result.ClientConfig)
   223  	if err != nil {
   224  		t.Fatalf("error creating clientset: %v", err)
   225  	}
   226  
   227  	// Helpful stuff for testing CRD.
   228  	apiExtensionClient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
   229  	if err != nil {
   230  		t.Fatalf("error creating extension clientset: %v", err)
   231  	}
   232  	// CreateCRDUsingRemovedAPI wants to use this namespace for verifying
   233  	// namespace-scoped CRD creation.
   234  	createNamespaceOrDie("aval", clientSet, t)
   235  
   236  	discoveryClient := cacheddiscovery.NewMemCacheClient(clientSet.Discovery())
   237  	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
   238  	restMapper.Reset()
   239  	config := *result.ClientConfig
   240  	metadataClient, err := metadata.NewForConfig(&config)
   241  	if err != nil {
   242  		t.Fatalf("failed to create metadataClient: %v", err)
   243  	}
   244  	dynamicClient, err := dynamic.NewForConfig(&config)
   245  	if err != nil {
   246  		t.Fatalf("failed to create dynamicClient: %v", err)
   247  	}
   248  	sharedInformers := informers.NewSharedInformerFactory(clientSet, 0)
   249  	metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, 0)
   250  
   251  	tCtx := ktesting.Init(t)
   252  	logger := tCtx.Logger()
   253  	alwaysStarted := make(chan struct{})
   254  	close(alwaysStarted)
   255  	gc, err := garbagecollector.NewGarbageCollector(
   256  		tCtx,
   257  		clientSet,
   258  		metadataClient,
   259  		restMapper,
   260  		garbagecollector.DefaultIgnoredResources(),
   261  		informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
   262  		alwaysStarted,
   263  	)
   264  	if err != nil {
   265  		t.Fatalf("failed to create garbage collector: %v", err)
   266  	}
   267  
   268  	tearDown := func() {
   269  		tCtx.Cancel("tearing down")
   270  		result.TearDownFn()
   271  	}
   272  	syncPeriod := 5 * time.Second
   273  	startGC := func(workers int) {
   274  		go wait.Until(func() {
   275  			// Resetting the REST mapper will also invalidate the underlying discovery
   276  			// client. This is a leaky abstraction and assumes behavior about the REST
   277  			// mapper, but we'll deal with it for now.
   278  			restMapper.Reset()
   279  		}, syncPeriod, tCtx.Done())
   280  		go gc.Run(tCtx, workers)
   281  		go gc.Sync(tCtx, clientSet.Discovery(), syncPeriod)
   282  	}
   283  
   284  	if workerCount > 0 {
   285  		startGC(workerCount)
   286  	}
   287  
   288  	return &testContext{
   289  		logger:             logger,
   290  		tearDown:           tearDown,
   291  		gc:                 gc,
   292  		clientSet:          clientSet,
   293  		apiExtensionClient: apiExtensionClient,
   294  		dynamicClient:      dynamicClient,
   295  		metadataClient:     metadataClient,
   296  		startGC:            startGC,
   297  		syncPeriod:         syncPeriod,
   298  	}
   299  }
   300  
   301  func createNamespaceOrDie(name string, c clientset.Interface, t *testing.T) *v1.Namespace {
   302  	ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}
   303  	if _, err := c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil {
   304  		t.Fatalf("failed to create namespace: %v", err)
   305  	}
   306  	falseVar := false
   307  	_, err := c.CoreV1().ServiceAccounts(ns.Name).Create(context.TODO(), &v1.ServiceAccount{
   308  		ObjectMeta:                   metav1.ObjectMeta{Name: "default"},
   309  		AutomountServiceAccountToken: &falseVar,
   310  	}, metav1.CreateOptions{})
   311  	if err != nil {
   312  		t.Fatalf("failed to create service account: %v", err)
   313  	}
   314  	return ns
   315  }
   316  
   317  func deleteNamespaceOrDie(name string, c clientset.Interface, t *testing.T) {
   318  	zero := int64(0)
   319  	background := metav1.DeletePropagationBackground
   320  	err := c.CoreV1().Namespaces().Delete(context.TODO(), name, metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &background})
   321  	if err != nil {
   322  		t.Fatalf("failed to delete namespace %q: %v", name, err)
   323  	}
   324  }
   325  
   326  func TestCrossNamespaceReferencesWithWatchCache(t *testing.T) {
   327  	testCrossNamespaceReferences(t, true)
   328  }
   329  func TestCrossNamespaceReferencesWithoutWatchCache(t *testing.T) {
   330  	testCrossNamespaceReferences(t, false)
   331  }
   332  
   333  func testCrossNamespaceReferences(t *testing.T, watchCache bool) {
   334  	var (
   335  		workers            = 5
   336  		validChildrenCount = 10
   337  		namespaceB         = "b"
   338  		namespaceA         = "a"
   339  	)
   340  
   341  	// Start the server
   342  	testServer := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{fmt.Sprintf("--watch-cache=%v", watchCache)}, framework.SharedEtcd())
   343  	defer func() {
   344  		if testServer != nil {
   345  			testServer.TearDownFn()
   346  		}
   347  	}()
   348  	clientSet, err := clientset.NewForConfig(testServer.ClientConfig)
   349  	if err != nil {
   350  		t.Fatalf("error creating clientset: %v", err)
   351  	}
   352  
   353  	createNamespaceOrDie(namespaceB, clientSet, t)
   354  	parent, err := clientSet.CoreV1().ConfigMaps(namespaceB).Create(context.TODO(), &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "parent"}}, metav1.CreateOptions{})
   355  	if err != nil {
   356  		t.Fatal(err)
   357  	}
   358  	for i := 0; i < validChildrenCount; i++ {
   359  		_, err := clientSet.CoreV1().Secrets(namespaceB).Create(context.TODO(), &v1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "child-", OwnerReferences: []metav1.OwnerReference{
   360  			{Name: "parent", Kind: "ConfigMap", APIVersion: "v1", UID: parent.UID, Controller: ptr.To(false)},
   361  		}}}, metav1.CreateOptions{})
   362  		if err != nil {
   363  			t.Fatal(err)
   364  		}
   365  	}
   366  
   367  	createNamespaceOrDie(namespaceA, clientSet, t)
   368  
   369  	// Construct invalid owner references:
   370  	invalidOwnerReferences := []metav1.OwnerReference{}
   371  	for i := 0; i < 25; i++ {
   372  		invalidOwnerReferences = append(invalidOwnerReferences, metav1.OwnerReference{Name: "invalid", UID: types.UID(fmt.Sprintf("invalid-%d", i)), APIVersion: "test/v1", Kind: fmt.Sprintf("invalid%d", i)})
   373  	}
   374  	invalidOwnerReferences = append(invalidOwnerReferences, metav1.OwnerReference{Name: "invalid", UID: parent.UID, APIVersion: "v1", Kind: "Pod", Controller: ptr.To(false)})
   375  
   376  	for i := 0; i < workers; i++ {
   377  		_, err := clientSet.CoreV1().ConfigMaps(namespaceA).Create(context.TODO(), &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{GenerateName: "invalid-child-", OwnerReferences: invalidOwnerReferences}}, metav1.CreateOptions{})
   378  		if err != nil {
   379  			t.Fatal(err)
   380  		}
   381  		_, err = clientSet.CoreV1().Secrets(namespaceA).Create(context.TODO(), &v1.Secret{ObjectMeta: metav1.ObjectMeta{GenerateName: "invalid-child-a-", OwnerReferences: invalidOwnerReferences}}, metav1.CreateOptions{})
   382  		if err != nil {
   383  			t.Fatal(err)
   384  		}
   385  		_, err = clientSet.CoreV1().Secrets(namespaceA).Create(context.TODO(), &v1.Secret{
   386  			ObjectMeta: metav1.ObjectMeta{
   387  				Labels:          map[string]string{"single-bad-reference": "true"},
   388  				GenerateName:    "invalid-child-b-",
   389  				OwnerReferences: []metav1.OwnerReference{{Name: "invalid", UID: parent.UID, APIVersion: "v1", Kind: "Pod", Controller: ptr.To(false)}},
   390  			},
   391  		}, metav1.CreateOptions{})
   392  		if err != nil {
   393  			t.Fatal(err)
   394  		}
   395  	}
   396  
   397  	// start GC with existing objects in place to simulate controller-manager restart
   398  	ctx := setupWithServer(t, testServer, workers)
   399  	defer ctx.tearDown()
   400  	testServer = nil
   401  
   402  	// Wait for the invalid children to be garbage collected
   403  	if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
   404  		children, err := clientSet.CoreV1().Secrets(namespaceA).List(context.TODO(), metav1.ListOptions{LabelSelector: "single-bad-reference=true"})
   405  		if err != nil {
   406  			return false, err
   407  		}
   408  		if len(children.Items) > 0 {
   409  			t.Logf("expected 0 invalid children, got %d, will wait and relist", len(children.Items))
   410  			return false, nil
   411  		}
   412  		return true, nil
   413  	}); err != nil && err != wait.ErrWaitTimeout {
   414  		t.Error(err)
   415  	}
   416  
   417  	// Wait for a little while to make sure they didn't trigger deletion of the valid children
   418  	if err := wait.Poll(time.Second, 5*time.Second, func() (bool, error) {
   419  		children, err := clientSet.CoreV1().Secrets(namespaceB).List(context.TODO(), metav1.ListOptions{})
   420  		if err != nil {
   421  			return false, err
   422  		}
   423  		if len(children.Items) != validChildrenCount {
   424  			return false, fmt.Errorf("expected %d valid children, got %d", validChildrenCount, len(children.Items))
   425  		}
   426  		return false, nil
   427  	}); err != nil && err != wait.ErrWaitTimeout {
   428  		t.Error(err)
   429  	}
   430  
   431  	if !ctx.gc.GraphHasUID(parent.UID) {
   432  		t.Errorf("valid parent UID no longer exists in the graph")
   433  	}
   434  
   435  	// Now that our graph has correct data in it, add a new invalid child and see if it gets deleted
   436  	invalidChild, err := clientSet.CoreV1().Secrets(namespaceA).Create(context.TODO(), &v1.Secret{
   437  		ObjectMeta: metav1.ObjectMeta{
   438  			GenerateName:    "invalid-child-c-",
   439  			OwnerReferences: []metav1.OwnerReference{{Name: "invalid", UID: parent.UID, APIVersion: "v1", Kind: "Pod", Controller: ptr.To(false)}},
   440  		},
   441  	}, metav1.CreateOptions{})
   442  	if err != nil {
   443  		t.Fatal(err)
   444  	}
   445  	// Wait for the invalid child to be garbage collected
   446  	if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
   447  		_, err := clientSet.CoreV1().Secrets(namespaceA).Get(context.TODO(), invalidChild.Name, metav1.GetOptions{})
   448  		if apierrors.IsNotFound(err) {
   449  			return true, nil
   450  		}
   451  		if err != nil {
   452  			return false, err
   453  		}
   454  		t.Logf("%s remains, waiting for deletion", invalidChild.Name)
   455  		return false, nil
   456  	}); err != nil {
   457  		t.Fatal(err)
   458  	}
   459  }
   460  
   461  // This test simulates the cascading deletion.
   462  func TestCascadingDeletion(t *testing.T) {
   463  	ctx := setup(t, 5)
   464  	defer ctx.tearDown()
   465  
   466  	gc, clientSet := ctx.gc, ctx.clientSet
   467  
   468  	ns := createNamespaceOrDie("gc-cascading-deletion", clientSet, t)
   469  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   470  
   471  	rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name)
   472  	podClient := clientSet.CoreV1().Pods(ns.Name)
   473  
   474  	toBeDeletedRC, err := rcClient.Create(context.TODO(), newOwnerRC(toBeDeletedRCName, ns.Name), metav1.CreateOptions{})
   475  	if err != nil {
   476  		t.Fatalf("Failed to create replication controller: %v", err)
   477  	}
   478  	remainingRC, err := rcClient.Create(context.TODO(), newOwnerRC(remainingRCName, ns.Name), metav1.CreateOptions{})
   479  	if err != nil {
   480  		t.Fatalf("Failed to create replication controller: %v", err)
   481  	}
   482  
   483  	rcs, err := rcClient.List(context.TODO(), metav1.ListOptions{})
   484  	if err != nil {
   485  		t.Fatalf("Failed to list replication controllers: %v", err)
   486  	}
   487  	if len(rcs.Items) != 2 {
   488  		t.Fatalf("Expect only 2 replication controller")
   489  	}
   490  
   491  	// this pod should be cascadingly deleted.
   492  	pod := newPod(garbageCollectedPodName, ns.Name, []metav1.OwnerReference{{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRCName}})
   493  	_, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   494  	if err != nil {
   495  		t.Fatalf("Failed to create Pod: %v", err)
   496  	}
   497  
   498  	// this pod shouldn't be cascadingly deleted, because it has a valid reference.
   499  	pod = newPod(oneValidOwnerPodName, ns.Name, []metav1.OwnerReference{
   500  		{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRCName},
   501  		{UID: remainingRC.ObjectMeta.UID, Name: remainingRCName},
   502  	})
   503  	_, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   504  	if err != nil {
   505  		t.Fatalf("Failed to create Pod: %v", err)
   506  	}
   507  
   508  	// this pod shouldn't be cascadingly deleted, because it doesn't have an owner.
   509  	pod = newPod(independentPodName, ns.Name, []metav1.OwnerReference{})
   510  	_, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   511  	if err != nil {
   512  		t.Fatalf("Failed to create Pod: %v", err)
   513  	}
   514  
   515  	// set up watch
   516  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   517  	if err != nil {
   518  		t.Fatalf("Failed to list pods: %v", err)
   519  	}
   520  	if len(pods.Items) != 3 {
   521  		t.Fatalf("Expect only 3 pods")
   522  	}
   523  	// delete one of the replication controller
   524  	if err := rcClient.Delete(context.TODO(), toBeDeletedRCName, getNonOrphanOptions()); err != nil {
   525  		t.Fatalf("failed to delete replication controller: %v", err)
   526  	}
   527  	// sometimes the deletion of the RC takes long time to be observed by
   528  	// the gc, so wait for the garbage collector to observe the deletion of
   529  	// the toBeDeletedRC
   530  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
   531  		return !gc.GraphHasUID(toBeDeletedRC.ObjectMeta.UID), nil
   532  	}); err != nil {
   533  		t.Fatal(err)
   534  	}
   535  	if err := integration.WaitForPodToDisappear(podClient, garbageCollectedPodName, 1*time.Second, 30*time.Second); err != nil {
   536  		t.Fatalf("expect pod %s to be garbage collected, got err= %v", garbageCollectedPodName, err)
   537  	}
   538  	// checks the garbage collect doesn't delete pods it shouldn't delete.
   539  	if _, err := podClient.Get(context.TODO(), independentPodName, metav1.GetOptions{}); err != nil {
   540  		t.Fatal(err)
   541  	}
   542  	if _, err := podClient.Get(context.TODO(), oneValidOwnerPodName, metav1.GetOptions{}); err != nil {
   543  		t.Fatal(err)
   544  	}
   545  }
   546  
   547  // This test simulates the case where an object is created with an owner that
   548  // doesn't exist. It verifies the GC will delete such an object.
   549  func TestCreateWithNonExistentOwner(t *testing.T) {
   550  	ctx := setup(t, 5)
   551  	defer ctx.tearDown()
   552  
   553  	clientSet := ctx.clientSet
   554  
   555  	ns := createNamespaceOrDie("gc-non-existing-owner", clientSet, t)
   556  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   557  
   558  	podClient := clientSet.CoreV1().Pods(ns.Name)
   559  
   560  	pod := newPod(garbageCollectedPodName, ns.Name, []metav1.OwnerReference{{UID: "doesn't matter", Name: toBeDeletedRCName}})
   561  	_, err := podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   562  	if err != nil {
   563  		t.Fatalf("Failed to create Pod: %v", err)
   564  	}
   565  
   566  	// set up watch
   567  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   568  	if err != nil {
   569  		t.Fatalf("Failed to list pods: %v", err)
   570  	}
   571  	if len(pods.Items) > 1 {
   572  		t.Fatalf("Unexpected pod list: %v", pods.Items)
   573  	}
   574  	// wait for the garbage collector to delete the pod
   575  	if err := integration.WaitForPodToDisappear(podClient, garbageCollectedPodName, 1*time.Second, 30*time.Second); err != nil {
   576  		t.Fatalf("expect pod %s to be garbage collected, got err= %v", garbageCollectedPodName, err)
   577  	}
   578  }
   579  
   580  func setupRCsPods(t *testing.T, gc *garbagecollector.GarbageCollector, clientSet clientset.Interface, nameSuffix, namespace string, initialFinalizers []string, options metav1.DeleteOptions, wg *sync.WaitGroup, rcUIDs chan types.UID, errs chan string) {
   581  	defer wg.Done()
   582  	rcClient := clientSet.CoreV1().ReplicationControllers(namespace)
   583  	podClient := clientSet.CoreV1().Pods(namespace)
   584  	// create rc.
   585  	rcName := "test.rc." + nameSuffix
   586  	rc := newOwnerRC(rcName, namespace)
   587  	rc.ObjectMeta.Finalizers = initialFinalizers
   588  	rc, err := rcClient.Create(context.TODO(), rc, metav1.CreateOptions{})
   589  	if err != nil {
   590  		errs <- fmt.Sprintf("Failed to create replication controller: %v", err)
   591  		return
   592  	}
   593  	rcUIDs <- rc.ObjectMeta.UID
   594  	// create pods.
   595  	var podUIDs []types.UID
   596  	for j := 0; j < 3; j++ {
   597  		podName := "test.pod." + nameSuffix + "-" + strconv.Itoa(j)
   598  		pod := newPod(podName, namespace, []metav1.OwnerReference{{UID: rc.ObjectMeta.UID, Name: rc.ObjectMeta.Name}})
   599  		createdPod, err := podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   600  		if err != nil {
   601  			errs <- fmt.Sprintf("Failed to create Pod: %v", err)
   602  			return
   603  		}
   604  		podUIDs = append(podUIDs, createdPod.ObjectMeta.UID)
   605  	}
   606  	orphan := false
   607  	switch {
   608  	case options.OrphanDependents == nil && options.PropagationPolicy == nil && len(initialFinalizers) == 0: //nolint:staticcheck // SA1019 Keep testing deprecated OrphanDependents option until it's being removed
   609  		// if there are no deletion options, the default policy for replication controllers is orphan
   610  		orphan = true
   611  	case options.OrphanDependents != nil: //nolint:staticcheck // SA1019 Keep testing deprecated OrphanDependents option until it's being removed
   612  		// if the deletion options explicitly specify whether to orphan, that controls
   613  		orphan = *options.OrphanDependents //nolint:staticcheck // SA1019 Keep testing deprecated OrphanDependents option until it's being removed
   614  	case options.PropagationPolicy != nil:
   615  		// if the deletion options explicitly specify whether to orphan, that controls
   616  		orphan = *options.PropagationPolicy == metav1.DeletePropagationOrphan
   617  	case len(initialFinalizers) != 0 && initialFinalizers[0] == metav1.FinalizerOrphanDependents:
   618  		// if the orphan finalizer is explicitly added, we orphan
   619  		orphan = true
   620  	}
   621  	// if we intend to orphan the pods, we need wait for the gc to observe the
   622  	// creation of the pods, otherwise if the deletion of RC is observed before
   623  	// the creation of the pods, the pods will not be orphaned.
   624  	if orphan {
   625  		err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
   626  			for _, u := range podUIDs {
   627  				if !gc.GraphHasUID(u) {
   628  					return false, nil
   629  				}
   630  			}
   631  			return true, nil
   632  		})
   633  		if err != nil {
   634  			errs <- fmt.Sprintf("failed to observe the expected pods in the GC graph for rc %s", rcName)
   635  			return
   636  		}
   637  	}
   638  	// delete the rc
   639  	if err := rcClient.Delete(context.TODO(), rc.ObjectMeta.Name, options); err != nil {
   640  		errs <- fmt.Sprintf("failed to delete replication controller: %v", err)
   641  		return
   642  	}
   643  }
   644  
   645  func verifyRemainingObjects(t *testing.T, clientSet clientset.Interface, namespace string, rcNum, podNum int) (bool, error) {
   646  	rcClient := clientSet.CoreV1().ReplicationControllers(namespace)
   647  	podClient := clientSet.CoreV1().Pods(namespace)
   648  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   649  	if err != nil {
   650  		return false, fmt.Errorf("Failed to list pods: %v", err)
   651  	}
   652  	var ret = true
   653  	if len(pods.Items) != podNum {
   654  		ret = false
   655  		t.Logf("expect %d pods, got %d pods", podNum, len(pods.Items))
   656  	}
   657  	rcs, err := rcClient.List(context.TODO(), metav1.ListOptions{})
   658  	if err != nil {
   659  		return false, fmt.Errorf("Failed to list replication controllers: %v", err)
   660  	}
   661  	if len(rcs.Items) != rcNum {
   662  		ret = false
   663  		t.Logf("expect %d RCs, got %d RCs", rcNum, len(rcs.Items))
   664  	}
   665  	return ret, nil
   666  }
   667  
   668  // The stress test is not very stressful, because we need to control the running
   669  // time of our pre-submit tests to increase submit-queue throughput. We'll add
   670  // e2e tests that put more stress.
   671  func TestStressingCascadingDeletion(t *testing.T) {
   672  	ctx := setup(t, 5)
   673  	defer ctx.tearDown()
   674  
   675  	gc, clientSet := ctx.gc, ctx.clientSet
   676  
   677  	ns := createNamespaceOrDie("gc-stressing-cascading-deletion", clientSet, t)
   678  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   679  
   680  	const collections = 10
   681  	var wg sync.WaitGroup
   682  	wg.Add(collections * 5)
   683  	rcUIDs := make(chan types.UID, collections*5)
   684  	errs := make(chan string, 5)
   685  	for i := 0; i < collections; i++ {
   686  		// rc is created with empty finalizers, deleted with nil delete options, pods will remain.
   687  		go setupRCsPods(t, gc, clientSet, "collection1-"+strconv.Itoa(i), ns.Name, []string{}, metav1.DeleteOptions{}, &wg, rcUIDs, errs)
   688  		// rc is created with the orphan finalizer, deleted with nil options, pods will remain.
   689  		go setupRCsPods(t, gc, clientSet, "collection2-"+strconv.Itoa(i), ns.Name, []string{metav1.FinalizerOrphanDependents}, metav1.DeleteOptions{}, &wg, rcUIDs, errs)
   690  		// rc is created with the orphan finalizer, deleted with DeleteOptions.OrphanDependents=false, pods will be deleted.
   691  		go setupRCsPods(t, gc, clientSet, "collection3-"+strconv.Itoa(i), ns.Name, []string{metav1.FinalizerOrphanDependents}, getNonOrphanOptions(), &wg, rcUIDs, errs)
   692  		// rc is created with empty finalizers, deleted with DeleteOptions.OrphanDependents=true, pods will remain.
   693  		go setupRCsPods(t, gc, clientSet, "collection4-"+strconv.Itoa(i), ns.Name, []string{}, getOrphanOptions(), &wg, rcUIDs, errs)
   694  		// rc is created with empty finalizers, deleted with DeleteOptions.PropagationPolicy=Orphan, pods will remain.
   695  		go setupRCsPods(t, gc, clientSet, "collection5-"+strconv.Itoa(i), ns.Name, []string{}, getPropagateOrphanOptions(), &wg, rcUIDs, errs)
   696  	}
   697  	wg.Wait()
   698  	close(errs)
   699  	for errString := range errs {
   700  		t.Fatalf(errString)
   701  	}
   702  	t.Logf("all pods are created, all replications controllers are created then deleted")
   703  	// wait for the RCs and Pods to reach the expected numbers.
   704  	if err := wait.Poll(1*time.Second, 300*time.Second, func() (bool, error) {
   705  		podsInEachCollection := 3
   706  		// see the comments on the calls to setupRCsPods for details
   707  		remainingGroups := 4
   708  		return verifyRemainingObjects(t, clientSet, ns.Name, 0, collections*podsInEachCollection*remainingGroups)
   709  	}); err != nil {
   710  		t.Fatal(err)
   711  	}
   712  	t.Logf("number of remaining replication controllers and pods are as expected")
   713  
   714  	// verify the remaining pods all have "orphan" in their names.
   715  	podClient := clientSet.CoreV1().Pods(ns.Name)
   716  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   717  	if err != nil {
   718  		t.Fatal(err)
   719  	}
   720  	for _, pod := range pods.Items {
   721  		if !strings.Contains(pod.ObjectMeta.Name, "collection1-") && !strings.Contains(pod.ObjectMeta.Name, "collection2-") && !strings.Contains(pod.ObjectMeta.Name, "collection4-") && !strings.Contains(pod.ObjectMeta.Name, "collection5-") {
   722  			t.Errorf("got unexpected remaining pod: %#v", pod)
   723  		}
   724  	}
   725  
   726  	// verify there is no node representing replication controllers in the gc's graph
   727  	for i := 0; i < collections; i++ {
   728  		uid := <-rcUIDs
   729  		if gc.GraphHasUID(uid) {
   730  			t.Errorf("Expect all nodes representing replication controllers are removed from the Propagator's graph")
   731  		}
   732  	}
   733  }
   734  
   735  func TestOrphaning(t *testing.T) {
   736  	ctx := setup(t, 5)
   737  	defer ctx.tearDown()
   738  
   739  	gc, clientSet := ctx.gc, ctx.clientSet
   740  
   741  	ns := createNamespaceOrDie("gc-orphaning", clientSet, t)
   742  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   743  
   744  	podClient := clientSet.CoreV1().Pods(ns.Name)
   745  	rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name)
   746  	// create the RC with the orphan finalizer set
   747  	toBeDeletedRC := newOwnerRC(toBeDeletedRCName, ns.Name)
   748  	toBeDeletedRC, err := rcClient.Create(context.TODO(), toBeDeletedRC, metav1.CreateOptions{})
   749  	if err != nil {
   750  		t.Fatalf("Failed to create replication controller: %v", err)
   751  	}
   752  
   753  	// these pods should be orphaned.
   754  	var podUIDs []types.UID
   755  	podsNum := 3
   756  	for i := 0; i < podsNum; i++ {
   757  		podName := garbageCollectedPodName + strconv.Itoa(i)
   758  		pod := newPod(podName, ns.Name, []metav1.OwnerReference{{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRCName}})
   759  		createdPod, err := podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   760  		if err != nil {
   761  			t.Fatalf("Failed to create Pod: %v", err)
   762  		}
   763  		podUIDs = append(podUIDs, createdPod.ObjectMeta.UID)
   764  	}
   765  
   766  	// we need wait for the gc to observe the creation of the pods, otherwise if
   767  	// the deletion of RC is observed before the creation of the pods, the pods
   768  	// will not be orphaned.
   769  	err = wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
   770  		for _, u := range podUIDs {
   771  			if !gc.GraphHasUID(u) {
   772  				return false, nil
   773  			}
   774  		}
   775  		return true, nil
   776  	})
   777  	if err != nil {
   778  		t.Fatalf("Failed to observe pods in GC graph for %s: %v", toBeDeletedRC.Name, err)
   779  	}
   780  
   781  	err = rcClient.Delete(context.TODO(), toBeDeletedRCName, getOrphanOptions())
   782  	if err != nil {
   783  		t.Fatalf("Failed to gracefully delete the rc: %v", err)
   784  	}
   785  	// verify the toBeDeleteRC is deleted
   786  	if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
   787  		rcs, err := rcClient.List(context.TODO(), metav1.ListOptions{})
   788  		if err != nil {
   789  			return false, err
   790  		}
   791  		if len(rcs.Items) == 0 {
   792  			t.Logf("Still has %d RCs", len(rcs.Items))
   793  			return true, nil
   794  		}
   795  		return false, nil
   796  	}); err != nil {
   797  		t.Errorf("unexpected error: %v", err)
   798  	}
   799  
   800  	// verify pods don't have the ownerPod as an owner anymore
   801  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   802  	if err != nil {
   803  		t.Fatalf("Failed to list pods: %v", err)
   804  	}
   805  	if len(pods.Items) != podsNum {
   806  		t.Errorf("Expect %d pod(s), but got %#v", podsNum, pods)
   807  	}
   808  	for _, pod := range pods.Items {
   809  		if len(pod.ObjectMeta.OwnerReferences) != 0 {
   810  			t.Errorf("pod %s still has non-empty OwnerReferences: %v", pod.ObjectMeta.Name, pod.ObjectMeta.OwnerReferences)
   811  		}
   812  	}
   813  }
   814  
   815  func TestSolidOwnerDoesNotBlockWaitingOwner(t *testing.T) {
   816  	ctx := setup(t, 5)
   817  	defer ctx.tearDown()
   818  
   819  	clientSet := ctx.clientSet
   820  
   821  	ns := createNamespaceOrDie("gc-foreground1", clientSet, t)
   822  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   823  
   824  	podClient := clientSet.CoreV1().Pods(ns.Name)
   825  	rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name)
   826  	// create the RC with the orphan finalizer set
   827  	toBeDeletedRC, err := rcClient.Create(context.TODO(), newOwnerRC(toBeDeletedRCName, ns.Name), metav1.CreateOptions{})
   828  	if err != nil {
   829  		t.Fatalf("Failed to create replication controller: %v", err)
   830  	}
   831  	remainingRC, err := rcClient.Create(context.TODO(), newOwnerRC(remainingRCName, ns.Name), metav1.CreateOptions{})
   832  	if err != nil {
   833  		t.Fatalf("Failed to create replication controller: %v", err)
   834  	}
   835  	trueVar := true
   836  	pod := newPod("pod", ns.Name, []metav1.OwnerReference{
   837  		{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRC.Name, BlockOwnerDeletion: &trueVar},
   838  		{UID: remainingRC.ObjectMeta.UID, Name: remainingRC.Name},
   839  	})
   840  	_, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
   841  	if err != nil {
   842  		t.Fatalf("Failed to create Pod: %v", err)
   843  	}
   844  
   845  	err = rcClient.Delete(context.TODO(), toBeDeletedRCName, getForegroundOptions())
   846  	if err != nil {
   847  		t.Fatalf("Failed to delete the rc: %v", err)
   848  	}
   849  	// verify the toBeDeleteRC is deleted
   850  	if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
   851  		_, err := rcClient.Get(context.TODO(), toBeDeletedRC.Name, metav1.GetOptions{})
   852  		if err != nil {
   853  			if apierrors.IsNotFound(err) {
   854  				return true, nil
   855  			}
   856  			return false, err
   857  		}
   858  		return false, nil
   859  	}); err != nil {
   860  		t.Errorf("unexpected error: %v", err)
   861  	}
   862  
   863  	// verify pods don't have the toBeDeleteRC as an owner anymore
   864  	pod, err = podClient.Get(context.TODO(), "pod", metav1.GetOptions{})
   865  	if err != nil {
   866  		t.Fatalf("Failed to list pods: %v", err)
   867  	}
   868  	if len(pod.ObjectMeta.OwnerReferences) != 1 {
   869  		t.Errorf("expect pod to have only one ownerReference: got %#v", pod.ObjectMeta.OwnerReferences)
   870  	} else if pod.ObjectMeta.OwnerReferences[0].Name != remainingRC.Name {
   871  		t.Errorf("expect pod to have an ownerReference pointing to %s, got %#v", remainingRC.Name, pod.ObjectMeta.OwnerReferences)
   872  	}
   873  }
   874  
   875  func TestNonBlockingOwnerRefDoesNotBlock(t *testing.T) {
   876  	ctx := setup(t, 5)
   877  	defer ctx.tearDown()
   878  
   879  	clientSet := ctx.clientSet
   880  
   881  	ns := createNamespaceOrDie("gc-foreground2", clientSet, t)
   882  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   883  
   884  	podClient := clientSet.CoreV1().Pods(ns.Name)
   885  	rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name)
   886  	// create the RC with the orphan finalizer set
   887  	toBeDeletedRC, err := rcClient.Create(context.TODO(), newOwnerRC(toBeDeletedRCName, ns.Name), metav1.CreateOptions{})
   888  	if err != nil {
   889  		t.Fatalf("Failed to create replication controller: %v", err)
   890  	}
   891  	// BlockingOwnerDeletion is not set
   892  	pod1 := newPod("pod1", ns.Name, []metav1.OwnerReference{
   893  		{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRC.Name},
   894  	})
   895  	// adding finalizer that no controller handles, so that the pod won't be deleted
   896  	pod1.ObjectMeta.Finalizers = []string{"x/y"}
   897  	// BlockingOwnerDeletion is false
   898  	falseVar := false
   899  	pod2 := newPod("pod2", ns.Name, []metav1.OwnerReference{
   900  		{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRC.Name, BlockOwnerDeletion: &falseVar},
   901  	})
   902  	// adding finalizer that no controller handles, so that the pod won't be deleted
   903  	pod2.ObjectMeta.Finalizers = []string{"x/y"}
   904  	_, err = podClient.Create(context.TODO(), pod1, metav1.CreateOptions{})
   905  	if err != nil {
   906  		t.Fatalf("Failed to create Pod: %v", err)
   907  	}
   908  	_, err = podClient.Create(context.TODO(), pod2, metav1.CreateOptions{})
   909  	if err != nil {
   910  		t.Fatalf("Failed to create Pod: %v", err)
   911  	}
   912  
   913  	err = rcClient.Delete(context.TODO(), toBeDeletedRCName, getForegroundOptions())
   914  	if err != nil {
   915  		t.Fatalf("Failed to delete the rc: %v", err)
   916  	}
   917  	// verify the toBeDeleteRC is deleted
   918  	if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) {
   919  		_, err := rcClient.Get(context.TODO(), toBeDeletedRC.Name, metav1.GetOptions{})
   920  		if err != nil {
   921  			if apierrors.IsNotFound(err) {
   922  				return true, nil
   923  			}
   924  			return false, err
   925  		}
   926  		return false, nil
   927  	}); err != nil {
   928  		t.Errorf("unexpected error: %v", err)
   929  	}
   930  
   931  	// verify pods are still there
   932  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
   933  	if err != nil {
   934  		t.Fatalf("Failed to list pods: %v", err)
   935  	}
   936  	if len(pods.Items) != 2 {
   937  		t.Errorf("expect there to be 2 pods, got %#v", pods.Items)
   938  	}
   939  }
   940  
   941  func TestDoubleDeletionWithFinalizer(t *testing.T) {
   942  	// test setup
   943  	ctx := setup(t, 5)
   944  	defer ctx.tearDown()
   945  	clientSet := ctx.clientSet
   946  	ns := createNamespaceOrDie("gc-double-foreground", clientSet, t)
   947  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
   948  
   949  	// step 1: creates a pod with a custom finalizer and deletes it, then waits until gc removes its finalizer
   950  	podClient := clientSet.CoreV1().Pods(ns.Name)
   951  	pod := newPod("lucy", ns.Name, nil)
   952  	pod.ObjectMeta.Finalizers = []string{"x/y"}
   953  	if _, err := podClient.Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil {
   954  		t.Fatalf("Failed to create pod: %v", err)
   955  	}
   956  	if err := podClient.Delete(context.TODO(), pod.Name, getForegroundOptions()); err != nil {
   957  		t.Fatalf("Failed to delete pod: %v", err)
   958  	}
   959  	if err := wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
   960  		returnedPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{})
   961  		if err != nil {
   962  			return false, err
   963  		}
   964  		if len(returnedPod.Finalizers) != 1 || returnedPod.Finalizers[0] != "x/y" {
   965  			t.Logf("waiting for pod %q to have only one finalizer %q at step 1, got %v", returnedPod.Name, "x/y", returnedPod.Finalizers)
   966  			return false, nil
   967  		}
   968  		return true, nil
   969  	}); err != nil {
   970  		t.Fatalf("Failed waiting for pod to have only one filanizer at step 1, error: %v", err)
   971  	}
   972  
   973  	// step 2: deletes the pod one more time and checks if there's only the custom finalizer left
   974  	if err := podClient.Delete(context.TODO(), pod.Name, getForegroundOptions()); err != nil {
   975  		t.Fatalf("Failed to delete pod: %v", err)
   976  	}
   977  	if err := wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) {
   978  		returnedPod, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{})
   979  		if err != nil {
   980  			return false, err
   981  		}
   982  		if len(returnedPod.Finalizers) != 1 || returnedPod.Finalizers[0] != "x/y" {
   983  			t.Logf("waiting for pod %q to have only one finalizer %q at step 2, got %v", returnedPod.Name, "x/y", returnedPod.Finalizers)
   984  			return false, nil
   985  		}
   986  		return true, nil
   987  	}); err != nil {
   988  		t.Fatalf("Failed waiting for pod to have only one finalizer at step 2, gc hasn't removed its finalzier?, error: %v", err)
   989  	}
   990  
   991  	// step 3: removes the custom finalizer and checks if the pod was removed
   992  	patch := []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`)
   993  	if _, err := podClient.Patch(context.TODO(), pod.Name, types.JSONPatchType, patch, metav1.PatchOptions{}); err != nil {
   994  		t.Fatalf("Failed to update pod: %v", err)
   995  	}
   996  	if err := wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) {
   997  		_, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{})
   998  		return apierrors.IsNotFound(err), nil
   999  	}); err != nil {
  1000  		t.Fatalf("Failed waiting for pod %q to be deleted", pod.Name)
  1001  	}
  1002  }
  1003  
  1004  func TestBlockingOwnerRefDoesBlock(t *testing.T) {
  1005  	ctx := setup(t, 0)
  1006  	defer ctx.tearDown()
  1007  	gc, clientSet := ctx.gc, ctx.clientSet
  1008  
  1009  	ns := createNamespaceOrDie("foo", clientSet, t)
  1010  	defer deleteNamespaceOrDie(ns.Name, clientSet, t)
  1011  
  1012  	podClient := clientSet.CoreV1().Pods(ns.Name)
  1013  	rcClient := clientSet.CoreV1().ReplicationControllers(ns.Name)
  1014  	// create the RC with the orphan finalizer set
  1015  	toBeDeletedRC, err := rcClient.Create(context.TODO(), newOwnerRC(toBeDeletedRCName, ns.Name), metav1.CreateOptions{})
  1016  	if err != nil {
  1017  		t.Fatalf("Failed to create replication controller: %v", err)
  1018  	}
  1019  	trueVar := true
  1020  	pod := newPod("pod", ns.Name, []metav1.OwnerReference{
  1021  		{UID: toBeDeletedRC.ObjectMeta.UID, Name: toBeDeletedRC.Name, BlockOwnerDeletion: &trueVar},
  1022  	})
  1023  	// adding finalizer that no controller handles, so that the pod won't be deleted
  1024  	pod.ObjectMeta.Finalizers = []string{"x/y"}
  1025  	_, err = podClient.Create(context.TODO(), pod, metav1.CreateOptions{})
  1026  	if err != nil {
  1027  		t.Fatalf("Failed to create Pod: %v", err)
  1028  	}
  1029  
  1030  	// this makes sure the garbage collector will have added the pod to its
  1031  	// dependency graph before handling the foreground deletion of the rc.
  1032  	ctx.startGC(5)
  1033  	timeout := make(chan struct{})
  1034  	time.AfterFunc(5*time.Second, func() { close(timeout) })
  1035  	if !cache.WaitForCacheSync(timeout, func() bool {
  1036  		return gc.IsSynced(ctx.logger)
  1037  	}) {
  1038  		t.Fatalf("failed to wait for garbage collector to be synced")
  1039  	}
  1040  
  1041  	err = rcClient.Delete(context.TODO(), toBeDeletedRCName, getForegroundOptions())
  1042  	if err != nil {
  1043  		t.Fatalf("Failed to delete the rc: %v", err)
  1044  	}
  1045  	time.Sleep(15 * time.Second)
  1046  	// verify the toBeDeleteRC is NOT deleted
  1047  	_, err = rcClient.Get(context.TODO(), toBeDeletedRC.Name, metav1.GetOptions{})
  1048  	if err != nil {
  1049  		t.Errorf("unexpected error: %v", err)
  1050  	}
  1051  
  1052  	// verify pods are still there
  1053  	pods, err := podClient.List(context.TODO(), metav1.ListOptions{})
  1054  	if err != nil {
  1055  		t.Fatalf("Failed to list pods: %v", err)
  1056  	}
  1057  	if len(pods.Items) != 1 {
  1058  		t.Errorf("expect there to be 1 pods, got %#v", pods.Items)
  1059  	}
  1060  }
  1061  
  1062  // TestCustomResourceCascadingDeletion ensures the basic cascading delete
  1063  // behavior supports custom resources.
  1064  func TestCustomResourceCascadingDeletion(t *testing.T) {
  1065  	ctx := setup(t, 5)
  1066  	defer ctx.tearDown()
  1067  
  1068  	clientSet, apiExtensionClient, dynamicClient := ctx.clientSet, ctx.apiExtensionClient, ctx.dynamicClient
  1069  
  1070  	ns := createNamespaceOrDie("crd-cascading", clientSet, t)
  1071  
  1072  	definition, resourceClient := createRandomCustomResourceDefinition(t, apiExtensionClient, dynamicClient, ns.Name)
  1073  
  1074  	// Create a custom owner resource.
  1075  	owner := newCRDInstance(definition, ns.Name, names.SimpleNameGenerator.GenerateName("owner"))
  1076  	owner, err := resourceClient.Create(context.TODO(), owner, metav1.CreateOptions{})
  1077  	if err != nil {
  1078  		t.Fatalf("failed to create owner resource %q: %v", owner.GetName(), err)
  1079  	}
  1080  	t.Logf("created owner resource %q", owner.GetName())
  1081  
  1082  	// Create a custom dependent resource.
  1083  	dependent := newCRDInstance(definition, ns.Name, names.SimpleNameGenerator.GenerateName("dependent"))
  1084  	link(t, owner, dependent)
  1085  
  1086  	dependent, err = resourceClient.Create(context.TODO(), dependent, metav1.CreateOptions{})
  1087  	if err != nil {
  1088  		t.Fatalf("failed to create dependent resource %q: %v", dependent.GetName(), err)
  1089  	}
  1090  	t.Logf("created dependent resource %q", dependent.GetName())
  1091  
  1092  	// Delete the owner.
  1093  	foreground := metav1.DeletePropagationForeground
  1094  	err = resourceClient.Delete(context.TODO(), owner.GetName(), metav1.DeleteOptions{PropagationPolicy: &foreground})
  1095  	if err != nil {
  1096  		t.Fatalf("failed to delete owner resource %q: %v", owner.GetName(), err)
  1097  	}
  1098  
  1099  	// Ensure the owner is deleted.
  1100  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
  1101  		_, err := resourceClient.Get(context.TODO(), owner.GetName(), metav1.GetOptions{})
  1102  		return apierrors.IsNotFound(err), nil
  1103  	}); err != nil {
  1104  		t.Fatalf("failed waiting for owner resource %q to be deleted", owner.GetName())
  1105  	}
  1106  
  1107  	// Ensure the dependent is deleted.
  1108  	_, err = resourceClient.Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
  1109  	if err == nil {
  1110  		t.Fatalf("expected dependent %q to be deleted", dependent.GetName())
  1111  	} else {
  1112  		if !apierrors.IsNotFound(err) {
  1113  			t.Fatalf("unexpected error getting dependent %q: %v", dependent.GetName(), err)
  1114  		}
  1115  	}
  1116  }
  1117  
  1118  // TestMixedRelationships ensures that owner/dependent relationships work
  1119  // between core and custom resources.
  1120  //
  1121  // TODO: Consider how this could be represented with table-style tests (e.g. a
  1122  // before/after expected object graph given a delete operation targeting a
  1123  // specific node in the before graph with certain delete options).
  1124  func TestMixedRelationships(t *testing.T) {
  1125  	ctx := setup(t, 5)
  1126  	defer ctx.tearDown()
  1127  
  1128  	clientSet, apiExtensionClient, dynamicClient := ctx.clientSet, ctx.apiExtensionClient, ctx.dynamicClient
  1129  
  1130  	ns := createNamespaceOrDie("crd-mixed", clientSet, t)
  1131  
  1132  	configMapClient := clientSet.CoreV1().ConfigMaps(ns.Name)
  1133  
  1134  	definition, resourceClient := createRandomCustomResourceDefinition(t, apiExtensionClient, dynamicClient, ns.Name)
  1135  
  1136  	// Create a custom owner resource.
  1137  	customOwner, err := resourceClient.Create(context.TODO(), newCRDInstance(definition, ns.Name, names.SimpleNameGenerator.GenerateName("owner")), metav1.CreateOptions{})
  1138  	if err != nil {
  1139  		t.Fatalf("failed to create owner: %v", err)
  1140  	}
  1141  	t.Logf("created custom owner %q", customOwner.GetName())
  1142  
  1143  	// Create a core dependent resource.
  1144  	coreDependent := newConfigMap(ns.Name, names.SimpleNameGenerator.GenerateName("dependent"))
  1145  	link(t, customOwner, coreDependent)
  1146  	coreDependent, err = configMapClient.Create(context.TODO(), coreDependent, metav1.CreateOptions{})
  1147  	if err != nil {
  1148  		t.Fatalf("failed to create dependent: %v", err)
  1149  	}
  1150  	t.Logf("created core dependent %q", coreDependent.GetName())
  1151  
  1152  	// Create a core owner resource.
  1153  	coreOwner, err := configMapClient.Create(context.TODO(), newConfigMap(ns.Name, names.SimpleNameGenerator.GenerateName("owner")), metav1.CreateOptions{})
  1154  	if err != nil {
  1155  		t.Fatalf("failed to create owner: %v", err)
  1156  	}
  1157  	t.Logf("created core owner %q: %#v", coreOwner.GetName(), coreOwner)
  1158  
  1159  	// Create a custom dependent resource.
  1160  	customDependent := newCRDInstance(definition, ns.Name, names.SimpleNameGenerator.GenerateName("dependent"))
  1161  	coreOwner.TypeMeta.Kind = "ConfigMap"
  1162  	coreOwner.TypeMeta.APIVersion = "v1"
  1163  	link(t, coreOwner, customDependent)
  1164  	customDependent, err = resourceClient.Create(context.TODO(), customDependent, metav1.CreateOptions{})
  1165  	if err != nil {
  1166  		t.Fatalf("failed to create dependent: %v", err)
  1167  	}
  1168  	t.Logf("created custom dependent %q", customDependent.GetName())
  1169  
  1170  	// Delete the custom owner.
  1171  	foreground := metav1.DeletePropagationForeground
  1172  	err = resourceClient.Delete(context.TODO(), customOwner.GetName(), metav1.DeleteOptions{PropagationPolicy: &foreground})
  1173  	if err != nil {
  1174  		t.Fatalf("failed to delete owner resource %q: %v", customOwner.GetName(), err)
  1175  	}
  1176  
  1177  	// Ensure the owner is deleted.
  1178  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
  1179  		_, err := resourceClient.Get(context.TODO(), customOwner.GetName(), metav1.GetOptions{})
  1180  		return apierrors.IsNotFound(err), nil
  1181  	}); err != nil {
  1182  		t.Fatalf("failed waiting for owner resource %q to be deleted", customOwner.GetName())
  1183  	}
  1184  
  1185  	// Ensure the dependent is deleted.
  1186  	_, err = resourceClient.Get(context.TODO(), coreDependent.GetName(), metav1.GetOptions{})
  1187  	if err == nil {
  1188  		t.Fatalf("expected dependent %q to be deleted", coreDependent.GetName())
  1189  	} else {
  1190  		if !apierrors.IsNotFound(err) {
  1191  			t.Fatalf("unexpected error getting dependent %q: %v", coreDependent.GetName(), err)
  1192  		}
  1193  	}
  1194  
  1195  	// Delete the core owner.
  1196  	err = configMapClient.Delete(context.TODO(), coreOwner.GetName(), metav1.DeleteOptions{PropagationPolicy: &foreground})
  1197  	if err != nil {
  1198  		t.Fatalf("failed to delete owner resource %q: %v", coreOwner.GetName(), err)
  1199  	}
  1200  
  1201  	// Ensure the owner is deleted.
  1202  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
  1203  		_, err := configMapClient.Get(context.TODO(), coreOwner.GetName(), metav1.GetOptions{})
  1204  		return apierrors.IsNotFound(err), nil
  1205  	}); err != nil {
  1206  		t.Fatalf("failed waiting for owner resource %q to be deleted", coreOwner.GetName())
  1207  	}
  1208  
  1209  	// Ensure the dependent is deleted.
  1210  	_, err = resourceClient.Get(context.TODO(), customDependent.GetName(), metav1.GetOptions{})
  1211  	if err == nil {
  1212  		t.Fatalf("expected dependent %q to be deleted", customDependent.GetName())
  1213  	} else {
  1214  		if !apierrors.IsNotFound(err) {
  1215  			t.Fatalf("unexpected error getting dependent %q: %v", customDependent.GetName(), err)
  1216  		}
  1217  	}
  1218  }
  1219  
  1220  // TestCRDDeletionCascading ensures propagating deletion of a custom resource
  1221  // definition with an instance that owns a core resource.
  1222  func TestCRDDeletionCascading(t *testing.T) {
  1223  	ctx := setup(t, 5)
  1224  	defer ctx.tearDown()
  1225  
  1226  	clientSet, apiExtensionClient, dynamicClient := ctx.clientSet, ctx.apiExtensionClient, ctx.dynamicClient
  1227  
  1228  	ns := createNamespaceOrDie("crd-mixed", clientSet, t)
  1229  
  1230  	t.Logf("First pass CRD cascading deletion")
  1231  	definition, resourceClient := createRandomCustomResourceDefinition(t, apiExtensionClient, dynamicClient, ns.Name)
  1232  	testCRDDeletion(t, ctx, ns, definition, resourceClient)
  1233  
  1234  	t.Logf("Second pass CRD cascading deletion")
  1235  	accessor := meta.NewAccessor()
  1236  	accessor.SetResourceVersion(definition, "")
  1237  	_, err := apiextensionstestserver.CreateNewV1CustomResourceDefinition(definition, apiExtensionClient, dynamicClient)
  1238  	if err != nil {
  1239  		t.Fatalf("failed to create CustomResourceDefinition: %v", err)
  1240  	}
  1241  	testCRDDeletion(t, ctx, ns, definition, resourceClient)
  1242  }
  1243  
  1244  func testCRDDeletion(t *testing.T, ctx *testContext, ns *v1.Namespace, definition *apiextensionsv1.CustomResourceDefinition, resourceClient dynamic.ResourceInterface) {
  1245  	clientSet, apiExtensionClient := ctx.clientSet, ctx.apiExtensionClient
  1246  
  1247  	configMapClient := clientSet.CoreV1().ConfigMaps(ns.Name)
  1248  
  1249  	// Create a custom owner resource.
  1250  	owner, err := resourceClient.Create(context.TODO(), newCRDInstance(definition, ns.Name, names.SimpleNameGenerator.GenerateName("owner")), metav1.CreateOptions{})
  1251  	if err != nil {
  1252  		t.Fatalf("failed to create owner: %v", err)
  1253  	}
  1254  	t.Logf("created owner %q", owner.GetName())
  1255  
  1256  	// Create a core dependent resource.
  1257  	dependent := newConfigMap(ns.Name, names.SimpleNameGenerator.GenerateName("dependent"))
  1258  	link(t, owner, dependent)
  1259  	dependent, err = configMapClient.Create(context.TODO(), dependent, metav1.CreateOptions{})
  1260  	if err != nil {
  1261  		t.Fatalf("failed to create dependent: %v", err)
  1262  	}
  1263  	t.Logf("created dependent %q", dependent.GetName())
  1264  
  1265  	time.Sleep(ctx.syncPeriod + 5*time.Second)
  1266  
  1267  	// Delete the definition, which should cascade to the owner and ultimately its dependents.
  1268  	if err := apiextensionstestserver.DeleteV1CustomResourceDefinition(definition, apiExtensionClient); err != nil {
  1269  		t.Fatalf("failed to delete %q: %v", definition.Name, err)
  1270  	}
  1271  
  1272  	// Ensure the owner is deleted.
  1273  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
  1274  		_, err := resourceClient.Get(context.TODO(), owner.GetName(), metav1.GetOptions{})
  1275  		return apierrors.IsNotFound(err), nil
  1276  	}); err != nil {
  1277  		t.Fatalf("failed waiting for owner %q to be deleted", owner.GetName())
  1278  	}
  1279  
  1280  	// Ensure the dependent is deleted.
  1281  	if err := wait.Poll(1*time.Second, 60*time.Second, func() (bool, error) {
  1282  		_, err := configMapClient.Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
  1283  		return apierrors.IsNotFound(err), nil
  1284  	}); err != nil {
  1285  		t.Fatalf("failed waiting for dependent %q (owned by %q) to be deleted", dependent.GetName(), owner.GetName())
  1286  	}
  1287  }
  1288  

View as plain text