Source file src/k8s.io/kubernetes/pkg/controller/namespace/deletion/namespaced_resources_deleter.go

Documentation: k8s.io/kubernetes/pkg/controller/namespace/deletion

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package deletion
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"reflect"
    23  	"sync"
    24  	"time"
    25  
    26  	"k8s.io/klog/v2"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	"k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime/schema"
    32  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    33  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    34  	"k8s.io/apimachinery/pkg/util/sets"
    35  	"k8s.io/client-go/discovery"
    36  	v1clientset "k8s.io/client-go/kubernetes/typed/core/v1"
    37  	"k8s.io/client-go/metadata"
    38  )
    39  
    40  // NamespacedResourcesDeleterInterface is the interface to delete a namespace with all resources in it.
    41  type NamespacedResourcesDeleterInterface interface {
    42  	Delete(ctx context.Context, nsName string) error
    43  }
    44  
    45  // NewNamespacedResourcesDeleter returns a new NamespacedResourcesDeleter.
    46  func NewNamespacedResourcesDeleter(ctx context.Context, nsClient v1clientset.NamespaceInterface,
    47  	metadataClient metadata.Interface, podsGetter v1clientset.PodsGetter,
    48  	discoverResourcesFn func() ([]*metav1.APIResourceList, error),
    49  	finalizerToken v1.FinalizerName) NamespacedResourcesDeleterInterface {
    50  	d := &namespacedResourcesDeleter{
    51  		nsClient:       nsClient,
    52  		metadataClient: metadataClient,
    53  		podsGetter:     podsGetter,
    54  		opCache: &operationNotSupportedCache{
    55  			m: make(map[operationKey]bool),
    56  		},
    57  		discoverResourcesFn: discoverResourcesFn,
    58  		finalizerToken:      finalizerToken,
    59  	}
    60  	d.initOpCache(ctx)
    61  	return d
    62  }
    63  
    64  var _ NamespacedResourcesDeleterInterface = &namespacedResourcesDeleter{}
    65  
    66  // namespacedResourcesDeleter is used to delete all resources in a given namespace.
    67  type namespacedResourcesDeleter struct {
    68  	// Client to manipulate the namespace.
    69  	nsClient v1clientset.NamespaceInterface
    70  	// Dynamic client to list and delete all namespaced resources.
    71  	metadataClient metadata.Interface
    72  	// Interface to get PodInterface.
    73  	podsGetter v1clientset.PodsGetter
    74  	// Cache of what operations are not supported on each group version resource.
    75  	opCache             *operationNotSupportedCache
    76  	discoverResourcesFn func() ([]*metav1.APIResourceList, error)
    77  	// The finalizer token that should be removed from the namespace
    78  	// when all resources in that namespace have been deleted.
    79  	finalizerToken v1.FinalizerName
    80  }
    81  
    82  // Delete deletes all resources in the given namespace.
    83  // Before deleting resources:
    84  //   - It ensures that deletion timestamp is set on the
    85  //     namespace (does nothing if deletion timestamp is missing).
    86  //   - Verifies that the namespace is in the "terminating" phase
    87  //     (updates the namespace phase if it is not yet marked terminating)
    88  //
    89  // After deleting the resources:
    90  // * It removes finalizer token from the given namespace.
    91  //
    92  // Returns an error if any of those steps fail.
    93  // Returns ResourcesRemainingError if it deleted some resources but needs
    94  // to wait for them to go away.
    95  // Caller is expected to keep calling this until it succeeds.
    96  func (d *namespacedResourcesDeleter) Delete(ctx context.Context, nsName string) error {
    97  	// Multiple controllers may edit a namespace during termination
    98  	// first get the latest state of the namespace before proceeding
    99  	// if the namespace was deleted already, don't do anything
   100  	namespace, err := d.nsClient.Get(ctx, nsName, metav1.GetOptions{})
   101  	if err != nil {
   102  		if errors.IsNotFound(err) {
   103  			return nil
   104  		}
   105  		return err
   106  	}
   107  	if namespace.DeletionTimestamp == nil {
   108  		return nil
   109  	}
   110  
   111  	klog.FromContext(ctx).V(5).Info("Namespace controller - syncNamespace", "namespace", namespace.Name, "finalizerToken", d.finalizerToken)
   112  
   113  	// ensure that the status is up to date on the namespace
   114  	// if we get a not found error, we assume the namespace is truly gone
   115  	namespace, err = d.retryOnConflictError(ctx, namespace, d.updateNamespaceStatusFunc)
   116  	if err != nil {
   117  		if errors.IsNotFound(err) {
   118  			return nil
   119  		}
   120  		return err
   121  	}
   122  
   123  	// the latest view of the namespace asserts that namespace is no longer deleting..
   124  	if namespace.DeletionTimestamp.IsZero() {
   125  		return nil
   126  	}
   127  
   128  	// return if it is already finalized.
   129  	if finalized(namespace) {
   130  		return nil
   131  	}
   132  
   133  	// there may still be content for us to remove
   134  	estimate, err := d.deleteAllContent(ctx, namespace)
   135  	if err != nil {
   136  		return err
   137  	}
   138  	if estimate > 0 {
   139  		return &ResourcesRemainingError{estimate}
   140  	}
   141  
   142  	// we have removed content, so mark it finalized by us
   143  	_, err = d.retryOnConflictError(ctx, namespace, d.finalizeNamespace)
   144  	if err != nil {
   145  		// in normal practice, this should not be possible, but if a deployment is running
   146  		// two controllers to do namespace deletion that share a common finalizer token it's
   147  		// possible that a not found could occur since the other controller would have finished the delete.
   148  		if errors.IsNotFound(err) {
   149  			return nil
   150  		}
   151  		return err
   152  	}
   153  	return nil
   154  }
   155  
   156  func (d *namespacedResourcesDeleter) initOpCache(ctx context.Context) {
   157  	// pre-fill opCache with the discovery info
   158  	//
   159  	// TODO(sttts): get rid of opCache and http 405 logic around it and trust discovery info
   160  	resources, err := d.discoverResourcesFn()
   161  	if err != nil {
   162  		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
   163  	}
   164  	logger := klog.FromContext(ctx)
   165  	if len(resources) == 0 {
   166  		logger.Error(err, "Unable to get any supported resources from server")
   167  		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
   168  	}
   169  
   170  	for _, rl := range resources {
   171  		gv, err := schema.ParseGroupVersion(rl.GroupVersion)
   172  		if err != nil {
   173  			logger.Error(err, "Failed to parse GroupVersion, skipping", "groupVersion", rl.GroupVersion)
   174  			continue
   175  		}
   176  
   177  		for _, r := range rl.APIResources {
   178  			gvr := schema.GroupVersionResource{Group: gv.Group, Version: gv.Version, Resource: r.Name}
   179  			verbs := sets.NewString([]string(r.Verbs)...)
   180  
   181  			if !verbs.Has("delete") {
   182  				logger.V(6).Info("Skipping resource because it cannot be deleted", "resource", gvr)
   183  			}
   184  
   185  			for _, op := range []operation{operationList, operationDeleteCollection} {
   186  				if !verbs.Has(string(op)) {
   187  					d.opCache.setNotSupported(operationKey{operation: op, gvr: gvr})
   188  				}
   189  			}
   190  		}
   191  	}
   192  }
   193  
   194  // ResourcesRemainingError is used to inform the caller that all resources are not yet fully removed from the namespace.
   195  type ResourcesRemainingError struct {
   196  	Estimate int64
   197  }
   198  
   199  func (e *ResourcesRemainingError) Error() string {
   200  	return fmt.Sprintf("some content remains in the namespace, estimate %d seconds before it is removed", e.Estimate)
   201  }
   202  
   203  // operation is used for caching if an operation is supported on a dynamic client.
   204  type operation string
   205  
   206  const (
   207  	operationDeleteCollection operation = "deletecollection"
   208  	operationList             operation = "list"
   209  	// assume a default estimate for finalizers to complete when found on items pending deletion.
   210  	finalizerEstimateSeconds = int64(15)
   211  )
   212  
   213  // operationKey is an entry in a cache.
   214  type operationKey struct {
   215  	operation operation
   216  	gvr       schema.GroupVersionResource
   217  }
   218  
   219  // operationNotSupportedCache is a simple cache to remember if an operation is not supported for a resource.
   220  // if the operationKey maps to true, it means the operation is not supported.
   221  type operationNotSupportedCache struct {
   222  	lock sync.RWMutex
   223  	m    map[operationKey]bool
   224  }
   225  
   226  // isSupported returns true if the operation is supported
   227  func (o *operationNotSupportedCache) isSupported(key operationKey) bool {
   228  	o.lock.RLock()
   229  	defer o.lock.RUnlock()
   230  	return !o.m[key]
   231  }
   232  
   233  func (o *operationNotSupportedCache) setNotSupported(key operationKey) {
   234  	o.lock.Lock()
   235  	defer o.lock.Unlock()
   236  	o.m[key] = true
   237  }
   238  
   239  // updateNamespaceFunc is a function that makes an update to a namespace
   240  type updateNamespaceFunc func(ctx context.Context, namespace *v1.Namespace) (*v1.Namespace, error)
   241  
   242  // retryOnConflictError retries the specified fn if there was a conflict error
   243  // it will return an error if the UID for an object changes across retry operations.
   244  // TODO RetryOnConflict should be a generic concept in client code
   245  func (d *namespacedResourcesDeleter) retryOnConflictError(ctx context.Context, namespace *v1.Namespace, fn updateNamespaceFunc) (result *v1.Namespace, err error) {
   246  	latestNamespace := namespace
   247  	for {
   248  		result, err = fn(ctx, latestNamespace)
   249  		if err == nil {
   250  			return result, nil
   251  		}
   252  		if !errors.IsConflict(err) {
   253  			return nil, err
   254  		}
   255  		prevNamespace := latestNamespace
   256  		latestNamespace, err = d.nsClient.Get(ctx, latestNamespace.Name, metav1.GetOptions{})
   257  		if err != nil {
   258  			return nil, err
   259  		}
   260  		if prevNamespace.UID != latestNamespace.UID {
   261  			return nil, fmt.Errorf("namespace uid has changed across retries")
   262  		}
   263  	}
   264  }
   265  
   266  // updateNamespaceStatusFunc will verify that the status of the namespace is correct
   267  func (d *namespacedResourcesDeleter) updateNamespaceStatusFunc(ctx context.Context, namespace *v1.Namespace) (*v1.Namespace, error) {
   268  	if namespace.DeletionTimestamp.IsZero() || namespace.Status.Phase == v1.NamespaceTerminating {
   269  		return namespace, nil
   270  	}
   271  	newNamespace := namespace.DeepCopy()
   272  	newNamespace.Status.Phase = v1.NamespaceTerminating
   273  	return d.nsClient.UpdateStatus(ctx, newNamespace, metav1.UpdateOptions{})
   274  }
   275  
   276  // finalized returns true if the namespace.Spec.Finalizers is an empty list
   277  func finalized(namespace *v1.Namespace) bool {
   278  	return len(namespace.Spec.Finalizers) == 0
   279  }
   280  
   281  // finalizeNamespace removes the specified finalizerToken and finalizes the namespace
   282  func (d *namespacedResourcesDeleter) finalizeNamespace(ctx context.Context, namespace *v1.Namespace) (*v1.Namespace, error) {
   283  	namespaceFinalize := v1.Namespace{}
   284  	namespaceFinalize.ObjectMeta = namespace.ObjectMeta
   285  	namespaceFinalize.Spec = namespace.Spec
   286  	finalizerSet := sets.NewString()
   287  	for i := range namespace.Spec.Finalizers {
   288  		if namespace.Spec.Finalizers[i] != d.finalizerToken {
   289  			finalizerSet.Insert(string(namespace.Spec.Finalizers[i]))
   290  		}
   291  	}
   292  	namespaceFinalize.Spec.Finalizers = make([]v1.FinalizerName, 0, len(finalizerSet))
   293  	for _, value := range finalizerSet.List() {
   294  		namespaceFinalize.Spec.Finalizers = append(namespaceFinalize.Spec.Finalizers, v1.FinalizerName(value))
   295  	}
   296  	namespace, err := d.nsClient.Finalize(ctx, &namespaceFinalize, metav1.UpdateOptions{})
   297  	if err != nil {
   298  		// it was removed already, so life is good
   299  		if errors.IsNotFound(err) {
   300  			return namespace, nil
   301  		}
   302  	}
   303  	return namespace, err
   304  }
   305  
   306  // deleteCollection is a helper function that will delete the collection of resources
   307  // it returns true if the operation was supported on the server.
   308  // it returns an error if the operation was supported on the server but was unable to complete.
   309  func (d *namespacedResourcesDeleter) deleteCollection(ctx context.Context, gvr schema.GroupVersionResource, namespace string) (bool, error) {
   310  	logger := klog.FromContext(ctx)
   311  	logger.V(5).Info("Namespace controller - deleteCollection", "namespace", namespace, "resource", gvr)
   312  
   313  	key := operationKey{operation: operationDeleteCollection, gvr: gvr}
   314  	if !d.opCache.isSupported(key) {
   315  		logger.V(5).Info("Namespace controller - deleteCollection ignored since not supported", "namespace", namespace, "resource", gvr)
   316  		return false, nil
   317  	}
   318  
   319  	// namespace controller does not want the garbage collector to insert the orphan finalizer since it calls
   320  	// resource deletions generically.  it will ensure all resources in the namespace are purged prior to releasing
   321  	// namespace itself.
   322  	background := metav1.DeletePropagationBackground
   323  	opts := metav1.DeleteOptions{PropagationPolicy: &background}
   324  	err := d.metadataClient.Resource(gvr).Namespace(namespace).DeleteCollection(ctx, opts, metav1.ListOptions{})
   325  	if err == nil {
   326  		return true, nil
   327  	}
   328  
   329  	// this is strange, but we need to special case for both MethodNotSupported and NotFound errors
   330  	// TODO: https://github.com/kubernetes/kubernetes/issues/22413
   331  	// we have a resource returned in the discovery API that supports no top-level verbs:
   332  	//  /apis/extensions/v1beta1/namespaces/default/replicationcontrollers
   333  	// when working with this resource type, we will get a literal not found error rather than expected method not supported
   334  	if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) {
   335  		logger.V(5).Info("Namespace controller - deleteCollection not supported", "namespace", namespace, "resource", gvr)
   336  		return false, nil
   337  	}
   338  
   339  	logger.V(5).Info("Namespace controller - deleteCollection unexpected error", "namespace", namespace, "resource", gvr, "err", err)
   340  	return true, err
   341  }
   342  
   343  // listCollection will list the items in the specified namespace
   344  // it returns the following:
   345  //
   346  //	the list of items in the collection (if found)
   347  //	a boolean if the operation is supported
   348  //	an error if the operation is supported but could not be completed.
   349  func (d *namespacedResourcesDeleter) listCollection(ctx context.Context, gvr schema.GroupVersionResource, namespace string) (*metav1.PartialObjectMetadataList, bool, error) {
   350  	logger := klog.FromContext(ctx)
   351  	logger.V(5).Info("Namespace controller - listCollection", "namespace", namespace, "resource", gvr)
   352  
   353  	key := operationKey{operation: operationList, gvr: gvr}
   354  	if !d.opCache.isSupported(key) {
   355  		logger.V(5).Info("Namespace controller - listCollection ignored since not supported", "namespace", namespace, "resource", gvr)
   356  		return nil, false, nil
   357  	}
   358  
   359  	partialList, err := d.metadataClient.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{})
   360  	if err == nil {
   361  		return partialList, true, nil
   362  	}
   363  
   364  	// this is strange, but we need to special case for both MethodNotSupported and NotFound errors
   365  	// TODO: https://github.com/kubernetes/kubernetes/issues/22413
   366  	// we have a resource returned in the discovery API that supports no top-level verbs:
   367  	//  /apis/extensions/v1beta1/namespaces/default/replicationcontrollers
   368  	// when working with this resource type, we will get a literal not found error rather than expected method not supported
   369  	if errors.IsMethodNotSupported(err) || errors.IsNotFound(err) {
   370  		logger.V(5).Info("Namespace controller - listCollection not supported", "namespace", namespace, "resource", gvr)
   371  		return nil, false, nil
   372  	}
   373  
   374  	return nil, true, err
   375  }
   376  
   377  // deleteEachItem is a helper function that will list the collection of resources and delete each item 1 by 1.
   378  func (d *namespacedResourcesDeleter) deleteEachItem(ctx context.Context, gvr schema.GroupVersionResource, namespace string) error {
   379  	klog.FromContext(ctx).V(5).Info("Namespace controller - deleteEachItem", "namespace", namespace, "resource", gvr)
   380  
   381  	partialList, listSupported, err := d.listCollection(ctx, gvr, namespace)
   382  	if err != nil {
   383  		return err
   384  	}
   385  	if !listSupported {
   386  		return nil
   387  	}
   388  	for _, item := range partialList.Items {
   389  		background := metav1.DeletePropagationBackground
   390  		opts := metav1.DeleteOptions{PropagationPolicy: &background}
   391  		if err = d.metadataClient.Resource(gvr).Namespace(namespace).Delete(ctx, item.GetName(), opts); err != nil && !errors.IsNotFound(err) && !errors.IsMethodNotSupported(err) {
   392  			return err
   393  		}
   394  	}
   395  	return nil
   396  }
   397  
   398  type gvrDeletionMetadata struct {
   399  	// finalizerEstimateSeconds is an estimate of how much longer to wait.  zero means that no estimate has made and does not
   400  	// mean that all content has been removed.
   401  	finalizerEstimateSeconds int64
   402  	// numRemaining is how many instances of the gvr remain
   403  	numRemaining int
   404  	// finalizersToNumRemaining maps finalizers to how many resources are stuck on them
   405  	finalizersToNumRemaining map[string]int
   406  }
   407  
   408  // deleteAllContentForGroupVersionResource will use the dynamic client to delete each resource identified in gvr.
   409  // It returns an estimate of the time remaining before the remaining resources are deleted.
   410  // If estimate > 0, not all resources are guaranteed to be gone.
   411  func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource(
   412  	ctx context.Context,
   413  	gvr schema.GroupVersionResource, namespace string,
   414  	namespaceDeletedAt metav1.Time) (gvrDeletionMetadata, error) {
   415  	logger := klog.FromContext(ctx)
   416  	logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource", "namespace", namespace, "resource", gvr)
   417  
   418  	// estimate how long it will take for the resource to be deleted (needed for objects that support graceful delete)
   419  	estimate, err := d.estimateGracefulTermination(ctx, gvr, namespace, namespaceDeletedAt)
   420  	if err != nil {
   421  		logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - unable to estimate", "namespace", namespace, "resource", gvr, "err", err)
   422  		return gvrDeletionMetadata{}, err
   423  	}
   424  	logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - estimate", "namespace", namespace, "resource", gvr, "estimate", estimate)
   425  
   426  	// first try to delete the entire collection
   427  	deleteCollectionSupported, err := d.deleteCollection(ctx, gvr, namespace)
   428  	if err != nil {
   429  		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
   430  	}
   431  
   432  	// delete collection was not supported, so we list and delete each item...
   433  	if !deleteCollectionSupported {
   434  		err = d.deleteEachItem(ctx, gvr, namespace)
   435  		if err != nil {
   436  			return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
   437  		}
   438  	}
   439  
   440  	// verify there are no more remaining items
   441  	// it is not an error condition for there to be remaining items if local estimate is non-zero
   442  	logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - checking for no more items in namespace", "namespace", namespace, "resource", gvr)
   443  	unstructuredList, listSupported, err := d.listCollection(ctx, gvr, namespace)
   444  	if err != nil {
   445  		logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - error verifying no items in namespace", "namespace", namespace, "resource", gvr, "err", err)
   446  		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
   447  	}
   448  	if !listSupported {
   449  		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, nil
   450  	}
   451  	logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - items remaining", "namespace", namespace, "resource", gvr, "items", len(unstructuredList.Items))
   452  	if len(unstructuredList.Items) == 0 {
   453  		// we're done
   454  		return gvrDeletionMetadata{finalizerEstimateSeconds: 0, numRemaining: 0}, nil
   455  	}
   456  
   457  	// use the list to find the finalizers
   458  	finalizersToNumRemaining := map[string]int{}
   459  	for _, item := range unstructuredList.Items {
   460  		for _, finalizer := range item.GetFinalizers() {
   461  			finalizersToNumRemaining[finalizer] = finalizersToNumRemaining[finalizer] + 1
   462  		}
   463  	}
   464  
   465  	if estimate != int64(0) {
   466  		logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - estimate is present", "namespace", namespace, "resource", gvr, "finalizers", finalizersToNumRemaining)
   467  		return gvrDeletionMetadata{
   468  			finalizerEstimateSeconds: estimate,
   469  			numRemaining:             len(unstructuredList.Items),
   470  			finalizersToNumRemaining: finalizersToNumRemaining,
   471  		}, nil
   472  	}
   473  
   474  	// if any item has a finalizer, we treat that as a normal condition, and use a default estimation to allow for GC to complete.
   475  	if len(finalizersToNumRemaining) > 0 {
   476  		logger.V(5).Info("Namespace controller - deleteAllContentForGroupVersionResource - items remaining with finalizers", "namespace", namespace, "resource", gvr, "finalizers", finalizersToNumRemaining)
   477  		return gvrDeletionMetadata{
   478  			finalizerEstimateSeconds: finalizerEstimateSeconds,
   479  			numRemaining:             len(unstructuredList.Items),
   480  			finalizersToNumRemaining: finalizersToNumRemaining,
   481  		}, nil
   482  	}
   483  
   484  	// nothing reported a finalizer, so something was unexpected as it should have been deleted.
   485  	return gvrDeletionMetadata{
   486  		finalizerEstimateSeconds: estimate,
   487  		numRemaining:             len(unstructuredList.Items),
   488  	}, fmt.Errorf("unexpected items still remain in namespace: %s for gvr: %v", namespace, gvr)
   489  }
   490  
   491  type allGVRDeletionMetadata struct {
   492  	// gvrToNumRemaining is how many instances of the gvr remain
   493  	gvrToNumRemaining map[schema.GroupVersionResource]int
   494  	// finalizersToNumRemaining maps finalizers to how many resources are stuck on them
   495  	finalizersToNumRemaining map[string]int
   496  }
   497  
   498  // deleteAllContent will use the dynamic client to delete each resource identified in groupVersionResources.
   499  // It returns an estimate of the time remaining before the remaining resources are deleted.
   500  // If estimate > 0, not all resources are guaranteed to be gone.
   501  func (d *namespacedResourcesDeleter) deleteAllContent(ctx context.Context, ns *v1.Namespace) (int64, error) {
   502  	namespace := ns.Name
   503  	namespaceDeletedAt := *ns.DeletionTimestamp
   504  	var errs []error
   505  	conditionUpdater := namespaceConditionUpdater{}
   506  	estimate := int64(0)
   507  	logger := klog.FromContext(ctx)
   508  	logger.V(4).Info("namespace controller - deleteAllContent", "namespace", namespace)
   509  
   510  	resources, err := d.discoverResourcesFn()
   511  	if err != nil {
   512  		// discovery errors are not fatal.  We often have some set of resources we can operate against even if we don't have a complete list
   513  		errs = append(errs, err)
   514  		conditionUpdater.ProcessDiscoverResourcesErr(err)
   515  	}
   516  	// TODO(sttts): get rid of opCache and pass the verbs (especially "deletecollection") down into the deleter
   517  	deletableResources := discovery.FilteredBy(discovery.SupportsAllVerbs{Verbs: []string{"delete"}}, resources)
   518  	groupVersionResources, err := discovery.GroupVersionResources(deletableResources)
   519  	if err != nil {
   520  		// discovery errors are not fatal.  We often have some set of resources we can operate against even if we don't have a complete list
   521  		errs = append(errs, err)
   522  		conditionUpdater.ProcessGroupVersionErr(err)
   523  	}
   524  
   525  	numRemainingTotals := allGVRDeletionMetadata{
   526  		gvrToNumRemaining:        map[schema.GroupVersionResource]int{},
   527  		finalizersToNumRemaining: map[string]int{},
   528  	}
   529  	for gvr := range groupVersionResources {
   530  		gvrDeletionMetadata, err := d.deleteAllContentForGroupVersionResource(ctx, gvr, namespace, namespaceDeletedAt)
   531  		if err != nil {
   532  			// If there is an error, hold on to it but proceed with all the remaining
   533  			// groupVersionResources.
   534  			errs = append(errs, err)
   535  			conditionUpdater.ProcessDeleteContentErr(err)
   536  		}
   537  		if gvrDeletionMetadata.finalizerEstimateSeconds > estimate {
   538  			estimate = gvrDeletionMetadata.finalizerEstimateSeconds
   539  		}
   540  		if gvrDeletionMetadata.numRemaining > 0 {
   541  			numRemainingTotals.gvrToNumRemaining[gvr] = gvrDeletionMetadata.numRemaining
   542  			for finalizer, numRemaining := range gvrDeletionMetadata.finalizersToNumRemaining {
   543  				if numRemaining == 0 {
   544  					continue
   545  				}
   546  				numRemainingTotals.finalizersToNumRemaining[finalizer] = numRemainingTotals.finalizersToNumRemaining[finalizer] + numRemaining
   547  			}
   548  		}
   549  	}
   550  	conditionUpdater.ProcessContentTotals(numRemainingTotals)
   551  
   552  	// we always want to update the conditions because if we have set a condition to "it worked" after it was previously, "it didn't work",
   553  	// we need to reflect that information.  Recall that additional finalizers can be set on namespaces, so this finalizer may clear itself and
   554  	// NOT remove the resource instance.
   555  	if hasChanged := conditionUpdater.Update(ns); hasChanged {
   556  		if _, err = d.nsClient.UpdateStatus(ctx, ns, metav1.UpdateOptions{}); err != nil {
   557  			utilruntime.HandleError(fmt.Errorf("couldn't update status condition for namespace %q: %v", namespace, err))
   558  		}
   559  	}
   560  
   561  	// if len(errs)==0, NewAggregate returns nil.
   562  	err = utilerrors.NewAggregate(errs)
   563  	logger.V(4).Info("namespace controller - deleteAllContent", "namespace", namespace, "estimate", estimate, "err", err)
   564  	return estimate, err
   565  }
   566  
   567  // estimateGracefulTermination will estimate the graceful termination required for the specific entity in the namespace
   568  func (d *namespacedResourcesDeleter) estimateGracefulTermination(ctx context.Context, gvr schema.GroupVersionResource, ns string, namespaceDeletedAt metav1.Time) (int64, error) {
   569  	groupResource := gvr.GroupResource()
   570  	klog.FromContext(ctx).V(5).Info("Namespace controller - estimateGracefulTermination", "group", groupResource.Group, "resource", groupResource.Resource)
   571  	estimate := int64(0)
   572  	var err error
   573  	switch groupResource {
   574  	case schema.GroupResource{Group: "", Resource: "pods"}:
   575  		estimate, err = d.estimateGracefulTerminationForPods(ctx, ns)
   576  	}
   577  	if err != nil {
   578  		return 0, err
   579  	}
   580  	// determine if the estimate is greater than the deletion timestamp
   581  	duration := time.Since(namespaceDeletedAt.Time)
   582  	allowedEstimate := time.Duration(estimate) * time.Second
   583  	if duration >= allowedEstimate {
   584  		estimate = int64(0)
   585  	}
   586  	return estimate, nil
   587  }
   588  
   589  // estimateGracefulTerminationForPods determines the graceful termination period for pods in the namespace
   590  func (d *namespacedResourcesDeleter) estimateGracefulTerminationForPods(ctx context.Context, ns string) (int64, error) {
   591  	klog.FromContext(ctx).V(5).Info("Namespace controller - estimateGracefulTerminationForPods", "namespace", ns)
   592  	estimate := int64(0)
   593  	podsGetter := d.podsGetter
   594  	if podsGetter == nil || reflect.ValueOf(podsGetter).IsNil() {
   595  		return 0, fmt.Errorf("unexpected: podsGetter is nil. Cannot estimate grace period seconds for pods")
   596  	}
   597  	items, err := podsGetter.Pods(ns).List(ctx, metav1.ListOptions{})
   598  	if err != nil {
   599  		return 0, err
   600  	}
   601  	for i := range items.Items {
   602  		pod := items.Items[i]
   603  		// filter out terminal pods
   604  		phase := pod.Status.Phase
   605  		if v1.PodSucceeded == phase || v1.PodFailed == phase {
   606  			continue
   607  		}
   608  		if pod.Spec.TerminationGracePeriodSeconds != nil {
   609  			grace := *pod.Spec.TerminationGracePeriodSeconds
   610  			if grace > estimate {
   611  				estimate = grace
   612  			}
   613  		}
   614  	}
   615  	return estimate, nil
   616  }
   617  

View as plain text