admission.go

Documentation: k8s.io/kubernetes/plugin/pkg/admission/limitranger

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package limitranger
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"io"
    23  	"sort"
    24  	"strings"
    25  	"time"
    26  
    27  	"golang.org/x/sync/singleflight"
    28  	corev1 "k8s.io/api/core/v1"
    29  	"k8s.io/apimachinery/pkg/api/meta"
    30  	"k8s.io/apimachinery/pkg/api/resource"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/labels"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    35  	"k8s.io/apiserver/pkg/admission"
    36  	genericadmissioninitailizer "k8s.io/apiserver/pkg/admission/initializer"
    37  	"k8s.io/apiserver/pkg/util/feature"
    38  	"k8s.io/client-go/informers"
    39  	"k8s.io/client-go/kubernetes"
    40  	corev1listers "k8s.io/client-go/listers/core/v1"
    41  	"k8s.io/utils/lru"
    42  
    43  	api "k8s.io/kubernetes/pkg/apis/core"
    44  	"k8s.io/kubernetes/pkg/features"
    45  )
    46  
    47  const (
    48  	limitRangerAnnotation = "kubernetes.io/limit-ranger"
    49  	// PluginName indicates name of admission plugin.
    50  	PluginName = "LimitRanger"
    51  )
    52  
    53  // Register registers a plugin
    54  func Register(plugins *admission.Plugins) {
    55  	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
    56  		return NewLimitRanger(&DefaultLimitRangerActions{})
    57  	})
    58  }
    59  
    60  // LimitRanger enforces usage limits on a per resource basis in the namespace
    61  type LimitRanger struct {
    62  	*admission.Handler
    63  	client  kubernetes.Interface
    64  	actions LimitRangerActions
    65  	lister  corev1listers.LimitRangeLister
    66  
    67  	// liveLookups holds the last few live lookups we've done to help ammortize cost on repeated lookup failures.
    68  	// This let's us handle the case of latent caches, by looking up actual results for a namespace on cache miss/no results.
    69  	// We track the lookup result here so that for repeated requests, we don't look it up very often.
    70  	liveLookupCache *lru.Cache
    71  	group           singleflight.Group
    72  	liveTTL         time.Duration
    73  }
    74  
    75  var _ admission.MutationInterface = &LimitRanger{}
    76  var _ admission.ValidationInterface = &LimitRanger{}
    77  
    78  var _ genericadmissioninitailizer.WantsExternalKubeInformerFactory = &LimitRanger{}
    79  var _ genericadmissioninitailizer.WantsExternalKubeClientSet = &LimitRanger{}
    80  
    81  type liveLookupEntry struct {
    82  	expiry time.Time
    83  	items  []*corev1.LimitRange
    84  }
    85  
    86  // SetExternalKubeInformerFactory registers an informer factory into the LimitRanger
    87  func (l *LimitRanger) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
    88  	limitRangeInformer := f.Core().V1().LimitRanges()
    89  	l.SetReadyFunc(limitRangeInformer.Informer().HasSynced)
    90  	l.lister = limitRangeInformer.Lister()
    91  }
    92  
    93  // SetExternalKubeClientSet registers the client into LimitRanger
    94  func (l *LimitRanger) SetExternalKubeClientSet(client kubernetes.Interface) {
    95  	l.client = client
    96  }
    97  
    98  // ValidateInitialization verifies the LimitRanger object has been properly initialized
    99  func (l *LimitRanger) ValidateInitialization() error {
   100  	if l.lister == nil {
   101  		return fmt.Errorf("missing limitRange lister")
   102  	}
   103  	if l.client == nil {
   104  		return fmt.Errorf("missing client")
   105  	}
   106  	return nil
   107  }
   108  
   109  // Admit admits resources into cluster that do not violate any defined LimitRange in the namespace
   110  func (l *LimitRanger) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
   111  	return l.runLimitFunc(a, l.actions.MutateLimit)
   112  }
   113  
   114  // Validate admits resources into cluster that do not violate any defined LimitRange in the namespace
   115  func (l *LimitRanger) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) {
   116  	return l.runLimitFunc(a, l.actions.ValidateLimit)
   117  }
   118  
   119  func (l *LimitRanger) runLimitFunc(a admission.Attributes, limitFn func(limitRange *corev1.LimitRange, kind string, obj runtime.Object) error) (err error) {
   120  	if !l.actions.SupportsAttributes(a) {
   121  		return nil
   122  	}
   123  
   124  	// ignore all objects marked for deletion
   125  	oldObj := a.GetOldObject()
   126  	if oldObj != nil {
   127  		oldAccessor, err := meta.Accessor(oldObj)
   128  		if err != nil {
   129  			return admission.NewForbidden(a, err)
   130  		}
   131  		if oldAccessor.GetDeletionTimestamp() != nil {
   132  			return nil
   133  		}
   134  	}
   135  
   136  	items, err := l.GetLimitRanges(a)
   137  	if err != nil {
   138  		return err
   139  	}
   140  
   141  	// ensure it meets each prescribed min/max
   142  	for i := range items {
   143  		limitRange := items[i]
   144  
   145  		if !l.actions.SupportsLimit(limitRange) {
   146  			continue
   147  		}
   148  
   149  		err = limitFn(limitRange, a.GetResource().Resource, a.GetObject())
   150  		if err != nil {
   151  			return admission.NewForbidden(a, err)
   152  		}
   153  	}
   154  	return nil
   155  }
   156  
   157  // GetLimitRanges returns a LimitRange object with the items held in
   158  // the indexer if available, or do alive lookup of the value.
   159  func (l *LimitRanger) GetLimitRanges(a admission.Attributes) ([]*corev1.LimitRange, error) {
   160  	items, err := l.lister.LimitRanges(a.GetNamespace()).List(labels.Everything())
   161  	if err != nil {
   162  		return nil, admission.NewForbidden(a, fmt.Errorf("unable to %s %v at this time because there was an error enforcing limit ranges", a.GetOperation(), a.GetResource()))
   163  	}
   164  
   165  	// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
   166  	if len(items) == 0 {
   167  		lruItemObj, ok := l.liveLookupCache.Get(a.GetNamespace())
   168  		if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
   169  			// Fixed: #22422
   170  			// use singleflight to alleviate simultaneous calls to
   171  			lruItemObj, err, _ = l.group.Do(a.GetNamespace(), func() (interface{}, error) {
   172  				liveList, err := l.client.CoreV1().LimitRanges(a.GetNamespace()).List(context.TODO(), metav1.ListOptions{})
   173  				if err != nil {
   174  					return nil, admission.NewForbidden(a, err)
   175  				}
   176  				newEntry := liveLookupEntry{expiry: time.Now().Add(l.liveTTL)}
   177  				for i := range liveList.Items {
   178  					newEntry.items = append(newEntry.items, &liveList.Items[i])
   179  				}
   180  				l.liveLookupCache.Add(a.GetNamespace(), newEntry)
   181  				return newEntry, nil
   182  			})
   183  			if err != nil {
   184  				return nil, err
   185  			}
   186  		}
   187  		lruEntry := lruItemObj.(liveLookupEntry)
   188  
   189  		items = append(items, lruEntry.items...)
   190  
   191  	}
   192  
   193  	return items, nil
   194  }
   195  
   196  // NewLimitRanger returns an object that enforces limits based on the supplied limit function
   197  func NewLimitRanger(actions LimitRangerActions) (*LimitRanger, error) {
   198  	liveLookupCache := lru.New(10000)
   199  
   200  	if actions == nil {
   201  		actions = &DefaultLimitRangerActions{}
   202  	}
   203  
   204  	return &LimitRanger{
   205  		Handler:         admission.NewHandler(admission.Create, admission.Update),
   206  		actions:         actions,
   207  		liveLookupCache: liveLookupCache,
   208  		liveTTL:         time.Duration(30 * time.Second),
   209  	}, nil
   210  }
   211  
   212  // defaultContainerResourceRequirements returns the default requirements for a container
   213  // the requirement.Limits are taken from the LimitRange defaults (if specified)
   214  // the requirement.Requests are taken from the LimitRange default request (if specified)
   215  func defaultContainerResourceRequirements(limitRange *corev1.LimitRange) api.ResourceRequirements {
   216  	requirements := api.ResourceRequirements{}
   217  	requirements.Requests = api.ResourceList{}
   218  	requirements.Limits = api.ResourceList{}
   219  
   220  	for i := range limitRange.Spec.Limits {
   221  		limit := limitRange.Spec.Limits[i]
   222  		if limit.Type == corev1.LimitTypeContainer {
   223  			for k, v := range limit.DefaultRequest {
   224  				requirements.Requests[api.ResourceName(k)] = v.DeepCopy()
   225  			}
   226  			for k, v := range limit.Default {
   227  				requirements.Limits[api.ResourceName(k)] = v.DeepCopy()
   228  			}
   229  		}
   230  	}
   231  	return requirements
   232  }
   233  
   234  // mergeContainerResources handles defaulting all of the resources on a container.
   235  func mergeContainerResources(container *api.Container, defaultRequirements *api.ResourceRequirements, annotationPrefix string, annotations []string) []string {
   236  	setRequests := []string{}
   237  	setLimits := []string{}
   238  	if container.Resources.Limits == nil {
   239  		container.Resources.Limits = api.ResourceList{}
   240  	}
   241  	if container.Resources.Requests == nil {
   242  		container.Resources.Requests = api.ResourceList{}
   243  	}
   244  	for k, v := range defaultRequirements.Limits {
   245  		_, found := container.Resources.Limits[k]
   246  		if !found {
   247  			container.Resources.Limits[k] = v.DeepCopy()
   248  			setLimits = append(setLimits, string(k))
   249  		}
   250  	}
   251  	for k, v := range defaultRequirements.Requests {
   252  		_, found := container.Resources.Requests[k]
   253  		if !found {
   254  			container.Resources.Requests[k] = v.DeepCopy()
   255  			setRequests = append(setRequests, string(k))
   256  		}
   257  	}
   258  	if len(setRequests) > 0 {
   259  		sort.Strings(setRequests)
   260  		a := strings.Join(setRequests, ", ") + fmt.Sprintf(" request for %s %s", annotationPrefix, container.Name)
   261  		annotations = append(annotations, a)
   262  	}
   263  	if len(setLimits) > 0 {
   264  		sort.Strings(setLimits)
   265  		a := strings.Join(setLimits, ", ") + fmt.Sprintf(" limit for %s %s", annotationPrefix, container.Name)
   266  		annotations = append(annotations, a)
   267  	}
   268  	return annotations
   269  }
   270  
   271  // mergePodResourceRequirements merges enumerated requirements with default requirements
   272  // it annotates the pod with information about what requirements were modified
   273  func mergePodResourceRequirements(pod *api.Pod, defaultRequirements *api.ResourceRequirements) {
   274  	annotations := []string{}
   275  
   276  	for i := range pod.Spec.Containers {
   277  		annotations = mergeContainerResources(&pod.Spec.Containers[i], defaultRequirements, "container", annotations)
   278  	}
   279  
   280  	for i := range pod.Spec.InitContainers {
   281  		annotations = mergeContainerResources(&pod.Spec.InitContainers[i], defaultRequirements, "init container", annotations)
   282  	}
   283  
   284  	if len(annotations) > 0 {
   285  		if pod.ObjectMeta.Annotations == nil {
   286  			pod.ObjectMeta.Annotations = make(map[string]string)
   287  		}
   288  		val := "LimitRanger plugin set: " + strings.Join(annotations, "; ")
   289  		pod.ObjectMeta.Annotations[limitRangerAnnotation] = val
   290  	}
   291  }
   292  
   293  // requestLimitEnforcedValues returns the specified values at a common precision to support comparability
   294  func requestLimitEnforcedValues(requestQuantity, limitQuantity, enforcedQuantity resource.Quantity) (request, limit, enforced int64) {
   295  	request = requestQuantity.Value()
   296  	limit = limitQuantity.Value()
   297  	enforced = enforcedQuantity.Value()
   298  	// do a more precise comparison if possible (if the value won't overflow)
   299  	if request <= resource.MaxMilliValue && limit <= resource.MaxMilliValue && enforced <= resource.MaxMilliValue {
   300  		request = requestQuantity.MilliValue()
   301  		limit = limitQuantity.MilliValue()
   302  		enforced = enforcedQuantity.MilliValue()
   303  	}
   304  	return
   305  }
   306  
   307  // minConstraint enforces the min constraint over the specified resource
   308  func minConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
   309  	req, reqExists := request[api.ResourceName(resourceName)]
   310  	lim, limExists := limit[api.ResourceName(resourceName)]
   311  	observedReqValue, observedLimValue, enforcedValue := requestLimitEnforcedValues(req, lim, enforced)
   312  
   313  	if !reqExists {
   314  		return fmt.Errorf("minimum %s usage per %s is %s.  No request is specified", resourceName, limitType, enforced.String())
   315  	}
   316  	if observedReqValue < enforcedValue {
   317  		return fmt.Errorf("minimum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
   318  	}
   319  	if limExists && (observedLimValue < enforcedValue) {
   320  		return fmt.Errorf("minimum %s usage per %s is %s, but limit is %s", resourceName, limitType, enforced.String(), lim.String())
   321  	}
   322  	return nil
   323  }
   324  
   325  // maxRequestConstraint enforces the max constraint over the specified resource
   326  // use when specify LimitType resource doesn't recognize limit values
   327  func maxRequestConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList) error {
   328  	req, reqExists := request[api.ResourceName(resourceName)]
   329  	observedReqValue, _, enforcedValue := requestLimitEnforcedValues(req, resource.Quantity{}, enforced)
   330  
   331  	if !reqExists {
   332  		return fmt.Errorf("maximum %s usage per %s is %s.  No request is specified", resourceName, limitType, enforced.String())
   333  	}
   334  	if observedReqValue > enforcedValue {
   335  		return fmt.Errorf("maximum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
   336  	}
   337  	return nil
   338  }
   339  
   340  // maxConstraint enforces the max constraint over the specified resource
   341  func maxConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
   342  	req, reqExists := request[api.ResourceName(resourceName)]
   343  	lim, limExists := limit[api.ResourceName(resourceName)]
   344  	observedReqValue, observedLimValue, enforcedValue := requestLimitEnforcedValues(req, lim, enforced)
   345  
   346  	if !limExists {
   347  		return fmt.Errorf("maximum %s usage per %s is %s.  No limit is specified", resourceName, limitType, enforced.String())
   348  	}
   349  	if observedLimValue > enforcedValue {
   350  		return fmt.Errorf("maximum %s usage per %s is %s, but limit is %s", resourceName, limitType, enforced.String(), lim.String())
   351  	}
   352  	if reqExists && (observedReqValue > enforcedValue) {
   353  		return fmt.Errorf("maximum %s usage per %s is %s, but request is %s", resourceName, limitType, enforced.String(), req.String())
   354  	}
   355  	return nil
   356  }
   357  
   358  // limitRequestRatioConstraint enforces the limit to request ratio over the specified resource
   359  func limitRequestRatioConstraint(limitType string, resourceName string, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
   360  	req, reqExists := request[api.ResourceName(resourceName)]
   361  	lim, limExists := limit[api.ResourceName(resourceName)]
   362  	observedReqValue, observedLimValue, _ := requestLimitEnforcedValues(req, lim, enforced)
   363  
   364  	if !reqExists || (observedReqValue == int64(0)) {
   365  		return fmt.Errorf("%s max limit to request ratio per %s is %s, but no request is specified or request is 0", resourceName, limitType, enforced.String())
   366  	}
   367  	if !limExists || (observedLimValue == int64(0)) {
   368  		return fmt.Errorf("%s max limit to request ratio per %s is %s, but no limit is specified or limit is 0", resourceName, limitType, enforced.String())
   369  	}
   370  
   371  	observedRatio := float64(observedLimValue) / float64(observedReqValue)
   372  	displayObservedRatio := observedRatio
   373  	maxLimitRequestRatio := float64(enforced.Value())
   374  	if enforced.Value() <= resource.MaxMilliValue {
   375  		observedRatio = observedRatio * 1000
   376  		maxLimitRequestRatio = float64(enforced.MilliValue())
   377  	}
   378  
   379  	if observedRatio > maxLimitRequestRatio {
   380  		return fmt.Errorf("%s max limit to request ratio per %s is %s, but provided ratio is %f", resourceName, limitType, enforced.String(), displayObservedRatio)
   381  	}
   382  
   383  	return nil
   384  }
   385  
   386  // DefaultLimitRangerActions is the default implementation of LimitRangerActions.
   387  type DefaultLimitRangerActions struct{}
   388  
   389  // ensure DefaultLimitRangerActions implements the LimitRangerActions interface.
   390  var _ LimitRangerActions = &DefaultLimitRangerActions{}
   391  
   392  // MutateLimit enforces resource requirements of incoming resources
   393  // against enumerated constraints on the LimitRange.  It may modify
   394  // the incoming object to apply default resource requirements if not
   395  // specified, and enumerated on the LimitRange
   396  func (d *DefaultLimitRangerActions) MutateLimit(limitRange *corev1.LimitRange, resourceName string, obj runtime.Object) error {
   397  	switch resourceName {
   398  	case "pods":
   399  		return PodMutateLimitFunc(limitRange, obj.(*api.Pod))
   400  	}
   401  	return nil
   402  }
   403  
   404  // ValidateLimit verifies the resource requirements of incoming
   405  // resources against enumerated constraints on the LimitRange are
   406  // valid
   407  func (d *DefaultLimitRangerActions) ValidateLimit(limitRange *corev1.LimitRange, resourceName string, obj runtime.Object) error {
   408  	switch resourceName {
   409  	case "pods":
   410  		return PodValidateLimitFunc(limitRange, obj.(*api.Pod))
   411  	case "persistentvolumeclaims":
   412  		return PersistentVolumeClaimValidateLimitFunc(limitRange, obj.(*api.PersistentVolumeClaim))
   413  	}
   414  	return nil
   415  }
   416  
   417  // SupportsAttributes ignores all calls that do not deal with pod resources or storage requests (PVCs).
   418  // Also ignores any call that has a subresource defined.
   419  func (d *DefaultLimitRangerActions) SupportsAttributes(a admission.Attributes) bool {
   420  	if a.GetSubresource() != "" {
   421  		return false
   422  	}
   423  
   424  	// Since containers and initContainers cannot currently be added, removed, or updated, it is unnecessary
   425  	// to mutate and validate limitrange on pod updates. Trying to mutate containers or initContainers on a pod
   426  	// update request will always fail pod validation because those fields are immutable once the object is created.
   427  	if a.GetKind().GroupKind() == api.Kind("Pod") && a.GetOperation() == admission.Update {
   428  		return false
   429  	}
   430  
   431  	return a.GetKind().GroupKind() == api.Kind("Pod") || a.GetKind().GroupKind() == api.Kind("PersistentVolumeClaim")
   432  }
   433  
   434  // SupportsLimit always returns true.
   435  func (d *DefaultLimitRangerActions) SupportsLimit(limitRange *corev1.LimitRange) bool {
   436  	return true
   437  }
   438  
   439  // PersistentVolumeClaimValidateLimitFunc enforces storage limits for PVCs.
   440  // Users request storage via pvc.Spec.Resources.Requests.  Min/Max is enforced by an admin with LimitRange.
   441  // Claims will not be modified with default values because storage is a required part of pvc.Spec.
   442  // All storage enforced values *only* apply to pvc.Spec.Resources.Requests.
   443  func PersistentVolumeClaimValidateLimitFunc(limitRange *corev1.LimitRange, pvc *api.PersistentVolumeClaim) error {
   444  	var errs []error
   445  	for i := range limitRange.Spec.Limits {
   446  		limit := limitRange.Spec.Limits[i]
   447  		limitType := limit.Type
   448  		if limitType == corev1.LimitTypePersistentVolumeClaim {
   449  			for k, v := range limit.Min {
   450  				// normal usage of minConstraint. pvc.Spec.Resources.Limits is not recognized as user input
   451  				if err := minConstraint(string(limitType), string(k), v, pvc.Spec.Resources.Requests, api.ResourceList{}); err != nil {
   452  					errs = append(errs, err)
   453  				}
   454  			}
   455  			for k, v := range limit.Max {
   456  				// We want to enforce the max of the LimitRange against what
   457  				// the user requested.
   458  				if err := maxRequestConstraint(string(limitType), string(k), v, pvc.Spec.Resources.Requests); err != nil {
   459  					errs = append(errs, err)
   460  				}
   461  			}
   462  		}
   463  	}
   464  	return utilerrors.NewAggregate(errs)
   465  }
   466  
   467  // PodMutateLimitFunc sets resource requirements enumerated by the pod against
   468  // the specified LimitRange.  The pod may be modified to apply default resource
   469  // requirements if not specified, and enumerated on the LimitRange
   470  func PodMutateLimitFunc(limitRange *corev1.LimitRange, pod *api.Pod) error {
   471  	defaultResources := defaultContainerResourceRequirements(limitRange)
   472  	mergePodResourceRequirements(pod, &defaultResources)
   473  	return nil
   474  }
   475  
   476  // PodValidateLimitFunc enforces resource requirements enumerated by the pod against
   477  // the specified LimitRange.
   478  func PodValidateLimitFunc(limitRange *corev1.LimitRange, pod *api.Pod) error {
   479  	var errs []error
   480  
   481  	for i := range limitRange.Spec.Limits {
   482  		limit := limitRange.Spec.Limits[i]
   483  		limitType := limit.Type
   484  		// enforce container limits
   485  		if limitType == corev1.LimitTypeContainer {
   486  			for j := range pod.Spec.Containers {
   487  				container := &pod.Spec.Containers[j]
   488  				for k, v := range limit.Min {
   489  					if err := minConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   490  						errs = append(errs, err)
   491  					}
   492  				}
   493  				for k, v := range limit.Max {
   494  					if err := maxConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   495  						errs = append(errs, err)
   496  					}
   497  				}
   498  				for k, v := range limit.MaxLimitRequestRatio {
   499  					if err := limitRequestRatioConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   500  						errs = append(errs, err)
   501  					}
   502  				}
   503  			}
   504  			for j := range pod.Spec.InitContainers {
   505  				container := &pod.Spec.InitContainers[j]
   506  				for k, v := range limit.Min {
   507  					if err := minConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   508  						errs = append(errs, err)
   509  					}
   510  				}
   511  				for k, v := range limit.Max {
   512  					if err := maxConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   513  						errs = append(errs, err)
   514  					}
   515  				}
   516  				for k, v := range limit.MaxLimitRequestRatio {
   517  					if err := limitRequestRatioConstraint(string(limitType), string(k), v, container.Resources.Requests, container.Resources.Limits); err != nil {
   518  						errs = append(errs, err)
   519  					}
   520  				}
   521  			}
   522  		}
   523  
   524  		// enforce pod limits on init containers
   525  		if limitType == corev1.LimitTypePod {
   526  			opts := podResourcesOptions{
   527  				InPlacePodVerticalScalingEnabled: feature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling),
   528  			}
   529  			podRequests := podRequests(pod, opts)
   530  			podLimits := podLimits(pod, opts)
   531  			for k, v := range limit.Min {
   532  				if err := minConstraint(string(limitType), string(k), v, podRequests, podLimits); err != nil {
   533  					errs = append(errs, err)
   534  				}
   535  			}
   536  			for k, v := range limit.Max {
   537  				if err := maxConstraint(string(limitType), string(k), v, podRequests, podLimits); err != nil {
   538  					errs = append(errs, err)
   539  				}
   540  			}
   541  			for k, v := range limit.MaxLimitRequestRatio {
   542  				if err := limitRequestRatioConstraint(string(limitType), string(k), v, podRequests, podLimits); err != nil {
   543  					errs = append(errs, err)
   544  				}
   545  			}
   546  		}
   547  	}
   548  	return utilerrors.NewAggregate(errs)
   549  }
   550  
   551  type podResourcesOptions struct {
   552  	// InPlacePodVerticalScalingEnabled indicates that the in-place pod vertical scaling feature gate is enabled.
   553  	InPlacePodVerticalScalingEnabled bool
   554  }
   555  
   556  // podRequests is a simplified version of pkg/api/v1/resource/PodRequests that operates against the core version of
   557  // pod. Any changes to that calculation should be reflected here.
   558  // TODO: Maybe we can consider doing a partial conversion of the pod to a v1
   559  // type and then using the pkg/api/v1/resource/PodRequests.
   560  func podRequests(pod *api.Pod, opts podResourcesOptions) api.ResourceList {
   561  	reqs := api.ResourceList{}
   562  
   563  	var containerStatuses map[string]*api.ContainerStatus
   564  	if opts.InPlacePodVerticalScalingEnabled {
   565  		containerStatuses = map[string]*api.ContainerStatus{}
   566  		for i := range pod.Status.ContainerStatuses {
   567  			containerStatuses[pod.Status.ContainerStatuses[i].Name] = &pod.Status.ContainerStatuses[i]
   568  		}
   569  	}
   570  
   571  	for _, container := range pod.Spec.Containers {
   572  		containerReqs := container.Resources.Requests
   573  		if opts.InPlacePodVerticalScalingEnabled {
   574  			cs, found := containerStatuses[container.Name]
   575  			if found {
   576  				if pod.Status.Resize == api.PodResizeStatusInfeasible {
   577  					containerReqs = cs.AllocatedResources
   578  				} else {
   579  					containerReqs = max(container.Resources.Requests, cs.AllocatedResources)
   580  				}
   581  			}
   582  		}
   583  
   584  		addResourceList(reqs, containerReqs)
   585  	}
   586  
   587  	restartableInitCotnainerReqs := api.ResourceList{}
   588  	initContainerReqs := api.ResourceList{}
   589  	// init containers define the minimum of any resource
   590  	// Note: In-place resize is not allowed for InitContainers, so no need to check for ResizeStatus value
   591  	for _, container := range pod.Spec.InitContainers {
   592  		containerReqs := container.Resources.Requests
   593  
   594  		if container.RestartPolicy != nil && *container.RestartPolicy == api.ContainerRestartPolicyAlways {
   595  			// and add them to the resulting cumulative container requests
   596  			addResourceList(reqs, containerReqs)
   597  
   598  			// track our cumulative restartable init container resources
   599  			addResourceList(restartableInitCotnainerReqs, containerReqs)
   600  			containerReqs = restartableInitCotnainerReqs
   601  		} else {
   602  			tmp := api.ResourceList{}
   603  			addResourceList(tmp, containerReqs)
   604  			addResourceList(tmp, restartableInitCotnainerReqs)
   605  			containerReqs = tmp
   606  		}
   607  
   608  		maxResourceList(initContainerReqs, containerReqs)
   609  	}
   610  
   611  	maxResourceList(reqs, initContainerReqs)
   612  	return reqs
   613  }
   614  
   615  // podLimits is a simplified version of pkg/api/v1/resource/PodLimits that operates against the core version of
   616  // pod. Any changes to that calculation should be reflected here.
   617  // TODO: Maybe we can consider doing a partial conversion of the pod to a v1
   618  // type and then using the pkg/api/v1/resource/PodLimits.
   619  func podLimits(pod *api.Pod, opts podResourcesOptions) api.ResourceList {
   620  	limits := api.ResourceList{}
   621  
   622  	for _, container := range pod.Spec.Containers {
   623  		addResourceList(limits, container.Resources.Limits)
   624  	}
   625  
   626  	restartableInitContainerLimits := api.ResourceList{}
   627  	initContainerLimits := api.ResourceList{}
   628  	// init containers define the minimum of any resource
   629  	for _, container := range pod.Spec.InitContainers {
   630  		containerLimits := container.Resources.Limits
   631  		// Is the init container marked as a sidecar?
   632  		if container.RestartPolicy != nil && *container.RestartPolicy == api.ContainerRestartPolicyAlways {
   633  			addResourceList(limits, containerLimits)
   634  
   635  			// track our cumulative restartable init container resources
   636  			addResourceList(restartableInitContainerLimits, containerLimits)
   637  			containerLimits = restartableInitContainerLimits
   638  		} else {
   639  			tmp := api.ResourceList{}
   640  			addResourceList(tmp, containerLimits)
   641  			addResourceList(tmp, restartableInitContainerLimits)
   642  			containerLimits = tmp
   643  		}
   644  		maxResourceList(initContainerLimits, containerLimits)
   645  	}
   646  
   647  	maxResourceList(limits, initContainerLimits)
   648  
   649  	return limits
   650  }
   651  
   652  // addResourceList adds the resources in newList to list.
   653  func addResourceList(list, newList api.ResourceList) {
   654  	for name, quantity := range newList {
   655  		if value, ok := list[name]; !ok {
   656  			list[name] = quantity.DeepCopy()
   657  		} else {
   658  			value.Add(quantity)
   659  			list[name] = value
   660  		}
   661  	}
   662  }
   663  
   664  // maxResourceList sets list to the greater of list/newList for every resource in newList
   665  func maxResourceList(list, newList api.ResourceList) {
   666  	for name, quantity := range newList {
   667  		if value, ok := list[name]; !ok || quantity.Cmp(value) > 0 {
   668  			list[name] = quantity.DeepCopy()
   669  		}
   670  	}
   671  }
   672  
   673  // max returns the result of max(a, b) for each named resource and is only used if we can't
   674  // accumulate into an existing resource list
   675  func max(a api.ResourceList, b api.ResourceList) api.ResourceList {
   676  	result := api.ResourceList{}
   677  	for key, value := range a {
   678  		if other, found := b[key]; found {
   679  			if value.Cmp(other) <= 0 {
   680  				result[key] = other.DeepCopy()
   681  				continue
   682  			}
   683  		}
   684  		result[key] = value.DeepCopy()
   685  	}
   686  	for key, value := range b {
   687  		if _, found := result[key]; !found {
   688  			result[key] = value.DeepCopy()
   689  		}
   690  	}
   691  	return result
   692  }
   693
View as plain text