resource_quota_controller_test.go

Documentation: k8s.io/kubernetes/pkg/controller/resourcequota

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package resourcequota
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net/http"
    23  	"net/http/httptest"
    24  	"strings"
    25  	"sync"
    26  	"testing"
    27  	"time"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/apimachinery/pkg/api/resource"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/labels"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"k8s.io/apimachinery/pkg/runtime/schema"
    35  	"k8s.io/apimachinery/pkg/util/sets"
    36  	quota "k8s.io/apiserver/pkg/quota/v1"
    37  	"k8s.io/apiserver/pkg/quota/v1/generic"
    38  	"k8s.io/client-go/discovery"
    39  	"k8s.io/client-go/informers"
    40  	"k8s.io/client-go/kubernetes"
    41  	"k8s.io/client-go/kubernetes/fake"
    42  	"k8s.io/client-go/rest"
    43  	core "k8s.io/client-go/testing"
    44  	"k8s.io/client-go/tools/cache"
    45  	"k8s.io/klog/v2/ktesting"
    46  	"k8s.io/kubernetes/pkg/controller"
    47  	"k8s.io/kubernetes/pkg/quota/v1/install"
    48  )
    49  
    50  func getResourceList(cpu, memory string) v1.ResourceList {
    51  	res := v1.ResourceList{}
    52  	if cpu != "" {
    53  		res[v1.ResourceCPU] = resource.MustParse(cpu)
    54  	}
    55  	if memory != "" {
    56  		res[v1.ResourceMemory] = resource.MustParse(memory)
    57  	}
    58  	return res
    59  }
    60  
    61  func getResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements {
    62  	res := v1.ResourceRequirements{}
    63  	res.Requests = requests
    64  	res.Limits = limits
    65  	return res
    66  }
    67  
    68  func mockDiscoveryFunc() ([]*metav1.APIResourceList, error) {
    69  	return []*metav1.APIResourceList{}, nil
    70  }
    71  
    72  func mockListerForResourceFunc(listersForResource map[schema.GroupVersionResource]cache.GenericLister) quota.ListerForResourceFunc {
    73  	return func(gvr schema.GroupVersionResource) (cache.GenericLister, error) {
    74  		lister, found := listersForResource[gvr]
    75  		if !found {
    76  			return nil, fmt.Errorf("no lister found for resource")
    77  		}
    78  		return lister, nil
    79  	}
    80  }
    81  
    82  func newGenericLister(groupResource schema.GroupResource, items []runtime.Object) cache.GenericLister {
    83  	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{"namespace": cache.MetaNamespaceIndexFunc})
    84  	for _, item := range items {
    85  		store.Add(item)
    86  	}
    87  	return cache.NewGenericLister(store, groupResource)
    88  }
    89  
    90  func newErrorLister() cache.GenericLister {
    91  	return errorLister{}
    92  }
    93  
    94  type errorLister struct {
    95  }
    96  
    97  func (errorLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
    98  	return nil, fmt.Errorf("error listing")
    99  }
   100  func (errorLister) Get(name string) (runtime.Object, error) {
   101  	return nil, fmt.Errorf("error getting")
   102  }
   103  func (errorLister) ByNamespace(namespace string) cache.GenericNamespaceLister {
   104  	return errorLister{}
   105  }
   106  
   107  type quotaController struct {
   108  	*Controller
   109  	stop chan struct{}
   110  }
   111  
   112  func setupQuotaController(t *testing.T, kubeClient kubernetes.Interface, lister quota.ListerForResourceFunc, discoveryFunc NamespacedResourcesFunc) quotaController {
   113  	informerFactory := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
   114  	quotaConfiguration := install.NewQuotaConfigurationForControllers(lister)
   115  	alwaysStarted := make(chan struct{})
   116  	close(alwaysStarted)
   117  	resourceQuotaControllerOptions := &ControllerOptions{
   118  		QuotaClient:               kubeClient.CoreV1(),
   119  		ResourceQuotaInformer:     informerFactory.Core().V1().ResourceQuotas(),
   120  		ResyncPeriod:              controller.NoResyncPeriodFunc,
   121  		ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc,
   122  		IgnoredResourcesFunc:      quotaConfiguration.IgnoredResources,
   123  		DiscoveryFunc:             discoveryFunc,
   124  		Registry:                  generic.NewRegistry(quotaConfiguration.Evaluators()),
   125  		InformersStarted:          alwaysStarted,
   126  		InformerFactory:           informerFactory,
   127  	}
   128  	_, ctx := ktesting.NewTestContext(t)
   129  	qc, err := NewController(ctx, resourceQuotaControllerOptions)
   130  	if err != nil {
   131  		t.Fatal(err)
   132  	}
   133  	stop := make(chan struct{})
   134  	informerFactory.Start(stop)
   135  	return quotaController{qc, stop}
   136  }
   137  
   138  func newTestPods() []runtime.Object {
   139  	return []runtime.Object{
   140  		&v1.Pod{
   141  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running", Namespace: "testing"},
   142  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   143  			Spec: v1.PodSpec{
   144  				Volumes:    []v1.Volume{{Name: "vol"}},
   145  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   146  			},
   147  		},
   148  		&v1.Pod{
   149  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running-2", Namespace: "testing"},
   150  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   151  			Spec: v1.PodSpec{
   152  				Volumes:    []v1.Volume{{Name: "vol"}},
   153  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   154  			},
   155  		},
   156  		&v1.Pod{
   157  			ObjectMeta: metav1.ObjectMeta{Name: "pod-failed", Namespace: "testing"},
   158  			Status:     v1.PodStatus{Phase: v1.PodFailed},
   159  			Spec: v1.PodSpec{
   160  				Volumes:    []v1.Volume{{Name: "vol"}},
   161  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   162  			},
   163  		},
   164  	}
   165  }
   166  
   167  func newBestEffortTestPods() []runtime.Object {
   168  	return []runtime.Object{
   169  		&v1.Pod{
   170  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running", Namespace: "testing"},
   171  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   172  			Spec: v1.PodSpec{
   173  				Volumes:    []v1.Volume{{Name: "vol"}},
   174  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", ""))}},
   175  			},
   176  		},
   177  		&v1.Pod{
   178  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running-2", Namespace: "testing"},
   179  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   180  			Spec: v1.PodSpec{
   181  				Volumes:    []v1.Volume{{Name: "vol"}},
   182  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", ""))}},
   183  			},
   184  		},
   185  		&v1.Pod{
   186  			ObjectMeta: metav1.ObjectMeta{Name: "pod-failed", Namespace: "testing"},
   187  			Status:     v1.PodStatus{Phase: v1.PodFailed},
   188  			Spec: v1.PodSpec{
   189  				Volumes:    []v1.Volume{{Name: "vol"}},
   190  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   191  			},
   192  		},
   193  	}
   194  }
   195  
   196  func newTestPodsWithPriorityClasses() []runtime.Object {
   197  	return []runtime.Object{
   198  		&v1.Pod{
   199  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running", Namespace: "testing"},
   200  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   201  			Spec: v1.PodSpec{
   202  				Volumes:           []v1.Volume{{Name: "vol"}},
   203  				Containers:        []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("500m", "50Gi"), getResourceList("", ""))}},
   204  				PriorityClassName: "high",
   205  			},
   206  		},
   207  		&v1.Pod{
   208  			ObjectMeta: metav1.ObjectMeta{Name: "pod-running-2", Namespace: "testing"},
   209  			Status:     v1.PodStatus{Phase: v1.PodRunning},
   210  			Spec: v1.PodSpec{
   211  				Volumes:           []v1.Volume{{Name: "vol"}},
   212  				Containers:        []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   213  				PriorityClassName: "low",
   214  			},
   215  		},
   216  		&v1.Pod{
   217  			ObjectMeta: metav1.ObjectMeta{Name: "pod-failed", Namespace: "testing"},
   218  			Status:     v1.PodStatus{Phase: v1.PodFailed},
   219  			Spec: v1.PodSpec{
   220  				Volumes:    []v1.Volume{{Name: "vol"}},
   221  				Containers: []v1.Container{{Name: "ctr", Image: "image", Resources: getResourceRequirements(getResourceList("100m", "1Gi"), getResourceList("", ""))}},
   222  			},
   223  		},
   224  	}
   225  }
   226  
   227  func TestSyncResourceQuota(t *testing.T) {
   228  	testCases := map[string]struct {
   229  		gvr               schema.GroupVersionResource
   230  		errorGVR          schema.GroupVersionResource
   231  		items             []runtime.Object
   232  		quota             v1.ResourceQuota
   233  		status            v1.ResourceQuotaStatus
   234  		expectedError     string
   235  		expectedActionSet sets.String
   236  	}{
   237  		"non-matching-best-effort-scoped-quota": {
   238  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   239  			quota: v1.ResourceQuota{
   240  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   241  				Spec: v1.ResourceQuotaSpec{
   242  					Hard: v1.ResourceList{
   243  						v1.ResourceCPU:    resource.MustParse("3"),
   244  						v1.ResourceMemory: resource.MustParse("100Gi"),
   245  						v1.ResourcePods:   resource.MustParse("5"),
   246  					},
   247  					Scopes: []v1.ResourceQuotaScope{v1.ResourceQuotaScopeBestEffort},
   248  				},
   249  			},
   250  			status: v1.ResourceQuotaStatus{
   251  				Hard: v1.ResourceList{
   252  					v1.ResourceCPU:    resource.MustParse("3"),
   253  					v1.ResourceMemory: resource.MustParse("100Gi"),
   254  					v1.ResourcePods:   resource.MustParse("5"),
   255  				},
   256  				Used: v1.ResourceList{
   257  					v1.ResourceCPU:    resource.MustParse("0"),
   258  					v1.ResourceMemory: resource.MustParse("0"),
   259  					v1.ResourcePods:   resource.MustParse("0"),
   260  				},
   261  			},
   262  			expectedActionSet: sets.NewString(
   263  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   264  			),
   265  			items: newTestPods(),
   266  		},
   267  		"matching-best-effort-scoped-quota": {
   268  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   269  			quota: v1.ResourceQuota{
   270  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   271  				Spec: v1.ResourceQuotaSpec{
   272  					Hard: v1.ResourceList{
   273  						v1.ResourceCPU:    resource.MustParse("3"),
   274  						v1.ResourceMemory: resource.MustParse("100Gi"),
   275  						v1.ResourcePods:   resource.MustParse("5"),
   276  					},
   277  					Scopes: []v1.ResourceQuotaScope{v1.ResourceQuotaScopeBestEffort},
   278  				},
   279  			},
   280  			status: v1.ResourceQuotaStatus{
   281  				Hard: v1.ResourceList{
   282  					v1.ResourceCPU:    resource.MustParse("3"),
   283  					v1.ResourceMemory: resource.MustParse("100Gi"),
   284  					v1.ResourcePods:   resource.MustParse("5"),
   285  				},
   286  				Used: v1.ResourceList{
   287  					v1.ResourceCPU:    resource.MustParse("0"),
   288  					v1.ResourceMemory: resource.MustParse("0"),
   289  					v1.ResourcePods:   resource.MustParse("2"),
   290  				},
   291  			},
   292  			expectedActionSet: sets.NewString(
   293  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   294  			),
   295  			items: newBestEffortTestPods(),
   296  		},
   297  		"non-matching-priorityclass-scoped-quota-OpExists": {
   298  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   299  			quota: v1.ResourceQuota{
   300  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   301  				Spec: v1.ResourceQuotaSpec{
   302  					Hard: v1.ResourceList{
   303  						v1.ResourceCPU:    resource.MustParse("3"),
   304  						v1.ResourceMemory: resource.MustParse("100Gi"),
   305  						v1.ResourcePods:   resource.MustParse("5"),
   306  					},
   307  					ScopeSelector: &v1.ScopeSelector{
   308  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   309  							{
   310  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   311  								Operator:  v1.ScopeSelectorOpExists},
   312  						},
   313  					},
   314  				},
   315  			},
   316  			status: v1.ResourceQuotaStatus{
   317  				Hard: v1.ResourceList{
   318  					v1.ResourceCPU:    resource.MustParse("3"),
   319  					v1.ResourceMemory: resource.MustParse("100Gi"),
   320  					v1.ResourcePods:   resource.MustParse("5"),
   321  				},
   322  				Used: v1.ResourceList{
   323  					v1.ResourceCPU:    resource.MustParse("0"),
   324  					v1.ResourceMemory: resource.MustParse("0"),
   325  					v1.ResourcePods:   resource.MustParse("0"),
   326  				},
   327  			},
   328  			expectedActionSet: sets.NewString(
   329  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   330  			),
   331  			items: newTestPods(),
   332  		},
   333  		"matching-priorityclass-scoped-quota-OpExists": {
   334  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   335  			quota: v1.ResourceQuota{
   336  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   337  				Spec: v1.ResourceQuotaSpec{
   338  					Hard: v1.ResourceList{
   339  						v1.ResourceCPU:    resource.MustParse("3"),
   340  						v1.ResourceMemory: resource.MustParse("100Gi"),
   341  						v1.ResourcePods:   resource.MustParse("5"),
   342  					},
   343  					ScopeSelector: &v1.ScopeSelector{
   344  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   345  							{
   346  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   347  								Operator:  v1.ScopeSelectorOpExists},
   348  						},
   349  					},
   350  				},
   351  			},
   352  			status: v1.ResourceQuotaStatus{
   353  				Hard: v1.ResourceList{
   354  					v1.ResourceCPU:    resource.MustParse("3"),
   355  					v1.ResourceMemory: resource.MustParse("100Gi"),
   356  					v1.ResourcePods:   resource.MustParse("5"),
   357  				},
   358  				Used: v1.ResourceList{
   359  					v1.ResourceCPU:    resource.MustParse("600m"),
   360  					v1.ResourceMemory: resource.MustParse("51Gi"),
   361  					v1.ResourcePods:   resource.MustParse("2"),
   362  				},
   363  			},
   364  			expectedActionSet: sets.NewString(
   365  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   366  			),
   367  			items: newTestPodsWithPriorityClasses(),
   368  		},
   369  		"matching-priorityclass-scoped-quota-OpIn": {
   370  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   371  			quota: v1.ResourceQuota{
   372  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   373  				Spec: v1.ResourceQuotaSpec{
   374  					Hard: v1.ResourceList{
   375  						v1.ResourceCPU:    resource.MustParse("3"),
   376  						v1.ResourceMemory: resource.MustParse("100Gi"),
   377  						v1.ResourcePods:   resource.MustParse("5"),
   378  					},
   379  					ScopeSelector: &v1.ScopeSelector{
   380  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   381  							{
   382  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   383  								Operator:  v1.ScopeSelectorOpIn,
   384  								Values:    []string{"high", "low"},
   385  							},
   386  						},
   387  					},
   388  				},
   389  			},
   390  			status: v1.ResourceQuotaStatus{
   391  				Hard: v1.ResourceList{
   392  					v1.ResourceCPU:    resource.MustParse("3"),
   393  					v1.ResourceMemory: resource.MustParse("100Gi"),
   394  					v1.ResourcePods:   resource.MustParse("5"),
   395  				},
   396  				Used: v1.ResourceList{
   397  					v1.ResourceCPU:    resource.MustParse("600m"),
   398  					v1.ResourceMemory: resource.MustParse("51Gi"),
   399  					v1.ResourcePods:   resource.MustParse("2"),
   400  				},
   401  			},
   402  			expectedActionSet: sets.NewString(
   403  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   404  			),
   405  			items: newTestPodsWithPriorityClasses(),
   406  		},
   407  		"matching-priorityclass-scoped-quota-OpIn-high": {
   408  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   409  			quota: v1.ResourceQuota{
   410  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   411  				Spec: v1.ResourceQuotaSpec{
   412  					Hard: v1.ResourceList{
   413  						v1.ResourceCPU:    resource.MustParse("3"),
   414  						v1.ResourceMemory: resource.MustParse("100Gi"),
   415  						v1.ResourcePods:   resource.MustParse("5"),
   416  					},
   417  					ScopeSelector: &v1.ScopeSelector{
   418  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   419  							{
   420  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   421  								Operator:  v1.ScopeSelectorOpIn,
   422  								Values:    []string{"high"},
   423  							},
   424  						},
   425  					},
   426  				},
   427  			},
   428  			status: v1.ResourceQuotaStatus{
   429  				Hard: v1.ResourceList{
   430  					v1.ResourceCPU:    resource.MustParse("3"),
   431  					v1.ResourceMemory: resource.MustParse("100Gi"),
   432  					v1.ResourcePods:   resource.MustParse("5"),
   433  				},
   434  				Used: v1.ResourceList{
   435  					v1.ResourceCPU:    resource.MustParse("500m"),
   436  					v1.ResourceMemory: resource.MustParse("50Gi"),
   437  					v1.ResourcePods:   resource.MustParse("1"),
   438  				},
   439  			},
   440  			expectedActionSet: sets.NewString(
   441  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   442  			),
   443  			items: newTestPodsWithPriorityClasses(),
   444  		},
   445  		"matching-priorityclass-scoped-quota-OpIn-low": {
   446  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   447  			quota: v1.ResourceQuota{
   448  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   449  				Spec: v1.ResourceQuotaSpec{
   450  					Hard: v1.ResourceList{
   451  						v1.ResourceCPU:    resource.MustParse("3"),
   452  						v1.ResourceMemory: resource.MustParse("100Gi"),
   453  						v1.ResourcePods:   resource.MustParse("5"),
   454  					},
   455  					ScopeSelector: &v1.ScopeSelector{
   456  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   457  							{
   458  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   459  								Operator:  v1.ScopeSelectorOpIn,
   460  								Values:    []string{"low"},
   461  							},
   462  						},
   463  					},
   464  				},
   465  			},
   466  			status: v1.ResourceQuotaStatus{
   467  				Hard: v1.ResourceList{
   468  					v1.ResourceCPU:    resource.MustParse("3"),
   469  					v1.ResourceMemory: resource.MustParse("100Gi"),
   470  					v1.ResourcePods:   resource.MustParse("5"),
   471  				},
   472  				Used: v1.ResourceList{
   473  					v1.ResourceCPU:    resource.MustParse("100m"),
   474  					v1.ResourceMemory: resource.MustParse("1Gi"),
   475  					v1.ResourcePods:   resource.MustParse("1"),
   476  				},
   477  			},
   478  			expectedActionSet: sets.NewString(
   479  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   480  			),
   481  			items: newTestPodsWithPriorityClasses(),
   482  		},
   483  		"matching-priorityclass-scoped-quota-OpNotIn-low": {
   484  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   485  			quota: v1.ResourceQuota{
   486  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   487  				Spec: v1.ResourceQuotaSpec{
   488  					Hard: v1.ResourceList{
   489  						v1.ResourceCPU:    resource.MustParse("3"),
   490  						v1.ResourceMemory: resource.MustParse("100Gi"),
   491  						v1.ResourcePods:   resource.MustParse("5"),
   492  					},
   493  					ScopeSelector: &v1.ScopeSelector{
   494  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   495  							{
   496  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   497  								Operator:  v1.ScopeSelectorOpNotIn,
   498  								Values:    []string{"high"},
   499  							},
   500  						},
   501  					},
   502  				},
   503  			},
   504  			status: v1.ResourceQuotaStatus{
   505  				Hard: v1.ResourceList{
   506  					v1.ResourceCPU:    resource.MustParse("3"),
   507  					v1.ResourceMemory: resource.MustParse("100Gi"),
   508  					v1.ResourcePods:   resource.MustParse("5"),
   509  				},
   510  				Used: v1.ResourceList{
   511  					v1.ResourceCPU:    resource.MustParse("100m"),
   512  					v1.ResourceMemory: resource.MustParse("1Gi"),
   513  					v1.ResourcePods:   resource.MustParse("1"),
   514  				},
   515  			},
   516  			expectedActionSet: sets.NewString(
   517  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   518  			),
   519  			items: newTestPodsWithPriorityClasses(),
   520  		},
   521  		"non-matching-priorityclass-scoped-quota-OpIn": {
   522  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   523  			quota: v1.ResourceQuota{
   524  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   525  				Spec: v1.ResourceQuotaSpec{
   526  					Hard: v1.ResourceList{
   527  						v1.ResourceCPU:    resource.MustParse("3"),
   528  						v1.ResourceMemory: resource.MustParse("100Gi"),
   529  						v1.ResourcePods:   resource.MustParse("5"),
   530  					},
   531  					ScopeSelector: &v1.ScopeSelector{
   532  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   533  							{
   534  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   535  								Operator:  v1.ScopeSelectorOpIn,
   536  								Values:    []string{"random"},
   537  							},
   538  						},
   539  					},
   540  				},
   541  			},
   542  			status: v1.ResourceQuotaStatus{
   543  				Hard: v1.ResourceList{
   544  					v1.ResourceCPU:    resource.MustParse("3"),
   545  					v1.ResourceMemory: resource.MustParse("100Gi"),
   546  					v1.ResourcePods:   resource.MustParse("5"),
   547  				},
   548  				Used: v1.ResourceList{
   549  					v1.ResourceCPU:    resource.MustParse("0"),
   550  					v1.ResourceMemory: resource.MustParse("0"),
   551  					v1.ResourcePods:   resource.MustParse("0"),
   552  				},
   553  			},
   554  			expectedActionSet: sets.NewString(
   555  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   556  			),
   557  			items: newTestPodsWithPriorityClasses(),
   558  		},
   559  		"non-matching-priorityclass-scoped-quota-OpNotIn": {
   560  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   561  			quota: v1.ResourceQuota{
   562  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   563  				Spec: v1.ResourceQuotaSpec{
   564  					Hard: v1.ResourceList{
   565  						v1.ResourceCPU:    resource.MustParse("3"),
   566  						v1.ResourceMemory: resource.MustParse("100Gi"),
   567  						v1.ResourcePods:   resource.MustParse("5"),
   568  					},
   569  					ScopeSelector: &v1.ScopeSelector{
   570  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   571  							{
   572  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   573  								Operator:  v1.ScopeSelectorOpNotIn,
   574  								Values:    []string{"random"},
   575  							},
   576  						},
   577  					},
   578  				},
   579  			},
   580  			status: v1.ResourceQuotaStatus{
   581  				Hard: v1.ResourceList{
   582  					v1.ResourceCPU:    resource.MustParse("3"),
   583  					v1.ResourceMemory: resource.MustParse("100Gi"),
   584  					v1.ResourcePods:   resource.MustParse("5"),
   585  				},
   586  				Used: v1.ResourceList{
   587  					v1.ResourceCPU:    resource.MustParse("200m"),
   588  					v1.ResourceMemory: resource.MustParse("2Gi"),
   589  					v1.ResourcePods:   resource.MustParse("2"),
   590  				},
   591  			},
   592  			expectedActionSet: sets.NewString(
   593  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   594  			),
   595  			items: newTestPods(),
   596  		},
   597  		"matching-priorityclass-scoped-quota-OpDoesNotExist": {
   598  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   599  			quota: v1.ResourceQuota{
   600  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   601  				Spec: v1.ResourceQuotaSpec{
   602  					Hard: v1.ResourceList{
   603  						v1.ResourceCPU:    resource.MustParse("3"),
   604  						v1.ResourceMemory: resource.MustParse("100Gi"),
   605  						v1.ResourcePods:   resource.MustParse("5"),
   606  					},
   607  					ScopeSelector: &v1.ScopeSelector{
   608  						MatchExpressions: []v1.ScopedResourceSelectorRequirement{
   609  							{
   610  								ScopeName: v1.ResourceQuotaScopePriorityClass,
   611  								Operator:  v1.ScopeSelectorOpDoesNotExist,
   612  							},
   613  						},
   614  					},
   615  				},
   616  			},
   617  			status: v1.ResourceQuotaStatus{
   618  				Hard: v1.ResourceList{
   619  					v1.ResourceCPU:    resource.MustParse("3"),
   620  					v1.ResourceMemory: resource.MustParse("100Gi"),
   621  					v1.ResourcePods:   resource.MustParse("5"),
   622  				},
   623  				Used: v1.ResourceList{
   624  					v1.ResourceCPU:    resource.MustParse("200m"),
   625  					v1.ResourceMemory: resource.MustParse("2Gi"),
   626  					v1.ResourcePods:   resource.MustParse("2"),
   627  				},
   628  			},
   629  			expectedActionSet: sets.NewString(
   630  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   631  			),
   632  			items: newTestPods(),
   633  		},
   634  		"pods": {
   635  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   636  			quota: v1.ResourceQuota{
   637  				ObjectMeta: metav1.ObjectMeta{Name: "quota", Namespace: "testing"},
   638  				Spec: v1.ResourceQuotaSpec{
   639  					Hard: v1.ResourceList{
   640  						v1.ResourceCPU:    resource.MustParse("3"),
   641  						v1.ResourceMemory: resource.MustParse("100Gi"),
   642  						v1.ResourcePods:   resource.MustParse("5"),
   643  					},
   644  				},
   645  			},
   646  			status: v1.ResourceQuotaStatus{
   647  				Hard: v1.ResourceList{
   648  					v1.ResourceCPU:    resource.MustParse("3"),
   649  					v1.ResourceMemory: resource.MustParse("100Gi"),
   650  					v1.ResourcePods:   resource.MustParse("5"),
   651  				},
   652  				Used: v1.ResourceList{
   653  					v1.ResourceCPU:    resource.MustParse("200m"),
   654  					v1.ResourceMemory: resource.MustParse("2Gi"),
   655  					v1.ResourcePods:   resource.MustParse("2"),
   656  				},
   657  			},
   658  			expectedActionSet: sets.NewString(
   659  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   660  			),
   661  			items: newTestPods(),
   662  		},
   663  		"quota-spec-hard-updated": {
   664  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   665  			quota: v1.ResourceQuota{
   666  				ObjectMeta: metav1.ObjectMeta{
   667  					Namespace: "default",
   668  					Name:      "rq",
   669  				},
   670  				Spec: v1.ResourceQuotaSpec{
   671  					Hard: v1.ResourceList{
   672  						v1.ResourceCPU: resource.MustParse("4"),
   673  					},
   674  				},
   675  				Status: v1.ResourceQuotaStatus{
   676  					Hard: v1.ResourceList{
   677  						v1.ResourceCPU: resource.MustParse("3"),
   678  					},
   679  					Used: v1.ResourceList{
   680  						v1.ResourceCPU: resource.MustParse("0"),
   681  					},
   682  				},
   683  			},
   684  			status: v1.ResourceQuotaStatus{
   685  				Hard: v1.ResourceList{
   686  					v1.ResourceCPU: resource.MustParse("4"),
   687  				},
   688  				Used: v1.ResourceList{
   689  					v1.ResourceCPU: resource.MustParse("0"),
   690  				},
   691  			},
   692  			expectedActionSet: sets.NewString(
   693  				strings.Join([]string{"update", "resourcequotas", "status"}, "-"),
   694  			),
   695  			items: []runtime.Object{},
   696  		},
   697  		"quota-unchanged": {
   698  			gvr: v1.SchemeGroupVersion.WithResource("pods"),
   699  			quota: v1.ResourceQuota{
   700  				ObjectMeta: metav1.ObjectMeta{
   701  					Namespace: "default",
   702  					Name:      "rq",
   703  				},
   704  				Spec: v1.ResourceQuotaSpec{
   705  					Hard: v1.ResourceList{
   706  						v1.ResourceCPU: resource.MustParse("4"),
   707  					},
   708  				},
   709  				Status: v1.ResourceQuotaStatus{
   710  					Hard: v1.ResourceList{
   711  						v1.ResourceCPU: resource.MustParse("0"),
   712  					},
   713  				},
   714  			},
   715  			status: v1.ResourceQuotaStatus{
   716  				Hard: v1.ResourceList{
   717  					v1.ResourceCPU: resource.MustParse("4"),
   718  				},
   719  				Used: v1.ResourceList{
   720  					v1.ResourceCPU: resource.MustParse("0"),
   721  				},
   722  			},
   723  			expectedActionSet: sets.NewString(),
   724  			items:             []runtime.Object{},
   725  		},
   726  		"quota-missing-status-with-calculation-error": {
   727  			errorGVR: v1.SchemeGroupVersion.WithResource("pods"),
   728  			quota: v1.ResourceQuota{
   729  				ObjectMeta: metav1.ObjectMeta{
   730  					Namespace: "default",
   731  					Name:      "rq",
   732  				},
   733  				Spec: v1.ResourceQuotaSpec{
   734  					Hard: v1.ResourceList{
   735  						v1.ResourcePods: resource.MustParse("1"),
   736  					},
   737  				},
   738  				Status: v1.ResourceQuotaStatus{},
   739  			},
   740  			status: v1.ResourceQuotaStatus{
   741  				Hard: v1.ResourceList{
   742  					v1.ResourcePods: resource.MustParse("1"),
   743  				},
   744  			},
   745  			expectedError:     "error listing",
   746  			expectedActionSet: sets.NewString("update-resourcequotas-status"),
   747  			items:             []runtime.Object{},
   748  		},
   749  		"quota-missing-status-with-partial-calculation-error": {
   750  			gvr:      v1.SchemeGroupVersion.WithResource("configmaps"),
   751  			errorGVR: v1.SchemeGroupVersion.WithResource("pods"),
   752  			quota: v1.ResourceQuota{
   753  				ObjectMeta: metav1.ObjectMeta{
   754  					Namespace: "default",
   755  					Name:      "rq",
   756  				},
   757  				Spec: v1.ResourceQuotaSpec{
   758  					Hard: v1.ResourceList{
   759  						v1.ResourcePods:       resource.MustParse("1"),
   760  						v1.ResourceConfigMaps: resource.MustParse("1"),
   761  					},
   762  				},
   763  				Status: v1.ResourceQuotaStatus{},
   764  			},
   765  			status: v1.ResourceQuotaStatus{
   766  				Hard: v1.ResourceList{
   767  					v1.ResourcePods:       resource.MustParse("1"),
   768  					v1.ResourceConfigMaps: resource.MustParse("1"),
   769  				},
   770  				Used: v1.ResourceList{
   771  					v1.ResourceConfigMaps: resource.MustParse("0"),
   772  				},
   773  			},
   774  			expectedError:     "error listing",
   775  			expectedActionSet: sets.NewString("update-resourcequotas-status"),
   776  			items:             []runtime.Object{},
   777  		},
   778  	}
   779  
   780  	for testName, testCase := range testCases {
   781  		kubeClient := fake.NewSimpleClientset(&testCase.quota)
   782  		listersForResourceConfig := map[schema.GroupVersionResource]cache.GenericLister{
   783  			testCase.gvr:      newGenericLister(testCase.gvr.GroupResource(), testCase.items),
   784  			testCase.errorGVR: newErrorLister(),
   785  		}
   786  		qc := setupQuotaController(t, kubeClient, mockListerForResourceFunc(listersForResourceConfig), mockDiscoveryFunc)
   787  		defer close(qc.stop)
   788  
   789  		if err := qc.syncResourceQuota(context.TODO(), &testCase.quota); err != nil {
   790  			if len(testCase.expectedError) == 0 || !strings.Contains(err.Error(), testCase.expectedError) {
   791  				t.Fatalf("test: %s, unexpected error: %v", testName, err)
   792  			}
   793  		} else if len(testCase.expectedError) > 0 {
   794  			t.Fatalf("test: %s, expected error %q, got none", testName, testCase.expectedError)
   795  		}
   796  
   797  		actionSet := sets.NewString()
   798  		for _, action := range kubeClient.Actions() {
   799  			actionSet.Insert(strings.Join([]string{action.GetVerb(), action.GetResource().Resource, action.GetSubresource()}, "-"))
   800  		}
   801  		if !actionSet.IsSuperset(testCase.expectedActionSet) {
   802  			t.Errorf("test: %s,\nExpected actions:\n%v\n but got:\n%v\nDifference:\n%v", testName, testCase.expectedActionSet, actionSet, testCase.expectedActionSet.Difference(actionSet))
   803  		}
   804  
   805  		var usage *v1.ResourceQuota
   806  		actions := kubeClient.Actions()
   807  		for i := len(actions) - 1; i >= 0; i-- {
   808  			if updateAction, ok := actions[i].(core.UpdateAction); ok {
   809  				usage = updateAction.GetObject().(*v1.ResourceQuota)
   810  				break
   811  			}
   812  		}
   813  		if usage == nil {
   814  			t.Fatalf("test: %s,\nExpected update action usage, got none: actions:\n%v", testName, actions)
   815  		}
   816  
   817  		// ensure usage is as expected
   818  		if len(usage.Status.Hard) != len(testCase.status.Hard) {
   819  			t.Errorf("test: %s, status hard lengths do not match", testName)
   820  		}
   821  		if len(usage.Status.Used) != len(testCase.status.Used) {
   822  			t.Errorf("test: %s, status used lengths do not match", testName)
   823  		}
   824  		for k, v := range testCase.status.Hard {
   825  			actual := usage.Status.Hard[k]
   826  			actualValue := actual.String()
   827  			expectedValue := v.String()
   828  			if expectedValue != actualValue {
   829  				t.Errorf("test: %s, Usage Hard: Key: %v, Expected: %v, Actual: %v", testName, k, expectedValue, actualValue)
   830  			}
   831  		}
   832  		for k, v := range testCase.status.Used {
   833  			actual := usage.Status.Used[k]
   834  			actualValue := actual.String()
   835  			expectedValue := v.String()
   836  			if expectedValue != actualValue {
   837  				t.Errorf("test: %s, Usage Used: Key: %v, Expected: %v, Actual: %v", testName, k, expectedValue, actualValue)
   838  			}
   839  		}
   840  	}
   841  }
   842  
   843  func TestAddQuota(t *testing.T) {
   844  	kubeClient := fake.NewSimpleClientset()
   845  	gvr := v1.SchemeGroupVersion.WithResource("pods")
   846  	listersForResourceConfig := map[schema.GroupVersionResource]cache.GenericLister{
   847  		gvr: newGenericLister(gvr.GroupResource(), newTestPods()),
   848  	}
   849  
   850  	qc := setupQuotaController(t, kubeClient, mockListerForResourceFunc(listersForResourceConfig), mockDiscoveryFunc)
   851  	defer close(qc.stop)
   852  
   853  	testCases := []struct {
   854  		name             string
   855  		quota            *v1.ResourceQuota
   856  		expectedPriority bool
   857  	}{
   858  		{
   859  			name:             "no status",
   860  			expectedPriority: true,
   861  			quota: &v1.ResourceQuota{
   862  				ObjectMeta: metav1.ObjectMeta{
   863  					Namespace: "default",
   864  					Name:      "rq",
   865  				},
   866  				Spec: v1.ResourceQuotaSpec{
   867  					Hard: v1.ResourceList{
   868  						v1.ResourceCPU: resource.MustParse("4"),
   869  					},
   870  				},
   871  			},
   872  		},
   873  		{
   874  			name:             "status, no usage",
   875  			expectedPriority: true,
   876  			quota: &v1.ResourceQuota{
   877  				ObjectMeta: metav1.ObjectMeta{
   878  					Namespace: "default",
   879  					Name:      "rq",
   880  				},
   881  				Spec: v1.ResourceQuotaSpec{
   882  					Hard: v1.ResourceList{
   883  						v1.ResourceCPU: resource.MustParse("4"),
   884  					},
   885  				},
   886  				Status: v1.ResourceQuotaStatus{
   887  					Hard: v1.ResourceList{
   888  						v1.ResourceCPU: resource.MustParse("4"),
   889  					},
   890  				},
   891  			},
   892  		},
   893  		{
   894  			name:             "status, no usage(to validate it works for extended resources)",
   895  			expectedPriority: true,
   896  			quota: &v1.ResourceQuota{
   897  				ObjectMeta: metav1.ObjectMeta{
   898  					Namespace: "default",
   899  					Name:      "rq",
   900  				},
   901  				Spec: v1.ResourceQuotaSpec{
   902  					Hard: v1.ResourceList{
   903  						"requests.example/foobars.example.com": resource.MustParse("4"),
   904  					},
   905  				},
   906  				Status: v1.ResourceQuotaStatus{
   907  					Hard: v1.ResourceList{
   908  						"requests.example/foobars.example.com": resource.MustParse("4"),
   909  					},
   910  				},
   911  			},
   912  		},
   913  		{
   914  			name:             "status, mismatch",
   915  			expectedPriority: true,
   916  			quota: &v1.ResourceQuota{
   917  				ObjectMeta: metav1.ObjectMeta{
   918  					Namespace: "default",
   919  					Name:      "rq",
   920  				},
   921  				Spec: v1.ResourceQuotaSpec{
   922  					Hard: v1.ResourceList{
   923  						v1.ResourceCPU: resource.MustParse("4"),
   924  					},
   925  				},
   926  				Status: v1.ResourceQuotaStatus{
   927  					Hard: v1.ResourceList{
   928  						v1.ResourceCPU: resource.MustParse("6"),
   929  					},
   930  					Used: v1.ResourceList{
   931  						v1.ResourceCPU: resource.MustParse("0"),
   932  					},
   933  				},
   934  			},
   935  		},
   936  		{
   937  			name:             "status, missing usage, but don't care (no informer)",
   938  			expectedPriority: false,
   939  			quota: &v1.ResourceQuota{
   940  				ObjectMeta: metav1.ObjectMeta{
   941  					Namespace: "default",
   942  					Name:      "rq",
   943  				},
   944  				Spec: v1.ResourceQuotaSpec{
   945  					Hard: v1.ResourceList{
   946  						"foobars.example.com": resource.MustParse("4"),
   947  					},
   948  				},
   949  				Status: v1.ResourceQuotaStatus{
   950  					Hard: v1.ResourceList{
   951  						"foobars.example.com": resource.MustParse("4"),
   952  					},
   953  				},
   954  			},
   955  		},
   956  		{
   957  			name:             "ready",
   958  			expectedPriority: false,
   959  			quota: &v1.ResourceQuota{
   960  				ObjectMeta: metav1.ObjectMeta{
   961  					Namespace: "default",
   962  					Name:      "rq",
   963  				},
   964  				Spec: v1.ResourceQuotaSpec{
   965  					Hard: v1.ResourceList{
   966  						v1.ResourceCPU: resource.MustParse("4"),
   967  					},
   968  				},
   969  				Status: v1.ResourceQuotaStatus{
   970  					Hard: v1.ResourceList{
   971  						v1.ResourceCPU: resource.MustParse("4"),
   972  					},
   973  					Used: v1.ResourceList{
   974  						v1.ResourceCPU: resource.MustParse("0"),
   975  					},
   976  				},
   977  			},
   978  		},
   979  	}
   980  
   981  	for _, tc := range testCases {
   982  		logger, _ := ktesting.NewTestContext(t)
   983  		qc.addQuota(logger, tc.quota)
   984  		if tc.expectedPriority {
   985  			if e, a := 1, qc.missingUsageQueue.Len(); e != a {
   986  				t.Errorf("%s: expected %v, got %v", tc.name, e, a)
   987  			}
   988  			if e, a := 0, qc.queue.Len(); e != a {
   989  				t.Errorf("%s: expected %v, got %v", tc.name, e, a)
   990  			}
   991  		} else {
   992  			if e, a := 0, qc.missingUsageQueue.Len(); e != a {
   993  				t.Errorf("%s: expected %v, got %v", tc.name, e, a)
   994  			}
   995  			if e, a := 1, qc.queue.Len(); e != a {
   996  				t.Errorf("%s: expected %v, got %v", tc.name, e, a)
   997  			}
   998  		}
   999  		for qc.missingUsageQueue.Len() > 0 {
  1000  			key, _ := qc.missingUsageQueue.Get()
  1001  			qc.missingUsageQueue.Done(key)
  1002  		}
  1003  		for qc.queue.Len() > 0 {
  1004  			key, _ := qc.queue.Get()
  1005  			qc.queue.Done(key)
  1006  		}
  1007  	}
  1008  }
  1009  
  1010  // TestDiscoverySync ensures that a discovery client error
  1011  // will not cause the quota controller to block infinitely.
  1012  func TestDiscoverySync(t *testing.T) {
  1013  	serverResources := []*metav1.APIResourceList{
  1014  		{
  1015  			GroupVersion: "v1",
  1016  			APIResources: []metav1.APIResource{
  1017  				{Name: "pods", Namespaced: true, Kind: "Pod", Verbs: metav1.Verbs{"create", "delete", "list", "watch"}},
  1018  			},
  1019  		},
  1020  		{
  1021  			GroupVersion: "apps/v1",
  1022  			APIResources: []metav1.APIResource{
  1023  				{Name: "deployments", Namespaced: true, Kind: "Deployment", Verbs: metav1.Verbs{"create", "delete", "list", "watch"}},
  1024  			},
  1025  		},
  1026  	}
  1027  	unsyncableServerResources := []*metav1.APIResourceList{
  1028  		{
  1029  			GroupVersion: "v1",
  1030  			APIResources: []metav1.APIResource{
  1031  				{Name: "pods", Namespaced: true, Kind: "Pod", Verbs: metav1.Verbs{"create", "delete", "list", "watch"}},
  1032  				{Name: "secrets", Namespaced: true, Kind: "Secret", Verbs: metav1.Verbs{"create", "delete", "list", "watch"}},
  1033  			},
  1034  		},
  1035  	}
  1036  	appsV1Resources := []*metav1.APIResourceList{
  1037  		{
  1038  			GroupVersion: "apps/v1",
  1039  			APIResources: []metav1.APIResource{
  1040  				{Name: "deployments", Namespaced: true, Kind: "Deployment", Verbs: metav1.Verbs{"create", "delete", "list", "watch"}},
  1041  			},
  1042  		},
  1043  	}
  1044  	appsV1Error := &discovery.ErrGroupDiscoveryFailed{Groups: map[schema.GroupVersion]error{{Group: "apps", Version: "v1"}: fmt.Errorf(":-/")}}
  1045  	coreV1Error := &discovery.ErrGroupDiscoveryFailed{Groups: map[schema.GroupVersion]error{{Group: "", Version: "v1"}: fmt.Errorf(":-/")}}
  1046  	fakeDiscoveryClient := &fakeServerResources{
  1047  		PreferredResources: serverResources,
  1048  		Error:              nil,
  1049  		Lock:               sync.Mutex{},
  1050  		InterfaceUsedCount: 0,
  1051  	}
  1052  
  1053  	testHandler := &fakeActionHandler{
  1054  		response: map[string]FakeResponse{
  1055  			"GET" + "/api/v1/pods": {
  1056  				200,
  1057  				[]byte("{}"),
  1058  			},
  1059  			"GET" + "/api/v1/secrets": {
  1060  				404,
  1061  				[]byte("{}"),
  1062  			},
  1063  			"GET" + "/apis/apps/v1/deployments": {
  1064  				200,
  1065  				[]byte("{}"),
  1066  			},
  1067  		},
  1068  	}
  1069  
  1070  	srv, clientConfig := testServerAndClientConfig(testHandler.ServeHTTP)
  1071  	defer srv.Close()
  1072  	clientConfig.ContentConfig.NegotiatedSerializer = nil
  1073  	kubeClient, err := kubernetes.NewForConfig(clientConfig)
  1074  	if err != nil {
  1075  		t.Fatal(err)
  1076  	}
  1077  
  1078  	pods := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
  1079  	secrets := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}
  1080  	deployments := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}
  1081  	listersForResourceConfig := map[schema.GroupVersionResource]cache.GenericLister{
  1082  		pods:        newGenericLister(pods.GroupResource(), []runtime.Object{}),
  1083  		secrets:     newGenericLister(secrets.GroupResource(), []runtime.Object{}),
  1084  		deployments: newGenericLister(deployments.GroupResource(), []runtime.Object{}),
  1085  	}
  1086  	qc := setupQuotaController(t, kubeClient, mockListerForResourceFunc(listersForResourceConfig), fakeDiscoveryClient.ServerPreferredNamespacedResources)
  1087  	defer close(qc.stop)
  1088  
  1089  	stopSync := make(chan struct{})
  1090  	defer close(stopSync)
  1091  	// The pseudo-code of Sync():
  1092  	// Sync(client, period, stopCh):
  1093  	//    wait.Until() loops with `period` until the `stopCh` is closed :
  1094  	//       GetQuotableResources()
  1095  	//       resyncMonitors()
  1096  	//       cache.WaitForNamedCacheSync() loops with `syncedPollPeriod` (hardcoded to 100ms), until either its stop channel is closed after `period`, or all caches synced.
  1097  	//
  1098  	// Setting the period to 200ms allows the WaitForCacheSync() to check
  1099  	// for cache sync ~2 times in every wait.Until() loop.
  1100  	//
  1101  	// The 1s sleep in the test allows GetQuotableResources and
  1102  	// resyncMonitors to run ~5 times to ensure the changes to the
  1103  	// fakeDiscoveryClient are picked up.
  1104  	_, ctx := ktesting.NewTestContext(t)
  1105  	go qc.Sync(ctx, fakeDiscoveryClient.ServerPreferredNamespacedResources, 200*time.Millisecond)
  1106  
  1107  	// Wait until the sync discovers the initial resources
  1108  	time.Sleep(1 * time.Second)
  1109  
  1110  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1111  	if err != nil {
  1112  		t.Fatalf("Expected quotacontroller.Sync to be running but it is blocked: %v", err)
  1113  	}
  1114  	assertMonitors(t, qc, "pods", "deployments")
  1115  
  1116  	// Simulate the discovery client returning an error
  1117  	fakeDiscoveryClient.setPreferredResources(nil, fmt.Errorf("error calling discoveryClient.ServerPreferredResources()"))
  1118  
  1119  	// Wait until sync discovers the change
  1120  	time.Sleep(1 * time.Second)
  1121  	// No monitors removed
  1122  	assertMonitors(t, qc, "pods", "deployments")
  1123  
  1124  	// Remove the error from being returned and see if the quota sync is still working
  1125  	fakeDiscoveryClient.setPreferredResources(serverResources, nil)
  1126  
  1127  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1128  	if err != nil {
  1129  		t.Fatalf("Expected quotacontroller.Sync to still be running but it is blocked: %v", err)
  1130  	}
  1131  	assertMonitors(t, qc, "pods", "deployments")
  1132  
  1133  	// Simulate the discovery client returning a resource the restmapper can resolve, but will not sync caches
  1134  	fakeDiscoveryClient.setPreferredResources(unsyncableServerResources, nil)
  1135  
  1136  	// Wait until sync discovers the change
  1137  	time.Sleep(1 * time.Second)
  1138  	// deployments removed, secrets added
  1139  	assertMonitors(t, qc, "pods", "secrets")
  1140  
  1141  	// Put the resources back to normal and ensure quota sync recovers
  1142  	fakeDiscoveryClient.setPreferredResources(serverResources, nil)
  1143  
  1144  	// Wait until sync discovers the change
  1145  	time.Sleep(1 * time.Second)
  1146  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1147  	if err != nil {
  1148  		t.Fatalf("Expected quotacontroller.Sync to still be running but it is blocked: %v", err)
  1149  	}
  1150  	// secrets removed, deployments readded
  1151  	assertMonitors(t, qc, "pods", "deployments")
  1152  
  1153  	// apps/v1 discovery failure
  1154  	fakeDiscoveryClient.setPreferredResources(unsyncableServerResources, appsV1Error)
  1155  	// Wait until sync discovers the change
  1156  	time.Sleep(1 * time.Second)
  1157  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1158  	if err != nil {
  1159  		t.Fatalf("Expected quotacontroller.Sync to still be running but it is blocked: %v", err)
  1160  	}
  1161  	// deployments remain due to appsv1 error, secrets added
  1162  	assertMonitors(t, qc, "pods", "deployments", "secrets")
  1163  
  1164  	// core/v1 discovery failure
  1165  	fakeDiscoveryClient.setPreferredResources(appsV1Resources, coreV1Error)
  1166  	// Wait until sync discovers the change
  1167  	time.Sleep(1 * time.Second)
  1168  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1169  	if err != nil {
  1170  		t.Fatalf("Expected quotacontroller.Sync to still be running but it is blocked: %v", err)
  1171  	}
  1172  	// pods and secrets remain due to corev1 error
  1173  	assertMonitors(t, qc, "pods", "deployments", "secrets")
  1174  
  1175  	// Put the resources back to normal and ensure quota sync recovers
  1176  	fakeDiscoveryClient.setPreferredResources(serverResources, nil)
  1177  
  1178  	err = expectSyncNotBlocked(fakeDiscoveryClient, &qc.workerLock)
  1179  	if err != nil {
  1180  		t.Fatalf("Expected quotacontroller.Sync to still be running but it is blocked: %v", err)
  1181  	}
  1182  	// secrets removed, deployments remain
  1183  	assertMonitors(t, qc, "pods", "deployments")
  1184  }
  1185  
  1186  func assertMonitors(t *testing.T, qc quotaController, resources ...string) {
  1187  	t.Helper()
  1188  	expected := sets.NewString(resources...)
  1189  	actual := sets.NewString()
  1190  	for m := range qc.Controller.quotaMonitor.monitors {
  1191  		actual.Insert(m.Resource)
  1192  	}
  1193  	if !actual.Equal(expected) {
  1194  		t.Fatalf("expected monitors %v, got %v", expected.List(), actual.List())
  1195  	}
  1196  }
  1197  
  1198  // testServerAndClientConfig returns a server that listens and a config that can reference it
  1199  func testServerAndClientConfig(handler func(http.ResponseWriter, *http.Request)) (*httptest.Server, *rest.Config) {
  1200  	srv := httptest.NewServer(http.HandlerFunc(handler))
  1201  	config := &rest.Config{
  1202  		Host: srv.URL,
  1203  	}
  1204  	return srv, config
  1205  }
  1206  
  1207  func expectSyncNotBlocked(fakeDiscoveryClient *fakeServerResources, workerLock *sync.RWMutex) error {
  1208  	before := fakeDiscoveryClient.getInterfaceUsedCount()
  1209  	t := 1 * time.Second
  1210  	time.Sleep(t)
  1211  	after := fakeDiscoveryClient.getInterfaceUsedCount()
  1212  	if before == after {
  1213  		return fmt.Errorf("discoveryClient.ServerPreferredResources() called %d times over %v", after-before, t)
  1214  	}
  1215  
  1216  	workerLockAcquired := make(chan struct{})
  1217  	go func() {
  1218  		workerLock.Lock()
  1219  		defer workerLock.Unlock()
  1220  		close(workerLockAcquired)
  1221  	}()
  1222  	select {
  1223  	case <-workerLockAcquired:
  1224  		return nil
  1225  	case <-time.After(t):
  1226  		return fmt.Errorf("workerLock blocked for at least %v", t)
  1227  	}
  1228  }
  1229  
  1230  type fakeServerResources struct {
  1231  	PreferredResources []*metav1.APIResourceList
  1232  	Error              error
  1233  	Lock               sync.Mutex
  1234  	InterfaceUsedCount int
  1235  }
  1236  
  1237  func (*fakeServerResources) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) {
  1238  	return nil, nil
  1239  }
  1240  
  1241  func (*fakeServerResources) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
  1242  	return nil, nil
  1243  }
  1244  
  1245  func (f *fakeServerResources) setPreferredResources(resources []*metav1.APIResourceList, err error) {
  1246  	f.Lock.Lock()
  1247  	defer f.Lock.Unlock()
  1248  	f.PreferredResources = resources
  1249  	f.Error = err
  1250  }
  1251  
  1252  func (f *fakeServerResources) getInterfaceUsedCount() int {
  1253  	f.Lock.Lock()
  1254  	defer f.Lock.Unlock()
  1255  	return f.InterfaceUsedCount
  1256  }
  1257  
  1258  func (f *fakeServerResources) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) {
  1259  	f.Lock.Lock()
  1260  	defer f.Lock.Unlock()
  1261  	f.InterfaceUsedCount++
  1262  	return f.PreferredResources, f.Error
  1263  }
  1264  
  1265  // fakeAction records information about requests to aid in testing.
  1266  type fakeAction struct {
  1267  	method string
  1268  	path   string
  1269  	query  string
  1270  }
  1271  
  1272  // String returns method=path to aid in testing
  1273  func (f *fakeAction) String() string {
  1274  	return strings.Join([]string{f.method, f.path}, "=")
  1275  }
  1276  
  1277  type FakeResponse struct {
  1278  	statusCode int
  1279  	content    []byte
  1280  }
  1281  
  1282  // fakeActionHandler holds a list of fakeActions received
  1283  type fakeActionHandler struct {
  1284  	// statusCode and content returned by this handler for different method + path.
  1285  	response map[string]FakeResponse
  1286  
  1287  	lock    sync.Mutex
  1288  	actions []fakeAction
  1289  }
  1290  
  1291  // ServeHTTP logs the action that occurred and always returns the associated status code
  1292  func (f *fakeActionHandler) ServeHTTP(response http.ResponseWriter, request *http.Request) {
  1293  	func() {
  1294  		f.lock.Lock()
  1295  		defer f.lock.Unlock()
  1296  
  1297  		f.actions = append(f.actions, fakeAction{method: request.Method, path: request.URL.Path, query: request.URL.RawQuery})
  1298  		fakeResponse, ok := f.response[request.Method+request.URL.Path]
  1299  		if !ok {
  1300  			fakeResponse.statusCode = 200
  1301  			fakeResponse.content = []byte("{\"kind\": \"List\"}")
  1302  		}
  1303  		response.Header().Set("Content-Type", "application/json")
  1304  		response.WriteHeader(fakeResponse.statusCode)
  1305  		response.Write(fakeResponse.content)
  1306  	}()
  1307  
  1308  	// This is to allow the fakeActionHandler to simulate a watch being opened
  1309  	if strings.Contains(request.URL.RawQuery, "watch=true") {
  1310  		hijacker, ok := response.(http.Hijacker)
  1311  		if !ok {
  1312  			return
  1313  		}
  1314  		connection, _, err := hijacker.Hijack()
  1315  		if err != nil {
  1316  			return
  1317  		}
  1318  		defer connection.Close()
  1319  		time.Sleep(30 * time.Second)
  1320  	}
  1321  }
  1322
View as plain text