instance_test.go

Documentation: k8s.io/kubernetes/pkg/controlplane

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"encoding/json"
    23  	"io"
    24  	"net"
    25  	"net/http"
    26  	"net/http/httptest"
    27  	"reflect"
    28  	"strings"
    29  	"testing"
    30  
    31  	autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1"
    32  	autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2"
    33  	batchapiv1beta1 "k8s.io/api/batch/v1beta1"
    34  	certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
    35  	discoveryv1beta1 "k8s.io/api/discovery/v1beta1"
    36  	eventsv1beta1 "k8s.io/api/events/v1beta1"
    37  	nodev1beta1 "k8s.io/api/node/v1beta1"
    38  	policyapiv1beta1 "k8s.io/api/policy/v1beta1"
    39  	storageapiv1beta1 "k8s.io/api/storage/v1beta1"
    40  	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
    41  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    42  	"k8s.io/apimachinery/pkg/runtime/schema"
    43  	utilnet "k8s.io/apimachinery/pkg/util/net"
    44  	"k8s.io/apimachinery/pkg/util/sets"
    45  	"k8s.io/apimachinery/pkg/version"
    46  	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
    47  	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
    48  	genericapiserver "k8s.io/apiserver/pkg/server"
    49  	"k8s.io/apiserver/pkg/server/options"
    50  	"k8s.io/apiserver/pkg/server/resourceconfig"
    51  	serverstorage "k8s.io/apiserver/pkg/server/storage"
    52  	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
    53  	"k8s.io/apiserver/pkg/util/openapi"
    54  	"k8s.io/client-go/discovery"
    55  	"k8s.io/client-go/informers"
    56  	"k8s.io/client-go/kubernetes"
    57  	restclient "k8s.io/client-go/rest"
    58  	kubeversion "k8s.io/component-base/version"
    59  	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
    60  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    61  	flowcontrolv1bet3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3"
    62  	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
    63  	"k8s.io/kubernetes/pkg/controlplane/storageversionhashdata"
    64  	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
    65  	"k8s.io/kubernetes/pkg/kubeapiserver"
    66  	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
    67  	certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
    68  	corerest "k8s.io/kubernetes/pkg/registry/core/rest"
    69  	"k8s.io/kubernetes/pkg/registry/registrytest"
    70  	netutils "k8s.io/utils/net"
    71  
    72  	"github.com/stretchr/testify/assert"
    73  )
    74  
    75  // setUp is a convenience function for setting up for (most) tests.
    76  func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertions) {
    77  	server, storageConfig := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
    78  
    79  	config := &Config{
    80  		GenericConfig: genericapiserver.NewConfig(legacyscheme.Codecs),
    81  		ExtraConfig: ExtraConfig{
    82  			APIResourceConfigSource: DefaultAPIResourceConfigSource(),
    83  			APIServerServicePort:    443,
    84  			MasterCount:             1,
    85  			EndpointReconcilerType:  reconcilers.MasterCountReconcilerType,
    86  			ServiceIPRange:          net.IPNet{IP: netutils.ParseIPSloppy("10.0.0.0"), Mask: net.CIDRMask(24, 32)},
    87  		},
    88  	}
    89  
    90  	storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
    91  	storageConfig.StorageObjectCountTracker = config.GenericConfig.StorageObjectCountTracker
    92  	resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides)
    93  	storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
    94  	etcdOptions := options.NewEtcdOptions(storageConfig)
    95  	// unit tests don't need watch cache and it leaks lots of goroutines with etcd testing functions during unit tests
    96  	etcdOptions.EnableWatchCache = false
    97  	err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.GenericConfig)
    98  	if err != nil {
    99  		t.Fatal(err)
   100  	}
   101  
   102  	kubeVersion := kubeversion.Get()
   103  	config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
   104  	config.GenericConfig.Version = &kubeVersion
   105  	config.ExtraConfig.StorageFactory = storageFactory
   106  	config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
   107  	config.GenericConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
   108  	config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
   109  	config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
   110  	config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
   111  		DialContext:     func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil },
   112  		TLSClientConfig: &tls.Config{},
   113  	})
   114  
   115  	// set fake SecureServingInfo because the listener port is needed for the kubernetes service
   116  	config.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}}
   117  
   118  	getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions)
   119  	namer := openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)
   120  	config.GenericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer)
   121  
   122  	clientset, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig)
   123  	if err != nil {
   124  		t.Fatalf("unable to create client set due to %v", err)
   125  	}
   126  	config.ExtraConfig.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.GenericConfig.LoopbackClientConfig.Timeout)
   127  
   128  	return server, *config, assert.New(t)
   129  }
   130  
   131  type fakeLocalhost443Listener struct{}
   132  
   133  func (fakeLocalhost443Listener) Accept() (net.Conn, error) {
   134  	return nil, nil
   135  }
   136  
   137  func (fakeLocalhost443Listener) Close() error {
   138  	return nil
   139  }
   140  
   141  func (fakeLocalhost443Listener) Addr() net.Addr {
   142  	return &net.TCPAddr{
   143  		IP:   net.IPv4(127, 0, 0, 1),
   144  		Port: 443,
   145  	}
   146  }
   147  
   148  // TestLegacyRestStorageStrategies ensures that all Storage objects which are using the generic registry Store have
   149  // their various strategies properly wired up. This surfaced as a bug where strategies defined Export functions, but
   150  // they were never used outside of unit tests because the export strategies were not assigned inside the Store.
   151  func TestLegacyRestStorageStrategies(t *testing.T) {
   152  	_, etcdserver, apiserverCfg, _ := newInstance(t)
   153  	defer etcdserver.Terminate(t)
   154  
   155  	storageProvider, err := corerest.New(corerest.Config{
   156  		GenericConfig: corerest.GenericConfig{
   157  			StorageFactory:       apiserverCfg.ExtraConfig.StorageFactory,
   158  			EventTTL:             apiserverCfg.ExtraConfig.EventTTL,
   159  			LoopbackClientConfig: apiserverCfg.GenericConfig.LoopbackClientConfig,
   160  			Informers:            apiserverCfg.ExtraConfig.VersionedInformers,
   161  		},
   162  		Proxy: corerest.ProxyConfig{
   163  			Transport:           apiserverCfg.ExtraConfig.ProxyTransport,
   164  			KubeletClientConfig: apiserverCfg.ExtraConfig.KubeletClientConfig,
   165  		},
   166  		Services: corerest.ServicesConfig{
   167  			ClusterIPRange: apiserverCfg.ExtraConfig.ServiceIPRange,
   168  			NodePortRange:  apiserverCfg.ExtraConfig.ServiceNodePortRange,
   169  		},
   170  	})
   171  	if err != nil {
   172  		t.Fatalf("unexpected error from REST storage: %v", err)
   173  	}
   174  
   175  	apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.GenericConfig.RESTOptionsGetter)
   176  	if err != nil {
   177  		t.Errorf("failed to create legacy REST storage: %v", err)
   178  	}
   179  
   180  	strategyErrors := registrytest.ValidateStorageStrategies(apiGroupInfo.VersionedResourcesStorageMap["v1"])
   181  	for _, err := range strategyErrors {
   182  		t.Error(err)
   183  	}
   184  }
   185  
   186  func TestCertificatesRestStorageStrategies(t *testing.T) {
   187  	_, etcdserver, apiserverCfg, _ := newInstance(t)
   188  	defer etcdserver.Terminate(t)
   189  
   190  	certStorageProvider := certificatesrest.RESTStorageProvider{}
   191  	apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ExtraConfig.APIResourceConfigSource, apiserverCfg.GenericConfig.RESTOptionsGetter)
   192  	if err != nil {
   193  		t.Fatalf("unexpected error from REST storage: %v", err)
   194  	}
   195  
   196  	strategyErrors := registrytest.ValidateStorageStrategies(
   197  		apiGroupInfo.VersionedResourcesStorageMap[certificatesapiv1beta1.SchemeGroupVersion.Version])
   198  	for _, err := range strategyErrors {
   199  		t.Error(err)
   200  	}
   201  }
   202  
   203  func newInstance(t *testing.T) (*Instance, *etcd3testing.EtcdTestServer, Config, *assert.Assertions) {
   204  	etcdserver, config, assert := setUp(t)
   205  
   206  	apiserver, err := config.Complete().New(genericapiserver.NewEmptyDelegate())
   207  	if err != nil {
   208  		t.Fatalf("Error in bringing up the master: %v", err)
   209  	}
   210  
   211  	return apiserver, etcdserver, config, assert
   212  }
   213  
   214  // TestVersion tests /version
   215  func TestVersion(t *testing.T) {
   216  	s, etcdserver, _, _ := newInstance(t)
   217  	defer etcdserver.Terminate(t)
   218  
   219  	req, _ := http.NewRequest("GET", "/version", nil)
   220  	resp := httptest.NewRecorder()
   221  	s.GenericAPIServer.Handler.ServeHTTP(resp, req)
   222  	if resp.Code != 200 {
   223  		t.Fatalf("expected http 200, got: %d", resp.Code)
   224  	}
   225  
   226  	var info version.Info
   227  	err := json.NewDecoder(resp.Body).Decode(&info)
   228  	if err != nil {
   229  		t.Errorf("unexpected error: %v", err)
   230  	}
   231  
   232  	if !reflect.DeepEqual(kubeversion.Get(), info) {
   233  		t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info)
   234  	}
   235  }
   236  
   237  func decodeResponse(resp *http.Response, obj interface{}) error {
   238  	defer resp.Body.Close()
   239  
   240  	data, err := io.ReadAll(resp.Body)
   241  	if err != nil {
   242  		return err
   243  	}
   244  	if err := json.Unmarshal(data, obj); err != nil {
   245  		return err
   246  	}
   247  	return nil
   248  }
   249  
   250  // Because we need to be backwards compatible with release 1.1, at endpoints
   251  // that exist in release 1.1, the responses should have empty APIVersion.
   252  func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) {
   253  	apiserver, etcdserver, _, assert := newInstance(t)
   254  	defer etcdserver.Terminate(t)
   255  
   256  	server := httptest.NewServer(apiserver.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
   257  
   258  	// /api exists in release-1.1
   259  	resp, err := http.Get(server.URL + "/api")
   260  	if err != nil {
   261  		t.Errorf("unexpected error: %v", err)
   262  	}
   263  	apiVersions := metav1.APIVersions{}
   264  	assert.NoError(decodeResponse(resp, &apiVersions))
   265  	assert.Equal(apiVersions.APIVersion, "")
   266  
   267  	// /api/v1 exists in release-1.1
   268  	resp, err = http.Get(server.URL + "/api/v1")
   269  	if err != nil {
   270  		t.Errorf("unexpected error: %v", err)
   271  	}
   272  	resourceList := metav1.APIResourceList{}
   273  	assert.NoError(decodeResponse(resp, &resourceList))
   274  	assert.Equal(resourceList.APIVersion, "")
   275  
   276  	// /apis exists in release-1.1
   277  	resp, err = http.Get(server.URL + "/apis")
   278  	if err != nil {
   279  		t.Errorf("unexpected error: %v", err)
   280  	}
   281  	groupList := metav1.APIGroupList{}
   282  	assert.NoError(decodeResponse(resp, &groupList))
   283  	assert.Equal(groupList.APIVersion, "")
   284  
   285  	// /apis/autoscaling doesn't exist in release-1.1, so the APIVersion field
   286  	// should be non-empty in the results returned by the server.
   287  	resp, err = http.Get(server.URL + "/apis/autoscaling")
   288  	if err != nil {
   289  		t.Errorf("unexpected error: %v", err)
   290  	}
   291  	group := metav1.APIGroup{}
   292  	assert.NoError(decodeResponse(resp, &group))
   293  	assert.Equal(group.APIVersion, "v1")
   294  
   295  	// apis/autoscaling/v1 doesn't exist in release-1.1, so the APIVersion field
   296  	// should be non-empty in the results returned by the server.
   297  
   298  	resp, err = http.Get(server.URL + "/apis/autoscaling/v1")
   299  	if err != nil {
   300  		t.Errorf("unexpected error: %v", err)
   301  	}
   302  	resourceList = metav1.APIResourceList{}
   303  	assert.NoError(decodeResponse(resp, &resourceList))
   304  	assert.Equal(resourceList.APIVersion, "v1")
   305  
   306  }
   307  
   308  // This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers.
   309  func TestStorageVersionHashes(t *testing.T) {
   310  	apiserver, etcdserver, _, _ := newInstance(t)
   311  	defer etcdserver.Terminate(t)
   312  
   313  	server := httptest.NewServer(apiserver.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
   314  
   315  	c := &restclient.Config{
   316  		Host:          server.URL,
   317  		APIPath:       "/api",
   318  		ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs},
   319  	}
   320  	discover := discovery.NewDiscoveryClientForConfigOrDie(c).WithLegacy()
   321  	_, all, err := discover.ServerGroupsAndResources()
   322  	if err != nil {
   323  		t.Error(err)
   324  	}
   325  	var count int
   326  	apiResources := sets.NewString()
   327  	for _, g := range all {
   328  		for _, r := range g.APIResources {
   329  			apiResources.Insert(g.GroupVersion + "/" + r.Name)
   330  			if strings.Contains(r.Name, "/") ||
   331  				storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) {
   332  				if r.StorageVersionHash != "" {
   333  					t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash)
   334  				}
   335  				continue
   336  			}
   337  			if r.StorageVersionHash == "" {
   338  				t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name)
   339  				continue
   340  			}
   341  			// Uncomment the following line if you want to update storageversionhash/data.go
   342  			// fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash)
   343  			expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name]
   344  			if r.StorageVersionHash != expected {
   345  				t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash)
   346  			}
   347  			count++
   348  		}
   349  	}
   350  	if count != len(storageversionhashdata.GVRToStorageVersionHash) {
   351  		knownResources := sets.StringKeySet(storageversionhashdata.GVRToStorageVersionHash)
   352  		t.Errorf("please remove the redundant entries from GVRToStorageVersionHash: %v", knownResources.Difference(apiResources).List())
   353  	}
   354  }
   355  
   356  func TestNoAlphaVersionsEnabledByDefault(t *testing.T) {
   357  	config := DefaultAPIResourceConfigSource()
   358  	for gv, enable := range config.GroupVersionConfigs {
   359  		if enable && strings.Contains(gv.Version, "alpha") {
   360  			t.Errorf("Alpha API version %s enabled by default", gv.String())
   361  		}
   362  	}
   363  
   364  	for gvr, enabled := range config.ResourceConfigs {
   365  		if !strings.Contains(gvr.Version, "alpha") || !enabled {
   366  			continue
   367  		}
   368  
   369  		// we have enabled an alpha api by resource {g,v,r}, we also expect the
   370  		// alpha api by version {g,v} to be disabled. This is so a programmer
   371  		// remembers to add the new alpha version to alphaAPIGroupVersionsDisabledByDefault.
   372  		gr := gvr.GroupVersion()
   373  		if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled {
   374  			t.Errorf("Alpha API version %q should be disabled by default", gr.String())
   375  		}
   376  	}
   377  }
   378  
   379  func TestNoBetaVersionsEnabledByDefault(t *testing.T) {
   380  	config := DefaultAPIResourceConfigSource()
   381  	for gv, enable := range config.GroupVersionConfigs {
   382  		if enable && strings.Contains(gv.Version, "beta") {
   383  			t.Errorf("Beta API version %s enabled by default", gv.String())
   384  		}
   385  	}
   386  
   387  	for gvr, enabled := range config.ResourceConfigs {
   388  		if !strings.Contains(gvr.Version, "beta") || !enabled {
   389  			continue
   390  		}
   391  
   392  		// we have enabled a beta api by resource {g,v,r}, we also expect the
   393  		// beta api by version {g,v} to be disabled. This is so a programmer
   394  		// remembers to add the new beta version to betaAPIGroupVersionsDisabledByDefault.
   395  		gr := gvr.GroupVersion()
   396  		if enabled, found := config.GroupVersionConfigs[gr]; !found || enabled {
   397  			t.Errorf("Beta API version %q should be disabled by default", gr.String())
   398  		}
   399  	}
   400  }
   401  
   402  func TestDefaultVars(t *testing.T) {
   403  	// stableAPIGroupVersionsEnabledByDefault should not contain beta or alpha
   404  	for i := range stableAPIGroupVersionsEnabledByDefault {
   405  		gv := stableAPIGroupVersionsEnabledByDefault[i]
   406  		if strings.Contains(gv.Version, "beta") || strings.Contains(gv.Version, "alpha") {
   407  			t.Errorf("stableAPIGroupVersionsEnabledByDefault should contain stable version, but found: %q", gv.String())
   408  		}
   409  	}
   410  
   411  	// legacyBetaEnabledByDefaultResources should contain only beta version
   412  	for i := range legacyBetaEnabledByDefaultResources {
   413  		gv := legacyBetaEnabledByDefaultResources[i]
   414  		if !strings.Contains(gv.Version, "beta") {
   415  			t.Errorf("legacyBetaEnabledByDefaultResources should contain beta version, but found: %q", gv.String())
   416  		}
   417  	}
   418  
   419  	// betaAPIGroupVersionsDisabledByDefault should contain only beta version
   420  	for i := range betaAPIGroupVersionsDisabledByDefault {
   421  		gv := betaAPIGroupVersionsDisabledByDefault[i]
   422  		if !strings.Contains(gv.Version, "beta") {
   423  			t.Errorf("betaAPIGroupVersionsDisabledByDefault should contain beta version, but found: %q", gv.String())
   424  		}
   425  	}
   426  
   427  	// alphaAPIGroupVersionsDisabledByDefault should contain only alpha version
   428  	for i := range alphaAPIGroupVersionsDisabledByDefault {
   429  		gv := alphaAPIGroupVersionsDisabledByDefault[i]
   430  		if !strings.Contains(gv.Version, "alpha") {
   431  			t.Errorf("alphaAPIGroupVersionsDisabledByDefault should contain alpha version, but found: %q", gv.String())
   432  		}
   433  	}
   434  }
   435  
   436  func TestNewBetaResourcesEnabledByDefault(t *testing.T) {
   437  	// legacyEnabledBetaResources is nearly a duplication from elsewhere.  This is intentional.  These types already have
   438  	// GA equivalents available and should therefore never have a beta enabled by default again.
   439  	legacyEnabledBetaResources := map[schema.GroupVersionResource]bool{
   440  		autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
   441  		autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
   442  		batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"):                       true,
   443  		discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"):                true,
   444  		eventsv1beta1.SchemeGroupVersion.WithResource("events"):                           true,
   445  		nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"):                     true,
   446  		policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"):          true,
   447  		policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"):           true,
   448  		storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"):                     true,
   449  	}
   450  
   451  	// legacyBetaResourcesWithoutStableEquivalents contains those groupresources that were enabled by default as beta
   452  	// before we changed that policy and do not have stable versions. These resources are allowed to have additional
   453  	// beta versions enabled by default.  Nothing new should be added here.  There are no future exceptions because there
   454  	// are no more beta resources enabled by default.
   455  	legacyBetaResourcesWithoutStableEquivalents := map[schema.GroupResource]bool{
   456  		flowcontrolv1bet3.SchemeGroupVersion.WithResource("flowschemas").GroupResource():                 true,
   457  		flowcontrolv1bet3.SchemeGroupVersion.WithResource("prioritylevelconfigurations").GroupResource(): true,
   458  	}
   459  
   460  	config := DefaultAPIResourceConfigSource()
   461  	for gvr, enable := range config.ResourceConfigs {
   462  		if !strings.Contains(gvr.Version, "beta") {
   463  			continue // only check beta things
   464  		}
   465  		if !enable {
   466  			continue // only check things that are enabled
   467  		}
   468  		if legacyEnabledBetaResources[gvr] {
   469  			continue // this is a legacy beta resource
   470  		}
   471  		if legacyBetaResourcesWithoutStableEquivalents[gvr.GroupResource()] {
   472  			continue // this is another beta of a legacy beta resource with no stable equivalent
   473  		}
   474  		t.Errorf("no new beta resources can be enabled by default, see https://github.com/kubernetes/enhancements/blob/0ad0fc8269165ca300d05ca51c7ce190a79976a5/keps/sig-architecture/3136-beta-apis-off-by-default/README.md: %v", gvr)
   475  	}
   476  }
   477
View as plain text