Source file src/k8s.io/kubernetes/test/integration/apiserver/discovery/discovery_test.go

Documentation: k8s.io/kubernetes/test/integration/apiserver/discovery

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package discovery
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"net/http"
    24  	"reflect"
    25  	"strings"
    26  	"testing"
    27  
    28  	"github.com/google/go-cmp/cmp"
    29  	"github.com/stretchr/testify/require"
    30  
    31  	apidiscoveryv2 "k8s.io/api/apidiscovery/v2"
    32  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    33  	apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    34  	"k8s.io/apimachinery/pkg/api/meta"
    35  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	"k8s.io/apimachinery/pkg/runtime"
    37  	"k8s.io/apimachinery/pkg/runtime/schema"
    38  	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
    39  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    40  	"k8s.io/apimachinery/pkg/util/sets"
    41  	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
    42  	genericfeatures "k8s.io/apiserver/pkg/features"
    43  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    44  	"k8s.io/client-go/discovery"
    45  	"k8s.io/client-go/dynamic"
    46  	kubernetes "k8s.io/client-go/kubernetes"
    47  	k8sscheme "k8s.io/client-go/kubernetes/scheme"
    48  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    49  	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
    50  	aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
    51  	aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
    52  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    53  
    54  	"k8s.io/kubernetes/test/integration/framework"
    55  )
    56  
    57  type kubeClientSet = kubernetes.Interface
    58  
    59  type aggegatorClientSet = aggregator.Interface
    60  
    61  type apiextensionsClientSet = apiextensions.Interface
    62  
    63  type dynamicClientset = dynamic.Interface
    64  type testClientSet struct {
    65  	kubeClientSet
    66  	aggegatorClientSet
    67  	apiextensionsClientSet
    68  	dynamicClientset
    69  }
    70  
    71  var _ testClient = testClientSet{}
    72  
    73  func (t testClientSet) Discovery() discovery.DiscoveryInterface {
    74  	return t.kubeClientSet.Discovery()
    75  }
    76  
    77  var (
    78  	scheme    = runtime.NewScheme()
    79  	codecs    = runtimeserializer.NewCodecFactory(scheme)
    80  	serialize runtime.NegotiatedSerializer
    81  
    82  	basicTestGroup = apidiscoveryv2.APIGroupDiscovery{
    83  		ObjectMeta: metav1.ObjectMeta{
    84  			Name: "stable.example.com",
    85  		},
    86  		Versions: []apidiscoveryv2.APIVersionDiscovery{
    87  			{
    88  				Version: "v1",
    89  				Resources: []apidiscoveryv2.APIResourceDiscovery{
    90  					{
    91  						Resource:   "jobs",
    92  						Verbs:      []string{"create", "list", "watch", "delete"},
    93  						ShortNames: []string{"jz"},
    94  						Categories: []string{"all"},
    95  					},
    96  				},
    97  				Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent,
    98  			},
    99  		},
   100  	}
   101  
   102  	basicTestGroupWithFixup = apidiscoveryv2.APIGroupDiscovery{
   103  		ObjectMeta: metav1.ObjectMeta{
   104  			Name: "stable.example.com",
   105  		},
   106  		Versions: []apidiscoveryv2.APIVersionDiscovery{
   107  			{
   108  				Version: "v1",
   109  				Resources: []apidiscoveryv2.APIResourceDiscovery{
   110  					{
   111  						Resource:   "jobs",
   112  						Verbs:      []string{"create", "list", "watch", "delete"},
   113  						ShortNames: []string{"jz"},
   114  						Categories: []string{"all"},
   115  						// aggregator will populate this with a non-nil value
   116  						ResponseKind: &metav1.GroupVersionKind{},
   117  					},
   118  				},
   119  				Freshness: apidiscoveryv2.DiscoveryFreshnessCurrent,
   120  			},
   121  		},
   122  	}
   123  
   124  	basicTestGroupStale = apidiscoveryv2.APIGroupDiscovery{
   125  		ObjectMeta: metav1.ObjectMeta{
   126  			Name: "stable.example.com",
   127  		},
   128  		Versions: []apidiscoveryv2.APIVersionDiscovery{
   129  			{
   130  				Version:   "v1",
   131  				Freshness: apidiscoveryv2.DiscoveryFreshnessStale,
   132  			},
   133  		},
   134  	}
   135  
   136  	stableGroup    = "stable.example.com"
   137  	stableV1       = metav1.GroupVersion{Group: stableGroup, Version: "v1"}
   138  	stableV1alpha1 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha1"}
   139  	stableV1alpha2 = metav1.GroupVersion{Group: stableGroup, Version: "v1alpha2"}
   140  	stableV1beta1  = metav1.GroupVersion{Group: stableGroup, Version: "v1beta1"}
   141  	stableV2       = metav1.GroupVersion{Group: stableGroup, Version: "v2"}
   142  )
   143  
   144  func init() {
   145  	// Add all builtin types to scheme
   146  	utilruntime.Must(k8sscheme.AddToScheme(scheme))
   147  	utilruntime.Must(aggregatorclientsetscheme.AddToScheme(scheme))
   148  	utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
   149  
   150  	info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON)
   151  	if !ok {
   152  		panic("failed to create serializer info")
   153  	}
   154  
   155  	serialize = runtime.NewSimpleNegotiatedSerializer(info)
   156  }
   157  
   158  // Spins up an api server which is cleaned up at the end up the test
   159  // Returns some kubernetes clients
   160  func setup(t *testing.T) (context.Context, testClientSet, context.CancelFunc) {
   161  	ctx, cancelCtx := context.WithCancel(context.Background())
   162  
   163  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
   164  	t.Cleanup(server.TearDownFn)
   165  
   166  	kubeClientSet, err := kubernetes.NewForConfig(server.ClientConfig)
   167  	require.NoError(t, err)
   168  
   169  	aggegatorClientSet, err := aggregator.NewForConfig(server.ClientConfig)
   170  	require.NoError(t, err)
   171  
   172  	apiextensionsClientSet, err := apiextensions.NewForConfig(server.ClientConfig)
   173  	require.NoError(t, err)
   174  
   175  	dynamicClientset, err := dynamic.NewForConfig(server.ClientConfig)
   176  	require.NoError(t, err)
   177  
   178  	client := testClientSet{
   179  		kubeClientSet:          kubeClientSet,
   180  		aggegatorClientSet:     aggegatorClientSet,
   181  		apiextensionsClientSet: apiextensionsClientSet,
   182  		dynamicClientset:       dynamicClientset,
   183  	}
   184  	return ctx, client, cancelCtx
   185  }
   186  
   187  func TestReadinessAggregatedAPIServiceDiscovery(t *testing.T) {
   188  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
   189  
   190  	// Keep any goroutines spawned from running past the execution of this test
   191  	ctx, client, cleanup := setup(t)
   192  	defer cleanup()
   193  
   194  	// Create a resource manager whichs serves our GroupVersion
   195  	resourceManager := discoveryendpoint.NewResourceManager("apis")
   196  	resourceManager.SetGroups([]apidiscoveryv2.APIGroupDiscovery{basicTestGroup})
   197  
   198  	apiServiceWaitCh := make(chan struct{})
   199  
   200  	// Install our ResourceManager as an Aggregated APIService to the
   201  	// test server
   202  	service := NewFakeService("test-server", client, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   203  		if strings.HasPrefix(r.URL.Path, "/apis/stable.example.com") {
   204  			// Return invalid response so APIService can be marked as "available"
   205  			w.WriteHeader(http.StatusOK)
   206  		} else if strings.HasPrefix(r.URL.Path, "/apis") {
   207  			select {
   208  			case <-apiServiceWaitCh:
   209  				// Hang responding to discovery until aggregated discovery document contains the aggregated group marked as Stale.
   210  				resourceManager.ServeHTTP(w, r)
   211  			case <-ctx.Done():
   212  				return
   213  			}
   214  		} else {
   215  			// reject openapi/v2, openapi/v3, apis/<group>/<version>
   216  			w.WriteHeader(http.StatusNotFound)
   217  		}
   218  	}))
   219  	go func() {
   220  		require.NoError(t, service.Run(ctx))
   221  	}()
   222  	require.NoError(t, service.WaitForReady(ctx))
   223  
   224  	// For each groupversion served by our resourcemanager, create an APIService
   225  	// object connected to our fake APIServer
   226  	for _, versionInfo := range basicTestGroup.Versions {
   227  		groupVersion := metav1.GroupVersion{
   228  			Group:   basicTestGroup.Name,
   229  			Version: versionInfo.Version,
   230  		}
   231  
   232  		require.NoError(t, registerAPIService(ctx, client, groupVersion, service))
   233  	}
   234  
   235  	// Keep repeatedly fetching document from aggregator.
   236  	// Check to see if it initially contains the aggregated group as stale
   237  	require.NoError(t, WaitForGroups(ctx, client, basicTestGroupStale))
   238  	require.NoError(t, WaitForRootPaths(t, ctx, client, sets.New("/apis/"+basicTestGroup.Name), nil))
   239  
   240  	// Allow the APIService to start responding and ensure that Freshness is updated when the APIService is reacheable.
   241  	close(apiServiceWaitCh)
   242  	require.NoError(t, WaitForGroups(ctx, client, basicTestGroupWithFixup))
   243  }
   244  
   245  func registerAPIService(ctx context.Context, client aggregator.Interface, gv metav1.GroupVersion, service FakeService) error {
   246  	port := service.Port()
   247  	if port == nil {
   248  		return errors.New("service not yet started")
   249  	}
   250  	// Register the APIService
   251  	patch := apiregistrationv1.APIService{
   252  		ObjectMeta: metav1.ObjectMeta{
   253  			Name: gv.Version + "." + gv.Group,
   254  		},
   255  		TypeMeta: metav1.TypeMeta{
   256  			Kind:       "APIService",
   257  			APIVersion: "apiregistration.k8s.io/v1",
   258  		},
   259  		Spec: apiregistrationv1.APIServiceSpec{
   260  			Group:                 gv.Group,
   261  			Version:               gv.Version,
   262  			InsecureSkipTLSVerify: true,
   263  			GroupPriorityMinimum:  1000,
   264  			VersionPriority:       15,
   265  			Service: &apiregistrationv1.ServiceReference{
   266  				Namespace: "default",
   267  				Name:      service.Name(),
   268  				Port:      port,
   269  			},
   270  		},
   271  	}
   272  
   273  	_, err := client.
   274  		ApiregistrationV1().
   275  		APIServices().
   276  		Create(context.TODO(), &patch, metav1.CreateOptions{FieldManager: "test-manager"})
   277  	return err
   278  }
   279  
   280  func unregisterAPIService(ctx context.Context, client aggregator.Interface, gv metav1.GroupVersion) error {
   281  	return client.ApiregistrationV1().APIServices().Delete(ctx, gv.Version+"."+gv.Group, metav1.DeleteOptions{})
   282  }
   283  
   284  func TestAggregatedAPIServiceDiscovery(t *testing.T) {
   285  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
   286  
   287  	// Keep any goroutines spawned from running past the execution of this test
   288  	ctx, client, cleanup := setup(t)
   289  	defer cleanup()
   290  
   291  	// Create a resource manager whichs serves our GroupVersion
   292  	resourceManager := discoveryendpoint.NewResourceManager("apis")
   293  	resourceManager.SetGroups([]apidiscoveryv2.APIGroupDiscovery{basicTestGroup})
   294  
   295  	// Install our ResourceManager as an Aggregated APIService to the
   296  	// test server
   297  	service := NewFakeService("test-server", client, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   298  		if strings.HasPrefix(r.URL.Path, "/apis") {
   299  			resourceManager.ServeHTTP(w, r)
   300  		} else if strings.HasPrefix(r.URL.Path, "/apis/stable.example.com") {
   301  			// Return invalid response so APIService can be marked as "available"
   302  			w.WriteHeader(http.StatusOK)
   303  		} else {
   304  			// reject openapi/v2, openapi/v3, apis/<group>/<version>
   305  			w.WriteHeader(http.StatusNotFound)
   306  		}
   307  	}))
   308  	go func() {
   309  		require.NoError(t, service.Run(ctx))
   310  	}()
   311  	require.NoError(t, service.WaitForReady(ctx))
   312  
   313  	// For each groupversion served by our resourcemanager, create an APIService
   314  	// object connected to our fake APIServer
   315  	var groupVersions []metav1.GroupVersion
   316  	for _, versionInfo := range basicTestGroup.Versions {
   317  		groupVersion := metav1.GroupVersion{
   318  			Group:   basicTestGroup.Name,
   319  			Version: versionInfo.Version,
   320  		}
   321  
   322  		require.NoError(t, registerAPIService(ctx, client, groupVersion, service))
   323  		groupVersions = append(groupVersions, groupVersion)
   324  	}
   325  
   326  	// Keep repeatedly fetching document from aggregator.
   327  	// Check to see if it contains our service within a reasonable amount of time
   328  	require.NoError(t, WaitForGroups(ctx, client, basicTestGroupWithFixup))
   329  	require.NoError(t, WaitForRootPaths(t, ctx, client, sets.New("/apis/"+basicTestGroup.Name), nil))
   330  
   331  	// Unregister and ensure the group gets dropped from root paths
   332  	for _, groupVersion := range groupVersions {
   333  		require.NoError(t, unregisterAPIService(ctx, client, groupVersion))
   334  	}
   335  	require.NoError(t, WaitForRootPaths(t, ctx, client, nil, sets.New("/apis/"+basicTestGroup.Name)))
   336  }
   337  
   338  func runTestCases(t *testing.T, cases []testCase) {
   339  	// Keep any goroutines spawned from running past the execution of this test
   340  	ctx, client, cleanup := setup(t)
   341  	defer cleanup()
   342  
   343  	// Fetch the original discovery information so we can wait for it to
   344  	// reset between tests
   345  	originalV1, err := FetchV1DiscoveryGroups(ctx, client)
   346  	require.NoError(t, err)
   347  
   348  	originalV2, err := FetchV2Discovery(ctx, client)
   349  	require.NoError(t, err)
   350  
   351  	for _, c := range cases {
   352  		t.Run(c.Name, func(t *testing.T) {
   353  			func() {
   354  				testContext, testDone := context.WithCancel(ctx)
   355  				defer testDone()
   356  
   357  				for i, a := range c.Actions {
   358  					if cleaning, ok := a.(cleaningAction); ok {
   359  						defer func() {
   360  							require.NoError(t, cleaning.Cleanup(testContext, client), "cleanup after \"%T\" step %v", a, i)
   361  						}()
   362  					}
   363  					require.NoError(t, a.Do(testContext, client), "running \"%T\" step %v", a, i)
   364  				}
   365  			}()
   366  
   367  			var diff string
   368  			err := WaitForV1GroupsWithCondition(ctx, client, func(result metav1.APIGroupList) bool {
   369  				diff = cmp.Diff(originalV1, result)
   370  				return reflect.DeepEqual(result, originalV1)
   371  			})
   372  			require.NoError(t, err, "v1 discovery must reset between tests: "+diff)
   373  
   374  			err = WaitForResultWithCondition(ctx, client, func(result apidiscoveryv2.APIGroupDiscoveryList) bool {
   375  				diff = cmp.Diff(originalV2, result)
   376  				return reflect.DeepEqual(result, originalV2)
   377  			})
   378  			require.NoError(t, err, "v2 discovery must reset between tests: "+diff)
   379  		})
   380  	}
   381  }
   382  
   383  // Declarative tests targeting CRD integration
   384  func TestCRD(t *testing.T) {
   385  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
   386  
   387  	runTestCases(t, []testCase{
   388  		{
   389  			// Show that when a CRD is added it gets included on the discovery doc
   390  			// within a reasonable amount of time
   391  			Name: "CRDInclusion",
   392  			Actions: []testAction{
   393  				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
   394  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   395  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   396  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   397  			},
   398  		},
   399  		{
   400  			// Show that a CRD added to the discovery doc can also be removed
   401  			Name: "CRDRemoval",
   402  			Actions: []testAction{
   403  				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
   404  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   405  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   406  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   407  				deleteObject{
   408  					GroupVersionResource: metav1.GroupVersionResource(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")),
   409  					Name:                 "foos.stable.example.com",
   410  				},
   411  				waitForAbsentGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   412  				waitForAbsentGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   413  			},
   414  		},
   415  		{
   416  			// Show that if CRD and APIService share a groupversion, and the
   417  			// APIService is deleted, and CRD updated, the APIService remains in
   418  			// discovery.
   419  			// This test simulates a resync of CRD controler to show that eventually
   420  			// APIService is recreated
   421  			Name: "CRDAPIServiceOverlap",
   422  			Actions: []testAction{
   423  				applyAPIService(
   424  					apiregistrationv1.APIServiceSpec{
   425  						Group:                 stableGroup,
   426  						Version:               "v1",
   427  						InsecureSkipTLSVerify: true,
   428  						GroupPriorityMinimum:  int32(1000),
   429  						VersionPriority:       int32(15),
   430  						Service: &apiregistrationv1.ServiceReference{
   431  							Name:      "unused",
   432  							Namespace: "default",
   433  						},
   434  					},
   435  				),
   436  
   437  				// Wait for GV to appear in both discovery documents
   438  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1}),
   439  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
   440  
   441  				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2"})),
   442  
   443  				// Show that we have v1 and v2 but v1 is stale
   444  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2}),
   445  				waitForStaleGroupVersionsV2([]metav1.GroupVersion{stableV1}),
   446  				waitForFreshGroupVersionsV2([]metav1.GroupVersion{stableV2}),
   447  
   448  				// Delete APIService shared by the aggregated apiservice and
   449  				// CRD
   450  				deleteObject{
   451  					GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")),
   452  					Name:                 "v1.stable.example.com",
   453  				},
   454  
   455  				// Update CRD to trigger a resync by adding a category and new groupversion
   456  				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2", "v1alpha1"}, "all")),
   457  
   458  				// Show that the groupversion is re-added back
   459  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
   460  				waitForFreshGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2, stableV1alpha1}),
   461  			},
   462  		},
   463  		{
   464  			// Show that if CRD and Aggregated APIservice share a groupversiom,
   465  			// The aggregated apiservice's discovery information is shown in both
   466  			// v1 and v2 discovery
   467  			Name: "CRDAPIServiceSameGroupDifferentVersions",
   468  			Actions: []testAction{
   469  				// Wait for CRD to apply
   470  				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v2", "v1alpha1"})),
   471  				// Wait for GV to appear in both discovery documents
   472  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   473  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   474  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   475  
   476  				applyAPIService(
   477  					apiregistrationv1.APIServiceSpec{
   478  						Group:                 stableGroup,
   479  						Version:               "v1",
   480  						InsecureSkipTLSVerify: true,
   481  						GroupPriorityMinimum:  int32(1000),
   482  						VersionPriority:       int32(100),
   483  						Service: &apiregistrationv1.ServiceReference{
   484  							Name:      "unused",
   485  							Namespace: "default",
   486  						},
   487  					},
   488  				),
   489  
   490  				// We should now have stable v1 available
   491  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1}),
   492  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
   493  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV1}),
   494  
   495  				// The CRD group-versions not served by the aggregated
   496  				// apiservice should still be availablee
   497  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   498  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   499  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   500  
   501  				// Remove API service. Show we have switched to CRD
   502  				deleteObject{
   503  					GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")),
   504  					Name:                 "v1.stable.example.com",
   505  				},
   506  
   507  				// Show that we still have stable v1 since it is in the CRD
   508  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   509  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   510  				waitForGroupVersionsV2Beta1([]metav1.GroupVersion{stableV2, stableV1alpha1}),
   511  
   512  				waitForAbsentGroupVersionsV1([]metav1.GroupVersion{stableV1}),
   513  				waitForAbsentGroupVersionsV2([]metav1.GroupVersion{stableV1}),
   514  				waitForAbsentGroupVersionsV2Beta1([]metav1.GroupVersion{stableV1}),
   515  			},
   516  		},
   517  		{
   518  			// Show that if CRD and a builtin share a group version,
   519  			// the builtin takes precedence in both versions of discovery
   520  			Name: "CRDBuiltinOverlapPrecence",
   521  			Actions: []testAction{
   522  				// Create CRD that overrides a builtin
   523  				applyCRD(makeCRDSpec("apiextensions.k8s.io", "Bar", true, []string{"v1", "v2", "vfake"})),
   524  
   525  				waitForGroupVersionsV1([]metav1.GroupVersion{{Group: "apiextensions.k8s.io", Version: "vfake"}}),
   526  				waitForGroupVersionsV2([]metav1.GroupVersion{{Group: "apiextensions.k8s.io", Version: "vfake"}}),
   527  
   528  				// Show that the builtin group-version is still used for V1
   529  				// By showing presence of v1.CustomResourceDefinition
   530  				// and absence of v1.Bar
   531  				waitForResourcesV1([]metav1.GroupVersionResource{
   532  					{
   533  						Group:    "apiextensions.k8s.io",
   534  						Version:  "v1",
   535  						Resource: "customresourcedefinitions",
   536  					},
   537  					{
   538  						Group:    "apiextensions.k8s.io",
   539  						Version:  "vfake",
   540  						Resource: "bars",
   541  					},
   542  				}),
   543  				waitForResourcesV2([]metav1.GroupVersionResource{
   544  					{
   545  						Group:    "apiextensions.k8s.io",
   546  						Version:  "v1",
   547  						Resource: "customresourcedefinitions",
   548  					},
   549  					{
   550  						Group:    "apiextensions.k8s.io",
   551  						Version:  "vfake",
   552  						Resource: "bars",
   553  					},
   554  				}),
   555  
   556  				waitForResourcesAbsentV1([]metav1.GroupVersionResource{
   557  					{
   558  						Group:    "apiextensions.k8s.io",
   559  						Version:  "v1",
   560  						Resource: "bars",
   561  					},
   562  				}),
   563  				waitForResourcesAbsentV2([]metav1.GroupVersionResource{
   564  					{
   565  						Group:    "apiextensions.k8s.io",
   566  						Version:  "v1",
   567  						Resource: "bars",
   568  					},
   569  				}),
   570  			},
   571  		},
   572  		{
   573  			// Tests that a race discovered during alpha phase of the feature is fixed.
   574  			// Rare race would occur if a CRD was synced before the removal of an aggregated
   575  			// APIService could be synced.
   576  			// To test this we:
   577  			//  1. Add CRD to apiserver
   578  			// 	2. Wait for it to sync
   579  			//  3. Add aggregated APIService with same groupversion
   580  			//  4. Remove aggregated apiservice
   581  			//  5. Check that we have CRD GVs in discovery document
   582  			// Show that if CRD and APIService share a groupversion, and the
   583  			// APIService is deleted, and CRD updated, the groupversion from
   584  			// the CRD remains in discovery.
   585  			Name: "Race",
   586  			Actions: []testAction{
   587  				// Create CRD with the same GV as the aggregated APIService
   588  				applyCRD(makeCRDSpec(stableGroup, "Bar", false, []string{"v1", "v2"})),
   589  
   590  				// only CRD has stable v2,  this will show that CRD has been synced
   591  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2}),
   592  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2}),
   593  
   594  				// Add Aggregated APIService that overlaps the CRD.
   595  				applyAPIService(
   596  					apiregistrationv1.APIServiceSpec{
   597  						Group:                 stableGroup,
   598  						Version:               "v1",
   599  						InsecureSkipTLSVerify: true,
   600  						GroupPriorityMinimum:  int32(1000),
   601  						VersionPriority:       int32(100),
   602  						Service: &apiregistrationv1.ServiceReference{
   603  							Name:      "fake",
   604  							Namespace: "default",
   605  						},
   606  					},
   607  				),
   608  
   609  				// Delete APIService shared by the aggregated apiservice and
   610  				// CRD
   611  				deleteObject{
   612  					GroupVersionResource: metav1.GroupVersionResource(apiregistrationv1.SchemeGroupVersion.WithResource("apiservices")),
   613  					Name:                 "v1.stable.example.com",
   614  				},
   615  
   616  				// Show the CRD (with stablev2) is the one which is now advertised
   617  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV2}),
   618  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV2}),
   619  			},
   620  		},
   621  	})
   622  }
   623  
   624  func TestFreshness(t *testing.T) {
   625  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
   626  
   627  	requireStaleGVs := func(gvs ...metav1.GroupVersion) inlineAction {
   628  		return inlineAction(func(ctx context.Context, client testClient) error {
   629  			document, err := FetchV2Discovery(ctx, client)
   630  			if err != nil {
   631  				return nil
   632  			}
   633  
   634  			// Track the stale gvs in array for nice diff output upon test failure
   635  			staleGVs := []metav1.GroupVersion{}
   636  
   637  			// Iterate through input so order does not matter
   638  			for _, targetGv := range gvs {
   639  				entry := FindGroupVersionV2(document, targetGv)
   640  				if entry == nil {
   641  					continue
   642  				}
   643  
   644  				switch entry.Freshness {
   645  				case apidiscoveryv2.DiscoveryFreshnessCurrent:
   646  					// Skip
   647  				case apidiscoveryv2.DiscoveryFreshnessStale:
   648  					staleGVs = append(staleGVs, targetGv)
   649  				default:
   650  					return fmt.Errorf("unrecognized freshness '%v' on gv '%v'", entry.Freshness, targetGv)
   651  				}
   652  			}
   653  
   654  			if !(len(staleGVs) == 0 && len(gvs) == 0) && !reflect.DeepEqual(staleGVs, gvs) {
   655  				diff := cmp.Diff(staleGVs, gvs)
   656  				return fmt.Errorf("expected sets of stale gvs to be equal:\n%v", diff)
   657  			}
   658  
   659  			return nil
   660  		})
   661  	}
   662  
   663  	runTestCases(t, []testCase{
   664  		{
   665  			Name: "BuiltinsFresh",
   666  			Actions: []testAction{
   667  				// Wait for discovery ready
   668  				waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
   669  				// Require there are no stale groupversions and no unrecognized
   670  				// GVs
   671  				requireStaleGVs(),
   672  			},
   673  		},
   674  		{
   675  			// CRD freshness is always current
   676  			Name: "CRDFresh",
   677  			Actions: []testAction{
   678  				// Add a CRD and wait for it to appear in discovery
   679  				applyCRD(makeCRDSpec(stableGroup, "Foo", false, []string{"v1", "v1alpha1", "v1beta1", "v2"})),
   680  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   681  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1beta1, stableV2}),
   682  
   683  				// Test CRD is current by requiring there is nothing stale
   684  				requireStaleGVs(),
   685  			},
   686  		},
   687  		{
   688  			// Make an aggregated APIService that's unreachable and show
   689  			// that its groupversion is included in the discovery document as
   690  			// stale
   691  			Name: "AggregatedUnreachable",
   692  			Actions: []testAction{
   693  				applyAPIService{
   694  					Group:                stableGroup,
   695  					Version:              "v1",
   696  					GroupPriorityMinimum: 1000,
   697  					VersionPriority:      15,
   698  					Service: &apiregistrationv1.ServiceReference{
   699  						Name:      "doesnt-exist",
   700  						Namespace: "default",
   701  					},
   702  				},
   703  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1}),
   704  				// Require there is one and only one stale GV and it is stableV1
   705  				requireStaleGVs(stableV1),
   706  			},
   707  		},
   708  	})
   709  
   710  }
   711  
   712  // Shows a group for which multiple APIServices specify a GroupPriorityMinimum,
   713  // it is sorted the same in both versions of discovery
   714  func TestGroupPriority(t *testing.T) {
   715  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.AggregatedDiscoveryEndpoint, true)()
   716  
   717  	makeApiServiceSpec := func(gv metav1.GroupVersion, groupPriorityMin, versionPriority int) apiregistrationv1.APIServiceSpec {
   718  		return apiregistrationv1.APIServiceSpec{
   719  			Group:                 gv.Group,
   720  			Version:               gv.Version,
   721  			InsecureSkipTLSVerify: true,
   722  			GroupPriorityMinimum:  int32(groupPriorityMin),
   723  			VersionPriority:       int32(versionPriority),
   724  			Service: &apiregistrationv1.ServiceReference{
   725  				Name:      "unused",
   726  				Namespace: "default",
   727  			},
   728  		}
   729  	}
   730  
   731  	checkGVOrder := inlineAction(func(ctx context.Context, client testClient) (err error) {
   732  		// Fetch v1 document and v2 document, and ensure they have
   733  		// equal orderings of groupversions. and nothing missing or
   734  		// extra.
   735  		v1GroupsAndVersions, err := FetchV1DiscoveryGroups(ctx, client)
   736  		if err != nil {
   737  			return err
   738  		}
   739  		v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
   740  		if err != nil {
   741  			return err
   742  		}
   743  
   744  		v1Gvs := []metav1.GroupVersion{}
   745  		v2Gvs := []metav1.GroupVersion{}
   746  
   747  		for _, group := range v1GroupsAndVersions.Groups {
   748  			for _, version := range group.Versions {
   749  				v1Gvs = append(v1Gvs, metav1.GroupVersion{
   750  					Group:   group.Name,
   751  					Version: version.Version,
   752  				})
   753  			}
   754  		}
   755  
   756  		for _, group := range v2GroupsAndVersions.Items {
   757  			for _, version := range group.Versions {
   758  				v2Gvs = append(v2Gvs, metav1.GroupVersion{
   759  					Group:   group.Name,
   760  					Version: version.Version,
   761  				})
   762  			}
   763  		}
   764  
   765  		if !reflect.DeepEqual(v1Gvs, v2Gvs) {
   766  			return fmt.Errorf("expected equal orderings and lists of groupversions in both v1 and v2 discovery:\n%v", cmp.Diff(v1Gvs, v2Gvs))
   767  		}
   768  
   769  		return nil
   770  	})
   771  
   772  	runTestCases(t, []testCase{
   773  		{
   774  			// Show that the legacy and aggregated discovery docs have the same
   775  			// set of builtin groupversions
   776  			Name: "BuiltinsAndOrdering",
   777  			Actions: []testAction{
   778  				waitForGroupVersionsV1{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
   779  				waitForGroupVersionsV2{metav1.GroupVersion(apiregistrationv1.SchemeGroupVersion)},
   780  				checkGVOrder,
   781  			},
   782  		},
   783  		{
   784  			// Show that a very high priority group is sorted first (below apiregistration v1)
   785  			// Also show the ordering is same for both v1 and v2 discovery apis
   786  			// Does not vary version priority
   787  			Name: "HighGroupPriority",
   788  			Actions: []testAction{
   789  				// A VERY high priority which should take precedence
   790  				// 20000 is highest possible priority
   791  				applyAPIService(makeApiServiceSpec(stableV1, 20000, 15)),
   792  				// A VERY low priority which should be ignored
   793  				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
   794  				// A medium-high priority (that conflicts with k8s) which should be ignored
   795  				applyAPIService(makeApiServiceSpec(stableV1alpha2, 17300, 15)),
   796  				// Wait for all the added group-versions to appear in both discovery documents
   797  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
   798  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
   799  				// Check that both v1 and v2 endpoints have exactly the same
   800  				// sets of groupversions
   801  				checkGVOrder,
   802  				// Check that the first group-version is the one with the highest
   803  				// priority
   804  				inlineAction(func(ctx context.Context, client testClient) error {
   805  					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
   806  					if err != nil {
   807  						return err
   808  					}
   809  
   810  					// First group should always be apiregistration.k8s.io
   811  					secondGV := metav1.GroupVersion{
   812  						Group:   v2GroupsAndVersions.Items[1].Name,
   813  						Version: v2GroupsAndVersions.Items[1].Versions[0].Version,
   814  					}
   815  
   816  					if !reflect.DeepEqual(&stableV1, &secondGV) {
   817  						return fmt.Errorf("expected second group's first version to be %v, not %v", stableV1, secondGV)
   818  					}
   819  
   820  					return nil
   821  				}),
   822  			},
   823  		},
   824  		{
   825  			// Show that a very low group priority is ordered last
   826  			Name: "LowGroupPriority",
   827  			Actions: []testAction{
   828  				// A minimal priority
   829  				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1, 15)),
   830  				// Wait for all the added group-versions to appear in v2 discovery
   831  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1alpha1}),
   832  				// Check that the last group-version is the one with the lowest
   833  				// priority
   834  				inlineAction(func(ctx context.Context, client testClient) error {
   835  					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
   836  					if err != nil {
   837  						return err
   838  					}
   839  					lastGroup := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
   840  
   841  					lastGV := metav1.GroupVersion{
   842  						Group:   lastGroup.Name,
   843  						Version: lastGroup.Versions[0].Version,
   844  					}
   845  
   846  					if !reflect.DeepEqual(&stableV1alpha1, &lastGV) {
   847  						return fmt.Errorf("expected last group to be %v, not %v", stableV1alpha1, lastGV)
   848  					}
   849  
   850  					return nil
   851  				}),
   852  				// Wait for all the added group-versions to appear in both discovery documents
   853  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1alpha1}),
   854  				// Check that both v1 and v2 endpoints have exactly the same
   855  				// sets of groupversions
   856  				checkGVOrder,
   857  			},
   858  		},
   859  		{
   860  			// Show that versions within a group are sorted by priority
   861  			Name: "VersionPriority",
   862  			Actions: []testAction{
   863  				applyAPIService(makeApiServiceSpec(stableV1, 1000, 2)),
   864  				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 1)),
   865  				applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 3)),
   866  				// Wait for all the added group-versions to appear in both discovery documents
   867  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
   868  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2}),
   869  				// Check that both v1 and v2 endpoints have exactly the same
   870  				// sets of groupversions
   871  				checkGVOrder,
   872  				inlineAction(func(ctx context.Context, client testClient) error {
   873  					// Find the entry for stable.example.com
   874  					// and show the versions are ordered how we expect
   875  					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
   876  					if err != nil {
   877  						return err
   878  					}
   879  
   880  					// Should be ordered last for this test
   881  					group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
   882  					if group.Name != stableGroup {
   883  						return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
   884  					}
   885  
   886  					versionOrder := []string{}
   887  					for _, version := range group.Versions {
   888  						versionOrder = append(versionOrder, version.Version)
   889  					}
   890  
   891  					expectedOrder := []string{
   892  						stableV1alpha2.Version,
   893  						stableV1.Version,
   894  						stableV1alpha1.Version,
   895  					}
   896  
   897  					if !reflect.DeepEqual(expectedOrder, versionOrder) {
   898  						return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
   899  					}
   900  
   901  					return nil
   902  				}),
   903  			},
   904  		},
   905  		{
   906  			// Show that versions within a group are sorted by priority
   907  			// and that equal versions will be sorted by a kube-aware version
   908  			// comparator
   909  			Name: "VersionPriorityTiebreaker",
   910  			Actions: []testAction{
   911  				applyAPIService(makeApiServiceSpec(stableV1, 1000, 15)),
   912  				applyAPIService(makeApiServiceSpec(stableV1alpha1, 1000, 15)),
   913  				applyAPIService(makeApiServiceSpec(stableV1alpha2, 1000, 15)),
   914  				applyAPIService(makeApiServiceSpec(stableV1beta1, 1000, 15)),
   915  				applyAPIService(makeApiServiceSpec(stableV2, 1000, 15)),
   916  				// Wait for all the added group-versions to appear in both discovery documents
   917  				waitForGroupVersionsV1([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
   918  				waitForGroupVersionsV2([]metav1.GroupVersion{stableV1, stableV1alpha1, stableV1alpha2, stableV1beta1, stableV2}),
   919  				// Check that both v1 and v2 endpoints have exactly the same
   920  				// sets of groupversions
   921  				checkGVOrder,
   922  				inlineAction(func(ctx context.Context, client testClient) error {
   923  					// Find the entry for stable.example.com
   924  					// and show the versions are ordered how we expect
   925  					v2GroupsAndVersions, err := FetchV2Discovery(ctx, client)
   926  					if err != nil {
   927  						return err
   928  					}
   929  
   930  					// Should be ordered last for this test
   931  					group := v2GroupsAndVersions.Items[len(v2GroupsAndVersions.Items)-1]
   932  					if group.Name != stableGroup {
   933  						return fmt.Errorf("group is not where we expect: found %v, expected %v", group.Name, stableGroup)
   934  					}
   935  
   936  					versionOrder := []string{}
   937  					for _, version := range group.Versions {
   938  						versionOrder = append(versionOrder, version.Version)
   939  					}
   940  
   941  					expectedOrder := []string{
   942  						stableV2.Version,
   943  						stableV1.Version,
   944  						stableV1beta1.Version,
   945  						stableV1alpha2.Version,
   946  						stableV1alpha1.Version,
   947  					}
   948  
   949  					if !reflect.DeepEqual(expectedOrder, versionOrder) {
   950  						return fmt.Errorf("version in wrong order: %v", cmp.Diff(expectedOrder, versionOrder))
   951  					}
   952  
   953  					return nil
   954  				}),
   955  			},
   956  		},
   957  	})
   958  }
   959  
   960  func TestSingularNames(t *testing.T) {
   961  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--runtime-config=api/all=true"}, framework.SharedEtcd())
   962  	t.Cleanup(server.TearDownFn)
   963  
   964  	kubeClientSet, err := kubernetes.NewForConfig(server.ClientConfig)
   965  	require.NoError(t, err)
   966  
   967  	_, resources, err := kubeClientSet.Discovery().ServerGroupsAndResources()
   968  	require.NoError(t, err)
   969  
   970  	for _, rr := range resources {
   971  		for _, r := range rr.APIResources {
   972  			if strings.Contains(r.Name, "/") {
   973  				continue
   974  			}
   975  			if r.SingularName == "" {
   976  				t.Errorf("missing singularName for resource %q in %q", r.Name, rr.GroupVersion)
   977  				continue
   978  			}
   979  			if r.SingularName != strings.ToLower(r.Kind) {
   980  				t.Errorf("expected singularName for resource %q in %q to be %q, got %q", r.Name, rr.GroupVersion, strings.ToLower(r.Kind), r.SingularName)
   981  				continue
   982  			}
   983  		}
   984  	}
   985  }
   986  
   987  func makeCRDSpec(group string, kind string, namespaced bool, versions []string, categories ...string) apiextensionsv1.CustomResourceDefinitionSpec {
   988  	scope := apiextensionsv1.NamespaceScoped
   989  	if !namespaced {
   990  		scope = apiextensionsv1.ClusterScoped
   991  	}
   992  
   993  	plural, singular := meta.UnsafeGuessKindToResource(schema.GroupVersionKind{Kind: kind})
   994  	res := apiextensionsv1.CustomResourceDefinitionSpec{
   995  		Group: group,
   996  		Scope: scope,
   997  		Names: apiextensionsv1.CustomResourceDefinitionNames{
   998  			Plural:     plural.Resource,
   999  			Singular:   singular.Resource,
  1000  			Kind:       kind,
  1001  			Categories: categories,
  1002  		},
  1003  	}
  1004  
  1005  	for i, version := range versions {
  1006  		res.Versions = append(res.Versions, apiextensionsv1.CustomResourceDefinitionVersion{
  1007  			Name:    version,
  1008  			Served:  true,
  1009  			Storage: i == 0,
  1010  			Schema: &apiextensionsv1.CustomResourceValidation{
  1011  				OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
  1012  					Type: "object",
  1013  					Properties: map[string]apiextensionsv1.JSONSchemaProps{
  1014  						"data": {
  1015  							Type: "string",
  1016  						},
  1017  					},
  1018  				},
  1019  			},
  1020  		})
  1021  	}
  1022  	return res
  1023  }
  1024  

View as plain text