crd_overlap_storage_test.go

Documentation: k8s.io/kubernetes/test/integration/etcd

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package etcd
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"strings"
    23  	"testing"
    24  	"time"
    25  
    26  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    27  	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
    28  	"k8s.io/apiextensions-apiserver/pkg/controller/finalizer"
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime/schema"
    32  	"k8s.io/apimachinery/pkg/types"
    33  	"k8s.io/apimachinery/pkg/util/sets"
    34  	"k8s.io/apimachinery/pkg/util/wait"
    35  	"k8s.io/client-go/dynamic"
    36  	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
    37  	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
    38  )
    39  
    40  // TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date
    41  func TestOverlappingBuiltInResources(t *testing.T) {
    42  	// Verify built-in resources that overlap with computed CRD storage paths are listed in OverlappingBuiltInResources()
    43  	detectedOverlappingResources := map[schema.GroupResource]bool{}
    44  	for gvr, gvrData := range GetEtcdStorageData() {
    45  		if !strings.HasSuffix(gvr.Group, ".k8s.io") {
    46  			// only fully-qualified group names can exist as CRDs
    47  			continue
    48  		}
    49  		if !strings.Contains(gvrData.ExpectedEtcdPath, "/"+gvr.Group+"/"+gvr.Resource+"/") {
    50  			// CRDs persist in storage under .../<group>/<resource>/...
    51  			continue
    52  		}
    53  		detectedOverlappingResources[gvr.GroupResource()] = true
    54  	}
    55  
    56  	for detected := range detectedOverlappingResources {
    57  		if !finalizer.OverlappingBuiltInResources()[detected] {
    58  			t.Errorf("built-in resource %#v would overlap with custom resource storage if a CRD was created for the same group/resource", detected)
    59  			t.Errorf("add %#v to the OverlappingBuiltInResources() list to prevent deletion by the CRD finalizer", detected)
    60  		}
    61  	}
    62  	for skip := range finalizer.OverlappingBuiltInResources() {
    63  		if !detectedOverlappingResources[skip] {
    64  			t.Errorf("resource %#v does not overlap with any built-in resources in storage, but is skipped for CRD finalization by OverlappingBuiltInResources()", skip)
    65  			t.Errorf("remove %#v from OverlappingBuiltInResources() to ensure CRD finalization cleans up stored custom resources", skip)
    66  		}
    67  	}
    68  }
    69  
    70  // TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data
    71  func TestOverlappingCustomResourceAPIService(t *testing.T) {
    72  	apiServer := StartRealAPIServerOrDie(t)
    73  	defer apiServer.Cleanup()
    74  
    75  	apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config)
    76  	if err != nil {
    77  		t.Fatal(err)
    78  	}
    79  	crdClient, err := crdclient.NewForConfig(apiServer.Config)
    80  	if err != nil {
    81  		t.Fatal(err)
    82  	}
    83  	dynamicClient, err := dynamic.NewForConfig(apiServer.Config)
    84  	if err != nil {
    85  		t.Fatal(err)
    86  	}
    87  
    88  	// Verify APIServices can be listed
    89  	apiServices, err := apiServiceClient.APIServices().List(context.TODO(), metav1.ListOptions{})
    90  	if err != nil {
    91  		t.Fatal(err)
    92  	}
    93  	apiServiceNames := sets.NewString()
    94  	for _, s := range apiServices.Items {
    95  		apiServiceNames.Insert(s.Name)
    96  	}
    97  	if len(apiServices.Items) == 0 {
    98  		t.Fatal("expected APIService objects, got none")
    99  	}
   100  
   101  	// Create a CRD defining an overlapping apiregistration.k8s.io apiservices resource with an incompatible schema
   102  	crdCRD, err := crdClient.CustomResourceDefinitions().Create(context.TODO(), &apiextensionsv1.CustomResourceDefinition{
   103  		ObjectMeta: metav1.ObjectMeta{
   104  			Name:        "apiservices.apiregistration.k8s.io",
   105  			Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, testing only"},
   106  		},
   107  		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
   108  			Group: "apiregistration.k8s.io",
   109  			Scope: apiextensionsv1.ClusterScoped,
   110  			Names: apiextensionsv1.CustomResourceDefinitionNames{Plural: "apiservices", Singular: "customapiservice", Kind: "CustomAPIService", ListKind: "CustomAPIServiceList"},
   111  			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
   112  				{
   113  					Name:    "v1",
   114  					Served:  true,
   115  					Storage: true,
   116  					Schema: &apiextensionsv1.CustomResourceValidation{
   117  						OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
   118  							Type:     "object",
   119  							Required: []string{"foo"},
   120  							Properties: map[string]apiextensionsv1.JSONSchemaProps{
   121  								"foo": {Type: "string"},
   122  								"bar": {Type: "string", Default: &apiextensionsv1.JSON{Raw: []byte(`"default"`)}},
   123  							},
   124  						},
   125  					},
   126  				},
   127  			},
   128  		},
   129  	}, metav1.CreateOptions{})
   130  	if err != nil {
   131  		t.Fatal(err)
   132  	}
   133  
   134  	// Wait until it is established
   135  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   136  		crd, err := crdClient.CustomResourceDefinitions().Get(context.TODO(), crdCRD.Name, metav1.GetOptions{})
   137  		if err != nil {
   138  			return false, err
   139  		}
   140  		for _, condition := range crd.Status.Conditions {
   141  			if condition.Status == apiextensionsv1.ConditionTrue && condition.Type == apiextensionsv1.Established {
   142  				return true, nil
   143  			}
   144  		}
   145  		conditionJSON, _ := json.Marshal(crd.Status.Conditions)
   146  		t.Logf("waiting for establishment (conditions: %s)", string(conditionJSON))
   147  		return false, nil
   148  	}); err != nil {
   149  		t.Fatal(err)
   150  	}
   151  
   152  	// Make sure API requests are still handled by the built-in handler (and return built-in kinds)
   153  
   154  	// Listing v1 succeeds
   155  	v1DynamicList, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "apiregistration.k8s.io", Version: "v1", Resource: "apiservices"}).List(context.TODO(), metav1.ListOptions{})
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  	// Result was served by built-in handler, not CR handler
   160  	if _, hasDefaultedCRField := v1DynamicList.Items[0].Object["spec"].(map[string]interface{})["bar"]; hasDefaultedCRField {
   161  		t.Fatalf("expected no CR defaulting, got %#v", v1DynamicList.Items[0].Object)
   162  	}
   163  
   164  	// Creating v1 succeeds (built-in validation, not CR validation)
   165  	testAPIService, err := apiServiceClient.APIServices().Create(context.TODO(), &apiregistrationv1.APIService{
   166  		ObjectMeta: metav1.ObjectMeta{Name: "v1.example.com"},
   167  		Spec: apiregistrationv1.APIServiceSpec{
   168  			Group:                "example.com",
   169  			Version:              "v1",
   170  			VersionPriority:      100,
   171  			GroupPriorityMinimum: 100,
   172  		},
   173  	}, metav1.CreateOptions{})
   174  	if err != nil {
   175  		t.Fatal(err)
   176  	}
   177  	err = apiServiceClient.APIServices().Delete(context.TODO(), testAPIService.Name, metav1.DeleteOptions{})
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  
   182  	// discovery is handled by the built-in handler
   183  	v1Resources, err := apiServer.Client.Discovery().ServerResourcesForGroupVersion("apiregistration.k8s.io/v1")
   184  	if err != nil {
   185  		t.Fatal(err)
   186  	}
   187  	for _, r := range v1Resources.APIResources {
   188  		if r.Name == "apiservices" {
   189  			if r.Kind != "APIService" {
   190  				t.Errorf("expected kind=APIService in discovery, got %s", r.Kind)
   191  			}
   192  		}
   193  	}
   194  	v2Resources, err := apiServer.Client.Discovery().ServerResourcesForGroupVersion("apiregistration.k8s.io/v2")
   195  	if err == nil {
   196  		t.Fatalf("expected error looking up apiregistration.k8s.io/v2 discovery, got %#v", v2Resources)
   197  	}
   198  
   199  	// Delete the overlapping CRD
   200  	err = crdClient.CustomResourceDefinitions().Delete(context.TODO(), crdCRD.Name, metav1.DeleteOptions{})
   201  	if err != nil {
   202  		t.Fatal(err)
   203  	}
   204  
   205  	// Make sure the CRD deletion succeeds
   206  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   207  		crd, err := crdClient.CustomResourceDefinitions().Get(context.TODO(), crdCRD.Name, metav1.GetOptions{})
   208  		if apierrors.IsNotFound(err) {
   209  			return true, nil
   210  		}
   211  		if err != nil {
   212  			return false, err
   213  		}
   214  		conditionJSON, _ := json.Marshal(crd.Status.Conditions)
   215  		t.Logf("waiting for deletion (conditions: %s)", string(conditionJSON))
   216  		return false, nil
   217  	}); err != nil {
   218  		t.Fatal(err)
   219  	}
   220  
   221  	// Make sure APIService objects are not removed
   222  	time.Sleep(5 * time.Second)
   223  	finalAPIServices, err := apiServiceClient.APIServices().List(context.TODO(), metav1.ListOptions{})
   224  	if err != nil {
   225  		t.Fatal(err)
   226  	}
   227  	if len(finalAPIServices.Items) != len(apiServices.Items) {
   228  		t.Fatalf("expected %d APIService objects, got %d", len(apiServices.Items), len(finalAPIServices.Items))
   229  	}
   230  }
   231  
   232  // TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data
   233  func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) {
   234  	apiServer := StartRealAPIServerOrDie(t)
   235  	defer apiServer.Cleanup()
   236  
   237  	crdClient, err := crdclient.NewForConfig(apiServer.Config)
   238  	if err != nil {
   239  		t.Fatal(err)
   240  	}
   241  	dynamicClient, err := dynamic.NewForConfig(apiServer.Config)
   242  	if err != nil {
   243  		t.Fatal(err)
   244  	}
   245  
   246  	// Verify CustomResourceDefinitions can be listed
   247  	crds, err := crdClient.CustomResourceDefinitions().List(context.TODO(), metav1.ListOptions{})
   248  	if err != nil {
   249  		t.Fatal(err)
   250  	}
   251  	crdNames := sets.NewString()
   252  	for _, s := range crds.Items {
   253  		crdNames.Insert(s.Name)
   254  	}
   255  	if len(crds.Items) == 0 {
   256  		t.Fatal("expected CustomResourceDefinition objects, got none")
   257  	}
   258  
   259  	// Create a CRD defining an overlapping apiregistration.k8s.io apiservices resource with an incompatible schema
   260  	crdCRD, err := crdClient.CustomResourceDefinitions().Create(context.TODO(), &apiextensionsv1.CustomResourceDefinition{
   261  		ObjectMeta: metav1.ObjectMeta{
   262  			Name:        "customresourcedefinitions.apiextensions.k8s.io",
   263  			Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, testing only"},
   264  		},
   265  		Spec: apiextensionsv1.CustomResourceDefinitionSpec{
   266  			Group: "apiextensions.k8s.io",
   267  			Scope: apiextensionsv1.ClusterScoped,
   268  			Names: apiextensionsv1.CustomResourceDefinitionNames{
   269  				Plural:   "customresourcedefinitions",
   270  				Singular: "customcustomresourcedefinition",
   271  				Kind:     "CustomCustomResourceDefinition",
   272  				ListKind: "CustomAPIServiceList",
   273  			},
   274  			Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
   275  				{
   276  					Name:    "v1",
   277  					Served:  true,
   278  					Storage: true,
   279  					Schema: &apiextensionsv1.CustomResourceValidation{
   280  						OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
   281  							Type:     "object",
   282  							Required: []string{"foo"},
   283  							Properties: map[string]apiextensionsv1.JSONSchemaProps{
   284  								"foo": {Type: "string"},
   285  								"bar": {Type: "string", Default: &apiextensionsv1.JSON{Raw: []byte(`"default"`)}},
   286  							},
   287  						},
   288  					},
   289  				},
   290  			},
   291  		},
   292  	}, metav1.CreateOptions{})
   293  	if err != nil {
   294  		t.Fatal(err)
   295  	}
   296  
   297  	// Wait until it is established
   298  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   299  		crd, err := crdClient.CustomResourceDefinitions().Get(context.TODO(), crdCRD.Name, metav1.GetOptions{})
   300  		if err != nil {
   301  			return false, err
   302  		}
   303  		for _, condition := range crd.Status.Conditions {
   304  			if condition.Status == apiextensionsv1.ConditionTrue && condition.Type == apiextensionsv1.Established {
   305  				return true, nil
   306  			}
   307  		}
   308  		conditionJSON, _ := json.Marshal(crd.Status.Conditions)
   309  		t.Logf("waiting for establishment (conditions: %s)", string(conditionJSON))
   310  		return false, nil
   311  	}); err != nil {
   312  		t.Fatal(err)
   313  	}
   314  
   315  	// Make sure API requests are still handled by the built-in handler (and return built-in kinds)
   316  
   317  	// Listing v1 succeeds
   318  	v1DynamicList, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"}).List(context.TODO(), metav1.ListOptions{})
   319  	if err != nil {
   320  		t.Fatal(err)
   321  	}
   322  	// Result was served by built-in handler, not CR handler
   323  	if _, hasDefaultedCRField := v1DynamicList.Items[0].Object["spec"].(map[string]interface{})["bar"]; hasDefaultedCRField {
   324  		t.Fatalf("expected no CR defaulting, got %#v", v1DynamicList.Items[0].Object)
   325  	}
   326  
   327  	// Updating v1 succeeds (built-in validation, not CR validation)
   328  	_, err = crdClient.CustomResourceDefinitions().Patch(context.TODO(), crdCRD.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"updated"}}}`), metav1.PatchOptions{})
   329  	if err != nil {
   330  		t.Fatal(err)
   331  	}
   332  
   333  	// discovery is handled by the built-in handler
   334  	v1Resources, err := apiServer.Client.Discovery().ServerResourcesForGroupVersion("apiextensions.k8s.io/v1")
   335  	if err != nil {
   336  		t.Fatal(err)
   337  	}
   338  	for _, r := range v1Resources.APIResources {
   339  		if r.Name == "customresourcedefinitions" {
   340  			if r.Kind != "CustomResourceDefinition" {
   341  				t.Errorf("expected kind=CustomResourceDefinition in discovery, got %s", r.Kind)
   342  			}
   343  		}
   344  	}
   345  	v2Resources, err := apiServer.Client.Discovery().ServerResourcesForGroupVersion("apiextensions.k8s.io/v2")
   346  	if err == nil {
   347  		t.Fatalf("expected error looking up apiregistration.k8s.io/v2 discovery, got %#v", v2Resources)
   348  	}
   349  
   350  	// Delete the overlapping CRD
   351  	err = crdClient.CustomResourceDefinitions().Delete(context.TODO(), crdCRD.Name, metav1.DeleteOptions{})
   352  	if err != nil {
   353  		t.Fatal(err)
   354  	}
   355  
   356  	// Make sure the CRD deletion succeeds
   357  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   358  		crd, err := crdClient.CustomResourceDefinitions().Get(context.TODO(), crdCRD.Name, metav1.GetOptions{})
   359  		if apierrors.IsNotFound(err) {
   360  			return true, nil
   361  		}
   362  		if err != nil {
   363  			return false, err
   364  		}
   365  		conditionJSON, _ := json.Marshal(crd.Status.Conditions)
   366  		t.Logf("waiting for deletion (conditions: %s)", string(conditionJSON))
   367  		return false, nil
   368  	}); err != nil {
   369  		t.Fatal(err)
   370  	}
   371  
   372  	// Make sure other CustomResourceDefinition objects are not removed
   373  	time.Sleep(5 * time.Second)
   374  	finalCRDs, err := crdClient.CustomResourceDefinitions().List(context.TODO(), metav1.ListOptions{})
   375  	if err != nil {
   376  		t.Fatal(err)
   377  	}
   378  	if len(finalCRDs.Items) != len(crds.Items) {
   379  		t.Fatalf("expected %d APIService objects, got %d", len(crds.Items), len(finalCRDs.Items))
   380  	}
   381  }
   382
View as plain text