custom_resource_definition.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/google/go-cmp/cmp"
    25  	"github.com/onsi/ginkgo/v2"
    26  	"github.com/onsi/gomega"
    27  
    28  	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    29  	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
    30  	"k8s.io/apiextensions-apiserver/test/integration/fixtures"
    31  	"k8s.io/apimachinery/pkg/api/equality"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  	"k8s.io/apimachinery/pkg/runtime/schema"
    36  	"k8s.io/apimachinery/pkg/types"
    37  	"k8s.io/apimachinery/pkg/util/uuid"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/apiserver/pkg/storage/names"
    40  	"k8s.io/client-go/dynamic"
    41  	"k8s.io/client-go/util/retry"
    42  	"k8s.io/kubernetes/test/e2e/framework"
    43  	admissionapi "k8s.io/pod-security-admission/api"
    44  )
    45  
    46  var _ = SIGDescribe("CustomResourceDefinition resources [Privileged:ClusterAdmin]", func() {
    47  
    48  	f := framework.NewDefaultFramework("custom-resource-definition")
    49  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    50  
    51  	ginkgo.Context("Simple CustomResourceDefinition", func() {
    52  		/*
    53  			Release: v1.9
    54  			Testname: Custom Resource Definition, create
    55  			Description: Create a API extension client and define a random custom resource definition.
    56  			Create the custom resource definition and then delete it. The creation and deletion MUST
    57  			be successful.
    58  		*/
    59  		framework.ConformanceIt("creating/deleting custom resource definition objects works", func(ctx context.Context) {
    60  
    61  			config, err := framework.LoadConfig()
    62  			framework.ExpectNoError(err, "loading config")
    63  			apiExtensionClient, err := clientset.NewForConfig(config)
    64  			framework.ExpectNoError(err, "initializing apiExtensionClient")
    65  
    66  			randomDefinition := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped)
    67  
    68  			// Create CRD and waits for the resource to be recognized and available.
    69  			randomDefinition, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(randomDefinition, apiExtensionClient)
    70  			framework.ExpectNoError(err, "creating CustomResourceDefinition")
    71  
    72  			defer func() {
    73  				err = fixtures.DeleteV1CustomResourceDefinition(randomDefinition, apiExtensionClient)
    74  				framework.ExpectNoError(err, "deleting CustomResourceDefinition")
    75  			}()
    76  		})
    77  
    78  		/*
    79  			Release: v1.16
    80  			Testname: Custom Resource Definition, list
    81  			Description: Create a API extension client, define 10 labeled custom resource definitions and list them using
    82  			a label selector; the list result MUST contain only the labeled custom resource definitions. Delete the labeled
    83  			custom resource definitions via delete collection; the delete MUST be successful and MUST delete only the
    84  			labeled custom resource definitions.
    85  		*/
    86  		framework.ConformanceIt("listing custom resource definition objects works", func(ctx context.Context) {
    87  			testListSize := 10
    88  			config, err := framework.LoadConfig()
    89  			framework.ExpectNoError(err, "loading config")
    90  			apiExtensionClient, err := clientset.NewForConfig(config)
    91  			framework.ExpectNoError(err, "initializing apiExtensionClient")
    92  
    93  			// Label the CRDs we create so we can list only them even though they are cluster scoped
    94  			testUUID := string(uuid.NewUUID())
    95  
    96  			// Create CRD and wait for the resource to be recognized and available.
    97  			crds := make([]*v1.CustomResourceDefinition, testListSize)
    98  			for i := 0; i < testListSize; i++ {
    99  				crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped)
   100  				crd.Labels = map[string]string{"e2e-list-test-uuid": testUUID}
   101  				crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
   102  				framework.ExpectNoError(err, "creating CustomResourceDefinition")
   103  				crds[i] = crd
   104  			}
   105  
   106  			// Create a crd w/o the label to ensure the label selector matching works correctly
   107  			crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped)
   108  			crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
   109  			framework.ExpectNoError(err, "creating CustomResourceDefinition")
   110  			defer func() {
   111  				err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
   112  				framework.ExpectNoError(err, "deleting CustomResourceDefinition")
   113  			}()
   114  
   115  			selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID}
   116  			list, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, selectorListOpts)
   117  			framework.ExpectNoError(err, "listing CustomResourceDefinitions")
   118  			gomega.Expect(list.Items).To(gomega.HaveLen(testListSize))
   119  			for _, actual := range list.Items {
   120  				var expected *v1.CustomResourceDefinition
   121  				for _, e := range crds {
   122  					if e.Name == actual.Name && e.Namespace == actual.Namespace {
   123  						expected = e
   124  					}
   125  				}
   126  				gomega.Expect(expected).ToNot(gomega.BeNil())
   127  				if !equality.Semantic.DeepEqual(actual.Spec, expected.Spec) {
   128  					framework.Failf("Expected CustomResourceDefinition in list with name %s to match crd created with same name, but got different specs:\n%s",
   129  						actual.Name, cmp.Diff(expected.Spec, actual.Spec))
   130  				}
   131  			}
   132  
   133  			// Use delete collection to remove the CRDs
   134  			err = fixtures.DeleteV1CustomResourceDefinitions(selectorListOpts, apiExtensionClient)
   135  			framework.ExpectNoError(err, "deleting CustomResourceDefinitions")
   136  			_, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
   137  			framework.ExpectNoError(err, "getting remaining CustomResourceDefinition")
   138  		})
   139  
   140  		/*
   141  			Release: v1.16
   142  			Testname: Custom Resource Definition, status sub-resource
   143  			Description: Create a custom resource definition. Attempt to read, update and patch its status sub-resource;
   144  			all mutating sub-resource operations MUST be visible to subsequent reads.
   145  		*/
   146  		framework.ConformanceIt("getting/updating/patching custom resource definition status sub-resource works", func(ctx context.Context) {
   147  			config, err := framework.LoadConfig()
   148  			framework.ExpectNoError(err, "loading config")
   149  			apiExtensionClient, err := clientset.NewForConfig(config)
   150  			framework.ExpectNoError(err, "initializing apiExtensionClient")
   151  			dynamicClient, err := dynamic.NewForConfig(config)
   152  			framework.ExpectNoError(err, "initializing dynamic client")
   153  			gvr := v1.SchemeGroupVersion.WithResource("customresourcedefinitions")
   154  			resourceClient := dynamicClient.Resource(gvr)
   155  
   156  			// Create CRD and waits for the resource to be recognized and available.
   157  			crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped)
   158  			crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
   159  			framework.ExpectNoError(err, "creating CustomResourceDefinition")
   160  			defer func() {
   161  				err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
   162  				framework.ExpectNoError(err, "deleting CustomResourceDefinition")
   163  			}()
   164  
   165  			var updated *v1.CustomResourceDefinition
   166  			updateCondition := v1.CustomResourceDefinitionCondition{Message: "updated"}
   167  			err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   168  				// Use dynamic client to read the status sub-resource since typed client does not expose it.
   169  				u, err := resourceClient.Get(ctx, crd.GetName(), metav1.GetOptions{}, "status")
   170  				framework.ExpectNoError(err, "getting CustomResourceDefinition status")
   171  				status := unstructuredToCRD(u)
   172  				if !equality.Semantic.DeepEqual(status.Spec, crd.Spec) {
   173  					framework.Failf("Expected CustomResourceDefinition Spec to match status sub-resource Spec, but got:\n%s", cmp.Diff(status.Spec, crd.Spec))
   174  				}
   175  				status.Status.Conditions = append(status.Status.Conditions, updateCondition)
   176  				updated, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().UpdateStatus(ctx, status, metav1.UpdateOptions{})
   177  				return err
   178  			})
   179  			framework.ExpectNoError(err, "updating CustomResourceDefinition status")
   180  			expectCondition(updated.Status.Conditions, updateCondition)
   181  
   182  			patchCondition := v1.CustomResourceDefinitionCondition{Message: "patched"}
   183  			patched, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.GetName(),
   184  				types.JSONPatchType,
   185  				[]byte(`[{"op": "add", "path": "/status/conditions", "value": [{"message": "patched"}]}]`), metav1.PatchOptions{},
   186  				"status")
   187  			framework.ExpectNoError(err, "patching CustomResourceDefinition status")
   188  			expectCondition(updated.Status.Conditions, updateCondition)
   189  			expectCondition(patched.Status.Conditions, patchCondition)
   190  		})
   191  	})
   192  
   193  	/*
   194  		Release: v1.16
   195  		Testname: Custom Resource Definition, discovery
   196  		Description: Fetch /apis, /apis/apiextensions.k8s.io, and /apis/apiextensions.k8s.io/v1 discovery documents,
   197  		and ensure they indicate CustomResourceDefinition apiextensions.k8s.io/v1 resources are available.
   198  	*/
   199  	framework.ConformanceIt("should include custom resource definition resources in discovery documents", func(ctx context.Context) {
   200  		{
   201  			ginkgo.By("fetching the /apis discovery document")
   202  			apiGroupList := &metav1.APIGroupList{}
   203  			err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Into(apiGroupList)
   204  			framework.ExpectNoError(err, "fetching /apis")
   205  
   206  			ginkgo.By("finding the apiextensions.k8s.io API group in the /apis discovery document")
   207  			var group *metav1.APIGroup
   208  			for _, g := range apiGroupList.Groups {
   209  				if g.Name == v1.GroupName {
   210  					group = &g
   211  					break
   212  				}
   213  			}
   214  			gomega.Expect(group).ToNot(gomega.BeNil(), "apiextensions.k8s.io API group not found in /apis discovery document")
   215  
   216  			ginkgo.By("finding the apiextensions.k8s.io/v1 API group/version in the /apis discovery document")
   217  			var version *metav1.GroupVersionForDiscovery
   218  			for _, v := range group.Versions {
   219  				if v.Version == v1.SchemeGroupVersion.Version {
   220  					version = &v
   221  					break
   222  				}
   223  			}
   224  			gomega.Expect(version).ToNot(gomega.BeNil(), "apiextensions.k8s.io/v1 API group version not found in /apis discovery document")
   225  		}
   226  
   227  		{
   228  			ginkgo.By("fetching the /apis/apiextensions.k8s.io discovery document")
   229  			group := &metav1.APIGroup{}
   230  			err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io").Do(ctx).Into(group)
   231  			framework.ExpectNoError(err, "fetching /apis/apiextensions.k8s.io")
   232  			gomega.Expect(group.Name).To(gomega.Equal(v1.GroupName), "verifying API group name in /apis/apiextensions.k8s.io discovery document")
   233  
   234  			ginkgo.By("finding the apiextensions.k8s.io/v1 API group/version in the /apis/apiextensions.k8s.io discovery document")
   235  			var version *metav1.GroupVersionForDiscovery
   236  			for _, v := range group.Versions {
   237  				if v.Version == v1.SchemeGroupVersion.Version {
   238  					version = &v
   239  					break
   240  				}
   241  			}
   242  			gomega.Expect(version).ToNot(gomega.BeNil(), "apiextensions.k8s.io/v1 API group version not found in /apis/apiextensions.k8s.io discovery document")
   243  		}
   244  
   245  		{
   246  			ginkgo.By("fetching the /apis/apiextensions.k8s.io/v1 discovery document")
   247  			apiResourceList := &metav1.APIResourceList{}
   248  			err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io/v1").Do(ctx).Into(apiResourceList)
   249  			framework.ExpectNoError(err, "fetching /apis/apiextensions.k8s.io/v1")
   250  			gomega.Expect(apiResourceList.GroupVersion).To(gomega.Equal(v1.SchemeGroupVersion.String()), "verifying API group/version in /apis/apiextensions.k8s.io/v1 discovery document")
   251  
   252  			ginkgo.By("finding customresourcedefinitions resources in the /apis/apiextensions.k8s.io/v1 discovery document")
   253  			var crdResource *metav1.APIResource
   254  			for i := range apiResourceList.APIResources {
   255  				if apiResourceList.APIResources[i].Name == "customresourcedefinitions" {
   256  					crdResource = &apiResourceList.APIResources[i]
   257  				}
   258  			}
   259  			gomega.Expect(crdResource).ToNot(gomega.BeNil(), "customresourcedefinitions resource not found in /apis/apiextensions.k8s.io/v1 discovery document")
   260  		}
   261  	})
   262  
   263  	/*
   264  		Release: v1.17
   265  		Testname: Custom Resource Definition, defaulting
   266  		Description: Create a custom resource definition without default. Create CR. Add default and read CR until
   267  		the default is applied. Create another CR. Remove default, add default for another field and read CR until
   268  		new field is defaulted, but old default stays.
   269  	*/
   270  	framework.ConformanceIt("custom resource defaulting for requests and from storage works", func(ctx context.Context) {
   271  		config, err := framework.LoadConfig()
   272  		framework.ExpectNoError(err, "loading config")
   273  		apiExtensionClient, err := clientset.NewForConfig(config)
   274  		framework.ExpectNoError(err, "initializing apiExtensionClient")
   275  		dynamicClient, err := dynamic.NewForConfig(config)
   276  		framework.ExpectNoError(err, "initializing dynamic client")
   277  
   278  		// Create CRD without default and waits for the resource to be recognized and available.
   279  		crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped)
   280  		if crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties == nil {
   281  			crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties = map[string]v1.JSONSchemaProps{}
   282  		}
   283  		crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["a"] = v1.JSONSchemaProps{Type: "string"}
   284  		crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["b"] = v1.JSONSchemaProps{Type: "string"}
   285  		crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
   286  		framework.ExpectNoError(err, "creating CustomResourceDefinition")
   287  		defer func() {
   288  			err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient)
   289  			framework.ExpectNoError(err, "deleting CustomResourceDefinition")
   290  		}()
   291  
   292  		// create CR without default in storage
   293  		name1 := names.SimpleNameGenerator.GenerateName("cr-1")
   294  		gvr := schema.GroupVersionResource{
   295  			Group:    crd.Spec.Group,
   296  			Version:  crd.Spec.Versions[0].Name,
   297  			Resource: crd.Spec.Names.Plural,
   298  		}
   299  		crClient := dynamicClient.Resource(gvr)
   300  		_, err = crClient.Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{
   301  			"apiVersion": gvr.Group + "/" + gvr.Version,
   302  			"kind":       crd.Spec.Names.Kind,
   303  			"metadata": map[string]interface{}{
   304  				"name": name1,
   305  			},
   306  		}}, metav1.CreateOptions{})
   307  		framework.ExpectNoError(err, "creating CR")
   308  
   309  		// Setting default for a to "A" and waiting for the CR to get defaulted on read
   310  		crd, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.Name, types.JSONPatchType, []byte(`[
   311  			{"op":"add","path":"/spec/versions/0/schema/openAPIV3Schema/properties/a/default", "value": "A"}
   312  		]`), metav1.PatchOptions{})
   313  		framework.ExpectNoError(err, "setting default for a to \"A\" in schema")
   314  
   315  		err = wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
   316  			u1, err := crClient.Get(ctx, name1, metav1.GetOptions{})
   317  			if err != nil {
   318  				return false, err
   319  			}
   320  			a, found, err := unstructured.NestedFieldNoCopy(u1.Object, "a")
   321  			if err != nil {
   322  				return false, err
   323  			}
   324  			if !found {
   325  				return false, nil
   326  			}
   327  			if a != "A" {
   328  				return false, fmt.Errorf("expected a:\"A\", but got a:%q", a)
   329  			}
   330  			return true, nil
   331  		})
   332  		framework.ExpectNoError(err, "waiting for CR to be defaulted on read")
   333  
   334  		// create CR with default in storage
   335  		name2 := names.SimpleNameGenerator.GenerateName("cr-2")
   336  		u2, err := crClient.Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{
   337  			"apiVersion": gvr.Group + "/" + gvr.Version,
   338  			"kind":       crd.Spec.Names.Kind,
   339  			"metadata": map[string]interface{}{
   340  				"name": name2,
   341  			},
   342  		}}, metav1.CreateOptions{})
   343  		framework.ExpectNoError(err, "creating CR")
   344  		v, found, err := unstructured.NestedFieldNoCopy(u2.Object, "a")
   345  		if !found {
   346  			framework.Failf("field `a` should have been defaulted in %+v", u2.Object)
   347  		}
   348  		gomega.Expect(v).To(gomega.Equal("A"), "\"a\" is defaulted to \"A\"")
   349  
   350  		// Deleting default for a, adding default "B" for b and waiting for the CR to get defaulted on read for b
   351  		crd, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.Name, types.JSONPatchType, []byte(`[
   352  			{"op":"remove","path":"/spec/versions/0/schema/openAPIV3Schema/properties/a/default"},
   353  			{"op":"add","path":"/spec/versions/0/schema/openAPIV3Schema/properties/b/default", "value": "B"}
   354  		]`), metav1.PatchOptions{})
   355  		framework.ExpectNoError(err, "setting default for b to \"B\" and remove default for a")
   356  
   357  		err = wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
   358  			u2, err := crClient.Get(ctx, name2, metav1.GetOptions{})
   359  			if err != nil {
   360  				return false, err
   361  			}
   362  			b, found, err := unstructured.NestedFieldNoCopy(u2.Object, "b")
   363  			if err != nil {
   364  				return false, err
   365  			}
   366  			if !found {
   367  				return false, nil
   368  			}
   369  			if b != "B" {
   370  				return false, fmt.Errorf("expected b:\"B\", but got b:%q", b)
   371  			}
   372  			a, found, err := unstructured.NestedFieldNoCopy(u2.Object, "a")
   373  			if err != nil {
   374  				return false, err
   375  			}
   376  			if !found {
   377  				return false, fmt.Errorf("expected a:\"A\" to be unchanged, but it was removed")
   378  			}
   379  			if a != "A" {
   380  				return false, fmt.Errorf("expected a:\"A\" to be unchanged, but it changed to %q", a)
   381  			}
   382  			return true, nil
   383  		})
   384  		framework.ExpectNoError(err, "waiting for CR to be defaulted on read for b and a staying the same")
   385  	})
   386  
   387  })
   388  
   389  func unstructuredToCRD(obj *unstructured.Unstructured) *v1.CustomResourceDefinition {
   390  	crd := new(v1.CustomResourceDefinition)
   391  	err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, crd)
   392  	framework.ExpectNoError(err, "converting unstructured to CustomResourceDefinition")
   393  	return crd
   394  }
   395  
   396  func expectCondition(conditions []v1.CustomResourceDefinitionCondition, expected v1.CustomResourceDefinitionCondition) {
   397  	for _, c := range conditions {
   398  		if equality.Semantic.DeepEqual(c, expected) {
   399  			return
   400  		}
   401  	}
   402  	framework.Failf("Condition %#v not found in conditions %#v", expected, conditions)
   403  }
   404
View as plain text