Source file src/k8s.io/kubernetes/test/e2e/apimachinery/crd_publish_openapi.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"io"
    24  	"net/http"
    25  	"regexp"
    26  	"strings"
    27  	"time"
    28  
    29  	"github.com/onsi/ginkgo/v2"
    30  	"sigs.k8s.io/yaml"
    31  
    32  	openapiutil "k8s.io/kube-openapi/pkg/util"
    33  	"k8s.io/utils/pointer"
    34  
    35  	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
    36  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    37  	"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
    38  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    39  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    40  	"k8s.io/apimachinery/pkg/types"
    41  	"k8s.io/apimachinery/pkg/util/wait"
    42  	k8sclientset "k8s.io/client-go/kubernetes"
    43  	"k8s.io/client-go/rest"
    44  	"k8s.io/kube-openapi/pkg/validation/spec"
    45  	"k8s.io/kubernetes/test/e2e/framework"
    46  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    47  	"k8s.io/kubernetes/test/utils/crd"
    48  	admissionapi "k8s.io/pod-security-admission/api"
    49  )
    50  
    51  var (
    52  	metaPattern = `"kind":"%s","apiVersion":"%s/%s","metadata":{"name":"%s"}`
    53  )
    54  
    55  var _ = SIGDescribe("CustomResourcePublishOpenAPI [Privileged:ClusterAdmin]", func() {
    56  	f := framework.NewDefaultFramework("crd-publish-openapi")
    57  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    58  
    59  	/*
    60  		Release: v1.16
    61  		Testname: Custom Resource OpenAPI Publish, with validation schema
    62  		Description: Register a custom resource definition with a validating schema consisting of objects, arrays and
    63  		primitives. Attempt to create and apply a change a custom resource using valid properties, via kubectl;
    64  		kubectl validation MUST pass. Attempt both operations with unknown properties and without required
    65  		properties; kubectl validation MUST reject the operations. Attempt kubectl explain; the output MUST
    66  		explain the custom resource properties. Attempt kubectl explain on custom resource properties; the output MUST
    67  		explain the nested custom resource properties.
    68  		All validation should be the same.
    69  	*/
    70  	framework.ConformanceIt("works for CRD with validation schema", func(ctx context.Context) {
    71  		crd, err := setupCRD(f, schemaFoo, "foo", "v1")
    72  		if err != nil {
    73  			framework.Failf("%v", err)
    74  		}
    75  
    76  		meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-foo")
    77  		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
    78  
    79  		ginkgo.By("kubectl validation (kubectl create and apply) allows request with known and required properties")
    80  		validCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}]}}`, meta)
    81  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, validCR, ns, "create", "-f", "-"); err != nil {
    82  			framework.Failf("failed to create valid CR %s: %v", validCR, err)
    83  		}
    84  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-foo"); err != nil {
    85  			framework.Failf("failed to delete valid CR: %v", err)
    86  		}
    87  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, validCR, ns, "apply", "-f", "-"); err != nil {
    88  			framework.Failf("failed to apply valid CR %s: %v", validCR, err)
    89  		}
    90  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-foo"); err != nil {
    91  			framework.Failf("failed to delete valid CR: %v", err)
    92  		}
    93  
    94  		ginkgo.By("kubectl validation (kubectl create and apply) rejects request with value outside defined enum values")
    95  		badEnumValueCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar", "feeling":"NonExistentValue"}]}}`, meta)
    96  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, badEnumValueCR, ns, "create", "-f", "-"); err == nil || !strings.Contains(err.Error(), `Unsupported value: "NonExistentValue"`) {
    97  			framework.Failf("unexpected no error when creating CR with unknown enum value: %v", err)
    98  		}
    99  
   100  		// TODO: server-side validation and client-side validation produce slightly different error messages.
   101  		// Because server-side is default in beta but not GA yet, we will produce different behaviors in the default vs GA only conformance tests. We have made the error generic enough to pass both, but should go back and make the error more specific once server-side validation goes GA.
   102  		ginkgo.By("kubectl validation (kubectl create and apply) rejects request with unknown properties when disallowed by the schema")
   103  		unknownCR := fmt.Sprintf(`{%s,"spec":{"foo":true}}`, meta)
   104  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, unknownCR, ns, "create", "-f", "-"); err == nil || (!strings.Contains(err.Error(), `unknown field "foo"`) && !strings.Contains(err.Error(), `unknown field "spec.foo"`)) {
   105  			framework.Failf("unexpected no error when creating CR with unknown field: %v", err)
   106  		}
   107  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, unknownCR, ns, "apply", "-f", "-"); err == nil || (!strings.Contains(err.Error(), `unknown field "foo"`) && !strings.Contains(err.Error(), `unknown field "spec.foo"`)) {
   108  			framework.Failf("unexpected no error when applying CR with unknown field: %v", err)
   109  		}
   110  
   111  		// TODO: see above note, we should check the value of the error once server-side validation is GA.
   112  		ginkgo.By("kubectl validation (kubectl create and apply) rejects request without required properties")
   113  		noRequireCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"age":"10"}]}}`, meta)
   114  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, noRequireCR, ns, "create", "-f", "-"); err == nil || (!strings.Contains(err.Error(), `missing required field "name"`) && !strings.Contains(err.Error(), `spec.bars[0].name: Required value`)) {
   115  			framework.Failf("unexpected no error when creating CR without required field: %v", err)
   116  		}
   117  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, noRequireCR, ns, "apply", "-f", "-"); err == nil || (!strings.Contains(err.Error(), `missing required field "name"`) && !strings.Contains(err.Error(), `spec.bars[0].name: Required value`)) {
   118  			framework.Failf("unexpected no error when applying CR without required field: %v", err)
   119  		}
   120  
   121  		ginkgo.By("kubectl explain works to explain CR properties")
   122  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural, `(?s)DESCRIPTION:.*Foo CRD for Testing.*FIELDS:.*apiVersion.*<string>.*APIVersion defines.*spec.*<Object>.*Specification of Foo`); err != nil {
   123  			framework.Failf("%v", err)
   124  		}
   125  
   126  		ginkgo.By("kubectl explain works to explain CR properties recursively")
   127  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural+".metadata", `(?s)DESCRIPTION:.*Standard object's metadata.*FIELDS:.*creationTimestamp.*<string>.*CreationTimestamp is a timestamp`); err != nil {
   128  			framework.Failf("%v", err)
   129  		}
   130  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural+".spec", `(?s)DESCRIPTION:.*Specification of Foo.*FIELDS:.*bars.*<\[\]Object>.*List of Bars and their specs`); err != nil {
   131  			framework.Failf("%v", err)
   132  		}
   133  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural+".spec.bars", `(?s)(FIELD|RESOURCE):.*bars.*<\[\]Object>.*DESCRIPTION:.*List of Bars and their specs.*FIELDS:.*bazs.*<\[\]string>.*List of Bazs.*name.*<string>.*Name of Bar`); err != nil {
   134  			framework.Failf("%v", err)
   135  		}
   136  
   137  		ginkgo.By("kubectl explain works to return error when explain is called on property that doesn't exist")
   138  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, "explain", crd.Crd.Spec.Names.Plural+".spec.bars2"); err == nil || !strings.Contains(err.Error(), `field "bars2" does not exist`) {
   139  			framework.Failf("unexpected no error when explaining property that doesn't exist: %v", err)
   140  		}
   141  
   142  		if err := cleanupCRD(ctx, f, crd); err != nil {
   143  			framework.Failf("%v", err)
   144  		}
   145  	})
   146  
   147  	/*
   148  		Release: v1.16
   149  		Testname: Custom Resource OpenAPI Publish, with x-kubernetes-preserve-unknown-fields in object
   150  		Description: Register a custom resource definition with x-kubernetes-preserve-unknown-fields in the top level object.
   151  		Attempt to create and apply a change a custom resource, via kubectl; kubectl validation MUST accept unknown
   152  		properties. Attempt kubectl explain; the output MUST contain a valid DESCRIPTION stanza.
   153  	*/
   154  	framework.ConformanceIt("works for CRD without validation schema", func(ctx context.Context) {
   155  		crd, err := setupCRD(f, nil, "empty", "v1")
   156  		if err != nil {
   157  			framework.Failf("%v", err)
   158  		}
   159  
   160  		meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr")
   161  		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
   162  
   163  		ginkgo.By("kubectl validation (kubectl create and apply) allows request with any unknown properties")
   164  		randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta)
   165  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "create", "-f", "-"); err != nil {
   166  			framework.Failf("failed to create random CR %s for CRD without schema: %v", randomCR, err)
   167  		}
   168  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   169  			framework.Failf("failed to delete random CR: %v", err)
   170  		}
   171  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "apply", "-f", "-"); err != nil {
   172  			framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err)
   173  		}
   174  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   175  			framework.Failf("failed to delete random CR: %v", err)
   176  		}
   177  
   178  		ginkgo.By("kubectl explain works to explain CR without validation schema")
   179  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural, `(?s)DESCRIPTION:.*<empty>`); err != nil {
   180  			framework.Failf("%v", err)
   181  		}
   182  
   183  		if err := cleanupCRD(ctx, f, crd); err != nil {
   184  			framework.Failf("%v", err)
   185  		}
   186  	})
   187  
   188  	/*
   189  		Release: v1.16
   190  		Testname: Custom Resource OpenAPI Publish, with x-kubernetes-preserve-unknown-fields at root
   191  		Description: Register a custom resource definition with x-kubernetes-preserve-unknown-fields in the schema root.
   192  		Attempt to create and apply a change a custom resource, via kubectl; kubectl validation MUST accept unknown
   193  		properties. Attempt kubectl explain; the output MUST show the custom resource KIND.
   194  	*/
   195  	framework.ConformanceIt("works for CRD preserving unknown fields at the schema root", func(ctx context.Context) {
   196  		crd, err := setupCRDAndVerifySchema(f, schemaPreserveRoot, nil, "unknown-at-root", "v1")
   197  		if err != nil {
   198  			framework.Failf("%v", err)
   199  		}
   200  
   201  		meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr")
   202  		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
   203  
   204  		ginkgo.By("kubectl validation (kubectl create and apply) allows request with any unknown properties")
   205  		randomCR := fmt.Sprintf(`{%s,"a":{"b":[{"c":"d"}]}}`, meta)
   206  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "create", "-f", "-"); err != nil {
   207  			framework.Failf("failed to create random CR %s for CRD that allows unknown properties at the root: %v", randomCR, err)
   208  		}
   209  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   210  			framework.Failf("failed to delete random CR: %v", err)
   211  		}
   212  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "apply", "-f", "-"); err != nil {
   213  			framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err)
   214  		}
   215  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   216  			framework.Failf("failed to delete random CR: %v", err)
   217  		}
   218  
   219  		ginkgo.By("kubectl explain works to explain CR")
   220  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural, fmt.Sprintf(`(?s)KIND:.*%s`, crd.Crd.Spec.Names.Kind)); err != nil {
   221  			framework.Failf("%v", err)
   222  		}
   223  
   224  		if err := cleanupCRD(ctx, f, crd); err != nil {
   225  			framework.Failf("%v", err)
   226  		}
   227  	})
   228  
   229  	/*
   230  		Release: v1.16
   231  		Testname: Custom Resource OpenAPI Publish, with x-kubernetes-preserve-unknown-fields in embedded object
   232  		Description: Register a custom resource definition with x-kubernetes-preserve-unknown-fields in an embedded object.
   233  		Attempt to create and apply a change a custom resource, via kubectl; kubectl validation MUST accept unknown
   234  		properties. Attempt kubectl explain; the output MUST show that x-preserve-unknown-properties is used on the
   235  		nested field.
   236  	*/
   237  	framework.ConformanceIt("works for CRD preserving unknown fields in an embedded object", func(ctx context.Context) {
   238  		crd, err := setupCRDAndVerifySchema(f, schemaPreserveNested, nil, "unknown-in-nested", "v1")
   239  		if err != nil {
   240  			framework.Failf("%v", err)
   241  		}
   242  
   243  		meta := fmt.Sprintf(metaPattern, crd.Crd.Spec.Names.Kind, crd.Crd.Spec.Group, crd.Crd.Spec.Versions[0].Name, "test-cr")
   244  		ns := fmt.Sprintf("--namespace=%v", f.Namespace.Name)
   245  
   246  		ginkgo.By("kubectl validation (kubectl create and apply) allows request with any unknown properties")
   247  		randomCR := fmt.Sprintf(`{%s,"spec":{"a":null,"b":[{"c":"d"}]}}`, meta)
   248  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "create", "-f", "-"); err != nil {
   249  			framework.Failf("failed to create random CR %s for CRD that allows unknown properties in a nested object: %v", randomCR, err)
   250  		}
   251  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   252  			framework.Failf("failed to delete random CR: %v", err)
   253  		}
   254  		if _, err := e2ekubectl.RunKubectlInput(f.Namespace.Name, randomCR, ns, "apply", "-f", "-"); err != nil {
   255  			framework.Failf("failed to apply random CR %s for CRD without schema: %v", randomCR, err)
   256  		}
   257  		if _, err := e2ekubectl.RunKubectl(f.Namespace.Name, ns, "delete", crd.Crd.Spec.Names.Plural, "test-cr"); err != nil {
   258  			framework.Failf("failed to delete random CR: %v", err)
   259  		}
   260  
   261  		ginkgo.By("kubectl explain works to explain CR")
   262  		if err := verifyKubectlExplain(f.Namespace.Name, crd.Crd.Spec.Names.Plural, `(?s)DESCRIPTION:.*preserve-unknown-properties in nested field for Testing`); err != nil {
   263  			framework.Failf("%v", err)
   264  		}
   265  
   266  		if err := cleanupCRD(ctx, f, crd); err != nil {
   267  			framework.Failf("%v", err)
   268  		}
   269  	})
   270  
   271  	/*
   272  		Release: v1.16
   273  		Testname: Custom Resource OpenAPI Publish, varying groups
   274  		Description: Register multiple custom resource definitions spanning different groups and versions;
   275  		OpenAPI definitions MUST be published for custom resource definitions.
   276  	*/
   277  	framework.ConformanceIt("works for multiple CRDs of different groups", func(ctx context.Context) {
   278  		ginkgo.By("CRs in different groups (two CRDs) show up in OpenAPI documentation")
   279  		crdFoo, err := setupCRD(f, schemaFoo, "foo", "v1")
   280  		if err != nil {
   281  			framework.Failf("%v", err)
   282  		}
   283  		crdWaldo, err := setupCRD(f, schemaWaldo, "waldo", "v1beta1")
   284  		if err != nil {
   285  			framework.Failf("%v", err)
   286  		}
   287  		if crdFoo.Crd.Spec.Group == crdWaldo.Crd.Spec.Group {
   288  			framework.Failf("unexpected: CRDs should be of different group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group)
   289  		}
   290  		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v1beta1"), schemaWaldo); err != nil {
   291  			framework.Failf("%v", err)
   292  		}
   293  		if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v1"), schemaFoo); err != nil {
   294  			framework.Failf("%v", err)
   295  		}
   296  		if err := cleanupCRD(ctx, f, crdFoo); err != nil {
   297  			framework.Failf("%v", err)
   298  		}
   299  		if err := cleanupCRD(ctx, f, crdWaldo); err != nil {
   300  			framework.Failf("%v", err)
   301  		}
   302  	})
   303  
   304  	/*
   305  		Release: v1.16
   306  		Testname: Custom Resource OpenAPI Publish, varying versions
   307  		Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published
   308  		for custom resource definitions.
   309  	*/
   310  	framework.ConformanceIt("works for multiple CRDs of same group but different versions", func(ctx context.Context) {
   311  		ginkgo.By("CRs in the same group but different versions (one multiversion CRD) show up in OpenAPI documentation")
   312  		crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
   313  		if err != nil {
   314  			framework.Failf("%v", err)
   315  		}
   316  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil {
   317  			framework.Failf("%v", err)
   318  		}
   319  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
   320  			framework.Failf("%v", err)
   321  		}
   322  		if err := cleanupCRD(ctx, f, crdMultiVer); err != nil {
   323  			framework.Failf("%v", err)
   324  		}
   325  
   326  		ginkgo.By("CRs in the same group but different versions (two CRDs) show up in OpenAPI documentation")
   327  		crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v4")
   328  		if err != nil {
   329  			framework.Failf("%v", err)
   330  		}
   331  		crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v5")
   332  		if err != nil {
   333  			framework.Failf("%v", err)
   334  		}
   335  		if crdFoo.Crd.Spec.Group != crdWaldo.Crd.Spec.Group {
   336  			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group)
   337  		}
   338  		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v5"), schemaWaldo); err != nil {
   339  			framework.Failf("%v", err)
   340  		}
   341  		if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v4"), schemaFoo); err != nil {
   342  			framework.Failf("%v", err)
   343  		}
   344  		if err := cleanupCRD(ctx, f, crdFoo); err != nil {
   345  			framework.Failf("%v", err)
   346  		}
   347  		if err := cleanupCRD(ctx, f, crdWaldo); err != nil {
   348  			framework.Failf("%v", err)
   349  		}
   350  	})
   351  
   352  	/*
   353  		Release: v1.16
   354  		Testname: Custom Resource OpenAPI Publish, varying kinds
   355  		Description: Register multiple custom resource definitions in the same group and version but spanning different kinds;
   356  		OpenAPI definitions MUST be published for custom resource definitions.
   357  	*/
   358  	framework.ConformanceIt("works for multiple CRDs of same group and version but different kinds", func(ctx context.Context) {
   359  		ginkgo.By("CRs in the same group and version but different kinds (two CRDs) show up in OpenAPI documentation")
   360  		crdFoo, err := setupCRD(f, schemaFoo, "common-group", "v6")
   361  		if err != nil {
   362  			framework.Failf("%v", err)
   363  		}
   364  		crdWaldo, err := setupCRD(f, schemaWaldo, "common-group", "v6")
   365  		if err != nil {
   366  			framework.Failf("%v", err)
   367  		}
   368  		if crdFoo.Crd.Spec.Group != crdWaldo.Crd.Spec.Group {
   369  			framework.Failf("unexpected: CRDs should be of the same group %v, %v", crdFoo.Crd.Spec.Group, crdWaldo.Crd.Spec.Group)
   370  		}
   371  		if err := waitForDefinition(f.ClientSet, definitionName(crdWaldo, "v6"), schemaWaldo); err != nil {
   372  			framework.Failf("%v", err)
   373  		}
   374  		if err := waitForDefinition(f.ClientSet, definitionName(crdFoo, "v6"), schemaFoo); err != nil {
   375  			framework.Failf("%v", err)
   376  		}
   377  		if err := cleanupCRD(ctx, f, crdFoo); err != nil {
   378  			framework.Failf("%v", err)
   379  		}
   380  		if err := cleanupCRD(ctx, f, crdWaldo); err != nil {
   381  			framework.Failf("%v", err)
   382  		}
   383  	})
   384  
   385  	/*
   386  		Release: v1.16
   387  		Testname: Custom Resource OpenAPI Publish, version rename
   388  		Description: Register a custom resource definition with multiple versions; OpenAPI definitions MUST be published
   389  		for custom resource definitions. Rename one of the versions of the custom resource definition via a patch;
   390  		OpenAPI definitions MUST update to reflect the rename.
   391  	*/
   392  	framework.ConformanceIt("updates the published spec when one version gets renamed", func(ctx context.Context) {
   393  		ginkgo.By("set up a multi version CRD")
   394  		crdMultiVer, err := setupCRD(f, schemaFoo, "multi-ver", "v2", "v3")
   395  		if err != nil {
   396  			framework.Failf("%v", err)
   397  		}
   398  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v3"), schemaFoo); err != nil {
   399  			framework.Failf("%v", err)
   400  		}
   401  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
   402  			framework.Failf("%v", err)
   403  		}
   404  
   405  		ginkgo.By("rename a version")
   406  		patch := []byte(`[
   407  			{"op":"test","path":"/spec/versions/1/name","value":"v3"},
   408  			{"op": "replace", "path": "/spec/versions/1/name", "value": "v4"}
   409  		]`)
   410  		crdMultiVer.Crd, err = crdMultiVer.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crdMultiVer.Crd.Name, types.JSONPatchType, patch, metav1.PatchOptions{})
   411  		if err != nil {
   412  			framework.Failf("%v", err)
   413  		}
   414  
   415  		ginkgo.By("check the new version name is served")
   416  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v4"), schemaFoo); err != nil {
   417  			framework.Failf("%v", err)
   418  		}
   419  		ginkgo.By("check the old version name is removed")
   420  		if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crdMultiVer, "v3")); err != nil {
   421  			framework.Failf("%v", err)
   422  		}
   423  		ginkgo.By("check the other version is not changed")
   424  		if err := waitForDefinition(f.ClientSet, definitionName(crdMultiVer, "v2"), schemaFoo); err != nil {
   425  			framework.Failf("%v", err)
   426  		}
   427  
   428  		// TestCrd.Versions is different from TestCrd.Crd.Versions, we have to manually
   429  		// update the name there. Used by cleanupCRD
   430  		crdMultiVer.Crd.Spec.Versions[1].Name = "v4"
   431  		if err := cleanupCRD(ctx, f, crdMultiVer); err != nil {
   432  			framework.Failf("%v", err)
   433  		}
   434  	})
   435  
   436  	/*
   437  		Release: v1.16
   438  		Testname: Custom Resource OpenAPI Publish, stop serving version
   439  		Description: Register a custom resource definition with multiple versions. OpenAPI definitions MUST be published
   440  		for custom resource definitions. Update the custom resource definition to not serve one of the versions. OpenAPI
   441  		definitions MUST be updated to not contain the version that is no longer served.
   442  	*/
   443  	framework.ConformanceIt("removes definition from spec when one version gets changed to not be served", func(ctx context.Context) {
   444  		ginkgo.By("set up a multi version CRD")
   445  		crd, err := setupCRD(f, schemaFoo, "multi-to-single-ver", "v5", "v6alpha1")
   446  		if err != nil {
   447  			framework.Failf("%v", err)
   448  		}
   449  		// just double check. setupCRD() checked this for us already
   450  		if err := waitForDefinition(f.ClientSet, definitionName(crd, "v6alpha1"), schemaFoo); err != nil {
   451  			framework.Failf("%v", err)
   452  		}
   453  		if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil {
   454  			framework.Failf("%v", err)
   455  		}
   456  
   457  		ginkgo.By("mark a version not serverd")
   458  		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Crd.Name, metav1.GetOptions{})
   459  		if err != nil {
   460  			framework.Failf("%v", err)
   461  		}
   462  		crd.Crd.Spec.Versions[1].Served = false
   463  		crd.Crd, err = crd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd.Crd, metav1.UpdateOptions{})
   464  		if err != nil {
   465  			framework.Failf("%v", err)
   466  		}
   467  
   468  		ginkgo.By("check the unserved version gets removed")
   469  		if err := waitForDefinitionCleanup(f.ClientSet, definitionName(crd, "v6alpha1")); err != nil {
   470  			framework.Failf("%v", err)
   471  		}
   472  		ginkgo.By("check the other version is not changed")
   473  		if err := waitForDefinition(f.ClientSet, definitionName(crd, "v5"), schemaFoo); err != nil {
   474  			framework.Failf("%v", err)
   475  		}
   476  
   477  		if err := cleanupCRD(ctx, f, crd); err != nil {
   478  			framework.Failf("%v", err)
   479  		}
   480  	})
   481  
   482  	// Marked as flaky until https://github.com/kubernetes/kubernetes/issues/65517 is solved.
   483  	f.It(f.WithFlaky(), "kubectl explain works for CR with the same resource name as built-in object.", func(ctx context.Context) {
   484  		customServiceShortName := fmt.Sprintf("ksvc-%d", time.Now().Unix()) // make short name unique
   485  		opt := func(crd *apiextensionsv1.CustomResourceDefinition) {
   486  			crd.ObjectMeta = metav1.ObjectMeta{Name: "services." + crd.Spec.Group}
   487  			crd.Spec.Names = apiextensionsv1.CustomResourceDefinitionNames{
   488  				Plural:     "services",
   489  				Singular:   "service",
   490  				ListKind:   "ServiceList",
   491  				Kind:       "Service",
   492  				ShortNames: []string{customServiceShortName},
   493  			}
   494  		}
   495  		crdSvc, err := setupCRDAndVerifySchemaWithOptions(f, schemaCustomService, schemaCustomService, "service", []string{"v1"}, opt)
   496  		if err != nil {
   497  			framework.Failf("%v", err)
   498  		}
   499  
   500  		if err := verifyKubectlExplain(f.Namespace.Name, customServiceShortName+".spec", `(?s)DESCRIPTION:.*Specification of CustomService.*FIELDS:.*dummy.*<string>.*Dummy property`); err != nil {
   501  			_ = cleanupCRD(ctx, f, crdSvc) // need to remove the crd since its name is unchanged
   502  			framework.Failf("%v", err)
   503  		}
   504  
   505  		if err := cleanupCRD(ctx, f, crdSvc); err != nil {
   506  			framework.Failf("%v", err)
   507  		}
   508  	})
   509  })
   510  
   511  func setupCRD(f *framework.Framework, schema []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) {
   512  	expect := schema
   513  	if schema == nil {
   514  		// to be backwards compatible, we expect CRD controller to treat
   515  		// CRD with nil schema specially and publish an empty schema
   516  		expect = []byte(`type: object`)
   517  	}
   518  	return setupCRDAndVerifySchema(f, schema, expect, groupSuffix, versions...)
   519  }
   520  
   521  func setupCRDAndVerifySchema(f *framework.Framework, schema, expect []byte, groupSuffix string, versions ...string) (*crd.TestCrd, error) {
   522  	return setupCRDAndVerifySchemaWithOptions(f, schema, expect, groupSuffix, versions)
   523  }
   524  
   525  func setupCRDAndVerifySchemaWithOptions(f *framework.Framework, schema, expect []byte, groupSuffix string, versions []string, options ...crd.Option) (*crd.TestCrd, error) {
   526  	group := fmt.Sprintf("%s-test-%s.example.com", f.BaseName, groupSuffix)
   527  	if len(versions) == 0 {
   528  		return nil, fmt.Errorf("require at least one version for CRD")
   529  	}
   530  
   531  	props := &apiextensionsv1.JSONSchemaProps{}
   532  	if schema != nil {
   533  		if err := yaml.Unmarshal(schema, props); err != nil {
   534  			return nil, err
   535  		}
   536  	}
   537  
   538  	options = append(options, func(crd *apiextensionsv1.CustomResourceDefinition) {
   539  		var apiVersions []apiextensionsv1.CustomResourceDefinitionVersion
   540  		for i, version := range versions {
   541  			version := apiextensionsv1.CustomResourceDefinitionVersion{
   542  				Name:    version,
   543  				Served:  true,
   544  				Storage: i == 0,
   545  			}
   546  			// set up validation when input schema isn't nil
   547  			if schema != nil {
   548  				version.Schema = &apiextensionsv1.CustomResourceValidation{
   549  					OpenAPIV3Schema: props,
   550  				}
   551  			} else {
   552  				version.Schema = &apiextensionsv1.CustomResourceValidation{
   553  					OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
   554  						XPreserveUnknownFields: pointer.BoolPtr(true),
   555  						Type:                   "object",
   556  					},
   557  				}
   558  			}
   559  			apiVersions = append(apiVersions, version)
   560  		}
   561  		crd.Spec.Versions = apiVersions
   562  	})
   563  	crd, err := crd.CreateMultiVersionTestCRD(f, group, options...)
   564  	if err != nil {
   565  		return nil, fmt.Errorf("failed to create CRD: %w", err)
   566  	}
   567  
   568  	for _, v := range crd.Crd.Spec.Versions {
   569  		if err := waitForDefinition(f.ClientSet, definitionName(crd, v.Name), expect); err != nil {
   570  			return nil, fmt.Errorf("%v", err)
   571  		}
   572  	}
   573  	return crd, nil
   574  }
   575  
   576  func cleanupCRD(ctx context.Context, f *framework.Framework, crd *crd.TestCrd) error {
   577  	_ = crd.CleanUp(ctx)
   578  	for _, v := range crd.Crd.Spec.Versions {
   579  		name := definitionName(crd, v.Name)
   580  		if err := waitForDefinitionCleanup(f.ClientSet, name); err != nil {
   581  			return fmt.Errorf("%v", err)
   582  		}
   583  	}
   584  	return nil
   585  }
   586  
   587  const waitSuccessThreshold = 10
   588  
   589  // mustSucceedMultipleTimes calls f multiple times on success and only returns true if all calls are successful.
   590  // This is necessary to avoid flaking tests where one call might hit a good apiserver while in HA other apiservers
   591  // might be lagging behind. Calling f multiple times reduces the chance exponentially.
   592  func mustSucceedMultipleTimes(n int, f func() (bool, error)) func() (bool, error) {
   593  	return func() (bool, error) {
   594  		for i := 0; i < n; i++ {
   595  			ok, err := f()
   596  			if err != nil || !ok {
   597  				return ok, err
   598  			}
   599  		}
   600  		return true, nil
   601  	}
   602  }
   603  
   604  // waitForDefinition waits for given definition showing up in swagger with given schema.
   605  // If schema is nil, only the existence of the given name is checked.
   606  func waitForDefinition(c k8sclientset.Interface, name string, schema []byte) error {
   607  	expect := spec.Schema{}
   608  	if err := convertJSONSchemaProps(schema, &expect); err != nil {
   609  		return err
   610  	}
   611  
   612  	err := waitForOpenAPISchema(c, func(spec *spec.Swagger) (bool, string) {
   613  		d, ok := spec.SwaggerProps.Definitions[name]
   614  		if !ok {
   615  			return false, fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not found", name)
   616  		}
   617  		if schema != nil {
   618  			// drop properties and extension that we added
   619  			dropDefaults(&d)
   620  			if !apiequality.Semantic.DeepEqual(expect, d) {
   621  				return false, fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] not match; expect: %v, actual: %v", name, expect, d)
   622  			}
   623  		}
   624  		return true, ""
   625  	})
   626  	if err != nil {
   627  		return fmt.Errorf("failed to wait for definition %q to be served with the right OpenAPI schema: %w", name, err)
   628  	}
   629  	return nil
   630  }
   631  
   632  // waitForDefinitionCleanup waits for given definition to be removed from swagger
   633  func waitForDefinitionCleanup(c k8sclientset.Interface, name string) error {
   634  	err := waitForOpenAPISchema(c, func(spec *spec.Swagger) (bool, string) {
   635  		if _, ok := spec.SwaggerProps.Definitions[name]; ok {
   636  			return false, fmt.Sprintf("spec.SwaggerProps.Definitions[\"%s\"] still exists", name)
   637  		}
   638  		return true, ""
   639  	})
   640  	if err != nil {
   641  		return fmt.Errorf("failed to wait for definition %q not to be served anymore: %w", name, err)
   642  	}
   643  	return nil
   644  }
   645  
   646  func waitForOpenAPISchema(c k8sclientset.Interface, pred func(*spec.Swagger) (bool, string)) error {
   647  	client := c.Discovery().RESTClient().(*rest.RESTClient).Client
   648  	url := c.Discovery().RESTClient().Get().AbsPath("openapi", "v2").URL()
   649  	lastMsg := ""
   650  	etag := ""
   651  	var etagSpec *spec.Swagger
   652  	if err := wait.Poll(500*time.Millisecond, 60*time.Second, mustSucceedMultipleTimes(waitSuccessThreshold, func() (bool, error) {
   653  		// download spec with etag support
   654  		spec := &spec.Swagger{}
   655  		req, err := http.NewRequest("GET", url.String(), nil)
   656  		if err != nil {
   657  			return false, err
   658  		}
   659  		req.Close = true // enforce a new connection to hit different HA API servers
   660  		if len(etag) > 0 {
   661  			req.Header.Set("If-None-Match", fmt.Sprintf(`"%s"`, etag))
   662  		}
   663  		resp, err := client.Do(req)
   664  		if err != nil {
   665  			return false, err
   666  		}
   667  		defer resp.Body.Close()
   668  		if resp.StatusCode == http.StatusNotModified {
   669  			spec = etagSpec
   670  		} else if resp.StatusCode != http.StatusOK {
   671  			return false, fmt.Errorf("unexpected response: %d", resp.StatusCode)
   672  		} else if bs, err := io.ReadAll(resp.Body); err != nil {
   673  			return false, err
   674  		} else if err := json.Unmarshal(bs, spec); err != nil {
   675  			return false, err
   676  		} else {
   677  			etag = strings.Trim(resp.Header.Get("ETag"), `"`)
   678  			etagSpec = spec
   679  		}
   680  
   681  		var ok bool
   682  		ok, lastMsg = pred(spec)
   683  		return ok, nil
   684  	})); err != nil {
   685  		return fmt.Errorf("failed to wait for OpenAPI spec validating condition: %v; lastMsg: %s", err, lastMsg)
   686  	}
   687  	return nil
   688  }
   689  
   690  // convertJSONSchemaProps converts JSONSchemaProps in YAML to spec.Schema
   691  func convertJSONSchemaProps(in []byte, out *spec.Schema) error {
   692  	external := apiextensionsv1.JSONSchemaProps{}
   693  	if err := yaml.UnmarshalStrict(in, &external); err != nil {
   694  		return err
   695  	}
   696  	internal := apiextensions.JSONSchemaProps{}
   697  	if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil {
   698  		return err
   699  	}
   700  	kubeOut := spec.Schema{}
   701  	if err := validation.ConvertJSONSchemaPropsWithPostProcess(&internal, &kubeOut, validation.StripUnsupportedFormatsPostProcess); err != nil {
   702  		return err
   703  	}
   704  	bs, err := json.Marshal(kubeOut)
   705  	if err != nil {
   706  		return err
   707  	}
   708  	return json.Unmarshal(bs, out)
   709  }
   710  
   711  // dropDefaults drops properties and extension that we added to a schema
   712  func dropDefaults(s *spec.Schema) {
   713  	delete(s.Properties, "metadata")
   714  	delete(s.Properties, "apiVersion")
   715  	delete(s.Properties, "kind")
   716  	delete(s.Extensions, "x-kubernetes-group-version-kind")
   717  	delete(s.Extensions, "x-kubernetes-selectable-fields")
   718  }
   719  
   720  func verifyKubectlExplain(ns, name, pattern string) error {
   721  	result, err := e2ekubectl.RunKubectl(ns, "explain", name)
   722  	if err != nil {
   723  		return fmt.Errorf("failed to explain %s: %w", name, err)
   724  	}
   725  	r := regexp.MustCompile(pattern)
   726  	if !r.Match([]byte(result)) {
   727  		return fmt.Errorf("kubectl explain %s result {%s} doesn't match pattern {%s}", name, result, pattern)
   728  	}
   729  	return nil
   730  }
   731  
   732  // definitionName returns the openapi definition name for given CRD in given version
   733  func definitionName(crd *crd.TestCrd, version string) string {
   734  	return openapiutil.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", crd.Crd.Spec.Group, version, crd.Crd.Spec.Names.Kind))
   735  }
   736  
   737  var schemaFoo = []byte(`description: Foo CRD for Testing
   738  type: object
   739  properties:
   740    spec:
   741      type: object
   742      description: Specification of Foo
   743      properties:
   744        bars:
   745          description: List of Bars and their specs.
   746          type: array
   747          items:
   748            type: object
   749            required:
   750            - name
   751            properties:
   752              name:
   753                description: Name of Bar.
   754                type: string
   755              age:
   756                description: Age of Bar.
   757                type: string
   758              feeling:
   759                description: Whether Bar is feeling great.
   760                type: string
   761                enum:
   762                - Great
   763                - Down
   764              bazs:
   765                description: List of Bazs.
   766                items:
   767                  type: string
   768                type: array
   769    status:
   770      description: Status of Foo
   771      type: object
   772      properties:
   773        bars:
   774          description: List of Bars and their statuses.
   775          type: array
   776          items:
   777            type: object
   778            properties:
   779              name:
   780                description: Name of Bar.
   781                type: string
   782              available:
   783                description: Whether the Bar is installed.
   784                type: boolean
   785              quxType:
   786                description: Indicates to external qux type.
   787                pattern: in-tree|out-of-tree
   788                type: string`)
   789  
   790  var schemaCustomService = []byte(`description: CustomService CRD for Testing
   791  type: object
   792  properties:
   793    spec:
   794      description: Specification of CustomService
   795      type: object
   796      properties:
   797        dummy:
   798          description: Dummy property.
   799          type: string
   800  `)
   801  
   802  var schemaWaldo = []byte(`description: Waldo CRD for Testing
   803  type: object
   804  properties:
   805    spec:
   806      description: Specification of Waldo
   807      type: object
   808      properties:
   809        dummy:
   810          description: Dummy property.
   811          type: object
   812    status:
   813      description: Status of Waldo
   814      type: object
   815      properties:
   816        bars:
   817          description: List of Bars and their statuses.
   818          type: array
   819          items:
   820            type: object`)
   821  
   822  var schemaPreserveRoot = []byte(`description: preserve-unknown-properties at root for Testing
   823  x-kubernetes-preserve-unknown-fields: true
   824  type: object
   825  properties:
   826    spec:
   827      description: Specification of Waldo
   828      type: object
   829      properties:
   830        dummy:
   831          description: Dummy property.
   832          type: object
   833    status:
   834      description: Status of Waldo
   835      type: object
   836      properties:
   837        bars:
   838          description: List of Bars and their statuses.
   839          type: array
   840          items:
   841            type: object`)
   842  
   843  var schemaPreserveNested = []byte(`description: preserve-unknown-properties in nested field for Testing
   844  type: object
   845  properties:
   846    spec:
   847      description: Specification of Waldo
   848      type: object
   849      x-kubernetes-preserve-unknown-fields: true
   850      properties:
   851        dummy:
   852          description: Dummy property.
   853          type: object
   854    status:
   855      description: Status of Waldo
   856      type: object
   857      properties:
   858        bars:
   859          description: List of Bars and their statuses.
   860          type: array
   861          items:
   862            type: object`)
   863  

View as plain text