openapi_test.go

Documentation: k8s.io/kube-openapi/pkg/schemaconv

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package schemaconv_test
    18  
    19  import (
    20  	"encoding/json"
    21  	"os"
    22  	"reflect"
    23  	"sort"
    24  	"strings"
    25  	"testing"
    26  
    27  	openapi_v2 "github.com/google/gnostic-models/openapiv2"
    28  	"github.com/stretchr/testify/require"
    29  
    30  	"k8s.io/kube-openapi/pkg/schemaconv"
    31  	"k8s.io/kube-openapi/pkg/spec3"
    32  	"k8s.io/kube-openapi/pkg/util/proto"
    33  	"k8s.io/kube-openapi/pkg/validation/spec"
    34  	"sigs.k8s.io/structured-merge-diff/v4/schema"
    35  )
    36  
    37  var swaggerJSONPath = "testdata/swagger.json"
    38  var testCRDPath = "testdata/crds"
    39  
    40  var deducedName string = "__untyped_deduced_"
    41  var untypedName string = "__untyped_atomic_"
    42  
    43  const (
    44  	quantityResource     = "io.k8s.apimachinery.pkg.api.resource.Quantity"
    45  	rawExtensionResource = "io.k8s.apimachinery.pkg.runtime.RawExtension"
    46  )
    47  
    48  func toPtrMap[T comparable, V any](m map[T]V) map[T]*V {
    49  	if m == nil {
    50  		return nil
    51  	}
    52  
    53  	res := map[T]*V{}
    54  	for k, v := range m {
    55  		vCopy := v
    56  		res[k] = &vCopy
    57  	}
    58  	return res
    59  }
    60  
    61  func normalizeTypeRef(tr *schema.TypeRef) {
    62  	var untypedScalar schema.Scalar = "untyped"
    63  
    64  	// Deduplicate deducedDef
    65  	if tr.Inlined.Equals(&schema.Atom{
    66  		Scalar: &untypedScalar,
    67  		List: &schema.List{
    68  			ElementType: schema.TypeRef{
    69  				NamedType: &untypedName,
    70  			},
    71  			ElementRelationship: schema.Atomic,
    72  		},
    73  		Map: &schema.Map{
    74  			ElementType: schema.TypeRef{
    75  				NamedType: &deducedName,
    76  			},
    77  			ElementRelationship: schema.Separable,
    78  		},
    79  	}) {
    80  		*tr = schema.TypeRef{
    81  			NamedType: &deducedName,
    82  		}
    83  	} else if tr.NamedType != nil && *tr.NamedType == rawExtensionResource {
    84  		// In old conversion all references to rawExtension were
    85  		// replaced with untyped. In new implementation we preserve
    86  		// the references and instead change the raw extension type
    87  		// to untyped.
    88  		// For normalization, just convert rawextension references
    89  		// to "untyped"
    90  		*tr = schema.TypeRef{
    91  			NamedType: &untypedName,
    92  		}
    93  	} else {
    94  		normalizeType(&tr.Inlined)
    95  	}
    96  }
    97  
    98  // There are minor differences in new API that are semantically equivalent:
    99  //  1. old openapi would replace refs to RawExtensoin with "untyped" and leave
   100  //     RawExtension with a non-referenced/nonworking definition. New implemenatation
   101  //     makes RawExtensoin "untyped", and leaves references to RawExtension.
   102  //  2. old openapi would include "separable" relationship with
   103  //     arbitrary/deduced maps where the new implementation leaves it unset
   104  //     if it is unset by the user.
   105  func normalizeType(typ *schema.Atom) {
   106  	if typ.List != nil {
   107  		if typ.List.ElementType.Inlined != (schema.Atom{}) {
   108  			typ.List = &*typ.List
   109  			normalizeTypeRef(&typ.List.ElementType)
   110  		}
   111  	}
   112  
   113  	if typ.Map != nil {
   114  		typ.Map = &*typ.Map
   115  
   116  		fields := make([]schema.StructField, 0)
   117  		copy(typ.Fields, fields)
   118  		typ.Fields = fields
   119  
   120  		for i, f := range typ.Fields {
   121  			// Known Difference: Old conversion parses "{}" as empty map[any]any.
   122  			// 					 New conversion parses it as empty map[string]any
   123  			if reflect.DeepEqual(f.Default, map[any]any{}) {
   124  				f.Default = map[string]any{}
   125  			}
   126  
   127  			normalizeTypeRef(&f.Type)
   128  			typ.Fields[i] = f
   129  		}
   130  
   131  		sort.SliceStable(typ.Fields, func(i, j int) bool {
   132  			return strings.Compare(typ.Fields[i].Name, typ.Fields[j].Name) < 0
   133  		})
   134  
   135  		// Current unions implementation is busted and not supported in new
   136  		// format. Do not include in comparison
   137  		typ.Unions = nil
   138  
   139  		if typ.Map.ElementType.NamedType != nil {
   140  			if len(typ.Map.ElementRelationship) == 0 && typ.Scalar != nil && typ.List != nil && *typ.Map.ElementType.NamedType == deducedName {
   141  				// In old implementation arbitrary/deduced map would always also
   142  				// include "separable".
   143  				// New implementation has some code paths that dont follow that
   144  				// (separable is default) so always attaach separable to deduced.
   145  				typ.Map.ElementRelationship = schema.Separable
   146  			}
   147  		}
   148  
   149  		normalizeTypeRef(&typ.Map.ElementType)
   150  	}
   151  }
   152  
   153  // Can't directly proto models conversion to direct conversion due to subtle,
   154  // expected differences between the two conversion methods.
   155  // i.e. toProtoModels preserves sort order of fields. Direct conversion does not
   156  // (due to spec.Schema using map for fields vs gnostic's slice)
   157  func normalizeTypes(types []schema.TypeDef) map[string]schema.TypeDef {
   158  	res := map[string]schema.TypeDef{}
   159  	for _, typ := range types {
   160  		if _, exists := res[typ.Name]; !exists {
   161  			normalizeType(&typ.Atom)
   162  			res[typ.Name] = typ
   163  		}
   164  	}
   165  
   166  	// Old conversion would leave broken raw-extension definition, and just replace
   167  	// references to it with an inlined __untyped_atomic_
   168  	// The new conversion leaves references to rawextension in place, and instead
   169  	// fixes the definition of RawExtension.
   170  	//
   171  	// This bit of code reverts the new conversion's fix, and puts in place the old
   172  	// broken raw extension definition.
   173  	res["io.k8s.apimachinery.pkg.runtime.RawExtension"] = schema.TypeDef{
   174  		Name: "io.k8s.apimachinery.pkg.runtime.RawExtension",
   175  		Atom: schema.Atom{
   176  			Map: &schema.Map{
   177  				ElementType: schema.TypeRef{
   178  					NamedType: &deducedName,
   179  				},
   180  			},
   181  		},
   182  	}
   183  
   184  	// v3 CRDs do not contain these orphaned, unnecessary definitions but v2 does
   185  	//!TODO: either bring v3 to parity, or just remove these from v2 samples
   186  	ignoreList := []string{
   187  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceColumnDefinition",
   188  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceConversion",
   189  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinition",
   190  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionCondition",
   191  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionNames",
   192  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionSpec",
   193  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionStatus",
   194  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionVersion",
   195  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceSubresourceScale",
   196  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceSubresourceStatus",
   197  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceSubresources",
   198  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceValidation",
   199  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.ExternalDocumentation",
   200  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSON",
   201  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaProps",
   202  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaPropsOrArray",
   203  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaPropsOrBool",
   204  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaPropsOrStringArray",
   205  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.ServiceReference",
   206  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.ValidationRule",
   207  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.WebhookClientConfig",
   208  		"io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.WebhookConversion",
   209  		"io.k8s.apimachinery.pkg.apis.meta.v1.DeleteOptions",
   210  		"io.k8s.apimachinery.pkg.apis.meta.v1.Patch",
   211  		"io.k8s.apimachinery.pkg.apis.meta.v1.Preconditions",
   212  		"io.k8s.apimachinery.pkg.apis.meta.v1.Status",
   213  		"io.k8s.apimachinery.pkg.apis.meta.v1.StatusCause",
   214  		"io.k8s.apimachinery.pkg.apis.meta.v1.StatusDetails",
   215  	}
   216  
   217  	for _, k := range ignoreList {
   218  		delete(res, k)
   219  	}
   220  
   221  	return res
   222  }
   223  
   224  func TestCRDOpenAPIConversion(t *testing.T) {
   225  	files, err := os.ReadDir("testdata/crds/openapiv2")
   226  	require.NoError(t, err)
   227  	for _, entry := range files {
   228  		t.Run(entry.Name(), func(t *testing.T) {
   229  			t.Parallel()
   230  			openAPIV2Contents, err := os.ReadFile("testdata/crds/openapiv2/" + entry.Name())
   231  			require.NoError(t, err)
   232  
   233  			openAPIV3Contents, err := os.ReadFile("testdata/crds/openapiv3/" + entry.Name())
   234  			require.NoError(t, err)
   235  
   236  			var v3 spec3.OpenAPI
   237  
   238  			err = json.Unmarshal(openAPIV3Contents, &v3)
   239  			require.NoError(t, err)
   240  
   241  			v2Types, err := specToSchemaViaProtoModels(openAPIV2Contents)
   242  			require.NoError(t, err)
   243  			v3Types, err := schemaconv.ToSchemaFromOpenAPI(v3.Components.Schemas, false)
   244  			require.NoError(t, err)
   245  
   246  			require.Equal(t, normalizeTypes(v2Types.Types), normalizeTypes(v3Types.Types))
   247  		})
   248  	}
   249  }
   250  
   251  // Using all models defined in swagger.json
   252  // Convert to SMD using two methods:
   253  //  1. Spec -> SMD
   254  //  2. Spec -> JSON -> gnostic -> SMD
   255  //
   256  // Compare YAML forms. We have some allowed differences...
   257  func TestOpenAPIImplementation(t *testing.T) {
   258  	swaggerJSON, err := os.ReadFile(swaggerJSONPath)
   259  	require.NoError(t, err)
   260  
   261  	protoModels, err := specToSchemaViaProtoModels(swaggerJSON)
   262  	require.NoError(t, err)
   263  
   264  	var swag spec.Swagger
   265  	err = json.Unmarshal(swaggerJSON, &swag)
   266  	require.NoError(t, err)
   267  
   268  	newConversionTypes, err := schemaconv.ToSchemaFromOpenAPI(toPtrMap(swag.Definitions), false)
   269  	require.NoError(t, err)
   270  
   271  	require.Equal(t, normalizeTypes(protoModels.Types), normalizeTypes(newConversionTypes.Types))
   272  }
   273  
   274  func specToSchemaViaProtoModels(input []byte) (*schema.Schema, error) {
   275  	document, err := openapi_v2.ParseDocument(input)
   276  	if err != nil {
   277  		return nil, err
   278  	}
   279  
   280  	models, err := proto.NewOpenAPIData(document)
   281  	if err != nil {
   282  		return nil, err
   283  	}
   284  
   285  	newSchema, err := schemaconv.ToSchema(models)
   286  	if err != nil {
   287  		return nil, err
   288  	}
   289  
   290  	return newSchema, nil
   291  }
   292  
   293  func BenchmarkOpenAPIConversion(b *testing.B) {
   294  	swaggerJSON, err := os.ReadFile(swaggerJSONPath)
   295  	require.NoError(b, err)
   296  
   297  	doc := spec.Swagger{}
   298  	require.NoError(b, doc.UnmarshalJSON(swaggerJSON))
   299  
   300  	// Beginning the benchmark from spec.Schema, since that is the format
   301  	// stored by the kube-apiserver
   302  	b.Run("spec.Schema->schema.Schema", func(b *testing.B) {
   303  		b.ReportAllocs()
   304  		for i := 0; i < b.N; i++ {
   305  			_, err := schemaconv.ToSchemaFromOpenAPI(toPtrMap(doc.Definitions), false)
   306  			require.NoError(b, err)
   307  		}
   308  	})
   309  
   310  	b.Run("spec.Schema->json->gnostic_v2->proto.Models->schema.Schema", func(b *testing.B) {
   311  		b.ReportAllocs()
   312  		for i := 0; i < b.N; i++ {
   313  			jsonText, err := doc.MarshalJSON()
   314  			require.NoError(b, err)
   315  
   316  			_, err = specToSchemaViaProtoModels(jsonText)
   317  			require.NoError(b, err)
   318  		}
   319  	})
   320  }
   321  
   322  func BenchmarkOpenAPICRDConversion(b *testing.B) {
   323  	files, err := os.ReadDir("testdata/crds/openapiv2")
   324  	require.NoError(b, err)
   325  	for _, entry := range files {
   326  		b.Run(entry.Name(), func(b *testing.B) {
   327  			openAPIV2Contents, err := os.ReadFile("testdata/crds/openapiv2/" + entry.Name())
   328  			require.NoError(b, err)
   329  
   330  			openAPIV3Contents, err := os.ReadFile("testdata/crds/openapiv3/" + entry.Name())
   331  			require.NoError(b, err)
   332  
   333  			var v2 spec.Swagger
   334  			var v3 spec3.OpenAPI
   335  
   336  			err = json.Unmarshal(openAPIV2Contents, &v2)
   337  			require.NoError(b, err)
   338  
   339  			err = json.Unmarshal(openAPIV3Contents, &v3)
   340  			require.NoError(b, err)
   341  
   342  			// Beginning the benchmark from spec.Schema, since that is the format
   343  			// stored by the kube-apiserver
   344  			b.Run("spec.Schema->schema.Schema", func(b *testing.B) {
   345  				b.ReportAllocs()
   346  				for i := 0; i < b.N; i++ {
   347  					_, err := schemaconv.ToSchemaFromOpenAPI(v3.Components.Schemas, false)
   348  					require.NoError(b, err)
   349  				}
   350  			})
   351  
   352  			b.Run("spec.Schema->json->gnostic_v2->proto.Models->schema.Schema", func(b *testing.B) {
   353  				b.ReportAllocs()
   354  				for i := 0; i < b.N; i++ {
   355  					jsonText, err := v2.MarshalJSON()
   356  					require.NoError(b, err)
   357  
   358  					_, err = specToSchemaViaProtoModels(jsonText)
   359  					require.NoError(b, err)
   360  				}
   361  			})
   362  		})
   363  	}
   364  }
   365
View as plain text