1
16
17 package controlplane
18
19 import (
20 "context"
21 "encoding/json"
22 "fmt"
23 "strings"
24 "testing"
25 "time"
26
27 "k8s.io/apiextensions-apiserver/test/integration/fixtures"
28
29 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
30
31 v1 "k8s.io/api/core/v1"
32 networkingv1 "k8s.io/api/networking/v1"
33 apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
34 apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
36 "k8s.io/apimachinery/pkg/runtime/schema"
37 "k8s.io/apimachinery/pkg/util/wait"
38 "k8s.io/client-go/dynamic"
39 "k8s.io/client-go/kubernetes"
40 "k8s.io/kube-openapi/pkg/validation/spec"
41 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
42 "k8s.io/kubernetes/test/integration/etcd"
43 "k8s.io/kubernetes/test/integration/framework"
44 )
45
46 func TestCRDShadowGroup(t *testing.T) {
47 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
48 defer result.TearDownFn()
49
50 testNamespace := "test-crd-shadow-group"
51 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
52 if err != nil {
53 t.Fatalf("Unexpected error: %v", err)
54 }
55 if _, err := kubeclient.CoreV1().Namespaces().Create(context.TODO(), (&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}), metav1.CreateOptions{}); err != nil {
56 t.Fatal(err)
57 }
58
59 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
60 if err != nil {
61 t.Fatalf("Unexpected error: %v", err)
62 }
63
64 t.Logf("Creating a NetworkPolicy")
65 nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies(testNamespace).Create(context.TODO(), &networkingv1.NetworkPolicy{
66 ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: testNamespace},
67 Spec: networkingv1.NetworkPolicySpec{
68 PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
69 Ingress: []networkingv1.NetworkPolicyIngressRule{},
70 },
71 }, metav1.CreateOptions{})
72 if err != nil {
73 t.Fatalf("Failed to create NetworkPolicy: %v", err)
74 }
75
76 t.Logf("Trying to shadow networking group")
77 crd := &apiextensionsv1.CustomResourceDefinition{
78 ObjectMeta: metav1.ObjectMeta{
79 Name: "foos." + networkingv1.GroupName,
80 Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"},
81 },
82 Spec: apiextensionsv1.CustomResourceDefinitionSpec{
83 Group: networkingv1.GroupName,
84 Scope: apiextensionsv1.ClusterScoped,
85 Names: apiextensionsv1.CustomResourceDefinitionNames{
86 Plural: "foos",
87 Kind: "Foo",
88 },
89 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
90 {
91 Name: networkingv1.SchemeGroupVersion.Version,
92 Served: true,
93 Storage: true,
94 Schema: fixtures.AllowAllSchema(),
95 },
96 },
97 },
98 }
99 etcd.CreateTestCRDs(t, apiextensionsclient, true, crd)
100
101
102 time.Sleep(2 * time.Second)
103
104 t.Logf("Checking that we still see the NetworkPolicy")
105 _, err = kubeclient.NetworkingV1().NetworkPolicies(nwPolicy.Namespace).Get(context.TODO(), nwPolicy.Name, metav1.GetOptions{})
106 if err != nil {
107 t.Errorf("Failed to get NetworkPolocy: %v", err)
108 }
109
110 t.Logf("Checking that crd resource does not show up in networking group")
111 if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) {
112 t.Errorf("CRD resource shows up in discovery, but shouldn't.")
113 }
114 }
115
116 func TestCRD(t *testing.T) {
117 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
118 defer result.TearDownFn()
119
120 testNamespace := "test-crd"
121 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
122 if err != nil {
123 t.Fatalf("Unexpected error: %v", err)
124 }
125 if _, err := kubeclient.CoreV1().Namespaces().Create(context.TODO(), (&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}), metav1.CreateOptions{}); err != nil {
126 t.Fatal(err)
127 }
128
129 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
130 if err != nil {
131 t.Fatalf("Unexpected error: %v", err)
132 }
133
134 t.Logf("Trying to create a custom resource without conflict")
135 crd := &apiextensionsv1.CustomResourceDefinition{
136 ObjectMeta: metav1.ObjectMeta{
137 Name: "foos.cr.bar.com",
138 },
139 Spec: apiextensionsv1.CustomResourceDefinitionSpec{
140 Group: "cr.bar.com",
141 Scope: apiextensionsv1.NamespaceScoped,
142 Names: apiextensionsv1.CustomResourceDefinitionNames{
143 Plural: "foos",
144 Kind: "Foo",
145 },
146 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
147 {
148 Name: networkingv1.SchemeGroupVersion.Version,
149 Served: true,
150 Storage: true,
151 Schema: fixtures.AllowAllSchema(),
152 },
153 },
154 },
155 }
156 etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
157
158 t.Logf("Trying to access foos.cr.bar.com with dynamic client")
159 dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
160 if err != nil {
161 t.Fatalf("Unexpected error: %v", err)
162 }
163 fooResource := schema.GroupVersionResource{Group: "cr.bar.com", Version: "v1", Resource: "foos"}
164 _, err = dynamicClient.Resource(fooResource).Namespace(testNamespace).List(context.TODO(), metav1.ListOptions{})
165 if err != nil {
166 t.Errorf("Failed to list foos.cr.bar.com instances: %v", err)
167 }
168 }
169
170 func TestCRDOpenAPI(t *testing.T) {
171 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
172 defer result.TearDownFn()
173 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
174 if err != nil {
175 t.Fatalf("Unexpected error: %v", err)
176 }
177 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
178 if err != nil {
179 t.Fatalf("Unexpected error: %v", err)
180 }
181 dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
182 if err != nil {
183 t.Fatalf("Unexpected error: %v", err)
184 }
185
186 t.Logf("Trying to create a CustomResourceDefinitions")
187 nonStructuralBetaCRD := &apiextensionsv1beta1.CustomResourceDefinition{
188 ObjectMeta: metav1.ObjectMeta{
189 Name: "foos.nonstructural.cr.bar.com",
190 },
191 Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
192 Group: "nonstructural.cr.bar.com",
193 Version: "v1",
194 Scope: apiextensionsv1beta1.NamespaceScoped,
195 Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
196 Plural: "foos",
197 Kind: "Foo",
198 },
199 Validation: &apiextensionsv1beta1.CustomResourceValidation{
200 OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
201 Type: "object",
202 Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
203 "foo": {},
204 },
205 },
206 },
207 },
208 }
209 structuralCRD := &apiextensionsv1.CustomResourceDefinition{
210 ObjectMeta: metav1.ObjectMeta{
211 Name: "foos.structural.cr.bar.com",
212 },
213 Spec: apiextensionsv1.CustomResourceDefinitionSpec{
214 Group: "structural.cr.bar.com",
215 Scope: apiextensionsv1.NamespaceScoped,
216 Names: apiextensionsv1.CustomResourceDefinitionNames{
217 Plural: "foos",
218 Kind: "Foo",
219 },
220 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
221 {
222 Name: "v1",
223 Served: true,
224 Storage: true,
225 Schema: &apiextensionsv1.CustomResourceValidation{
226 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
227 Type: "object",
228 Properties: map[string]apiextensionsv1.JSONSchemaProps{
229 "foo": {Type: "string"},
230 },
231 },
232 },
233 },
234 },
235 },
236 }
237 nonStructuralCRD, err := fixtures.CreateCRDUsingRemovedAPI(result.EtcdClient, result.EtcdStoragePrefix, nonStructuralBetaCRD, apiextensionsclient, dynamicClient)
238 if err != nil {
239 t.Fatal(err)
240 }
241 etcd.CreateTestCRDs(t, apiextensionsclient, false, structuralCRD)
242
243 getPublishedSchema := func(defName string) (*spec.Schema, error) {
244 bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw(context.TODO())
245 if err != nil {
246 return nil, err
247 }
248 spec := spec.Swagger{}
249 if err := json.Unmarshal(bs, &spec); err != nil {
250 return nil, err
251 }
252 if spec.SwaggerProps.Paths == nil {
253 return nil, nil
254 }
255 d, ok := spec.SwaggerProps.Definitions[defName]
256 if !ok {
257 return nil, nil
258 }
259 return &d, nil
260 }
261
262 waitForSpec := func(crd *apiextensionsv1.CustomResourceDefinition, expectedType string) {
263 t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType)
264 lastMsg := ""
265 if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
266 lastMsg = ""
267 defName := crdDefinitionName(crd)
268 schema, err := getPublishedSchema(defName)
269 if err != nil {
270 lastMsg = err.Error()
271 return false, nil
272 }
273 if schema == nil {
274 lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[%q] not found", defName)
275 return false, nil
276 }
277 p, ok := schema.Properties["foo"]
278 if !ok {
279 lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions[%q].Properties["foo"] not found`, defName)
280 return false, nil
281 }
282 if !p.Type.Contains(expectedType) {
283 lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions[%q].Properties["foo"].Type should be %q, but got: %q`, defName, expectedType, p.Type)
284 return false, nil
285 }
286 return true, nil
287 }); err != nil {
288 t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", structuralCRD.Name, err, lastMsg)
289 }
290 }
291
292 t.Logf("Check that structural schema is published")
293 waitForSpec(structuralCRD, "string")
294 structuralCRD, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), structuralCRD.Name, metav1.GetOptions{})
295 if err != nil {
296 t.Fatal(err)
297 }
298 prop := structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"]
299 prop.Type = "boolean"
300 structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"] = prop
301 if _, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), structuralCRD, metav1.UpdateOptions{}); err != nil {
302 t.Fatal(err)
303 }
304 waitForSpec(structuralCRD, "boolean")
305
306 t.Logf("Check that non-structural schema is not published")
307 schema, err := getPublishedSchema(crdDefinitionName(nonStructuralCRD))
308 if err != nil {
309 t.Fatal(err)
310 }
311 if schema == nil {
312 t.Fatal("expected a non-nil schema")
313 }
314 if foo, ok := schema.Properties["foo"]; ok {
315 t.Fatalf("unexpected published 'foo' property: %#v", foo)
316 }
317 }
318
319 func crdDefinitionName(crd *apiextensionsv1.CustomResourceDefinition) string {
320 sgmts := strings.Split(crd.Spec.Group, ".")
321 reverse(sgmts)
322 return strings.Join(append(sgmts, crd.Spec.Versions[0].Name, crd.Spec.Names.Kind), ".")
323 }
324
325 func reverse(s []string) {
326 for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
327 s[i], s[j] = s[j], s[i]
328 }
329 }
330
View as plain text