1
16
17 package integration
18
19 import (
20 "context"
21 "encoding/json"
22 "fmt"
23 "os"
24 "path"
25 "reflect"
26 "testing"
27 "time"
28
29 clientv3 "go.etcd.io/etcd/client/v3"
30
31 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
32 apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
33 "k8s.io/apiextensions-apiserver/test/integration/fixtures"
34 "k8s.io/apimachinery/pkg/api/errors"
35 "k8s.io/apimachinery/pkg/api/meta"
36 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
37 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
38 "k8s.io/apimachinery/pkg/watch"
39 "k8s.io/client-go/dynamic"
40 )
41
42 func TestMultipleResourceInstances(t *testing.T) {
43 tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
44 if err != nil {
45 t.Fatal(err)
46 }
47 defer tearDown()
48
49 ns := "not-the-default"
50 noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
51 noxuDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
52 if err != nil {
53 t.Fatal(err)
54 }
55 noxuNamespacedResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
56 noxuList, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
57 if err != nil {
58 t.Fatal(err)
59 }
60
61 noxuListListMeta, err := meta.ListAccessor(noxuList)
62 if err != nil {
63 t.Fatal(err)
64 }
65
66 noxuNamespacedWatch, err := noxuNamespacedResourceClient.Watch(context.TODO(), metav1.ListOptions{ResourceVersion: noxuListListMeta.GetResourceVersion()})
67 if err != nil {
68 t.Fatal(err)
69 }
70 defer noxuNamespacedWatch.Stop()
71
72 instances := map[string]*struct {
73 Added bool
74 Deleted bool
75 Instance *unstructured.Unstructured
76 }{
77 "foo": {},
78 "bar": {},
79 }
80
81 for key, val := range instances {
82 val.Instance, err = instantiateCustomResource(t, fixtures.NewNoxuInstance(ns, key), noxuNamespacedResourceClient, noxuDefinition)
83 if err != nil {
84 t.Fatalf("unable to create Noxu Instance %q:%v", key, err)
85 }
86 }
87
88 addEvents := 0
89 for addEvents < len(instances) {
90 select {
91 case watchEvent := <-noxuNamespacedWatch.ResultChan():
92 if e, a := watch.Added, watchEvent.Type; e != a {
93 t.Fatalf("expected %v, got %v", e, a)
94 }
95 name, err := meta.NewAccessor().Name(watchEvent.Object)
96 if err != nil {
97 t.Fatalf("unable to retrieve object name:%v", err)
98 }
99 if instances[name].Added {
100 t.Fatalf("Add event already registered for %q", name)
101 }
102 instances[name].Added = true
103 addEvents++
104 case <-time.After(5 * time.Second):
105 t.Fatalf("missing watch event")
106 }
107 }
108
109 for key, val := range instances {
110 gottenNoxuInstace, err := noxuNamespacedResourceClient.Get(context.TODO(), key, metav1.GetOptions{})
111 if err != nil {
112 t.Fatal(err)
113 }
114 if e, a := val.Instance, gottenNoxuInstace; !reflect.DeepEqual(e, a) {
115 t.Errorf("expected %v, got %v", e, a)
116 }
117 }
118 listWithItem, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
119 if err != nil {
120 t.Fatal(err)
121 }
122 if e, a := len(instances), len(listWithItem.Items); e != a {
123 t.Errorf("expected %v, got %v", e, a)
124 }
125 for _, a := range listWithItem.Items {
126 if e := instances[a.GetName()].Instance; !reflect.DeepEqual(e, &a) {
127 t.Errorf("expected %v, got %v", e, a)
128 }
129 }
130 for key := range instances {
131 if err := noxuNamespacedResourceClient.Delete(context.TODO(), key, metav1.DeleteOptions{}); err != nil {
132 t.Fatalf("unable to delete %s:%v", key, err)
133 }
134 }
135 listWithoutItem, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
136 if err != nil {
137 t.Fatal(err)
138 }
139 if e, a := 0, len(listWithoutItem.Items); e != a {
140 t.Errorf("expected %v, got %v", e, a)
141 }
142
143 deleteEvents := 0
144 for deleteEvents < len(instances) {
145 select {
146 case watchEvent := <-noxuNamespacedWatch.ResultChan():
147 if e, a := watch.Deleted, watchEvent.Type; e != a {
148 t.Errorf("expected %v, got %v", e, a)
149 break
150 }
151 name, err := meta.NewAccessor().Name(watchEvent.Object)
152 if err != nil {
153 t.Errorf("unable to retrieve object name:%v", err)
154 }
155 if instances[name].Deleted {
156 t.Errorf("Delete event already registered for %q", name)
157 }
158 instances[name].Deleted = true
159 deleteEvents++
160 case <-time.After(5 * time.Second):
161 t.Errorf("missing watch event")
162 }
163 }
164 }
165
166 func TestMultipleRegistration(t *testing.T) {
167 tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
168 if err != nil {
169 t.Fatal(err)
170 }
171 defer tearDown()
172
173 ns := "not-the-default"
174 sameInstanceName := "foo"
175 noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
176 noxuDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
177 if err != nil {
178 t.Fatal(err)
179 }
180 noxuNamespacedResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
181 createdNoxuInstance, err := instantiateCustomResource(t, fixtures.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition)
182 if err != nil {
183 t.Fatalf("unable to create noxu Instance:%v", err)
184 }
185
186 gottenNoxuInstance, err := noxuNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{})
187 if err != nil {
188 t.Fatal(err)
189 }
190 if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) {
191 t.Errorf("expected %v, got %v", e, a)
192 }
193
194 curletDefinition := fixtures.NewCurletV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
195 curletDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(curletDefinition, apiExtensionClient, dynamicClient)
196 if err != nil {
197 t.Fatal(err)
198 }
199 curletNamespacedResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, curletDefinition)
200 createdCurletInstance, err := instantiateCustomResource(t, fixtures.NewCurletInstance(ns, sameInstanceName), curletNamespacedResourceClient, curletDefinition)
201 if err != nil {
202 t.Fatalf("unable to create noxu Instance:%v", err)
203 }
204 gottenCurletInstance, err := curletNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{})
205 if err != nil {
206 t.Fatal(err)
207 }
208 if e, a := createdCurletInstance, gottenCurletInstance; !reflect.DeepEqual(e, a) {
209 t.Errorf("expected %v, got %v", e, a)
210 }
211
212
213 gottenNoxuInstance2, err := noxuNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{})
214 if err != nil {
215 t.Fatal(err)
216 }
217 if e, a := createdNoxuInstance, gottenNoxuInstance2; !reflect.DeepEqual(e, a) {
218 t.Errorf("expected %v, got %v", e, a)
219 }
220 }
221
222 func TestDeRegistrationAndReRegistration(t *testing.T) {
223 tearDown, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t)
224 if err != nil {
225 t.Fatal(err)
226 }
227 defer tearDown()
228 noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
229 ns := "not-the-default"
230 sameInstanceName := "foo"
231 func() {
232 noxuDefinition, err := fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
233 if err != nil {
234 t.Fatal(err)
235 }
236 noxuNamespacedResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
237 if _, err := instantiateCustomResource(t, fixtures.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition); err != nil {
238 t.Fatal(err)
239 }
240 if err := fixtures.DeleteV1CustomResourceDefinition(noxuDefinition, apiExtensionClient); err != nil {
241 t.Fatal(err)
242 }
243 if _, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), noxuDefinition.Name, metav1.GetOptions{}); err == nil || !errors.IsNotFound(err) {
244 t.Fatalf("expected a NotFound error, got:%v", err)
245 }
246 if _, err = noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{}); err == nil || !errors.IsNotFound(err) {
247 t.Fatalf("expected a NotFound error, got:%v", err)
248 }
249 if _, err = noxuNamespacedResourceClient.Get(context.TODO(), "foo", metav1.GetOptions{}); err == nil || !errors.IsNotFound(err) {
250 t.Fatalf("expected a NotFound error, got:%v", err)
251 }
252 }()
253
254 func() {
255 if _, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), noxuDefinition.Name, metav1.GetOptions{}); err == nil || !errors.IsNotFound(err) {
256 t.Fatalf("expected a NotFound error, got:%v", err)
257 }
258 noxuDefinition, err := fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
259 if err != nil {
260 t.Fatal(err)
261 }
262 noxuNamespacedResourceClient := newNamespacedCustomResourceClient(ns, dynamicClient, noxuDefinition)
263 initialList, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
264 if err != nil {
265 t.Fatal(err)
266 }
267 if _, err = noxuNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{}); err == nil || !errors.IsNotFound(err) {
268 t.Fatalf("expected a NotFound error, got:%v", err)
269 }
270 if e, a := 0, len(initialList.Items); e != a {
271 t.Fatalf("expected %v, got %v", e, a)
272 }
273 createdNoxuInstance, err := instantiateCustomResource(t, fixtures.NewNoxuInstance(ns, sameInstanceName), noxuNamespacedResourceClient, noxuDefinition)
274 if err != nil {
275 t.Fatal(err)
276 }
277 gottenNoxuInstance, err := noxuNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{})
278 if err != nil {
279 t.Fatal(err)
280 }
281 if e, a := createdNoxuInstance, gottenNoxuInstance; !reflect.DeepEqual(e, a) {
282 t.Fatalf("expected %v, got %v", e, a)
283 }
284 listWithItem, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
285 if err != nil {
286 t.Fatal(err)
287 }
288 if e, a := 1, len(listWithItem.Items); e != a {
289 t.Fatalf("expected %v, got %v", e, a)
290 }
291 if e, a := *createdNoxuInstance, listWithItem.Items[0]; !reflect.DeepEqual(e, a) {
292 t.Fatalf("expected %v, got %v", e, a)
293 }
294
295 if err := noxuNamespacedResourceClient.Delete(context.TODO(), sameInstanceName, metav1.DeleteOptions{}); err != nil {
296 t.Fatal(err)
297 }
298 if _, err = noxuNamespacedResourceClient.Get(context.TODO(), sameInstanceName, metav1.GetOptions{}); err == nil || !errors.IsNotFound(err) {
299 t.Fatalf("expected a NotFound error, got:%v", err)
300 }
301 listWithoutItem, err := noxuNamespacedResourceClient.List(context.TODO(), metav1.ListOptions{})
302 if err != nil {
303 t.Fatal(err)
304 }
305 if e, a := 0, len(listWithoutItem.Items); e != a {
306 t.Fatalf("expected %v, got %v", e, a)
307 }
308 }()
309 }
310
311 func TestEtcdStorage(t *testing.T) {
312 tearDown, clientConfig, s, err := fixtures.StartDefaultServer(t)
313 if err != nil {
314 t.Fatal(err)
315 }
316 defer tearDown()
317
318 apiExtensionClient, err := apiextensionsclientset.NewForConfig(clientConfig)
319 if err != nil {
320 t.Fatal(err)
321 }
322 dynamicClient, err := dynamic.NewForConfig(clientConfig)
323 if err != nil {
324 t.Fatal(err)
325 }
326
327 etcdPrefix := s.RecommendedOptions.Etcd.StorageConfig.Prefix
328
329 ns1 := "another-default-is-possible"
330 curletDefinition := fixtures.NewCurletV1CustomResourceDefinition(apiextensionsv1.ClusterScoped)
331 curletDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(curletDefinition, apiExtensionClient, dynamicClient)
332 if err != nil {
333 t.Fatal(err)
334 }
335 curletNamespacedResourceClient := newNamespacedCustomResourceClient(ns1, dynamicClient, curletDefinition)
336 if _, err := instantiateCustomResource(t, fixtures.NewCurletInstance(ns1, "bar"), curletNamespacedResourceClient, curletDefinition); err != nil {
337 t.Fatalf("unable to create curlet cluster scoped Instance:%v", err)
338 }
339
340 ns2 := "the-cruel-default"
341 noxuDefinition := fixtures.NewNoxuV1CustomResourceDefinition(apiextensionsv1.NamespaceScoped)
342 noxuDefinition, err = fixtures.CreateNewV1CustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
343 if err != nil {
344 t.Fatal(err)
345 }
346 noxuNamespacedResourceClient := newNamespacedCustomResourceClient(ns2, dynamicClient, noxuDefinition)
347 if _, err := instantiateCustomResource(t, fixtures.NewNoxuInstance(ns2, "foo"), noxuNamespacedResourceClient, noxuDefinition); err != nil {
348 t.Fatalf("unable to create noxu namespace scoped Instance:%v", err)
349 }
350
351 testcases := map[string]struct {
352 etcdPath string
353 expectedObject *metaObject
354 }{
355 "namespacedNoxuDefinition": {
356 etcdPath: "apiextensions.k8s.io/customresourcedefinitions/noxus.mygroup.example.com",
357 expectedObject: &metaObject{
358 Kind: "CustomResourceDefinition",
359 APIVersion: "apiextensions.k8s.io/v1beta1",
360 Metadata: Metadata{
361 Name: "noxus.mygroup.example.com",
362 Namespace: "",
363 },
364 },
365 },
366 "namespacedNoxuInstance": {
367 etcdPath: "mygroup.example.com/noxus/the-cruel-default/foo",
368 expectedObject: &metaObject{
369 Kind: "WishIHadChosenNoxu",
370 APIVersion: "mygroup.example.com/v1beta1",
371 Metadata: Metadata{
372 Name: "foo",
373 Namespace: "the-cruel-default",
374 },
375 },
376 },
377
378 "clusteredCurletDefinition": {
379 etcdPath: "apiextensions.k8s.io/customresourcedefinitions/curlets.mygroup.example.com",
380 expectedObject: &metaObject{
381 Kind: "CustomResourceDefinition",
382 APIVersion: "apiextensions.k8s.io/v1beta1",
383 Metadata: Metadata{
384 Name: "curlets.mygroup.example.com",
385 Namespace: "",
386 },
387 },
388 },
389
390 "clusteredCurletInstance": {
391 etcdPath: "mygroup.example.com/curlets/bar",
392 expectedObject: &metaObject{
393 Kind: "Curlet",
394 APIVersion: "mygroup.example.com/v1beta1",
395 Metadata: Metadata{
396 Name: "bar",
397 Namespace: "",
398 },
399 },
400 },
401 }
402
403 etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL")
404 if !ok {
405 etcdURL = "http://127.0.0.1:2379"
406 }
407 cfg := clientv3.Config{
408 Endpoints: []string{etcdURL},
409 }
410 c, err := clientv3.New(cfg)
411 if err != nil {
412 t.Fatal(err)
413 }
414 kv := clientv3.NewKV(c)
415 for testName, tc := range testcases {
416 output, err := getFromEtcd(kv, etcdPrefix, tc.etcdPath)
417 if err != nil {
418 t.Fatalf("%s - no path gotten from etcd:%v", testName, err)
419 }
420 if e, a := tc.expectedObject, output; !reflect.DeepEqual(e, a) {
421 t.Errorf("%s - expected %#v\n got %#v\n", testName, e, a)
422 }
423 }
424 }
425
426 func getFromEtcd(keys clientv3.KV, prefix, localPath string) (*metaObject, error) {
427 internalPath := path.Join("/", prefix, localPath)
428 response, err := keys.Get(context.Background(), internalPath)
429 if err != nil {
430 return nil, err
431 }
432 if response.More || response.Count != 1 || len(response.Kvs) != 1 {
433 return nil, fmt.Errorf("Invalid etcd response (not found == %v): %#v", response.Count == 0, response)
434 }
435 obj := &metaObject{}
436 if err := json.Unmarshal(response.Kvs[0].Value, obj); err != nil {
437 return nil, err
438 }
439 return obj, nil
440 }
441
442 type metaObject struct {
443 Kind string `json:"kind,omitempty" protobuf:"bytes,1,opt,name=kind"`
444 APIVersion string `json:"apiVersion,omitempty" protobuf:"bytes,2,opt,name=apiVersion"`
445 Metadata `json:"metadata,omitempty" protobuf:"bytes,3,opt,name=metadata"`
446 }
447
448 type Metadata struct {
449 Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"`
450 Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=namespace"`
451 }
452
View as plain text