1
16
17 package etcd
18
19 import (
20 "context"
21 "fmt"
22 "reflect"
23 "strconv"
24 "testing"
25 "time"
26
27 "github.com/pkg/errors"
28
29 pb "go.etcd.io/etcd/api/v3/etcdserverpb"
30 clientv3 "go.etcd.io/etcd/client/v3"
31 apierrors "k8s.io/apimachinery/pkg/api/errors"
32 "k8s.io/apimachinery/pkg/runtime"
33 clientsetfake "k8s.io/client-go/kubernetes/fake"
34 clienttesting "k8s.io/client-go/testing"
35
36 kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
37 "k8s.io/kubernetes/cmd/kubeadm/app/constants"
38 testresources "k8s.io/kubernetes/cmd/kubeadm/test/resources"
39 )
40
41 var errNotImplemented = errors.New("not implemented")
42
43 type fakeEtcdClient struct {
44 members []*pb.Member
45 endpoints []string
46 }
47
48
49 func (f *fakeEtcdClient) Close() error {
50 f.members = []*pb.Member{}
51 return nil
52 }
53
54
55 func (f *fakeEtcdClient) Endpoints() []string {
56 return f.endpoints
57 }
58
59
60 func (f *fakeEtcdClient) MemberList(_ context.Context) (*clientv3.MemberListResponse, error) {
61 return &clientv3.MemberListResponse{
62 Members: f.members,
63 }, nil
64 }
65
66
67 func (f *fakeEtcdClient) MemberAdd(_ context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) {
68 return nil, errNotImplemented
69 }
70
71
72 func (f *fakeEtcdClient) MemberAddAsLearner(_ context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) {
73 return nil, errNotImplemented
74 }
75
76
77 func (f *fakeEtcdClient) MemberRemove(_ context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) {
78 return nil, errNotImplemented
79 }
80
81
82 func (f *fakeEtcdClient) MemberPromote(_ context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) {
83 return nil, errNotImplemented
84 }
85
86
87 func (f *fakeEtcdClient) Status(_ context.Context, endpoint string) (*clientv3.StatusResponse, error) {
88 return nil, errNotImplemented
89 }
90
91
92 func (f *fakeEtcdClient) Sync(_ context.Context) error {
93 return errNotImplemented
94 }
95
96 func testGetURL(t *testing.T, getURLFunc func(*kubeadmapi.APIEndpoint) string, port int) {
97 portStr := strconv.Itoa(port)
98 tests := []struct {
99 name string
100 advertiseAddress string
101 expectedURL string
102 }{
103 {
104 name: "IPv4",
105 advertiseAddress: "10.10.10.10",
106 expectedURL: fmt.Sprintf("https://10.10.10.10:%s", portStr),
107 },
108 {
109 name: "IPv6",
110 advertiseAddress: "2001:db8::2",
111 expectedURL: fmt.Sprintf("https://[2001:db8::2]:%s", portStr),
112 },
113 {
114 name: "IPv4 localhost",
115 advertiseAddress: "127.0.0.1",
116 expectedURL: fmt.Sprintf("https://127.0.0.1:%s", portStr),
117 },
118 {
119 name: "IPv6 localhost",
120 advertiseAddress: "::1",
121 expectedURL: fmt.Sprintf("https://[::1]:%s", portStr),
122 },
123 }
124
125 for _, test := range tests {
126 url := getURLFunc(&kubeadmapi.APIEndpoint{AdvertiseAddress: test.advertiseAddress})
127 if url != test.expectedURL {
128 t.Errorf("expected %s, got %s", test.expectedURL, url)
129 }
130 }
131 }
132
133 func TestGetClientURL(t *testing.T) {
134 testGetURL(t, GetClientURL, constants.EtcdListenClientPort)
135 }
136
137 func TestGetPeerURL(t *testing.T) {
138 testGetURL(t, GetPeerURL, constants.EtcdListenPeerPort)
139 }
140
141 func TestGetClientURLByIP(t *testing.T) {
142 portStr := strconv.Itoa(constants.EtcdListenClientPort)
143 tests := []struct {
144 name string
145 ip string
146 expectedURL string
147 }{
148 {
149 name: "IPv4",
150 ip: "10.10.10.10",
151 expectedURL: fmt.Sprintf("https://10.10.10.10:%s", portStr),
152 },
153 {
154 name: "IPv6",
155 ip: "2001:db8::2",
156 expectedURL: fmt.Sprintf("https://[2001:db8::2]:%s", portStr),
157 },
158 {
159 name: "IPv4 localhost",
160 ip: "127.0.0.1",
161 expectedURL: fmt.Sprintf("https://127.0.0.1:%s", portStr),
162 },
163 {
164 name: "IPv6 localhost",
165 ip: "::1",
166 expectedURL: fmt.Sprintf("https://[::1]:%s", portStr),
167 },
168 }
169
170 for _, test := range tests {
171 url := GetClientURLByIP(test.ip)
172 if url != test.expectedURL {
173 t.Errorf("expected %s, got %s", test.expectedURL, url)
174 }
175 }
176 }
177
178 func TestGetEtcdEndpointsWithBackoff(t *testing.T) {
179 tests := []struct {
180 name string
181 pods []testresources.FakeStaticPod
182 expectedEndpoints []string
183 expectedErr bool
184 }{
185 {
186 name: "no pod annotations",
187 expectedEndpoints: []string{},
188 expectedErr: true,
189 },
190 {
191 name: "ipv4 endpoint in pod annotation; port is preserved",
192 pods: []testresources.FakeStaticPod{
193 {
194 Component: constants.Etcd,
195 Annotations: map[string]string{
196 constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:1234",
197 },
198 },
199 },
200 expectedEndpoints: []string{"https://1.2.3.4:1234"},
201 },
202 }
203 for _, rt := range tests {
204 t.Run(rt.name, func(t *testing.T) {
205 client := clientsetfake.NewSimpleClientset()
206 for _, pod := range rt.pods {
207 if err := pod.Create(client); err != nil {
208 t.Errorf("error setting up test creating pod for node %q", pod.NodeName)
209 }
210 }
211 endpoints, err := getEtcdEndpointsWithRetry(client, time.Microsecond*10, time.Millisecond*100)
212 if err != nil && !rt.expectedErr {
213 t.Errorf("got error %q; was expecting no errors", err)
214 return
215 } else if err == nil && rt.expectedErr {
216 t.Error("got no error; was expecting an error")
217 return
218 } else if err != nil && rt.expectedErr {
219 return
220 }
221
222 if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) {
223 t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints)
224 }
225 })
226 }
227 }
228
229 func TestGetRawEtcdEndpointsFromPodAnnotation(t *testing.T) {
230 tests := []struct {
231 name string
232 pods []testresources.FakeStaticPod
233 clientSetup func(*clientsetfake.Clientset)
234 expectedEndpoints []string
235 expectedErr bool
236 }{
237 {
238 name: "exactly one pod with annotation",
239 pods: []testresources.FakeStaticPod{
240 {
241 NodeName: "cp-0",
242 Component: constants.Etcd,
243 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
244 },
245 },
246 expectedEndpoints: []string{"https://1.2.3.4:2379"},
247 },
248 {
249 name: "two pods; one is missing annotation",
250 pods: []testresources.FakeStaticPod{
251 {
252 NodeName: "cp-0",
253 Component: constants.Etcd,
254 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
255 },
256 {
257 NodeName: "cp-1",
258 Component: constants.Etcd,
259 },
260 },
261 expectedEndpoints: []string{"https://1.2.3.4:2379"},
262 expectedErr: true,
263 },
264 {
265 name: "no pods with annotation",
266 expectedErr: true,
267 },
268 {
269 name: "exactly one pod with annotation; all requests fail",
270 pods: []testresources.FakeStaticPod{
271 {
272 NodeName: "cp-0",
273 Component: constants.Etcd,
274 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
275 },
276 },
277 clientSetup: func(clientset *clientsetfake.Clientset) {
278 clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
279 return true, nil, apierrors.NewInternalError(errors.New("API server down"))
280 })
281 },
282 expectedErr: true,
283 },
284 }
285 for _, rt := range tests {
286 t.Run(rt.name, func(t *testing.T) {
287 client := clientsetfake.NewSimpleClientset()
288 for i, pod := range rt.pods {
289 if err := pod.CreateWithPodSuffix(client, strconv.Itoa(i)); err != nil {
290 t.Errorf("error setting up test creating pod for node %q", pod.NodeName)
291 }
292 }
293 if rt.clientSetup != nil {
294 rt.clientSetup(client)
295 }
296 endpoints, err := getRawEtcdEndpointsFromPodAnnotation(client, time.Microsecond*10, time.Millisecond*100)
297 if err != nil && !rt.expectedErr {
298 t.Errorf("got error %v, but wasn't expecting any error", err)
299 return
300 } else if err == nil && rt.expectedErr {
301 t.Error("didn't get any error; but was expecting an error")
302 return
303 } else if err != nil && rt.expectedErr {
304 return
305 }
306 if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) {
307 t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints)
308 }
309 })
310 }
311 }
312
313 func TestGetRawEtcdEndpointsFromPodAnnotationWithoutRetry(t *testing.T) {
314 tests := []struct {
315 name string
316 pods []testresources.FakeStaticPod
317 clientSetup func(*clientsetfake.Clientset)
318 expectedEndpoints []string
319 expectedErr bool
320 }{
321 {
322 name: "no pods",
323 expectedEndpoints: []string{},
324 },
325 {
326 name: "exactly one pod with annotation",
327 pods: []testresources.FakeStaticPod{
328 {
329 NodeName: "cp-0",
330 Component: constants.Etcd,
331 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
332 },
333 },
334 expectedEndpoints: []string{"https://1.2.3.4:2379"},
335 },
336 {
337 name: "two pods; one is missing annotation",
338 pods: []testresources.FakeStaticPod{
339 {
340 NodeName: "cp-0",
341 Component: constants.Etcd,
342 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
343 },
344 {
345 NodeName: "cp-1",
346 Component: constants.Etcd,
347 },
348 },
349 expectedEndpoints: []string{"https://1.2.3.4:2379"},
350 },
351 {
352 name: "two pods with annotation",
353 pods: []testresources.FakeStaticPod{
354 {
355 NodeName: "cp-0",
356 Component: constants.Etcd,
357 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
358 },
359 {
360 NodeName: "cp-1",
361 Component: constants.Etcd,
362 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.5:2379"},
363 },
364 },
365 expectedEndpoints: []string{"https://1.2.3.4:2379", "https://1.2.3.5:2379"},
366 },
367 {
368 name: "exactly one pod with annotation; request fails",
369 pods: []testresources.FakeStaticPod{
370 {
371 NodeName: "cp-0",
372 Component: constants.Etcd,
373 Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"},
374 },
375 },
376 clientSetup: func(clientset *clientsetfake.Clientset) {
377 clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
378 return true, nil, apierrors.NewInternalError(errors.New("API server down"))
379 })
380 },
381 expectedErr: true,
382 },
383 }
384 for _, rt := range tests {
385 t.Run(rt.name, func(t *testing.T) {
386 client := clientsetfake.NewSimpleClientset()
387 for _, pod := range rt.pods {
388 if err := pod.Create(client); err != nil {
389 t.Errorf("error setting up test creating pod for node %q", pod.NodeName)
390 return
391 }
392 }
393 if rt.clientSetup != nil {
394 rt.clientSetup(client)
395 }
396 endpoints, _, err := getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client)
397 if err != nil && !rt.expectedErr {
398 t.Errorf("got error %v, but wasn't expecting any error", err)
399 return
400 } else if err == nil && rt.expectedErr {
401 t.Error("didn't get any error; but was expecting an error")
402 return
403 } else if err != nil && rt.expectedErr {
404 return
405 }
406 if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) {
407 t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints)
408 }
409 })
410 }
411 }
412
413 func TestClient_GetMemberID(t *testing.T) {
414 type fields struct {
415 Endpoints []string
416 newEtcdClient func(endpoints []string) (etcdClient, error)
417 }
418 type args struct {
419 peerURL string
420 }
421 tests := []struct {
422 name string
423 fields fields
424 args args
425 want uint64
426 wantErr error
427 }{
428 {
429 name: "member ID found",
430 fields: fields{
431 Endpoints: []string{},
432 newEtcdClient: func(endpoints []string) (etcdClient, error) {
433 f := &fakeEtcdClient{
434 members: []*pb.Member{
435 {
436 ID: 1,
437 Name: "member1",
438 PeerURLs: []string{
439 "https://member1:2380",
440 },
441 },
442 },
443 }
444 return f, nil
445 },
446 },
447 args: args{
448 peerURL: "https://member1:2380",
449 },
450 wantErr: nil,
451 want: 1,
452 },
453 {
454 name: "member ID not found",
455 fields: fields{
456 Endpoints: []string{},
457 newEtcdClient: func(endpoints []string) (etcdClient, error) {
458 f := &fakeEtcdClient{
459 members: []*pb.Member{
460 {
461 ID: 1,
462 Name: "member1",
463 PeerURLs: []string{
464 "https://member1:2380",
465 },
466 },
467 },
468 }
469 return f, nil
470 },
471 },
472 args: args{
473 peerURL: "https://member2:2380",
474 },
475 wantErr: ErrNoMemberIDForPeerURL,
476 want: 0,
477 },
478 }
479 for _, tt := range tests {
480 t.Run(tt.name, func(t *testing.T) {
481 c := &Client{
482 Endpoints: tt.fields.Endpoints,
483 newEtcdClient: tt.fields.newEtcdClient,
484 }
485 c.listMembersFunc = func(_ time.Duration) (*clientv3.MemberListResponse, error) {
486 f, _ := c.newEtcdClient([]string{})
487 resp, _ := f.MemberList(context.Background())
488 return resp, nil
489 }
490
491 got, err := c.GetMemberID(tt.args.peerURL)
492 if !errors.Is(tt.wantErr, err) {
493 t.Errorf("Client.GetMemberID() error = %v, wantErr %v", err, tt.wantErr)
494 return
495 }
496 if got != tt.want {
497 t.Errorf("Client.GetMemberID() = %v, want %v", got, tt.want)
498 }
499 })
500 }
501 }
502
503 func TestListMembers(t *testing.T) {
504 type fields struct {
505 Endpoints []string
506 newEtcdClient func(endpoints []string) (etcdClient, error)
507 listMembersFunc func(timeout time.Duration) (*clientv3.MemberListResponse, error)
508 }
509 tests := []struct {
510 name string
511 fields fields
512 want []Member
513 wantError bool
514 }{
515 {
516 name: "PeerURLs are empty",
517 fields: fields{
518 Endpoints: []string{},
519 newEtcdClient: func(endpoints []string) (etcdClient, error) {
520 f := &fakeEtcdClient{}
521 return f, nil
522 },
523 },
524 want: []Member{},
525 },
526 {
527 name: "PeerURLs are non-empty",
528 fields: fields{
529 Endpoints: []string{},
530 newEtcdClient: func(endpoints []string) (etcdClient, error) {
531 f := &fakeEtcdClient{
532 members: []*pb.Member{
533 {
534 ID: 1,
535 Name: "member1",
536 PeerURLs: []string{
537 "https://member1:2380",
538 },
539 },
540 {
541 ID: 2,
542 Name: "member2",
543 PeerURLs: []string{
544 "https://member2:2380",
545 },
546 },
547 },
548 }
549 return f, nil
550 },
551 },
552 want: []Member{
553 {
554 Name: "member1",
555 PeerURL: "https://member1:2380",
556 },
557 {
558 Name: "member2",
559 PeerURL: "https://member2:2380",
560 },
561 },
562 },
563 {
564 name: "PeerURLs has multiple urls",
565 fields: fields{
566 Endpoints: []string{},
567 newEtcdClient: func(endpoints []string) (etcdClient, error) {
568 f := &fakeEtcdClient{
569 members: []*pb.Member{
570 {
571 ID: 1,
572 Name: "member1",
573 PeerURLs: []string{
574 "https://member1:2380",
575 "https://member2:2380",
576 },
577 },
578 },
579 }
580 return f, nil
581 },
582 },
583 want: []Member{
584 {
585 Name: "member1",
586 PeerURL: "https://member1:2380",
587 },
588 },
589 },
590 {
591 name: "ListMembers return error",
592 fields: fields{
593 Endpoints: []string{},
594 newEtcdClient: func(endpoints []string) (etcdClient, error) {
595 f := &fakeEtcdClient{
596 members: []*pb.Member{
597 {
598 ID: 1,
599 Name: "member1",
600 PeerURLs: []string{
601 "https://member1:2380",
602 "https://member2:2380",
603 },
604 },
605 },
606 }
607 return f, nil
608 },
609 listMembersFunc: func(_ time.Duration) (*clientv3.MemberListResponse, error) {
610 return nil, errNotImplemented
611 },
612 },
613 want: nil,
614 wantError: true,
615 },
616 }
617 for _, tt := range tests {
618 t.Run(tt.name, func(t *testing.T) {
619 c := &Client{
620 Endpoints: tt.fields.Endpoints,
621 newEtcdClient: tt.fields.newEtcdClient,
622 listMembersFunc: tt.fields.listMembersFunc,
623 }
624 if c.listMembersFunc == nil {
625 c.listMembersFunc = func(_ time.Duration) (*clientv3.MemberListResponse, error) {
626 return c.listMembers(100 * time.Millisecond)
627 }
628 }
629 got, err := c.ListMembers()
630 if !reflect.DeepEqual(got, tt.want) {
631 t.Errorf("ListMembers() = %v, want %v", got, tt.want)
632 }
633 if (err != nil) != (tt.wantError) {
634 t.Errorf("ListMembers() error = %v, wantError %v", err, tt.wantError)
635 }
636 })
637 }
638 }
639
640 func TestIsLearner(t *testing.T) {
641 type fields struct {
642 Endpoints []string
643 newEtcdClient func(endpoints []string) (etcdClient, error)
644 listMembersFunc func(timeout time.Duration) (*clientv3.MemberListResponse, error)
645 }
646 tests := []struct {
647 name string
648 fields fields
649 memberID uint64
650 want bool
651 wantError bool
652 }{
653 {
654 name: "The specified member is not a learner",
655 fields: fields{
656 Endpoints: []string{},
657 newEtcdClient: func(endpoints []string) (etcdClient, error) {
658 f := &fakeEtcdClient{
659 members: []*pb.Member{
660 {
661 ID: 1,
662 Name: "member1",
663 PeerURLs: []string{
664 "https://member1:2380",
665 },
666 IsLearner: false,
667 },
668 },
669 }
670 return f, nil
671 },
672 },
673 memberID: 1,
674 want: false,
675 },
676 {
677 name: "The specified member is a learner",
678 fields: fields{
679 Endpoints: []string{},
680 newEtcdClient: func(endpoints []string) (etcdClient, error) {
681 f := &fakeEtcdClient{
682 members: []*pb.Member{
683 {
684 ID: 1,
685 Name: "member1",
686 PeerURLs: []string{
687 "https://member1:2380",
688 },
689 IsLearner: true,
690 },
691 {
692 ID: 2,
693 Name: "member2",
694 PeerURLs: []string{
695 "https://member2:2380",
696 },
697 },
698 },
699 }
700 return f, nil
701 },
702 },
703 memberID: 1,
704 want: true,
705 },
706 {
707 name: "The specified member does not exist",
708 fields: fields{
709 Endpoints: []string{},
710 newEtcdClient: func(endpoints []string) (etcdClient, error) {
711 f := &fakeEtcdClient{
712 members: []*pb.Member{},
713 }
714 return f, nil
715 },
716 },
717 memberID: 3,
718 want: false,
719 },
720 {
721 name: "Learner ID is empty",
722 fields: fields{
723 Endpoints: []string{},
724 newEtcdClient: func(endpoints []string) (etcdClient, error) {
725 f := &fakeEtcdClient{
726 members: []*pb.Member{
727 {
728 Name: "member2",
729 PeerURLs: []string{
730 "https://member2:2380",
731 },
732 IsLearner: true,
733 },
734 },
735 }
736 return f, nil
737 },
738 },
739 want: true,
740 },
741 {
742 name: "ListMembers returns an error",
743 fields: fields{
744 Endpoints: []string{},
745 newEtcdClient: func(endpoints []string) (etcdClient, error) {
746 f := &fakeEtcdClient{
747 members: []*pb.Member{
748 {
749 Name: "member2",
750 PeerURLs: []string{
751 "https://member2:2380",
752 },
753 IsLearner: true,
754 },
755 },
756 }
757 return f, nil
758 },
759 listMembersFunc: func(_ time.Duration) (*clientv3.MemberListResponse, error) {
760 return nil, errNotImplemented
761 },
762 },
763 want: false,
764 wantError: true,
765 },
766 }
767 for _, tt := range tests {
768 t.Run(tt.name, func(t *testing.T) {
769 c := &Client{
770 Endpoints: tt.fields.Endpoints,
771 newEtcdClient: tt.fields.newEtcdClient,
772 listMembersFunc: tt.fields.listMembersFunc,
773 }
774 if c.listMembersFunc == nil {
775 c.listMembersFunc = func(t_ time.Duration) (*clientv3.MemberListResponse, error) {
776 f, _ := c.newEtcdClient([]string{})
777 resp, _ := f.MemberList(context.Background())
778 return resp, nil
779 }
780 }
781 got, err := c.isLearner(tt.memberID)
782 if got != tt.want {
783 t.Errorf("isLearner() = %v, want %v", got, tt.want)
784 }
785 if (err != nil) != (tt.wantError) {
786 t.Errorf("isLearner() error = %v, wantError %v", err, tt.wantError)
787 }
788 })
789 }
790 }
791
View as plain text