1
16
17 package network
18
19 import (
20 "context"
21 "fmt"
22 "regexp"
23 "strings"
24 "time"
25
26 v1 "k8s.io/api/core/v1"
27 apierrors "k8s.io/apimachinery/pkg/api/errors"
28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29
30 "k8s.io/apimachinery/pkg/fields"
31 "k8s.io/apimachinery/pkg/labels"
32 "k8s.io/apimachinery/pkg/util/intstr"
33 "k8s.io/apimachinery/pkg/util/uuid"
34 "k8s.io/apimachinery/pkg/util/wait"
35 clientset "k8s.io/client-go/kubernetes"
36 "k8s.io/kubernetes/test/e2e/framework"
37 e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
38 imageutils "k8s.io/kubernetes/test/utils/image"
39 dnsclient "k8s.io/kubernetes/third_party/forked/golang/net"
40 admissionapi "k8s.io/pod-security-admission/api"
41
42 "github.com/onsi/ginkgo/v2"
43 "github.com/onsi/gomega"
44 )
45
46
47 var newLineRegexp = regexp.MustCompile("\r?\n")
48
49 type dnsTestCommon struct {
50 f *framework.Framework
51 c clientset.Interface
52 ns string
53 name string
54
55 dnsPod *v1.Pod
56 utilPod *v1.Pod
57 utilService *v1.Service
58 dnsServerPod *v1.Pod
59
60 cm *v1.ConfigMap
61 }
62
63 func newDNSTestCommon() dnsTestCommon {
64 framework := framework.NewDefaultFramework("dns-config-map")
65 framework.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
66 return dnsTestCommon{
67 f: framework,
68 ns: "kube-system",
69 }
70 }
71
72 func (t *dnsTestCommon) init(ctx context.Context) {
73 ginkgo.By("Finding a DNS pod")
74 label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
75 options := metav1.ListOptions{LabelSelector: label.String()}
76
77 namespace := "kube-system"
78 pods, err := t.f.ClientSet.CoreV1().Pods(namespace).List(ctx, options)
79 framework.ExpectNoError(err, "failed to list pods in namespace: %s", namespace)
80 gomega.Expect(pods.Items).ToNot(gomega.BeEmpty())
81
82 t.dnsPod = &pods.Items[0]
83 framework.Logf("Using DNS pod: %v", t.dnsPod.Name)
84
85 if strings.Contains(t.dnsPod.Name, "coredns") {
86 t.name = "coredns"
87 } else {
88 t.name = "kube-dns"
89 }
90 }
91
92 func (t *dnsTestCommon) checkDNSRecordFrom(name string, predicate func([]string) bool, target string, timeout time.Duration) {
93 var actual []string
94
95 err := wait.PollImmediate(
96 time.Duration(1)*time.Second,
97 timeout,
98 func() (bool, error) {
99 actual = t.runDig(name, target)
100 if predicate(actual) {
101 return true, nil
102 }
103 return false, nil
104 })
105
106 if err != nil {
107 framework.Failf("dig result did not match: %#v after %v",
108 actual, timeout)
109 }
110 }
111
112
113 func (t *dnsTestCommon) runDig(dnsName, target string) []string {
114 cmd := []string{"dig", "+short"}
115 switch target {
116 case "coredns":
117 cmd = append(cmd, "@"+t.dnsPod.Status.PodIP)
118 case "kube-dns":
119 cmd = append(cmd, "@"+t.dnsPod.Status.PodIP, "-p", "10053")
120 case "ptr-record":
121 cmd = append(cmd, "-x")
122 case "cluster-dns":
123 case "cluster-dns-ipv6":
124 cmd = append(cmd, "AAAA")
125 default:
126 panic(fmt.Errorf("invalid target: " + target))
127 }
128 cmd = append(cmd, dnsName)
129
130 stdout, stderr, err := e2epod.ExecWithOptions(t.f, e2epod.ExecOptions{
131 Command: cmd,
132 Namespace: t.f.Namespace.Name,
133 PodName: t.utilPod.Name,
134 ContainerName: t.utilPod.Spec.Containers[0].Name,
135 CaptureStdout: true,
136 CaptureStderr: true,
137 })
138
139 framework.Logf("Running dig: %v, stdout: %q, stderr: %q, err: %v",
140 cmd, stdout, stderr, err)
141
142 if stdout == "" {
143 return []string{}
144 }
145 return newLineRegexp.Split(stdout, -1)
146 }
147
148 func (t *dnsTestCommon) setConfigMap(ctx context.Context, cm *v1.ConfigMap) {
149 if t.cm != nil {
150 t.cm = cm
151 }
152
153 cm.ObjectMeta.Namespace = t.ns
154 cm.ObjectMeta.Name = t.name
155
156 options := metav1.ListOptions{
157 FieldSelector: fields.Set{
158 "metadata.namespace": t.ns,
159 "metadata.name": t.name,
160 }.AsSelector().String(),
161 }
162 cmList, err := t.c.CoreV1().ConfigMaps(t.ns).List(ctx, options)
163 framework.ExpectNoError(err, "failed to list ConfigMaps in namespace: %s", t.ns)
164
165 if len(cmList.Items) == 0 {
166 ginkgo.By(fmt.Sprintf("Creating the ConfigMap (%s:%s) %+v", t.ns, t.name, *cm))
167 _, err := t.c.CoreV1().ConfigMaps(t.ns).Create(ctx, cm, metav1.CreateOptions{})
168 framework.ExpectNoError(err, "failed to create ConfigMap (%s:%s) %+v", t.ns, t.name, *cm)
169 } else {
170 ginkgo.By(fmt.Sprintf("Updating the ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm))
171 _, err := t.c.CoreV1().ConfigMaps(t.ns).Update(ctx, cm, metav1.UpdateOptions{})
172 framework.ExpectNoError(err, "failed to update ConfigMap (%s:%s) to %+v", t.ns, t.name, *cm)
173 }
174 }
175
176 func (t *dnsTestCommon) fetchDNSConfigMapData(ctx context.Context) map[string]string {
177 if t.name == "coredns" {
178 pcm, err := t.c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, t.name, metav1.GetOptions{})
179 framework.ExpectNoError(err, "failed to get DNS ConfigMap: %s", t.name)
180 return pcm.Data
181 }
182 return nil
183 }
184
185 func (t *dnsTestCommon) restoreDNSConfigMap(ctx context.Context, configMapData map[string]string) {
186 if t.name == "coredns" {
187 t.setConfigMap(ctx, &v1.ConfigMap{Data: configMapData})
188 t.deleteCoreDNSPods(ctx)
189 } else {
190 err := t.c.CoreV1().ConfigMaps(t.ns).Delete(ctx, t.name, metav1.DeleteOptions{})
191 if err != nil && !apierrors.IsNotFound(err) {
192 framework.Failf("Unexpected error deleting configmap %s/%s", t.ns, t.name)
193 }
194 }
195 }
196
197 func (t *dnsTestCommon) createUtilPodLabel(ctx context.Context, baseName string) {
198
199 const servicePort = 10101
200 podName := fmt.Sprintf("%s-%s", baseName, string(uuid.NewUUID()))
201 ports := []v1.ContainerPort{{ContainerPort: servicePort, Protocol: v1.ProtocolTCP}}
202 t.utilPod = e2epod.NewAgnhostPod(t.f.Namespace.Name, podName, nil, nil, ports)
203
204 var err error
205 t.utilPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(ctx, t.utilPod, metav1.CreateOptions{})
206 framework.ExpectNoError(err, "failed to create pod: %v", t.utilPod)
207 framework.Logf("Created pod %v", t.utilPod)
208 err = e2epod.WaitForPodNameRunningInNamespace(ctx, t.f.ClientSet, t.utilPod.Name, t.f.Namespace.Name)
209 framework.ExpectNoError(err, "pod failed to start running: %v", t.utilPod)
210
211 t.utilService = &v1.Service{
212 TypeMeta: metav1.TypeMeta{
213 Kind: "Service",
214 },
215 ObjectMeta: metav1.ObjectMeta{
216 Namespace: t.f.Namespace.Name,
217 Name: baseName,
218 },
219 Spec: v1.ServiceSpec{
220 Selector: map[string]string{"app": baseName},
221 Ports: []v1.ServicePort{
222 {
223 Protocol: v1.ProtocolTCP,
224 Port: servicePort,
225 TargetPort: intstr.FromInt32(servicePort),
226 },
227 },
228 },
229 }
230
231 t.utilService, err = t.c.CoreV1().Services(t.f.Namespace.Name).Create(ctx, t.utilService, metav1.CreateOptions{})
232 framework.ExpectNoError(err, "failed to create service: %s/%s", t.f.Namespace.Name, t.utilService.ObjectMeta.Name)
233 framework.Logf("Created service %v", t.utilService)
234 }
235
236 func (t *dnsTestCommon) deleteUtilPod(ctx context.Context) {
237 podClient := t.c.CoreV1().Pods(t.f.Namespace.Name)
238 if err := podClient.Delete(ctx, t.utilPod.Name, *metav1.NewDeleteOptions(0)); err != nil {
239 framework.Logf("Delete of pod %v/%v failed: %v",
240 t.utilPod.Namespace, t.utilPod.Name, err)
241 }
242 }
243
244
245 func (t *dnsTestCommon) deleteCoreDNSPods(ctx context.Context) {
246
247 label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": "kube-dns"}))
248 options := metav1.ListOptions{LabelSelector: label.String()}
249
250 pods, err := t.f.ClientSet.CoreV1().Pods("kube-system").List(ctx, options)
251 framework.ExpectNoError(err, "failed to list pods of kube-system with label %q", label.String())
252 podClient := t.c.CoreV1().Pods(metav1.NamespaceSystem)
253
254 for _, pod := range pods.Items {
255 err = podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
256 framework.ExpectNoError(err, "failed to delete pod: %s", pod.Name)
257 }
258 }
259
260 func generateCoreDNSServerPod(corednsConfig *v1.ConfigMap) *v1.Pod {
261 podName := fmt.Sprintf("e2e-configmap-dns-server-%s", string(uuid.NewUUID()))
262 volumes := []v1.Volume{
263 {
264 Name: "coredns-config",
265 VolumeSource: v1.VolumeSource{
266 ConfigMap: &v1.ConfigMapVolumeSource{
267 LocalObjectReference: v1.LocalObjectReference{
268 Name: corednsConfig.Name,
269 },
270 },
271 },
272 },
273 }
274 mounts := []v1.VolumeMount{
275 {
276 Name: "coredns-config",
277 MountPath: "/etc/coredns",
278 ReadOnly: true,
279 },
280 }
281
282 pod := e2epod.NewAgnhostPod("", podName, volumes, mounts, nil, "-conf", "/etc/coredns/Corefile")
283 pod.Spec.Containers[0].Command = []string{"/coredns"}
284 pod.Spec.DNSPolicy = "Default"
285 return pod
286 }
287
288 func generateCoreDNSConfigmap(namespaceName string, aRecords map[string]string) *v1.ConfigMap {
289 entries := ""
290 for name, ip := range aRecords {
291 entries += fmt.Sprintf("\n\t\t%v %v", ip, name)
292 }
293
294 corefileData := fmt.Sprintf(`. {
295 hosts {%s
296 }
297 log
298 }`, entries)
299
300 return &v1.ConfigMap{
301 ObjectMeta: metav1.ObjectMeta{
302 Namespace: namespaceName,
303 GenerateName: "e2e-coredns-configmap-",
304 },
305 Data: map[string]string{
306 "Corefile": corefileData,
307 },
308 }
309 }
310
311 func (t *dnsTestCommon) createDNSPodFromObj(ctx context.Context, pod *v1.Pod) {
312 t.dnsServerPod = pod
313
314 var err error
315 t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Create(ctx, t.dnsServerPod, metav1.CreateOptions{})
316 framework.ExpectNoError(err, "failed to create pod: %v", t.dnsServerPod)
317 framework.Logf("Created pod %v", t.dnsServerPod)
318 err = e2epod.WaitForPodNameRunningInNamespace(ctx, t.f.ClientSet, t.dnsServerPod.Name, t.f.Namespace.Name)
319 framework.ExpectNoError(err, "pod failed to start running: %v", t.dnsServerPod)
320
321 t.dnsServerPod, err = t.c.CoreV1().Pods(t.f.Namespace.Name).Get(ctx, t.dnsServerPod.Name, metav1.GetOptions{})
322 framework.ExpectNoError(err, "failed to get pod: %s", t.dnsServerPod.Name)
323 }
324
325 func (t *dnsTestCommon) createDNSServer(ctx context.Context, namespace string, aRecords map[string]string) {
326 corednsConfig := generateCoreDNSConfigmap(namespace, aRecords)
327 corednsConfig, err := t.c.CoreV1().ConfigMaps(namespace).Create(ctx, corednsConfig, metav1.CreateOptions{})
328 if err != nil {
329 framework.Failf("unable to create test configMap %s: %v", corednsConfig.Name, err)
330 }
331
332 t.createDNSPodFromObj(ctx, generateCoreDNSServerPod(corednsConfig))
333 }
334
335 func (t *dnsTestCommon) createDNSServerWithPtrRecord(ctx context.Context, namespace string, isIPv6 bool) {
336
337
338 var aRecords map[string]string
339 if isIPv6 {
340 aRecords = map[string]string{"my.test": "2001:db8::29"}
341 } else {
342 aRecords = map[string]string{"my.test": "192.0.2.123"}
343 }
344 t.createDNSServer(ctx, namespace, aRecords)
345 }
346
347 func (t *dnsTestCommon) deleteDNSServerPod(ctx context.Context) {
348 podClient := t.c.CoreV1().Pods(t.f.Namespace.Name)
349 if err := podClient.Delete(ctx, t.dnsServerPod.Name, *metav1.NewDeleteOptions(0)); err != nil {
350 framework.Logf("Delete of pod %v/%v failed: %v",
351 t.utilPod.Namespace, t.dnsServerPod.Name, err)
352 }
353 }
354
355 func createDNSPod(namespace, wheezyProbeCmd, jessieProbeCmd, podHostName, serviceName string) *v1.Pod {
356 podName := "dns-test-" + string(uuid.NewUUID())
357 volumes := []v1.Volume{
358 {
359 Name: "results",
360 VolumeSource: v1.VolumeSource{
361 EmptyDir: &v1.EmptyDirVolumeSource{},
362 },
363 },
364 }
365 mounts := []v1.VolumeMount{
366 {
367 Name: "results",
368 MountPath: "/results",
369 },
370 }
371
372
373 dnsPod := e2epod.NewAgnhostPod(namespace, podName, volumes, mounts, nil, "test-webserver")
374 dnsPod.Spec.Containers[0].Name = "webserver"
375
376 querier := e2epod.NewAgnhostContainer("querier", mounts, nil, wheezyProbeCmd)
377 querier.Command = []string{"sh", "-c"}
378
379 jessieQuerier := v1.Container{
380 Name: "jessie-querier",
381 Image: imageutils.GetE2EImage(imageutils.JessieDnsutils),
382 Command: []string{"sh", "-c", jessieProbeCmd},
383 VolumeMounts: mounts,
384 }
385
386 dnsPod.Spec.Containers = append(dnsPod.Spec.Containers, querier, jessieQuerier)
387 dnsPod.Spec.Hostname = podHostName
388 dnsPod.Spec.Subdomain = serviceName
389
390 return dnsPod
391 }
392
393 func createProbeCommand(namesToResolve []string, hostEntries []string, ptrLookupIP string, fileNamePrefix, namespace, dnsDomain string, isIPv6 bool) (string, []string) {
394 fileNames := make([]string, 0, len(namesToResolve)*2)
395 probeCmd := "for i in `seq 1 600`; do "
396 dnsRecord := "A"
397 if isIPv6 {
398 dnsRecord = "AAAA"
399 }
400 for _, name := range namesToResolve {
401
402
403
404 lookup := fmt.Sprintf("%s %s", name, dnsRecord)
405 if strings.HasPrefix(name, "_") {
406 lookup = fmt.Sprintf("%s SRV", name)
407 }
408 fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, name)
409 fileNames = append(fileNames, fileName)
410 probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s)" && test -n "$$check" && echo OK > /results/%s;`, lookup, fileName)
411 fileName = fmt.Sprintf("%s_tcp@%s", fileNamePrefix, name)
412 fileNames = append(fileNames, fileName)
413 probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s)" && test -n "$$check" && echo OK > /results/%s;`, lookup, fileName)
414 }
415
416 hostEntryCmd := `test -n "$$(getent hosts %s)" && echo OK > /results/%s;`
417 if framework.NodeOSDistroIs("windows") {
418
419 hostEntryCmd = `test -n "$$(grep '%s' C:/Windows/System32/drivers/etc/hosts)" && echo OK > /results/%s;`
420 }
421 for _, name := range hostEntries {
422 fileName := fmt.Sprintf("%s_hosts@%s", fileNamePrefix, name)
423 fileNames = append(fileNames, fileName)
424 probeCmd += fmt.Sprintf(hostEntryCmd, name, fileName)
425 }
426
427 if len(ptrLookupIP) > 0 {
428 ptrLookup, err := dnsclient.Reverseaddr(ptrLookupIP)
429 if err != nil {
430 framework.Failf("Unable to obtain reverse IP address record from IP %s: %v", ptrLookupIP, err)
431 }
432 ptrRecByUDPFileName := fmt.Sprintf("%s_udp@PTR", ptrLookupIP)
433 ptrRecByTCPFileName := fmt.Sprintf("%s_tcp@PTR", ptrLookupIP)
434 probeCmd += fmt.Sprintf(`check="$$(dig +notcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > /results/%s;`, ptrLookup, ptrRecByUDPFileName)
435 probeCmd += fmt.Sprintf(`check="$$(dig +tcp +noall +answer +search %s PTR)" && test -n "$$check" && echo OK > /results/%s;`, ptrLookup, ptrRecByTCPFileName)
436 fileNames = append(fileNames, ptrRecByUDPFileName)
437 fileNames = append(fileNames, ptrRecByTCPFileName)
438 }
439
440 probeCmd += "sleep 1; done"
441 return probeCmd, fileNames
442 }
443
444
445 func createTargetedProbeCommand(nameToResolve string, lookup string, fileNamePrefix string) (string, string) {
446 fileName := fmt.Sprintf("%s_udp@%s", fileNamePrefix, nameToResolve)
447 nameLookup := fmt.Sprintf("%s %s", nameToResolve, lookup)
448 probeCmd := fmt.Sprintf("for i in `seq 1 30`; do dig +short %s > /results/%s; sleep 1; done", nameLookup, fileName)
449 return probeCmd, fileName
450 }
451
452 func assertFilesExist(ctx context.Context, fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface) {
453 assertFilesContain(ctx, fileNames, fileDir, pod, client, false, "")
454 }
455
456 func assertFilesContain(ctx context.Context, fileNames []string, fileDir string, pod *v1.Pod, client clientset.Interface, check bool, expected string) {
457 var failed []string
458
459 framework.ExpectNoError(wait.PollUntilContextTimeout(ctx, time.Second*5, time.Second*600, true, func(ctx context.Context) (bool, error) {
460 failed = []string{}
461
462 ctx, cancel := context.WithTimeout(ctx, framework.SingleCallTimeout)
463 defer cancel()
464
465 for _, fileName := range fileNames {
466 contents, err := client.CoreV1().RESTClient().Get().
467 Namespace(pod.Namespace).
468 Resource("pods").
469 SubResource("proxy").
470 Name(pod.Name).
471 Suffix(fileDir, fileName).
472 Do(ctx).Raw()
473
474 if err != nil {
475 if ctx.Err() != nil {
476 framework.Failf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err)
477 } else {
478 framework.Logf("Unable to read %s from pod %s/%s: %v", fileName, pod.Namespace, pod.Name, err)
479 }
480 failed = append(failed, fileName)
481 } else if check && strings.TrimSpace(string(contents)) != expected {
482 framework.Logf("File %s from pod %s/%s contains '%s' instead of '%s'", fileName, pod.Namespace, pod.Name, string(contents), expected)
483 failed = append(failed, fileName)
484 }
485 }
486 if len(failed) == 0 {
487 return true, nil
488 }
489 framework.Logf("Lookups using %s/%s failed for: %v\n", pod.Namespace, pod.Name, failed)
490
491
492 for _, container := range pod.Spec.Containers {
493 logs, err := e2epod.GetPodLogs(ctx, client, pod.Namespace, pod.Name, container.Name)
494 framework.ExpectNoError(err)
495 framework.Logf("Pod client logs for %s: %s", container.Name, logs)
496 }
497
498 return false, nil
499 }))
500 gomega.Expect(failed).To(gomega.BeEmpty())
501 }
502
503 func validateDNSResults(ctx context.Context, f *framework.Framework, pod *v1.Pod, fileNames []string) {
504 ginkgo.By("submitting the pod to kubernetes")
505 podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name)
506 ginkgo.DeferCleanup(func(ctx context.Context) error {
507 ginkgo.By("deleting the pod")
508 return podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
509 })
510 if _, err := podClient.Create(ctx, pod, metav1.CreateOptions{}); err != nil {
511 framework.Failf("ginkgo.Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err)
512 }
513
514 framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
515
516 ginkgo.By("retrieving the pod")
517 pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{})
518 if err != nil {
519 framework.Failf("ginkgo.Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err)
520 }
521
522 ginkgo.By("looking for the results for each expected name from probers")
523 assertFilesExist(ctx, fileNames, "results", pod, f.ClientSet)
524
525
526
527 framework.Logf("DNS probes using %s/%s succeeded\n", pod.Namespace, pod.Name)
528 }
529
530 func validateTargetedProbeOutput(ctx context.Context, f *framework.Framework, pod *v1.Pod, fileNames []string, value string) {
531 ginkgo.By("submitting the pod to kubernetes")
532 podClient := f.ClientSet.CoreV1().Pods(f.Namespace.Name)
533 ginkgo.DeferCleanup(func(ctx context.Context) error {
534 ginkgo.By("deleting the pod")
535 return podClient.Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
536 })
537 if _, err := podClient.Create(ctx, pod, metav1.CreateOptions{}); err != nil {
538 framework.Failf("ginkgo.Failed to create pod %s/%s: %v", pod.Namespace, pod.Name, err)
539 }
540
541 framework.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, f.ClientSet, pod.Name, f.Namespace.Name))
542
543 ginkgo.By("retrieving the pod")
544 pod, err := podClient.Get(ctx, pod.Name, metav1.GetOptions{})
545 if err != nil {
546 framework.Failf("ginkgo.Failed to get pod %s/%s: %v", pod.Namespace, pod.Name, err)
547 }
548
549 ginkgo.By("looking for the results for each expected name from probers")
550 assertFilesContain(ctx, fileNames, "results", pod, f.ClientSet, true, value)
551
552 framework.Logf("DNS probes using %s succeeded\n", pod.Name)
553 }
554
View as plain text