mirror_pod_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"path/filepath"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	apiequality "k8s.io/apimachinery/pkg/api/equality"
    28  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/types"
    31  	"k8s.io/apimachinery/pkg/util/uuid"
    32  	"k8s.io/apimachinery/pkg/util/wait"
    33  	clientset "k8s.io/client-go/kubernetes"
    34  	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	imageutils "k8s.io/kubernetes/test/utils/image"
    37  	admissionapi "k8s.io/pod-security-admission/api"
    38  
    39  	"github.com/google/go-cmp/cmp"
    40  	"github.com/onsi/ginkgo/v2"
    41  	"github.com/onsi/gomega"
    42  	"k8s.io/cli-runtime/pkg/printers"
    43  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    44  )
    45  
    46  var _ = SIGDescribe("MirrorPod", func() {
    47  	f := framework.NewDefaultFramework("mirror-pod")
    48  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    49  	ginkgo.Context("when create a mirror pod ", func() {
    50  		var ns, podPath, staticPodName, mirrorPodName string
    51  		ginkgo.BeforeEach(func(ctx context.Context) {
    52  			ns = f.Namespace.Name
    53  			staticPodName = "static-pod-" + string(uuid.NewUUID())
    54  			mirrorPodName = staticPodName + "-" + framework.TestContext.NodeName
    55  
    56  			podPath = kubeletCfg.StaticPodPath
    57  
    58  			ginkgo.By("create the static pod")
    59  			err := createStaticPod(podPath, staticPodName, ns,
    60  				imageutils.GetE2EImage(imageutils.Nginx), v1.RestartPolicyAlways)
    61  			framework.ExpectNoError(err)
    62  
    63  			ginkgo.By("wait for the mirror pod to be running")
    64  			gomega.Eventually(ctx, func(ctx context.Context) error {
    65  				return checkMirrorPodRunning(ctx, f.ClientSet, mirrorPodName, ns)
    66  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
    67  		})
    68  		/*
    69  			Release: v1.9
    70  			Testname: Mirror Pod, update
    71  			Description: Updating a static Pod MUST recreate an updated mirror Pod. Create a static pod, verify that a mirror pod is created. Update the static pod by changing the container image, the mirror pod MUST be re-created and updated with the new image.
    72  		*/
    73  		f.It("should be updated when static pod updated", f.WithNodeConformance(), func(ctx context.Context) {
    74  			ginkgo.By("get mirror pod uid")
    75  			pod, err := f.ClientSet.CoreV1().Pods(ns).Get(ctx, mirrorPodName, metav1.GetOptions{})
    76  			framework.ExpectNoError(err)
    77  			uid := pod.UID
    78  
    79  			ginkgo.By("update the static pod container image")
    80  			image := imageutils.GetPauseImageName()
    81  			err = createStaticPod(podPath, staticPodName, ns, image, v1.RestartPolicyAlways)
    82  			framework.ExpectNoError(err)
    83  
    84  			ginkgo.By("wait for the mirror pod to be updated")
    85  			gomega.Eventually(ctx, func(ctx context.Context) error {
    86  				return checkMirrorPodRecreatedAndRunning(ctx, f.ClientSet, mirrorPodName, ns, uid)
    87  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
    88  
    89  			ginkgo.By("check the mirror pod container image is updated")
    90  			pod, err = f.ClientSet.CoreV1().Pods(ns).Get(ctx, mirrorPodName, metav1.GetOptions{})
    91  			framework.ExpectNoError(err)
    92  			gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(1))
    93  			gomega.Expect(pod.Spec.Containers[0].Image).To(gomega.Equal(image))
    94  		})
    95  		/*
    96  			Release: v1.9
    97  			Testname: Mirror Pod, delete
    98  			Description:  When a mirror-Pod is deleted then the mirror pod MUST be re-created. Create a static pod, verify that a mirror pod is created. Delete the mirror pod, the mirror pod MUST be re-created and running.
    99  		*/
   100  		f.It("should be recreated when mirror pod gracefully deleted", f.WithNodeConformance(), func(ctx context.Context) {
   101  			ginkgo.By("get mirror pod uid")
   102  			pod, err := f.ClientSet.CoreV1().Pods(ns).Get(ctx, mirrorPodName, metav1.GetOptions{})
   103  			framework.ExpectNoError(err)
   104  			uid := pod.UID
   105  
   106  			ginkgo.By("delete the mirror pod with grace period 30s")
   107  			err = f.ClientSet.CoreV1().Pods(ns).Delete(ctx, mirrorPodName, *metav1.NewDeleteOptions(30))
   108  			framework.ExpectNoError(err)
   109  
   110  			ginkgo.By("wait for the mirror pod to be recreated")
   111  			gomega.Eventually(ctx, func(ctx context.Context) error {
   112  				return checkMirrorPodRecreatedAndRunning(ctx, f.ClientSet, mirrorPodName, ns, uid)
   113  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   114  		})
   115  		/*
   116  			Release: v1.9
   117  			Testname: Mirror Pod, force delete
   118  			Description: When a mirror-Pod is deleted, forcibly, then the mirror pod MUST be re-created. Create a static pod, verify that a mirror pod is created. Delete the mirror pod with delete wait time set to zero forcing immediate deletion, the mirror pod MUST be re-created and running.
   119  		*/
   120  		f.It("should be recreated when mirror pod forcibly deleted", f.WithNodeConformance(), func(ctx context.Context) {
   121  			ginkgo.By("get mirror pod uid")
   122  			pod, err := f.ClientSet.CoreV1().Pods(ns).Get(ctx, mirrorPodName, metav1.GetOptions{})
   123  			framework.ExpectNoError(err)
   124  			uid := pod.UID
   125  
   126  			ginkgo.By("delete the mirror pod with grace period 0s")
   127  			err = f.ClientSet.CoreV1().Pods(ns).Delete(ctx, mirrorPodName, *metav1.NewDeleteOptions(0))
   128  			framework.ExpectNoError(err)
   129  
   130  			ginkgo.By("wait for the mirror pod to be recreated")
   131  			gomega.Eventually(ctx, func(ctx context.Context) error {
   132  				return checkMirrorPodRecreatedAndRunning(ctx, f.ClientSet, mirrorPodName, ns, uid)
   133  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   134  		})
   135  		ginkgo.AfterEach(func(ctx context.Context) {
   136  			ginkgo.By("delete the static pod")
   137  			err := deleteStaticPod(podPath, staticPodName, ns)
   138  			framework.ExpectNoError(err)
   139  
   140  			ginkgo.By("wait for the mirror pod to disappear")
   141  			gomega.Eventually(ctx, func(ctx context.Context) error {
   142  				return checkMirrorPodDisappear(ctx, f.ClientSet, mirrorPodName, ns)
   143  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   144  		})
   145  	})
   146  	ginkgo.Context("when create a mirror pod without changes ", func() {
   147  		var ns, podPath, staticPodName, mirrorPodName string
   148  		ginkgo.BeforeEach(func() {
   149  		})
   150  		/*
   151  			Release: v1.23
   152  			Testname: Mirror Pod, recreate
   153  			Description: When a static pod's manifest is removed and readded, the mirror pod MUST successfully recreate. Create the static pod, verify it is running, remove its manifest and then add it back, and verify the static pod runs again.
   154  		*/
   155  		f.It("should successfully recreate when file is removed and recreated", f.WithNodeConformance(), func(ctx context.Context) {
   156  			ns = f.Namespace.Name
   157  			staticPodName = "static-pod-" + string(uuid.NewUUID())
   158  			mirrorPodName = staticPodName + "-" + framework.TestContext.NodeName
   159  
   160  			podPath = kubeletCfg.StaticPodPath
   161  			ginkgo.By("create the static pod")
   162  			err := createStaticPod(podPath, staticPodName, ns,
   163  				imageutils.GetE2EImage(imageutils.Nginx), v1.RestartPolicyAlways)
   164  			framework.ExpectNoError(err)
   165  
   166  			ginkgo.By("wait for the mirror pod to be running")
   167  			gomega.Eventually(ctx, func(ctx context.Context) error {
   168  				return checkMirrorPodRunning(ctx, f.ClientSet, mirrorPodName, ns)
   169  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   170  
   171  			ginkgo.By("delete the pod manifest from disk")
   172  			err = deleteStaticPod(podPath, staticPodName, ns)
   173  			framework.ExpectNoError(err)
   174  
   175  			ginkgo.By("recreate the file")
   176  			err = createStaticPod(podPath, staticPodName, ns,
   177  				imageutils.GetE2EImage(imageutils.Nginx), v1.RestartPolicyAlways)
   178  			framework.ExpectNoError(err)
   179  
   180  			ginkgo.By("mirror pod should restart with count 1")
   181  			gomega.Eventually(ctx, func(ctx context.Context) error {
   182  				return checkMirrorPodRunningWithRestartCount(ctx, 2*time.Second, 2*time.Minute, f.ClientSet, mirrorPodName, ns, 1)
   183  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   184  
   185  			ginkgo.By("mirror pod should stay running")
   186  			gomega.Consistently(ctx, func(ctx context.Context) error {
   187  				return checkMirrorPodRunning(ctx, f.ClientSet, mirrorPodName, ns)
   188  			}, time.Second*30, time.Second*4).Should(gomega.BeNil())
   189  
   190  			ginkgo.By("delete the static pod")
   191  			err = deleteStaticPod(podPath, staticPodName, ns)
   192  			framework.ExpectNoError(err)
   193  
   194  			ginkgo.By("wait for the mirror pod to disappear")
   195  			gomega.Eventually(ctx, func(ctx context.Context) error {
   196  				return checkMirrorPodDisappear(ctx, f.ClientSet, mirrorPodName, ns)
   197  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   198  		})
   199  	})
   200  	ginkgo.Context("when recreating a static pod", func() {
   201  		var ns, podPath, staticPodName, mirrorPodName string
   202  		f.It("it should launch successfully even if it temporarily failed termination due to volume failing to unmount", f.WithNodeConformance(), f.WithSerial(), func(ctx context.Context) {
   203  			node := getNodeName(ctx, f)
   204  			ns = f.Namespace.Name
   205  			c := f.ClientSet
   206  			nfsTestConfig, nfsServerPod, nfsServerHost := e2evolume.NewNFSServerWithNodeName(ctx, c, ns, []string{"-G", "777", "/exports"}, node)
   207  			ginkgo.DeferCleanup(func(ctx context.Context) {
   208  				framework.Logf("Cleaning up NFS server pod")
   209  				e2evolume.TestServerCleanup(ctx, f, nfsTestConfig)
   210  			})
   211  
   212  			podPath = kubeletCfg.StaticPodPath
   213  			staticPodName = "static-pod-nfs-test-pod" + string(uuid.NewUUID())
   214  			mirrorPodName = staticPodName + "-" + framework.TestContext.NodeName
   215  
   216  			ginkgo.By(fmt.Sprintf("Creating nfs test pod: %s", staticPodName))
   217  
   218  			err := createStaticPodUsingNfs(nfsServerHost, node, "sleep 999999", podPath, staticPodName, ns)
   219  			framework.ExpectNoError(err)
   220  			ginkgo.By(fmt.Sprintf("Wating for nfs test pod: %s to start running...", staticPodName))
   221  			gomega.Eventually(func() error {
   222  				return checkMirrorPodRunning(ctx, f.ClientSet, mirrorPodName, ns)
   223  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   224  
   225  			mirrorPod, err := c.CoreV1().Pods(ns).Get(ctx, mirrorPodName, metav1.GetOptions{})
   226  			framework.ExpectNoError(err)
   227  
   228  			hash, ok := mirrorPod.Annotations[kubetypes.ConfigHashAnnotationKey]
   229  			if !ok || hash == "" {
   230  				framework.Failf("Failed to get hash for mirrorPod")
   231  			}
   232  
   233  			ginkgo.By("Stopping the NFS server")
   234  			stopNfsServer(f, nfsServerPod)
   235  
   236  			ginkgo.By("Waiting for NFS server to stop...")
   237  			time.Sleep(30 * time.Second)
   238  
   239  			ginkgo.By(fmt.Sprintf("Deleting the static nfs test pod: %s", staticPodName))
   240  			err = deleteStaticPod(podPath, staticPodName, ns)
   241  			framework.ExpectNoError(err)
   242  
   243  			// Wait 5 mins for syncTerminatedPod to fail. We expect that the pod volume should not be cleaned up because the NFS server is down.
   244  			gomega.Consistently(func() bool {
   245  				return podVolumeDirectoryExists(types.UID(hash))
   246  			}, 5*time.Minute, 10*time.Second).Should(gomega.BeTrue(), "pod volume should exist while nfs server is stopped")
   247  
   248  			ginkgo.By("Start the NFS server")
   249  			restartNfsServer(f, nfsServerPod)
   250  
   251  			ginkgo.By("Waiting for the pod volume to deleted after the NFS server is started")
   252  			gomega.Eventually(func() bool {
   253  				return podVolumeDirectoryExists(types.UID(hash))
   254  			}, 5*time.Minute, 10*time.Second).Should(gomega.BeFalse(), "pod volume should be deleted after nfs server is started")
   255  
   256  			// Create the static pod again with the same config and expect it to start running
   257  			err = createStaticPodUsingNfs(nfsServerHost, node, "sleep 999999", podPath, staticPodName, ns)
   258  			framework.ExpectNoError(err)
   259  			ginkgo.By(fmt.Sprintf("Wating for nfs test pod: %s to start running (after being recreated)", staticPodName))
   260  			gomega.Eventually(func() error {
   261  				return checkMirrorPodRunning(ctx, f.ClientSet, mirrorPodName, ns)
   262  			}, 5*time.Minute, 5*time.Second).Should(gomega.BeNil())
   263  		})
   264  
   265  		ginkgo.AfterEach(func(ctx context.Context) {
   266  			ginkgo.By("delete the static pod")
   267  			err := deleteStaticPod(podPath, staticPodName, ns)
   268  			framework.ExpectNoError(err)
   269  
   270  			ginkgo.By("wait for the mirror pod to disappear")
   271  			gomega.Eventually(ctx, func(ctx context.Context) error {
   272  				return checkMirrorPodDisappear(ctx, f.ClientSet, mirrorPodName, ns)
   273  			}, 2*time.Minute, time.Second*4).Should(gomega.BeNil())
   274  
   275  		})
   276  
   277  	})
   278  
   279  })
   280  
   281  func podVolumeDirectoryExists(uid types.UID) bool {
   282  	podVolumePath := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/", uid)
   283  	var podVolumeDirectoryExists bool
   284  
   285  	if _, err := os.Stat(podVolumePath); !os.IsNotExist(err) {
   286  		podVolumeDirectoryExists = true
   287  	}
   288  
   289  	return podVolumeDirectoryExists
   290  }
   291  
   292  // Restart the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 1` command in the
   293  // pod's (only) container. This command changes the number of nfs server threads from
   294  // (presumably) zero back to 1, and therefore allows nfs to open connections again.
   295  func restartNfsServer(f *framework.Framework, serverPod *v1.Pod) {
   296  	const startcmd = "/usr/sbin/rpc.nfsd 1"
   297  	_, _, err := e2evolume.PodExec(f, serverPod, startcmd)
   298  	framework.ExpectNoError(err)
   299  
   300  }
   301  
   302  // Stop the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 0` command in the
   303  // pod's (only) container. This command changes the number of nfs server threads to 0,
   304  // thus closing all open nfs connections.
   305  func stopNfsServer(f *framework.Framework, serverPod *v1.Pod) {
   306  	const stopcmd = "/usr/sbin/rpc.nfsd 0"
   307  	_, _, err := e2evolume.PodExec(f, serverPod, stopcmd)
   308  	framework.ExpectNoError(err)
   309  }
   310  
   311  func createStaticPodUsingNfs(nfsIP string, nodeName string, cmd string, dir string, name string, ns string) error {
   312  	ginkgo.By("create pod using nfs volume")
   313  
   314  	isPrivileged := true
   315  	cmdLine := []string{"-c", cmd}
   316  	pod := &v1.Pod{
   317  		TypeMeta: metav1.TypeMeta{
   318  			Kind:       "Pod",
   319  			APIVersion: "v1",
   320  		},
   321  		ObjectMeta: metav1.ObjectMeta{
   322  			Name:      name,
   323  			Namespace: ns,
   324  		},
   325  		Spec: v1.PodSpec{
   326  			NodeName: nodeName,
   327  			Containers: []v1.Container{
   328  				{
   329  					Name:    "pod-nfs-vol",
   330  					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   331  					Command: []string{"/bin/sh"},
   332  					Args:    cmdLine,
   333  					VolumeMounts: []v1.VolumeMount{
   334  						{
   335  							Name:      "nfs-vol",
   336  							MountPath: "/mnt",
   337  						},
   338  					},
   339  					SecurityContext: &v1.SecurityContext{
   340  						Privileged: &isPrivileged,
   341  					},
   342  				},
   343  			},
   344  			RestartPolicy: v1.RestartPolicyNever, //don't restart pod
   345  			Volumes: []v1.Volume{
   346  				{
   347  					Name: "nfs-vol",
   348  					VolumeSource: v1.VolumeSource{
   349  						NFS: &v1.NFSVolumeSource{
   350  							Server:   nfsIP,
   351  							Path:     "/",
   352  							ReadOnly: false,
   353  						},
   354  					},
   355  				},
   356  			},
   357  		},
   358  	}
   359  
   360  	file := staticPodPath(dir, name, ns)
   361  	f, err := os.OpenFile(file, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666)
   362  	if err != nil {
   363  		return err
   364  	}
   365  	defer f.Close()
   366  
   367  	y := printers.YAMLPrinter{}
   368  	y.PrintObj(pod, f)
   369  
   370  	return nil
   371  }
   372  
   373  func staticPodPath(dir, name, namespace string) string {
   374  	return filepath.Join(dir, namespace+"-"+name+".yaml")
   375  }
   376  
   377  func createStaticPod(dir, name, namespace, image string, restart v1.RestartPolicy) error {
   378  	template := `
   379  apiVersion: v1
   380  kind: Pod
   381  metadata:
   382    name: %s
   383    namespace: %s
   384  spec:
   385    containers:
   386    - name: test
   387      image: %s
   388    restartPolicy: %s
   389  `
   390  	file := staticPodPath(dir, name, namespace)
   391  	podYaml := fmt.Sprintf(template, name, namespace, image, string(restart))
   392  
   393  	f, err := os.OpenFile(file, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666)
   394  	if err != nil {
   395  		return err
   396  	}
   397  	defer f.Close()
   398  
   399  	_, err = f.WriteString(podYaml)
   400  	return err
   401  }
   402  
   403  func deleteStaticPod(dir, name, namespace string) error {
   404  	file := staticPodPath(dir, name, namespace)
   405  	return os.Remove(file)
   406  }
   407  
   408  func checkMirrorPodDisappear(ctx context.Context, cl clientset.Interface, name, namespace string) error {
   409  	_, err := cl.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
   410  	if apierrors.IsNotFound(err) {
   411  		return nil
   412  	}
   413  	if err == nil {
   414  		return fmt.Errorf("mirror pod %v/%v still exists", namespace, name)
   415  	}
   416  	return fmt.Errorf("expect mirror pod %v/%v to not exist but got error: %w", namespace, name, err)
   417  }
   418  
   419  func checkMirrorPodRunning(ctx context.Context, cl clientset.Interface, name, namespace string) error {
   420  	pod, err := cl.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
   421  	if err != nil {
   422  		return fmt.Errorf("expected the mirror pod %q to appear: %w", name, err)
   423  	}
   424  	if pod.Status.Phase != v1.PodRunning {
   425  		return fmt.Errorf("expected the mirror pod %q to be running, got %q", name, pod.Status.Phase)
   426  	}
   427  	for i := range pod.Status.ContainerStatuses {
   428  		if pod.Status.ContainerStatuses[i].State.Running == nil {
   429  			return fmt.Errorf("expected the mirror pod %q with container %q to be running (got containers=%v)", name, pod.Status.ContainerStatuses[i].Name, pod.Status.ContainerStatuses[i].State)
   430  		}
   431  	}
   432  	return validateMirrorPod(ctx, cl, pod)
   433  }
   434  
   435  func checkMirrorPodRunningWithRestartCount(ctx context.Context, interval time.Duration, timeout time.Duration, cl clientset.Interface, name, namespace string, count int32) error {
   436  	var pod *v1.Pod
   437  	var err error
   438  	err = wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (bool, error) {
   439  		pod, err = cl.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
   440  		if err != nil {
   441  			return false, fmt.Errorf("expected the mirror pod %q to appear: %w", name, err)
   442  		}
   443  		if pod.Status.Phase != v1.PodRunning {
   444  			return false, fmt.Errorf("expected the mirror pod %q to be running, got %q", name, pod.Status.Phase)
   445  		}
   446  		for i := range pod.Status.ContainerStatuses {
   447  			if pod.Status.ContainerStatuses[i].State.Waiting != nil {
   448  				// retry if pod is in waiting state
   449  				return false, nil
   450  			}
   451  			if pod.Status.ContainerStatuses[i].State.Running == nil {
   452  				return false, fmt.Errorf("expected the mirror pod %q with container %q to be running (got containers=%v)", name, pod.Status.ContainerStatuses[i].Name, pod.Status.ContainerStatuses[i].State)
   453  			}
   454  			if pod.Status.ContainerStatuses[i].RestartCount == count {
   455  				// found the restart count
   456  				return true, nil
   457  			}
   458  		}
   459  		return false, nil
   460  	})
   461  	if err != nil {
   462  		return err
   463  	}
   464  	return validateMirrorPod(ctx, cl, pod)
   465  }
   466  
   467  func checkMirrorPodRecreatedAndRunning(ctx context.Context, cl clientset.Interface, name, namespace string, oUID types.UID) error {
   468  	pod, err := cl.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
   469  	if err != nil {
   470  		return fmt.Errorf("expected the mirror pod %q to appear: %w", name, err)
   471  	}
   472  	if pod.UID == oUID {
   473  		return fmt.Errorf("expected the uid of mirror pod %q to be changed, got %q", name, pod.UID)
   474  	}
   475  	if pod.Status.Phase != v1.PodRunning {
   476  		return fmt.Errorf("expected the mirror pod %q to be running, got %q", name, pod.Status.Phase)
   477  	}
   478  	return validateMirrorPod(ctx, cl, pod)
   479  }
   480  
   481  func validateMirrorPod(ctx context.Context, cl clientset.Interface, mirrorPod *v1.Pod) error {
   482  	hash, ok := mirrorPod.Annotations[kubetypes.ConfigHashAnnotationKey]
   483  	if !ok || hash == "" {
   484  		return fmt.Errorf("expected mirror pod %q to have a hash annotation", mirrorPod.Name)
   485  	}
   486  	mirrorHash, ok := mirrorPod.Annotations[kubetypes.ConfigMirrorAnnotationKey]
   487  	if !ok || mirrorHash == "" {
   488  		return fmt.Errorf("expected mirror pod %q to have a mirror pod annotation", mirrorPod.Name)
   489  	}
   490  	if hash != mirrorHash {
   491  		return fmt.Errorf("expected mirror pod %q to have a matching mirror pod hash: got %q; expected %q", mirrorPod.Name, mirrorHash, hash)
   492  	}
   493  	source, ok := mirrorPod.Annotations[kubetypes.ConfigSourceAnnotationKey]
   494  	if !ok {
   495  		return fmt.Errorf("expected mirror pod %q to have a source annotation", mirrorPod.Name)
   496  	}
   497  	if source == kubetypes.ApiserverSource {
   498  		return fmt.Errorf("expected mirror pod %q source to not be 'api'; got: %q", mirrorPod.Name, source)
   499  	}
   500  
   501  	if len(mirrorPod.OwnerReferences) != 1 {
   502  		return fmt.Errorf("expected mirror pod %q to have a single owner reference: got %d", mirrorPod.Name, len(mirrorPod.OwnerReferences))
   503  	}
   504  	node, err := cl.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{})
   505  	if err != nil {
   506  		return fmt.Errorf("failed to fetch test node: %w", err)
   507  	}
   508  
   509  	controller := true
   510  	expectedOwnerRef := metav1.OwnerReference{
   511  		APIVersion: "v1",
   512  		Kind:       "Node",
   513  		Name:       framework.TestContext.NodeName,
   514  		UID:        node.UID,
   515  		Controller: &controller,
   516  	}
   517  	ref := mirrorPod.OwnerReferences[0]
   518  	if !apiequality.Semantic.DeepEqual(ref, expectedOwnerRef) {
   519  		return fmt.Errorf("unexpected mirror pod %q owner ref: %v", mirrorPod.Name, cmp.Diff(expectedOwnerRef, ref))
   520  	}
   521  
   522  	return nil
   523  }
   524
View as plain text