reboot_node.go

Documentation: k8s.io/kubernetes/test/e2e/windows

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package windows
    18  
    19  import (
    20  	"context"
    21  	"time"
    22  
    23  	"github.com/onsi/ginkgo/v2"
    24  	"github.com/onsi/gomega"
    25  	v1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  
    28  	"k8s.io/apimachinery/pkg/util/uuid"
    29  	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
    30  	"k8s.io/kubernetes/test/e2e/feature"
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    33  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    34  	imageutils "k8s.io/kubernetes/test/utils/image"
    35  	admissionapi "k8s.io/pod-security-admission/api"
    36  )
    37  
    38  var _ = sigDescribe(feature.Windows, "[Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] RebootHost containers", framework.WithSerial(), framework.WithDisruptive(), framework.WithSlow(), skipUnlessWindows(func() {
    39  	ginkgo.BeforeEach(func() {
    40  		e2eskipper.SkipUnlessNodeOSDistroIs("windows")
    41  	})
    42  
    43  	f := framework.NewDefaultFramework("reboot-host-test-windows")
    44  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    45  
    46  	ginkgo.It("should run as a reboot process on the host/node", func(ctx context.Context) {
    47  
    48  		ginkgo.By("selecting a Windows node")
    49  		targetNode, err := findWindowsNode(ctx, f)
    50  		framework.ExpectNoError(err, "Error finding Windows node")
    51  		framework.Logf("Using node: %v", targetNode.Name)
    52  
    53  		windowsImage := imageutils.GetE2EImage(imageutils.Agnhost)
    54  
    55  		// Create Windows pod on the selected Windows node Using Agnhost
    56  		podName := "pod-" + string(uuid.NewUUID())
    57  		agnPod := &v1.Pod{
    58  			TypeMeta: metav1.TypeMeta{
    59  				Kind:       "Pod",
    60  				APIVersion: "v1",
    61  			},
    62  			ObjectMeta: metav1.ObjectMeta{
    63  				Name: podName,
    64  			},
    65  			Spec: v1.PodSpec{
    66  				Containers: []v1.Container{
    67  					{
    68  						Name:  "windows-container",
    69  						Image: windowsImage,
    70  						Ports: []v1.ContainerPort{{ContainerPort: 80}},
    71  					},
    72  				},
    73  				RestartPolicy: v1.RestartPolicyAlways,
    74  				NodeName:      targetNode.Name,
    75  			},
    76  		}
    77  		agnPod.Spec.Containers[0].Args = []string{"test-webserver"}
    78  		ginkgo.By("creating a windows pod and waiting for it to be running")
    79  		agnPod = e2epod.NewPodClient(f).CreateSync(ctx, agnPod)
    80  
    81  		// Create Linux pod to ping the windows pod
    82  		linuxBusyBoxImage := imageutils.GetE2EImage(imageutils.Nginx)
    83  		podName = "pod-" + string(uuid.NewUUID())
    84  		nginxPod := &v1.Pod{
    85  			TypeMeta: metav1.TypeMeta{
    86  				Kind:       "Pod",
    87  				APIVersion: "v1",
    88  			},
    89  			ObjectMeta: metav1.ObjectMeta{
    90  				Name: podName,
    91  			},
    92  			Spec: v1.PodSpec{
    93  				Containers: []v1.Container{
    94  					{
    95  						Name:  "linux-container",
    96  						Image: linuxBusyBoxImage,
    97  						Ports: []v1.ContainerPort{{ContainerPort: 80}},
    98  					},
    99  				},
   100  				NodeSelector: map[string]string{
   101  					"kubernetes.io/os": "linux",
   102  				},
   103  				Tolerations: []v1.Toleration{
   104  					{
   105  						Operator: v1.TolerationOpExists,
   106  						Effect:   v1.TaintEffectNoSchedule,
   107  					},
   108  				},
   109  			},
   110  		}
   111  		ginkgo.By("Waiting for the Linux pod to run")
   112  		nginxPod = e2epod.NewPodClient(f).CreateSync(ctx, nginxPod)
   113  
   114  		ginkgo.By("checking connectivity to 8.8.8.8 53 (google.com) from Linux")
   115  		assertConsistentConnectivity(ctx, f, nginxPod.ObjectMeta.Name, "linux", linuxCheck("8.8.8.8", 53), externalMaxTries)
   116  
   117  		ginkgo.By("checking connectivity to www.google.com from Windows")
   118  		assertConsistentConnectivity(ctx, f, agnPod.ObjectMeta.Name, "windows", windowsCheck("www.google.com"), externalMaxTries)
   119  
   120  		ginkgo.By("checking connectivity from Linux to Windows for the first time")
   121  		assertConsistentConnectivity(ctx, f, nginxPod.ObjectMeta.Name, "linux", linuxCheck(agnPod.Status.PodIP, 80), internalMaxTries)
   122  
   123  		initialRestartCount := podutil.GetExistingContainerStatus(agnPod.Status.ContainerStatuses, "windows-container").RestartCount
   124  
   125  		ginkgo.By("scheduling a pod with a container that verifies reboot selected node works as well")
   126  
   127  		trueVar := true
   128  		podName = "reboot-host-test-pod"
   129  		user := "NT AUTHORITY\\SYSTEM"
   130  		pod := &v1.Pod{
   131  			ObjectMeta: metav1.ObjectMeta{
   132  				Name: podName,
   133  			},
   134  			Spec: v1.PodSpec{
   135  				SecurityContext: &v1.PodSecurityContext{
   136  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   137  						HostProcess:   &trueVar,
   138  						RunAsUserName: &user,
   139  					},
   140  				},
   141  				HostNetwork: true,
   142  				Containers: []v1.Container{
   143  					{
   144  						Image: windowsImage,
   145  						Name:  "reboot-computer-test",
   146  						Command: []string{
   147  							"powershell.exe",
   148  							"-Command",
   149  							"$os = Get-WmiObject -Class win32_operatingsystem;",
   150  							"[Environment]::SetEnvironmentVariable(\"TMP_BOOT_DATE\", $os.LastBootUpTime, \"Machine\");",
   151  							"[Environment]::SetEnvironmentVariable(\"TMP_INSTALL_DATE\", $os.InstallDate, \"Machine\");",
   152  							"shutdown.exe -r -t 30",
   153  						},
   154  					},
   155  				},
   156  				RestartPolicy: v1.RestartPolicyNever,
   157  				NodeName:      targetNode.Name,
   158  			},
   159  		}
   160  
   161  		e2epod.NewPodClient(f).Create(ctx, pod)
   162  
   163  		ginkgo.By("Waiting for pod to run")
   164  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   165  
   166  		ginkgo.By("Then ensuring pod finished running successfully")
   167  		p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   168  			ctx,
   169  			podName,
   170  			metav1.GetOptions{})
   171  
   172  		framework.ExpectNoError(err, "Error retrieving pod")
   173  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   174  
   175  		ginkgo.By("Waiting for Windows worker rebooting")
   176  
   177  		restartCount := 0
   178  
   179  		timeout := time.After(time.Minute * 10)
   180  	FOR:
   181  		for {
   182  			select {
   183  			case <-timeout:
   184  				break FOR
   185  			default:
   186  				if restartCount > 0 {
   187  					break FOR
   188  				}
   189  				ginkgo.By("Then checking existed agn-test-pod is running on the rebooted host")
   190  				agnPodOut, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, agnPod.Name, metav1.GetOptions{})
   191  				if err == nil {
   192  					lastRestartCount := podutil.GetExistingContainerStatus(agnPodOut.Status.ContainerStatuses, "windows-container").RestartCount
   193  					restartCount = int(lastRestartCount - initialRestartCount)
   194  				}
   195  				time.Sleep(time.Second * 30)
   196  			}
   197  		}
   198  
   199  		ginkgo.By("Checking whether agn-test-pod is rebooted")
   200  		gomega.Expect(restartCount).To(gomega.Equal(1), "restart count of agn-test-pod is 1")
   201  
   202  		agnPodOut, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, agnPod.Name, metav1.GetOptions{})
   203  		gomega.Expect(agnPodOut.Status.Phase).To(gomega.Equal(v1.PodRunning))
   204  		framework.ExpectNoError(err, "getting pod info after reboot")
   205  		assertConsistentConnectivity(ctx, f, nginxPod.ObjectMeta.Name, "linux", linuxCheck(agnPodOut.Status.PodIP, 80), internalMaxTries)
   206  
   207  		// create another host process pod to check system boot time
   208  		checkPod := &v1.Pod{
   209  			ObjectMeta: metav1.ObjectMeta{
   210  				Name: "check-reboot-pod",
   211  			},
   212  			Spec: v1.PodSpec{
   213  				SecurityContext: &v1.PodSecurityContext{
   214  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   215  						HostProcess:   &trueVar,
   216  						RunAsUserName: &user,
   217  					},
   218  				},
   219  				HostNetwork: true,
   220  				Containers: []v1.Container{
   221  					{
   222  						Image: windowsImage,
   223  						Name:  "reboot-computer-check",
   224  						Command: []string{
   225  							"powershell.exe",
   226  							"-Command",
   227  							"$os = Get-WmiObject -Class win32_operatingsystem;",
   228  							"$lastBootTime = [Environment]::GetEnvironmentVariable(\"TMP_BOOT_DATE\", \"Machine\");",
   229  							"$lastInstallTime = [Environment]::GetEnvironmentVariable(\"TMP_INSTALL_DATE\", \"Machine\");",
   230  							"$timeInterval = $os.ConvertToDateTime($os.LastBootUpTime) -  $os.ConvertToDateTime($lastBootTime);",
   231  							"$installInterval = $os.ConvertToDateTime($os.InstallDate) -  $os.ConvertToDateTime($lastInstallTime);",
   232  							"if ( $timeInterval.TotalSeconds -le 0 ) {exit -1};",
   233  							"if ( $installInterval.TotalSeconds -ne 0 ) {exit -1};",
   234  							"[Environment]::SetEnvironmentVariable(\"TMP_BOOT_DATE\", $null, \"Machine\");",
   235  							"[Environment]::SetEnvironmentVariable(\"TMP_INSTALL_DATE\", $null, \"Machine\");",
   236  						},
   237  					},
   238  				},
   239  				RestartPolicy: v1.RestartPolicyNever,
   240  				NodeName:      targetNode.Name,
   241  			},
   242  		}
   243  
   244  		e2epod.NewPodClient(f).Create(ctx, checkPod)
   245  
   246  		ginkgo.By("Waiting for pod to run")
   247  		e2epod.NewPodClient(f).WaitForFinish(ctx, "check-reboot-pod", 3*time.Minute)
   248  
   249  		ginkgo.By("Then ensuring pod finished running successfully")
   250  		p, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   251  			ctx,
   252  			"check-reboot-pod",
   253  			metav1.GetOptions{})
   254  
   255  		framework.ExpectNoError(err, "Error retrieving pod")
   256  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   257  	})
   258  }))
   259
View as plain text