Source file src/k8s.io/kubernetes/test/e2e/windows/host_process.go

Documentation: k8s.io/kubernetes/test/e2e/windows

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package windows
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"time"
    24  
    25  	semver "github.com/blang/semver/v4"
    26  	"github.com/onsi/ginkgo/v2"
    27  	"github.com/onsi/gomega"
    28  	v1 "k8s.io/api/core/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/fields"
    31  	"k8s.io/apimachinery/pkg/util/uuid"
    32  	"k8s.io/apimachinery/pkg/util/wait"
    33  	clientset "k8s.io/client-go/kubernetes"
    34  	"k8s.io/kubernetes/test/e2e/feature"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
    37  	e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
    38  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    39  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    40  	imageutils "k8s.io/kubernetes/test/utils/image"
    41  	admissionapi "k8s.io/pod-security-admission/api"
    42  )
    43  
    44  const (
    45  	validation_script = `if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\emptydir)) {
    46  		throw "Cannot find emptydir volume"
    47  	}
    48  	if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\configmap\text.txt)) {
    49  		throw "Cannot find text.txt in configmap-volume"
    50  	}
    51  	$c = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\configmap\text.txt
    52  	if ($c -ne "Lorem ipsum dolor sit amet") {
    53  		throw "Contents of /etc/configmap/text.txt are not as expected"
    54  	}
    55  	if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\hostpath)) {
    56  		throw "Cannot find hostpath volume" 
    57  	}
    58  	if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\downwardapi\podname)) {
    59  		throw "Cannot find podname file in downward-api volume" 
    60  	}
    61  	$c = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\downwardapi\podname
    62  	if ($c -ne "host-process-volume-mounts") {
    63  		throw "Contents of /etc/downward-api/podname are not as expected"
    64  	}
    65  	if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\secret\foo.txt)) {
    66  		throw "Cannot find file foo.txt in secret volume"
    67  	}
    68  	$c = Get-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\secret\foo.txt
    69  	if ($c -ne "bar") {
    70  		Write-Output $c
    71  		throw "Contents of /etc/secret/foo.txt are not as expected"
    72  	}
    73  	# Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME
    74  	# can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead.
    75  	# The following comparison is case insensitive.
    76  	if ($env:NODE_NAME_TEST -ine "$(hostname)") {
    77  		throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname"
    78  	}
    79  	Write-Output "SUCCESS"`
    80  )
    81  
    82  var (
    83  	trueVar = true
    84  
    85  	User_NTAuthorityLocalService = "NT AUTHORITY\\Local Service"
    86  	User_NTAuthoritySystem       = "NT AUTHORITY\\SYSTEM"
    87  )
    88  
    89  var _ = sigDescribe(feature.WindowsHostProcessContainers, "[MinimumKubeletVersion:1.22] HostProcess containers", skipUnlessWindows(func() {
    90  	ginkgo.BeforeEach(func() {
    91  		e2eskipper.SkipUnlessNodeOSDistroIs("windows")
    92  	})
    93  
    94  	f := framework.NewDefaultFramework("host-process-test-windows")
    95  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    96  
    97  	ginkgo.It("should run as a process on the host/node", func(ctx context.Context) {
    98  
    99  		ginkgo.By("selecting a Windows node")
   100  		targetNode, err := findWindowsNode(ctx, f)
   101  		framework.ExpectNoError(err, "Error finding Windows node")
   102  		framework.Logf("Using node: %v", targetNode.Name)
   103  
   104  		ginkgo.By("scheduling a pod with a container that verifies %COMPUTERNAME% matches selected node name")
   105  		image := imageutils.GetConfig(imageutils.BusyBox)
   106  		podName := "host-process-test-pod"
   107  		// We're passing this to powershell.exe -Command. Inside, we must use apostrophes for strings.
   108  		command := fmt.Sprintf(`& {if ('%s' -ine "$(hostname)") { exit -1 }}`, targetNode.Name)
   109  		pod := &v1.Pod{
   110  			ObjectMeta: metav1.ObjectMeta{
   111  				Name: podName,
   112  			},
   113  			Spec: v1.PodSpec{
   114  				SecurityContext: &v1.PodSecurityContext{
   115  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   116  						HostProcess:   &trueVar,
   117  						RunAsUserName: &User_NTAuthoritySystem,
   118  					},
   119  				},
   120  				HostNetwork: true,
   121  				Containers: []v1.Container{
   122  					{
   123  						Image:   image.GetE2EImage(),
   124  						Name:    "computer-name-test",
   125  						Command: []string{"powershell.exe", "-Command", command},
   126  					},
   127  				},
   128  				RestartPolicy: v1.RestartPolicyNever,
   129  				NodeName:      targetNode.Name,
   130  			},
   131  		}
   132  
   133  		e2epod.NewPodClient(f).Create(ctx, pod)
   134  
   135  		ginkgo.By("Waiting for pod to run")
   136  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   137  
   138  		ginkgo.By("Then ensuring pod finished running successfully")
   139  		p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   140  			ctx,
   141  			podName,
   142  			metav1.GetOptions{})
   143  
   144  		framework.ExpectNoError(err, "Error retrieving pod")
   145  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   146  	})
   147  
   148  	ginkgo.It("should support init containers", func(ctx context.Context) {
   149  		ginkgo.By("scheduling a pod with a container that verifies init container can configure the node")
   150  		podName := "host-process-init-pods"
   151  		filename := fmt.Sprintf("/testfile%s.txt", string(uuid.NewUUID()))
   152  		pod := &v1.Pod{
   153  			ObjectMeta: metav1.ObjectMeta{
   154  				Name: podName,
   155  			},
   156  			Spec: v1.PodSpec{
   157  				SecurityContext: &v1.PodSecurityContext{
   158  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   159  						HostProcess:   &trueVar,
   160  						RunAsUserName: &User_NTAuthoritySystem,
   161  					},
   162  				},
   163  				HostNetwork: true,
   164  				InitContainers: []v1.Container{
   165  					{
   166  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   167  						Name:    "configure-node",
   168  						Command: []string{"powershell", "-c", "Set-content", "-Path", filename, "-V", "test"},
   169  					},
   170  				},
   171  				Containers: []v1.Container{
   172  					{
   173  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   174  						Name:    "read-configuration",
   175  						Command: []string{"powershell", "-c", "ls", filename},
   176  					},
   177  				},
   178  				RestartPolicy: v1.RestartPolicyNever,
   179  				NodeSelector: map[string]string{
   180  					"kubernetes.io/os": "windows",
   181  				},
   182  			},
   183  		}
   184  
   185  		e2epod.NewPodClient(f).Create(ctx, pod)
   186  
   187  		ginkgo.By("Waiting for pod to run")
   188  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   189  
   190  		ginkgo.By("Then ensuring pod finished running successfully")
   191  		p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   192  			ctx,
   193  			podName,
   194  			metav1.GetOptions{})
   195  
   196  		framework.ExpectNoError(err, "Error retrieving pod")
   197  
   198  		if p.Status.Phase != v1.PodSucceeded {
   199  			logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "read-configuration")
   200  			if err != nil {
   201  				framework.Logf("Error pulling logs: %v", err)
   202  			}
   203  			framework.Logf("Pod phase: %v\nlogs:\n%s", p.Status.Phase, logs)
   204  		}
   205  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   206  	})
   207  
   208  	ginkgo.It("container command path validation", func(ctx context.Context) {
   209  
   210  		// The way hostprocess containers are created is being updated in container
   211  		// v1.7 to better support volume mounts and part of these changes include
   212  		// updates how the container's starting process is invoked.
   213  		// These test cases are only valid for containerd v1.6.
   214  		// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/1981-windows-privileged-container-support/README.md
   215  		// for more details.
   216  		ginkgo.By("Ensuring Windows nodes are running containerd v1.6.x")
   217  		windowsNode, err := findWindowsNode(ctx, f)
   218  		framework.ExpectNoError(err, "error finding Windows node")
   219  		r, v, err := getNodeContainerRuntimeAndVersion(windowsNode)
   220  		framework.ExpectNoError(err, "error getting node container runtime and version")
   221  		framework.Logf("Got runtime: %s, version %v, node: %s", r, v, windowsNode.Name)
   222  
   223  		if !strings.EqualFold(r, "containerd") {
   224  			e2eskipper.Skipf("container runtime is not containerd")
   225  		}
   226  
   227  		v1dot7 := semver.MustParse("1.7.0")
   228  		if v.GTE(v1dot7) {
   229  			e2eskipper.Skipf("container runtime is >= 1.7.0")
   230  		}
   231  
   232  		// The following test cases are broken into batches to speed up the test.
   233  		// Each batch will be scheduled as a single pod with a container for each test case.
   234  		// Pods will be scheduled sequentially since the start-up cost of containers is high
   235  		// on Windows and ginkgo may also schedule test cases in parallel.
   236  		tests := [][]struct {
   237  			command    []string
   238  			args       []string
   239  			workingDir string
   240  		}{
   241  			{
   242  				{
   243  					command: []string{"cmd.exe", "/c", "ver"},
   244  				},
   245  				{
   246  					command:    []string{"System32\\cmd.exe", "/c", "ver"},
   247  					workingDir: "c:\\Windows",
   248  				},
   249  				{
   250  					command:    []string{"System32\\cmd.exe", "/c", "ver"},
   251  					workingDir: "c:\\Windows\\",
   252  				},
   253  				{
   254  					command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe", "-o"},
   255  				},
   256  			},
   257  			{
   258  				{
   259  					command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%/bin/uname.exe", "-o"},
   260  				},
   261  				{
   262  					command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin/uname.exe", "-o"},
   263  				},
   264  				{
   265  					command: []string{"bin/uname.exe", "-o"},
   266  				},
   267  				{
   268  					command:    []string{"bin/uname.exe", "-o"},
   269  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
   270  				},
   271  			},
   272  			{
   273  				{
   274  					command:    []string{"bin\\uname.exe", "-o"},
   275  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
   276  				},
   277  				{
   278  					command:    []string{"uname.exe", "-o"},
   279  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
   280  				},
   281  				{
   282  					command:    []string{"uname.exe", "-o"},
   283  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin/",
   284  				},
   285  				{
   286  					command:    []string{"uname.exe", "-o"},
   287  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\",
   288  				},
   289  			},
   290  			{
   291  				{
   292  					command: []string{"powershell", "cmd.exe", "/ver"},
   293  				},
   294  				{
   295  					command: []string{"powershell", "c:/Windows/System32/cmd.exe", "/c", "ver"},
   296  				},
   297  				{
   298  					command: []string{"powershell", "c:\\Windows\\System32/cmd.exe", "/c", "ver"},
   299  				},
   300  				{
   301  					command: []string{"powershell", "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe", "-o"},
   302  				},
   303  			},
   304  			{
   305  				{
   306  					command: []string{"powershell", "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\uname.exe", "-o"},
   307  				},
   308  				{
   309  					command: []string{"powershell", "%CONTAINER_SANDBOX_MOUNT_POINT%/bin/uname.exe", "-o"},
   310  				},
   311  				{
   312  					command: []string{"powershell", "$env:CONTAINER_SANDBOX_MOUNT_POINT/bin/uname.exe", "-o"},
   313  				},
   314  				{
   315  					command:    []string{"powershell", "bin/uname.exe", "-o"},
   316  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
   317  				},
   318  			},
   319  			{
   320  				{
   321  					command:    []string{"powershell", "bin/uname.exe", "-o"},
   322  					workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
   323  				},
   324  				{
   325  					command:    []string{"powershell", "bin\\uname.exe", "-o"},
   326  					workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
   327  				},
   328  				{
   329  					command:    []string{"powershell", ".\\uname.exe", "-o"},
   330  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
   331  				},
   332  				{
   333  					command:    []string{"powershell", "./uname.exe", "-o"},
   334  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
   335  				},
   336  			},
   337  			{
   338  				{
   339  					command:    []string{"powershell", "./uname.exe", "-o"},
   340  					workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\",
   341  				},
   342  				{
   343  					command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe"},
   344  					args:    []string{"-o"},
   345  				},
   346  				{
   347  					command:    []string{"bin\\uname.exe"},
   348  					args:       []string{"-o"},
   349  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
   350  				},
   351  				{
   352  					command:    []string{"uname.exe"},
   353  					args:       []string{"-o"},
   354  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin",
   355  				},
   356  			},
   357  			{
   358  				{
   359  					command: []string{"cmd.exe"},
   360  					args:    []string{"/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe"},
   361  				},
   362  				{
   363  					command:    []string{"cmd.exe"},
   364  					args:       []string{"/c", "dir", "bin\\uname.exe"},
   365  					workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
   366  				},
   367  				{
   368  					command: []string{"powershell"},
   369  					args:    []string{"Get-ChildItem", "-Path", "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\uname.exe"},
   370  				},
   371  				{
   372  					command:    []string{"powershell"},
   373  					args:       []string{"Get-ChildItem", "-Path", "bin\\uname.exe"},
   374  					workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
   375  				},
   376  			},
   377  			{
   378  				{
   379  					command:    []string{"powershell"},
   380  					args:       []string{"Get-ChildItem", "-Path", "uname.exe"},
   381  					workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/bin",
   382  				},
   383  			},
   384  		}
   385  
   386  		for podIndex, testCaseBatch := range tests {
   387  			image := imageutils.GetConfig(imageutils.BusyBox)
   388  			podName := fmt.Sprintf("host-process-command-%d", podIndex)
   389  			containers := []v1.Container{}
   390  			for containerIndex, testCase := range testCaseBatch {
   391  				containerName := fmt.Sprintf("host-process-command-%d-%d", podIndex, containerIndex)
   392  				ginkgo.By(fmt.Sprintf("Adding a container '%s' to pod '%s' with command: %s, args: %s, workingDir: %s", containerName, podName, strings.Join(testCase.command, " "), strings.Join(testCase.args, " "), testCase.workingDir))
   393  
   394  				container := v1.Container{
   395  					Image:      image.GetE2EImage(),
   396  					Name:       containerName,
   397  					Command:    testCase.command,
   398  					Args:       testCase.args,
   399  					WorkingDir: testCase.workingDir,
   400  				}
   401  				containers = append(containers, container)
   402  			}
   403  
   404  			pod := &v1.Pod{
   405  				ObjectMeta: metav1.ObjectMeta{
   406  					Name: podName,
   407  				},
   408  				Spec: v1.PodSpec{
   409  					SecurityContext: &v1.PodSecurityContext{
   410  						WindowsOptions: &v1.WindowsSecurityContextOptions{
   411  							HostProcess:   &trueVar,
   412  							RunAsUserName: &User_NTAuthorityLocalService,
   413  						},
   414  					},
   415  					HostNetwork:   true,
   416  					Containers:    containers,
   417  					RestartPolicy: v1.RestartPolicyNever,
   418  					NodeSelector: map[string]string{
   419  						"kubernetes.io/os": "windows",
   420  					},
   421  				},
   422  			}
   423  			e2epod.NewPodClient(f).Create(ctx, pod)
   424  
   425  			ginkgo.By(fmt.Sprintf("Waiting for pod '%s' to run", podName))
   426  			e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   427  
   428  			ginkgo.By("Then ensuring pod finished running successfully")
   429  			p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   430  				ctx,
   431  				podName,
   432  				metav1.GetOptions{})
   433  
   434  			framework.ExpectNoError(err, "Error retrieving pod")
   435  
   436  			if p.Status.Phase != v1.PodSucceeded {
   437  				framework.Logf("Getting pod events")
   438  				options := metav1.ListOptions{
   439  					FieldSelector: fields.Set{
   440  						"involvedObject.kind":      "Pod",
   441  						"involvedObject.name":      podName,
   442  						"involvedObject.namespace": f.Namespace.Name,
   443  					}.AsSelector().String(),
   444  				}
   445  				events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, options)
   446  				framework.ExpectNoError(err, "Error getting events for failed pod")
   447  				for _, event := range events.Items {
   448  					framework.Logf("%s: %s", event.Reason, event.Message)
   449  				}
   450  				framework.Failf("Pod '%s' did failed.", p.Name)
   451  			}
   452  		}
   453  
   454  	})
   455  
   456  	ginkgo.It("should support various volume mount types", func(ctx context.Context) {
   457  		ns := f.Namespace
   458  
   459  		ginkgo.By("Creating a configmap containing test data and a validation script")
   460  		configMap := &v1.ConfigMap{
   461  			TypeMeta: metav1.TypeMeta{
   462  				APIVersion: "v1",
   463  				Kind:       "ConfigMap",
   464  			},
   465  			ObjectMeta: metav1.ObjectMeta{
   466  				Name: "sample-config-map",
   467  			},
   468  			Data: map[string]string{
   469  				"text":              "Lorem ipsum dolor sit amet",
   470  				"validation-script": validation_script,
   471  			},
   472  		}
   473  		_, err := f.ClientSet.CoreV1().ConfigMaps(ns.Name).Create(ctx, configMap, metav1.CreateOptions{})
   474  		framework.ExpectNoError(err, "unable to create create configmap")
   475  
   476  		ginkgo.By("Creating a secret containing test data")
   477  		secret := &v1.Secret{
   478  			TypeMeta: metav1.TypeMeta{
   479  				APIVersion: "v1",
   480  				Kind:       "Secret",
   481  			},
   482  			ObjectMeta: metav1.ObjectMeta{
   483  				Name: "sample-secret",
   484  			},
   485  			Type: v1.SecretTypeOpaque,
   486  			Data: map[string][]byte{
   487  				"foo": []byte("bar"),
   488  			},
   489  		}
   490  		_, err = f.ClientSet.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{})
   491  		framework.ExpectNoError(err, "unable to create secret")
   492  
   493  		ginkgo.By("Creating a pod with a HostProcess container that uses various types of volume mounts")
   494  
   495  		podAndContainerName := "host-process-volume-mounts"
   496  		pod := makeTestPodWithVolumeMounts(podAndContainerName)
   497  
   498  		e2epod.NewPodClient(f).Create(ctx, pod)
   499  
   500  		ginkgo.By("Waiting for pod to run")
   501  		e2epod.NewPodClient(f).WaitForFinish(ctx, podAndContainerName, 3*time.Minute)
   502  
   503  		logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, ns.Name, podAndContainerName, podAndContainerName)
   504  		framework.ExpectNoError(err, "Error getting pod logs")
   505  		framework.Logf("Container logs: %s", logs)
   506  
   507  		ginkgo.By("Then ensuring pod finished running successfully")
   508  		p, err := f.ClientSet.CoreV1().Pods(ns.Name).Get(
   509  			ctx,
   510  			podAndContainerName,
   511  			metav1.GetOptions{})
   512  
   513  		framework.ExpectNoError(err, "Error retrieving pod")
   514  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   515  	})
   516  
   517  	ginkgo.It("metrics should report count of started and failed to start HostProcess containers", func(ctx context.Context) {
   518  		ginkgo.By("Selecting a Windows node")
   519  		targetNode, err := findWindowsNode(ctx, f)
   520  		framework.ExpectNoError(err, "Error finding Windows node")
   521  		framework.Logf("Using node: %v", targetNode.Name)
   522  
   523  		ginkgo.By("Getting initial kubelet metrics values")
   524  		beforeMetrics, err := getCurrentHostProcessMetrics(ctx, f, targetNode.Name)
   525  		framework.ExpectNoError(err, "Error getting initial kubelet metrics for node")
   526  		framework.Logf("Initial HostProcess container metrics -- StartedContainers: %v, StartedContainersErrors: %v, StartedInitContainers: %v, StartedInitContainersErrors: %v",
   527  			beforeMetrics.StartedContainersCount, beforeMetrics.StartedContainersErrorCount, beforeMetrics.StartedInitContainersCount, beforeMetrics.StartedInitContainersErrorCount)
   528  
   529  		ginkgo.By("Scheduling a pod with a HostProcess init container that will fail")
   530  
   531  		badUserName := "bad-user-name"
   532  
   533  		podName := "host-process-metrics-pod-failing-init-container"
   534  		pod := &v1.Pod{
   535  			ObjectMeta: metav1.ObjectMeta{
   536  				Name: podName,
   537  			},
   538  			Spec: v1.PodSpec{
   539  				SecurityContext: &v1.PodSecurityContext{
   540  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   541  						HostProcess:   &trueVar,
   542  						RunAsUserName: &User_NTAuthoritySystem,
   543  					},
   544  				},
   545  				HostNetwork: true,
   546  				InitContainers: []v1.Container{
   547  					{
   548  						SecurityContext: &v1.SecurityContext{
   549  							WindowsOptions: &v1.WindowsSecurityContextOptions{
   550  								RunAsUserName: &badUserName,
   551  							},
   552  						},
   553  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   554  						Name:    "failing-init-container",
   555  						Command: []string{"foobar.exe"},
   556  					},
   557  				},
   558  				Containers: []v1.Container{
   559  					{
   560  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   561  						Name:    "container",
   562  						Command: []string{"cmd.exe", "/c", "exit", "/b", "0"},
   563  					},
   564  				},
   565  				RestartPolicy: v1.RestartPolicyNever,
   566  				NodeName:      targetNode.Name,
   567  			},
   568  		}
   569  
   570  		e2epod.NewPodClient(f).Create(ctx, pod)
   571  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   572  
   573  		ginkgo.By("Scheduling a pod with a HostProcess container that will fail")
   574  		podName = "host-process-metrics-pod-failing-container"
   575  		pod = &v1.Pod{
   576  			ObjectMeta: metav1.ObjectMeta{
   577  				Name: podName,
   578  			},
   579  			Spec: v1.PodSpec{
   580  				SecurityContext: &v1.PodSecurityContext{
   581  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   582  						HostProcess:   &trueVar,
   583  						RunAsUserName: &User_NTAuthoritySystem,
   584  					},
   585  				},
   586  				HostNetwork: true,
   587  				Containers: []v1.Container{
   588  					{
   589  						SecurityContext: &v1.SecurityContext{
   590  							WindowsOptions: &v1.WindowsSecurityContextOptions{
   591  								RunAsUserName: &badUserName,
   592  							},
   593  						},
   594  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   595  						Name:    "failing-container",
   596  						Command: []string{"foobar.exe"},
   597  					},
   598  				},
   599  				RestartPolicy: v1.RestartPolicyNever,
   600  				NodeName:      targetNode.Name,
   601  			},
   602  		}
   603  
   604  		e2epod.NewPodClient(f).Create(ctx, pod)
   605  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   606  
   607  		ginkgo.By("Getting subsequent kubelet metrics values")
   608  
   609  		afterMetrics, err := getCurrentHostProcessMetrics(ctx, f, targetNode.Name)
   610  		framework.ExpectNoError(err, "Error getting subsequent kubelet metrics for node")
   611  		framework.Logf("Subsequent HostProcess container metrics -- StartedContainers: %v, StartedContainersErrors: %v, StartedInitContainers: %v, StartedInitContainersErrors: %v",
   612  			afterMetrics.StartedContainersCount, afterMetrics.StartedContainersErrorCount, afterMetrics.StartedInitContainersCount, afterMetrics.StartedInitContainersErrorCount)
   613  
   614  		// Note: This test performs relative comparisons to ensure metrics values were logged and does not validate specific values.
   615  		// This done so the test can be run in parallel with other tests which may start HostProcess containers on the same node.
   616  		ginkgo.By("Ensuring metrics were updated")
   617  		gomega.Expect(beforeMetrics.StartedContainersCount).To(gomega.BeNumerically("<", afterMetrics.StartedContainersCount), "Count of started HostProcess containers should increase")
   618  		gomega.Expect(beforeMetrics.StartedContainersErrorCount).To(gomega.BeNumerically("<", afterMetrics.StartedContainersErrorCount), "Count of started HostProcess errors containers should increase")
   619  		gomega.Expect(beforeMetrics.StartedInitContainersCount).To(gomega.BeNumerically("<", afterMetrics.StartedInitContainersCount), "Count of started HostProcess init containers should increase")
   620  		gomega.Expect(beforeMetrics.StartedInitContainersErrorCount).To(gomega.BeNumerically("<", afterMetrics.StartedInitContainersErrorCount), "Count of started HostProcess errors init containers should increase")
   621  	})
   622  
   623  	ginkgo.It("container stats validation", func(ctx context.Context) {
   624  		ginkgo.By("selecting a Windows node")
   625  		targetNode, err := findWindowsNode(ctx, f)
   626  		framework.ExpectNoError(err, "Error finding Windows node")
   627  		framework.Logf("Using node: %v", targetNode.Name)
   628  
   629  		ginkgo.By("schedule a pod with a HostProcess container")
   630  		image := imageutils.GetConfig(imageutils.BusyBox)
   631  		podName := "host-process-stats-pod"
   632  		pod := &v1.Pod{
   633  			ObjectMeta: metav1.ObjectMeta{
   634  				Name: podName,
   635  			},
   636  			Spec: v1.PodSpec{
   637  				SecurityContext: &v1.PodSecurityContext{
   638  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   639  						HostProcess:   &trueVar,
   640  						RunAsUserName: &User_NTAuthorityLocalService,
   641  					},
   642  				},
   643  				HostNetwork: true,
   644  				Containers: []v1.Container{
   645  					{
   646  						Image:   image.GetE2EImage(),
   647  						Name:    "host-process-stats",
   648  						Command: []string{"powershell.exe", "-Command", "Write-Host 'Hello'; sleep -Seconds 600"},
   649  					},
   650  				},
   651  				RestartPolicy: v1.RestartPolicyNever,
   652  				NodeName:      targetNode.Name,
   653  			},
   654  		}
   655  
   656  		e2epod.NewPodClient(f).Create(ctx, pod)
   657  
   658  		ginkgo.By("Waiting for the pod to start running")
   659  		timeout := 3 * time.Minute
   660  		e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, timeout)
   661  
   662  		ginkgo.By("Getting container stats for pod")
   663  		statsChecked := false
   664  
   665  		wait.Poll(2*time.Second, timeout, func() (bool, error) {
   666  			return ensurePodsStatsOnNode(ctx, f.ClientSet, f.Namespace.Name, targetNode.Name, &statsChecked)
   667  		})
   668  
   669  		if !statsChecked {
   670  			framework.Failf("Failed to get stats for pod %s/%s", f.Namespace.Name, podName)
   671  		}
   672  	})
   673  
   674  	ginkgo.It("should support querying api-server using in-cluster config", func(ctx context.Context) {
   675  		// This functionality is only support on containerd  v1.7+
   676  		skipUnlessContainerdOneSevenOrGreater(ctx, f)
   677  
   678  		ginkgo.By("Scheduling a pod that runs agnhost inclusterclient")
   679  		podName := "host-process-agnhost-icc"
   680  		pod := &v1.Pod{
   681  			ObjectMeta: metav1.ObjectMeta{
   682  				Name: podName,
   683  			},
   684  			Spec: v1.PodSpec{
   685  				SecurityContext: &v1.PodSecurityContext{
   686  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   687  						HostProcess:   &trueVar,
   688  						RunAsUserName: &User_NTAuthoritySystem,
   689  					},
   690  				},
   691  				HostNetwork: true,
   692  				Containers: []v1.Container{
   693  					{
   694  						Image: imageutils.GetE2EImage(imageutils.Agnhost),
   695  						Name:  "hpc-agnhost",
   696  						// TODO: Figure out why we need to copy agnhost as agnhost.exe here
   697  						// Possibly related to https://github.com/microsoft/hcsshim/issues/1128 and
   698  						// https://github.com/microsoft/hcsshim/pull/1174 updating PATHEXT here doesn't
   699  						// seem address the issue.
   700  						Command: []string{"cmd", "/C", "copy", "c:\\hpc\\agnhost", "c:\\hpc\\agnhost.exe", "&&", "c:\\hpc\\agnhost.exe", "inclusterclient"},
   701  					},
   702  				},
   703  				RestartPolicy: v1.RestartPolicyNever,
   704  				NodeSelector: map[string]string{
   705  					"kubernetes.io/os": "windows",
   706  				},
   707  			},
   708  		}
   709  
   710  		pc := e2epod.NewPodClient(f)
   711  		pc.Create(ctx, pod)
   712  
   713  		ginkgo.By("Waiting for pod to run")
   714  		e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, 3*time.Minute)
   715  
   716  		ginkgo.By("Waiting for 60 seconds")
   717  		// We wait an additional 60 seconds after the pod is Running because the
   718  		// `InClusterClient` app in the agnhost image waits 30 seconds before
   719  		// starting to make queries - https://github.com/kubernetes/kubernetes/blob/ceee3783ed963497db669504241af2ca6a11610f/test/images/agnhost/inclusterclient/main.go#L51
   720  		time.Sleep(60 * time.Second)
   721  
   722  		ginkgo.By("Ensuring the test app was able to successfully query the api-server")
   723  		logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "hpc-agnhost")
   724  		framework.ExpectNoError(err, "Error getting pod logs")
   725  
   726  		framework.Logf("Logs: %s\n", logs)
   727  
   728  		gomega.Expect(logs).Should(gomega.ContainSubstring("calling /healthz"), "app logs should contain 'calling /healthz'")
   729  
   730  		gomega.Expect(logs).ShouldNot(gomega.ContainSubstring("status=failed"), "app logs should not contain 'status=failed'")
   731  	})
   732  
   733  	ginkgo.It("should run as localgroup accounts", func(ctx context.Context) {
   734  		// This functionality is only supported on containerd v1.7+
   735  		skipUnlessContainerdOneSevenOrGreater(ctx, f)
   736  
   737  		ginkgo.By("Scheduling a pod that creates a localgroup from an init container then starts a container using that group")
   738  		localGroupName := getRandomUserGrounName()
   739  		podName := "host-process-localgroup-pod"
   740  		pod := &v1.Pod{
   741  			ObjectMeta: metav1.ObjectMeta{
   742  				Name: podName,
   743  			},
   744  			Spec: v1.PodSpec{
   745  				SecurityContext: &v1.PodSecurityContext{
   746  					WindowsOptions: &v1.WindowsSecurityContextOptions{
   747  						HostProcess:   &trueVar,
   748  						RunAsUserName: &User_NTAuthoritySystem,
   749  					},
   750  				},
   751  				HostNetwork: true,
   752  				InitContainers: []v1.Container{
   753  					{
   754  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   755  						Name:    "setup",
   756  						Command: []string{"cmd", "/C", "net", "localgroup", localGroupName, "/add"},
   757  					},
   758  				},
   759  				Containers: []v1.Container{
   760  					{
   761  						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   762  						Name:    "localgroup-container",
   763  						Command: []string{"cmd", "/C", "whoami"},
   764  						SecurityContext: &v1.SecurityContext{
   765  							WindowsOptions: &v1.WindowsSecurityContextOptions{
   766  								RunAsUserName: &localGroupName,
   767  							},
   768  						},
   769  					},
   770  				},
   771  				RestartPolicy: v1.RestartPolicyNever,
   772  				NodeSelector: map[string]string{
   773  					"kubernetes.io/os": "windows",
   774  				},
   775  			},
   776  		}
   777  
   778  		e2epod.NewPodClient(f).Create(ctx, pod)
   779  
   780  		ginkgo.By("Waiting for pod to run")
   781  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
   782  
   783  		ginkgo.By("Then ensuring pod finished running successfully")
   784  		p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
   785  			context.TODO(),
   786  			podName,
   787  			metav1.GetOptions{})
   788  
   789  		framework.ExpectNoError(err, "error retrieving pod")
   790  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodSucceeded))
   791  
   792  		// whoami will output %COMPUTER_NAME%/{randomly generated username} here.
   793  		// It is sufficient to just check that the logs do not container `nt authority`
   794  		// because all of the 'built-in' accounts that can be used with HostProcess
   795  		// are prefixed with this.
   796  		ginkgo.By("Then ensuring pod was not running as a system account")
   797  		logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "localgroup-container")
   798  		framework.ExpectNoError(err, "error retrieving container logs")
   799  		framework.Logf("Pod logs: %s", logs)
   800  		gomega.Expect(strings.ToLower(logs)).ShouldNot(gomega.ContainSubstring("nt authority"), "Container runs 'whoami' and logs should not contain 'nt authority'")
   801  	})
   802  
   803  }))
   804  
   805  func makeTestPodWithVolumeMounts(name string) *v1.Pod {
   806  	hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
   807  	return &v1.Pod{
   808  		ObjectMeta: metav1.ObjectMeta{
   809  			Name: name,
   810  		},
   811  		Spec: v1.PodSpec{
   812  			SecurityContext: &v1.PodSecurityContext{
   813  				WindowsOptions: &v1.WindowsSecurityContextOptions{
   814  					HostProcess:   &trueVar,
   815  					RunAsUserName: &User_NTAuthoritySystem,
   816  				},
   817  			},
   818  			HostNetwork: true,
   819  			Containers: []v1.Container{
   820  				{
   821  					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   822  					Name:    name,
   823  					Command: []string{"powershell.exe", "./etc/configmap/validationscript.ps1"},
   824  					Env: []v1.EnvVar{
   825  						{
   826  							Name: "NODE_NAME_TEST",
   827  							ValueFrom: &v1.EnvVarSource{
   828  								FieldRef: &v1.ObjectFieldSelector{
   829  									FieldPath: "spec.nodeName",
   830  								},
   831  							},
   832  						},
   833  					},
   834  					VolumeMounts: []v1.VolumeMount{
   835  						{
   836  							Name:      "emptydir-volume",
   837  							MountPath: "/etc/emptydir",
   838  						},
   839  						{
   840  							Name:      "configmap-volume",
   841  							MountPath: "/etc/configmap",
   842  						},
   843  						{
   844  							Name:      "hostpath-volume",
   845  							MountPath: "/etc/hostpath",
   846  						},
   847  						{
   848  							Name:      "downwardapi-volume",
   849  							MountPath: "/etc/downwardapi",
   850  						},
   851  						{
   852  							Name:      "secret-volume",
   853  							MountPath: "/etc/secret",
   854  						},
   855  					},
   856  				},
   857  			},
   858  			RestartPolicy: v1.RestartPolicyNever,
   859  			NodeSelector: map[string]string{
   860  				"kubernetes.io/os": "windows",
   861  			},
   862  			Volumes: []v1.Volume{
   863  				{
   864  					Name: "emptydir-volume",
   865  					VolumeSource: v1.VolumeSource{
   866  						EmptyDir: &v1.EmptyDirVolumeSource{
   867  							Medium: v1.StorageMediumDefault,
   868  						},
   869  					},
   870  				},
   871  				{
   872  					Name: "configmap-volume",
   873  					VolumeSource: v1.VolumeSource{
   874  						ConfigMap: &v1.ConfigMapVolumeSource{
   875  							LocalObjectReference: v1.LocalObjectReference{
   876  								Name: "sample-config-map",
   877  							},
   878  							Items: []v1.KeyToPath{
   879  								{
   880  									Key:  "text",
   881  									Path: "text.txt",
   882  								},
   883  								{
   884  									Key:  "validation-script",
   885  									Path: "validationscript.ps1",
   886  								},
   887  							},
   888  						},
   889  					},
   890  				},
   891  				{
   892  					Name: "hostpath-volume",
   893  					VolumeSource: v1.VolumeSource{
   894  						HostPath: &v1.HostPathVolumeSource{
   895  							Path: "/var/hostpath",
   896  							Type: &hostPathDirectoryOrCreate,
   897  						},
   898  					},
   899  				},
   900  				{
   901  					Name: "downwardapi-volume",
   902  					VolumeSource: v1.VolumeSource{
   903  						DownwardAPI: &v1.DownwardAPIVolumeSource{
   904  							Items: []v1.DownwardAPIVolumeFile{
   905  								{
   906  									Path: "podname",
   907  									FieldRef: &v1.ObjectFieldSelector{
   908  										FieldPath: "metadata.name",
   909  									},
   910  								},
   911  							},
   912  						},
   913  					},
   914  				},
   915  				{
   916  					Name: "secret-volume",
   917  					VolumeSource: v1.VolumeSource{
   918  						Secret: &v1.SecretVolumeSource{
   919  							SecretName: "sample-secret",
   920  							Items: []v1.KeyToPath{
   921  								{
   922  									Key:  "foo",
   923  									Path: "foo.txt",
   924  								},
   925  							},
   926  						},
   927  					},
   928  				},
   929  			},
   930  		},
   931  	}
   932  }
   933  
   934  type HostProcessContainersMetrics struct {
   935  	StartedContainersCount          int64
   936  	StartedContainersErrorCount     int64
   937  	StartedInitContainersCount      int64
   938  	StartedInitContainersErrorCount int64
   939  }
   940  
   941  // ensurePodsStatsOnNode returns a boolean after checking Pods statistics on a particular node
   942  func ensurePodsStatsOnNode(ctx context.Context, c clientset.Interface, namespace, nodeName string, statsChecked *bool) (bool, error) {
   943  	nodeStats, err := e2ekubelet.GetStatsSummary(ctx, c, nodeName)
   944  	framework.ExpectNoError(err, "Error getting node stats")
   945  
   946  	for _, podStats := range nodeStats.Pods {
   947  		if podStats.PodRef.Namespace != namespace {
   948  			continue
   949  		}
   950  
   951  		// check various pod stats
   952  		if *podStats.CPU.UsageCoreNanoSeconds <= 0 {
   953  			framework.Logf("Pod %s/%s stats report cpu usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
   954  			return false, nil
   955  		}
   956  		if *podStats.Memory.WorkingSetBytes <= 0 {
   957  			framework.Logf("Pod %s/%s stats report memory usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
   958  			return false, nil
   959  		}
   960  
   961  		for _, containerStats := range podStats.Containers {
   962  			*statsChecked = true
   963  
   964  			// check various container stats
   965  			if *containerStats.CPU.UsageCoreNanoSeconds <= 0 {
   966  				framework.Logf("Container %s stats report cpu usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.CPU.UsageCoreNanoSeconds)
   967  				return false, nil
   968  			}
   969  			if *containerStats.Memory.WorkingSetBytes <= 0 {
   970  				framework.Logf("Container %s stats report memory usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Memory.WorkingSetBytes)
   971  				return false, nil
   972  			}
   973  			if *containerStats.Logs.UsedBytes <= 0 {
   974  				framework.Logf("Container %s stats log usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Logs.UsedBytes)
   975  				return false, nil
   976  			}
   977  		}
   978  	}
   979  	return true, nil
   980  }
   981  
   982  // getCurrentHostProcessMetrics returns a HostPRocessContainersMetrics object. Any metrics that do not have any
   983  // values reported will be set to 0.
   984  func getCurrentHostProcessMetrics(ctx context.Context, f *framework.Framework, nodeName string) (HostProcessContainersMetrics, error) {
   985  	var result HostProcessContainersMetrics
   986  
   987  	metrics, err := e2emetrics.GetKubeletMetrics(ctx, f.ClientSet, nodeName)
   988  	if err != nil {
   989  		return result, err
   990  	}
   991  
   992  	for _, sample := range metrics["started_host_process_containers_total"] {
   993  		switch sample.Metric["container_type"] {
   994  		case "container":
   995  			result.StartedContainersCount = int64(sample.Value)
   996  		case "init_container":
   997  			result.StartedInitContainersCount = int64(sample.Value)
   998  		}
   999  	}
  1000  
  1001  	// note: accumulate failures of all types (ErrImagePull, RunContainerError, etc)
  1002  	// for each container type here.
  1003  	for _, sample := range metrics["started_host_process_containers_errors_total"] {
  1004  		switch sample.Metric["container_type"] {
  1005  		case "container":
  1006  			result.StartedContainersErrorCount += int64(sample.Value)
  1007  		case "init_container":
  1008  			result.StartedInitContainersErrorCount += int64(sample.Value)
  1009  		}
  1010  	}
  1011  
  1012  	return result, nil
  1013  }
  1014  

View as plain text