gmsa_full.go

Documentation: k8s.io/kubernetes/test/e2e/windows

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // This test ensures that the whole GMSA process works as intended.
    18  // However, it does require a pretty heavy weight set up to run correctly;
    19  // in particular, it does make a number of assumptions about the cluster it
    20  // runs against:
    21  //  * there exists a Windows worker node with the agentpool=windowsgmsa label on it
    22  //  * that node is joined to a working Active Directory domain.
    23  //  * a GMSA account has been created in that AD domain, and then installed on that
    24  //    same worker.
    25  //  * a valid k8s manifest file containing a single CRD definition has been generated using
    26  //    https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
    27  //    with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
    28  //    and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
    29  //    on that same worker node.
    30  //  * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
    31  //    admission controllers enabled.
    32  //  * the cluster comprises at least one Linux node that accepts workloads - it
    33  //    can be the master, but any other Linux node is fine too. This is needed for
    34  //    the webhook's pod.
    35  //  * in order to run "can read and write file to remote folder" test case, a folder (e.g. "write_test") need to be created
    36  //    in that AD domain and it should be shared with that GMSA account.
    37  // All these assumptions are fulfilled by an AKS extension when setting up the AKS
    38  // cluster we run daily e2e tests against, but they do make running this test
    39  // outside of that very specific context pretty hard.
    40  
    41  package windows
    42  
    43  import (
    44  	"context"
    45  	"fmt"
    46  	"os"
    47  	"regexp"
    48  	"strings"
    49  	"time"
    50  
    51  	"github.com/onsi/ginkgo/v2"
    52  	"github.com/onsi/gomega"
    53  	appsv1 "k8s.io/api/apps/v1"
    54  	v1 "k8s.io/api/core/v1"
    55  	rbacv1 "k8s.io/api/rbac/v1"
    56  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    57  	"k8s.io/apimachinery/pkg/util/uuid"
    58  	clientset "k8s.io/client-go/kubernetes"
    59  	"k8s.io/kubernetes/test/e2e/feature"
    60  	"k8s.io/kubernetes/test/e2e/framework"
    61  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    62  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    63  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    64  	imageutils "k8s.io/kubernetes/test/utils/image"
    65  	admissionapi "k8s.io/pod-security-admission/api"
    66  )
    67  
    68  const (
    69  	// gmsaFullNodeLabel is the label we expect to find on at least one node
    70  	// that is then expected to fulfill all the expectations explained above.
    71  	gmsaFullNodeLabel = "agentpool=windowsgmsa"
    72  
    73  	// gmsaCrdManifestPath is where we expect to find the manifest file for
    74  	// the GMSA cred spec on that node - see explanations above.
    75  	gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
    76  
    77  	// gmsaCustomResourceName is the expected name of the GMSA custom resource
    78  	// defined at gmsaCrdManifestPath
    79  	gmsaCustomResourceName = "gmsa-e2e"
    80  
    81  	// gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
    82  	gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
    83  
    84  	// output from the nltest /query command should have this in it
    85  	expectedQueryOutput = "The command completed successfully"
    86  
    87  	// The name of the expected domain
    88  	gmsaDomain = "k8sgmsa.lan"
    89  
    90  	// The shared folder on the expected domain for file-writing test
    91  	gmsaSharedFolder = "write_test"
    92  )
    93  
    94  var _ = sigDescribe(feature.Windows, "GMSA Full", framework.WithSerial(), framework.WithSlow(), skipUnlessWindows(func() {
    95  	f := framework.NewDefaultFramework("gmsa-full-test-windows")
    96  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    97  
    98  	ginkgo.Describe("GMSA support", func() {
    99  		ginkgo.It("works end to end", func(ctx context.Context) {
   100  			defer ginkgo.GinkgoRecover()
   101  
   102  			ginkgo.By("finding the worker node that fulfills this test's assumptions")
   103  			nodes := findPreconfiguredGmsaNodes(ctx, f.ClientSet)
   104  			if len(nodes) != 1 {
   105  				e2eskipper.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
   106  			}
   107  			node := nodes[0]
   108  
   109  			ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
   110  			crdManifestContents := retrieveCRDManifestFileContents(ctx, f, node)
   111  
   112  			ginkgo.By("deploying the GMSA webhook")
   113  			err := deployGmsaWebhook(ctx, f)
   114  			if err != nil {
   115  				framework.Failf(err.Error())
   116  			}
   117  
   118  			ginkgo.By("creating the GMSA custom resource")
   119  			err = createGmsaCustomResource(f.Namespace.Name, crdManifestContents)
   120  			if err != nil {
   121  				framework.Failf(err.Error())
   122  			}
   123  
   124  			ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
   125  			rbacRoleName, err := createRBACRoleForGmsa(ctx, f)
   126  			if err != nil {
   127  				framework.Failf(err.Error())
   128  			}
   129  
   130  			ginkgo.By("creating a service account")
   131  			serviceAccountName := createServiceAccount(ctx, f)
   132  
   133  			ginkgo.By("binding the RBAC role to the service account")
   134  			bindRBACRoleToServiceAccount(ctx, f, serviceAccountName, rbacRoleName)
   135  
   136  			ginkgo.By("creating a pod using the GMSA cred spec")
   137  			podName := createPodWithGmsa(ctx, f, serviceAccountName)
   138  
   139  			// nltest /QUERY will only return successfully if there is a GMSA
   140  			// identity configured, _and_ it succeeds in contacting the AD controller
   141  			// and authenticating with it.
   142  			ginkgo.By("checking that nltest /QUERY returns successfully")
   143  			var output string
   144  			gomega.Eventually(ctx, func() bool {
   145  				output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
   146  				if err != nil {
   147  					framework.Logf("unable to run command in container via exec: %s", err)
   148  					return false
   149  				}
   150  
   151  				if !isValidOutput(output) {
   152  					// try repairing the secure channel by running reset command
   153  					// https://kubernetes.io/docs/tasks/configure-pod-container/configure-gmsa/#troubleshooting
   154  					output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", fmt.Sprintf("/sc_reset:%s", gmsaDomain))
   155  					if err != nil {
   156  						framework.Logf("unable to run command in container via exec: %s", err)
   157  						return false
   158  					}
   159  					framework.Logf("failed to connect to domain; tried resetting the domain, output:\n%s", string(output))
   160  					return false
   161  				}
   162  				return true
   163  			}, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
   164  		})
   165  
   166  		ginkgo.It("can read and write file to remote SMB folder", func(ctx context.Context) {
   167  			defer ginkgo.GinkgoRecover()
   168  
   169  			ginkgo.By("finding the worker node that fulfills this test's assumptions")
   170  			nodes := findPreconfiguredGmsaNodes(ctx, f.ClientSet)
   171  			if len(nodes) != 1 {
   172  				e2eskipper.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
   173  			}
   174  			node := nodes[0]
   175  
   176  			ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
   177  			crdManifestContents := retrieveCRDManifestFileContents(ctx, f, node)
   178  
   179  			ginkgo.By("deploying the GMSA webhook")
   180  			err := deployGmsaWebhook(ctx, f)
   181  			if err != nil {
   182  				framework.Failf(err.Error())
   183  			}
   184  
   185  			ginkgo.By("creating the GMSA custom resource")
   186  			err = createGmsaCustomResource(f.Namespace.Name, crdManifestContents)
   187  			if err != nil {
   188  				framework.Failf(err.Error())
   189  			}
   190  
   191  			ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
   192  			rbacRoleName, err := createRBACRoleForGmsa(ctx, f)
   193  			if err != nil {
   194  				framework.Failf(err.Error())
   195  			}
   196  
   197  			ginkgo.By("creating a service account")
   198  			serviceAccountName := createServiceAccount(ctx, f)
   199  
   200  			ginkgo.By("binding the RBAC role to the service account")
   201  			bindRBACRoleToServiceAccount(ctx, f, serviceAccountName, rbacRoleName)
   202  
   203  			ginkgo.By("creating a pod using the GMSA cred spec")
   204  			podName := createPodWithGmsa(ctx, f, serviceAccountName)
   205  
   206  			ginkgo.By("getting the ip of GMSA domain")
   207  			gmsaDomainIP := getGmsaDomainIP(f, podName)
   208  
   209  			ginkgo.By("checking that file can be read and write from the remote folder successfully")
   210  			filePath := fmt.Sprintf("\\\\%s\\%s\\write-test-%s.txt", gmsaDomainIP, gmsaSharedFolder, string(uuid.NewUUID())[0:4])
   211  			gomega.Eventually(ctx, func() bool {
   212  				// The filePath is a remote folder, do not change the format of it
   213  				_, _ = runKubectlExecInNamespace(f.Namespace.Name, podName, "--", "powershell.exe", "-Command", "echo 'This is a test file.' > "+filePath)
   214  				output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell.exe", "--", "cat", filePath)
   215  				if err != nil {
   216  					framework.Logf("unable to get file from AD server: %s", err)
   217  					return false
   218  				}
   219  				return strings.Contains(output, "This is a test file.")
   220  			}, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
   221  
   222  		})
   223  	})
   224  }))
   225  
   226  func isValidOutput(output string) bool {
   227  	return strings.Contains(output, expectedQueryOutput) &&
   228  		!strings.Contains(output, "ERROR_NO_LOGON_SERVERS") &&
   229  		!strings.Contains(output, "RPC_S_SERVER_UNAVAILABLE")
   230  }
   231  
   232  // findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
   233  func findPreconfiguredGmsaNodes(ctx context.Context, c clientset.Interface) []v1.Node {
   234  	nodeOpts := metav1.ListOptions{
   235  		LabelSelector: gmsaFullNodeLabel,
   236  	}
   237  	nodes, err := c.CoreV1().Nodes().List(ctx, nodeOpts)
   238  	if err != nil {
   239  		framework.Failf("Unable to list nodes: %v", err)
   240  	}
   241  	return nodes.Items
   242  }
   243  
   244  // retrieveCRDManifestFileContents retrieves the contents of the file
   245  // at gmsaCrdManifestPath on node; it does so by scheduling a single pod
   246  // on nodes with the gmsaFullNodeLabel label with that file's directory
   247  // mounted on it, and then exec-ing into that pod to retrieve the file's
   248  // contents.
   249  func retrieveCRDManifestFileContents(ctx context.Context, f *framework.Framework, node v1.Node) string {
   250  	podName := "retrieve-gmsa-crd-contents"
   251  	// we can't use filepath.Dir here since the test itself runs on a Linux machine
   252  	splitPath := strings.Split(gmsaCrdManifestPath, `\`)
   253  	dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
   254  	volumeName := "retrieve-gmsa-crd-contents-volume"
   255  
   256  	pod := &v1.Pod{
   257  		ObjectMeta: metav1.ObjectMeta{
   258  			Name:      podName,
   259  			Namespace: f.Namespace.Name,
   260  		},
   261  		Spec: v1.PodSpec{
   262  			NodeSelector: node.Labels,
   263  			Containers: []v1.Container{
   264  				{
   265  					Name:  podName,
   266  					Image: imageutils.GetPauseImageName(),
   267  					VolumeMounts: []v1.VolumeMount{
   268  						{
   269  							Name:      volumeName,
   270  							MountPath: dirPath,
   271  						},
   272  					},
   273  				},
   274  			},
   275  			Volumes: []v1.Volume{
   276  				{
   277  					Name: volumeName,
   278  					VolumeSource: v1.VolumeSource{
   279  						HostPath: &v1.HostPathVolumeSource{
   280  							Path: dirPath,
   281  						},
   282  					},
   283  				},
   284  			},
   285  		},
   286  	}
   287  	e2epod.NewPodClient(f).CreateSync(ctx, pod)
   288  
   289  	output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "cmd", "/S", "/C", fmt.Sprintf("type %s", gmsaCrdManifestPath))
   290  	if err != nil {
   291  		framework.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
   292  	}
   293  
   294  	// Windows to linux new lines
   295  	return strings.ReplaceAll(output, "\r\n", "\n")
   296  }
   297  
   298  // deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
   299  // to be called when done with testing, that removes the temp files it's created
   300  // on disks as well as the API resources it's created.
   301  func deployGmsaWebhook(ctx context.Context, f *framework.Framework) error {
   302  	deployerName := "webhook-deployer"
   303  	deployerNamespace := f.Namespace.Name
   304  	webHookName := "gmsa-webhook"
   305  	webHookNamespace := deployerNamespace + "-webhook"
   306  
   307  	// regardless of whether the deployment succeeded, let's do a best effort at cleanup
   308  	ginkgo.DeferCleanup(func() {
   309  		framework.Logf("Best effort clean up of the webhook:\n")
   310  		stdout, err := e2ekubectl.RunKubectl("", "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
   311  		framework.Logf("stdout:%s\nerror:%s", stdout, err)
   312  
   313  		stdout, err = e2ekubectl.RunKubectl("", "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", webHookName, webHookNamespace))
   314  		framework.Logf("stdout:%s\nerror:%s", stdout, err)
   315  
   316  		stdout, err = runKubectlExecInNamespace(deployerNamespace, deployerName, "--", "kubectl", "delete", "-f", "/manifests.yml")
   317  		framework.Logf("stdout:%s\nerror:%s", stdout, err)
   318  	})
   319  
   320  	// ensure the deployer has ability to approve certificatesigningrequests to install the webhook
   321  	s := createServiceAccount(ctx, f)
   322  	bindClusterRBACRoleToServiceAccount(ctx, f, s, "cluster-admin")
   323  
   324  	installSteps := []string{
   325  		"echo \"@community http://dl-cdn.alpinelinux.org/alpine/edge/community/\" >> /etc/apk/repositories",
   326  		"&& apk add kubectl@community gettext openssl",
   327  		"&& apk add --update coreutils",
   328  		fmt.Sprintf("&& curl %s > gmsa.sh", gmsaWebhookDeployScriptURL),
   329  		"&& chmod +x gmsa.sh",
   330  		fmt.Sprintf("&& ./gmsa.sh --file %s --name %s --namespace %s --certs-dir %s --tolerate-master", "/manifests.yml", webHookName, webHookNamespace, "certs"),
   331  		"&& /agnhost pause",
   332  	}
   333  	installCommand := strings.Join(installSteps, " ")
   334  
   335  	pod := &v1.Pod{
   336  		ObjectMeta: metav1.ObjectMeta{
   337  			Name:      deployerName,
   338  			Namespace: deployerNamespace,
   339  		},
   340  		Spec: v1.PodSpec{
   341  			ServiceAccountName: s,
   342  			NodeSelector: map[string]string{
   343  				"kubernetes.io/os": "linux",
   344  			},
   345  			Containers: []v1.Container{
   346  				{
   347  					Name:    deployerName,
   348  					Image:   imageutils.GetE2EImage(imageutils.Agnhost),
   349  					Command: []string{"bash", "-c"},
   350  					Args:    []string{installCommand},
   351  				},
   352  			},
   353  			Tolerations: []v1.Toleration{
   354  				{
   355  					Operator: v1.TolerationOpExists,
   356  					Effect:   v1.TaintEffectNoSchedule,
   357  				},
   358  			},
   359  		},
   360  	}
   361  	e2epod.NewPodClient(f).CreateSync(ctx, pod)
   362  
   363  	// Wait for the Webhook deployment to become ready. The deployer pod takes a few seconds to initialize and create resources
   364  	err := waitForDeployment(func() (*appsv1.Deployment, error) {
   365  		return f.ClientSet.AppsV1().Deployments(webHookNamespace).Get(ctx, webHookName, metav1.GetOptions{})
   366  	}, 10*time.Second, f.Timeouts.PodStart)
   367  	if err == nil {
   368  		framework.Logf("GMSA webhook successfully deployed")
   369  	} else {
   370  		err = fmt.Errorf("GMSA webhook did not become ready: %w", err)
   371  	}
   372  
   373  	// Dump deployer logs
   374  	logs, _ := e2epod.GetPodLogs(ctx, f.ClientSet, deployerNamespace, deployerName, deployerName)
   375  	framework.Logf("GMSA deployment logs:\n%s", logs)
   376  
   377  	return err
   378  }
   379  
   380  // createGmsaCustomResource creates the GMSA API object from the contents
   381  // of the manifest file retrieved from the worker node.
   382  // It returns a function to clean up both the temp file it creates and
   383  // the API object it creates when done with testing.
   384  func createGmsaCustomResource(ns string, crdManifestContents string) error {
   385  	tempFile, err := os.CreateTemp("", "")
   386  	if err != nil {
   387  		return fmt.Errorf("unable to create temp file: %w", err)
   388  	}
   389  	defer tempFile.Close()
   390  
   391  	ginkgo.DeferCleanup(func() {
   392  		e2ekubectl.RunKubectl(ns, "delete", "--filename", tempFile.Name())
   393  		os.Remove(tempFile.Name())
   394  	})
   395  
   396  	_, err = tempFile.WriteString(crdManifestContents)
   397  	if err != nil {
   398  		err = fmt.Errorf("unable to write GMSA contents to %q: %w", tempFile.Name(), err)
   399  		return err
   400  	}
   401  
   402  	output, err := e2ekubectl.RunKubectl(ns, "apply", "--filename", tempFile.Name())
   403  	if err != nil {
   404  		err = fmt.Errorf("unable to create custom resource, output:\n%s: %w", output, err)
   405  	}
   406  
   407  	return err
   408  }
   409  
   410  // createRBACRoleForGmsa creates an RBAC cluster role to grant use
   411  // access to our test credential spec.
   412  // It returns the role's name, as well as a function to delete it when done.
   413  func createRBACRoleForGmsa(ctx context.Context, f *framework.Framework) (string, error) {
   414  	roleName := f.Namespace.Name + "-rbac-role"
   415  
   416  	role := &rbacv1.ClusterRole{
   417  		ObjectMeta: metav1.ObjectMeta{
   418  			Name: roleName,
   419  		},
   420  		Rules: []rbacv1.PolicyRule{
   421  			{
   422  				APIGroups:     []string{"windows.k8s.io"},
   423  				Resources:     []string{"gmsacredentialspecs"},
   424  				Verbs:         []string{"use"},
   425  				ResourceNames: []string{gmsaCustomResourceName},
   426  			},
   427  		},
   428  	}
   429  
   430  	ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.RbacV1().ClusterRoles().Delete), roleName, metav1.DeleteOptions{})
   431  	_, err := f.ClientSet.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{})
   432  	if err != nil {
   433  		err = fmt.Errorf("unable to create RBAC cluster role %q: %w", roleName, err)
   434  	}
   435  
   436  	return roleName, err
   437  }
   438  
   439  // createServiceAccount creates a service account, and returns its name.
   440  func createServiceAccount(ctx context.Context, f *framework.Framework) string {
   441  	accountName := f.Namespace.Name + "-sa-" + string(uuid.NewUUID())
   442  	account := &v1.ServiceAccount{
   443  		ObjectMeta: metav1.ObjectMeta{
   444  			Name:      accountName,
   445  			Namespace: f.Namespace.Name,
   446  		},
   447  	}
   448  	if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, account, metav1.CreateOptions{}); err != nil {
   449  		framework.Failf("unable to create service account %q: %v", accountName, err)
   450  	}
   451  	return accountName
   452  }
   453  
   454  // bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
   455  func bindRBACRoleToServiceAccount(ctx context.Context, f *framework.Framework, serviceAccountName, rbacRoleName string) {
   456  	binding := &rbacv1.RoleBinding{
   457  		ObjectMeta: metav1.ObjectMeta{
   458  			Name:      f.Namespace.Name + "-rbac-binding",
   459  			Namespace: f.Namespace.Name,
   460  		},
   461  		Subjects: []rbacv1.Subject{
   462  			{
   463  				Kind:      "ServiceAccount",
   464  				Name:      serviceAccountName,
   465  				Namespace: f.Namespace.Name,
   466  			},
   467  		},
   468  		RoleRef: rbacv1.RoleRef{
   469  			APIGroup: "rbac.authorization.k8s.io",
   470  			Kind:     "ClusterRole",
   471  			Name:     rbacRoleName,
   472  		},
   473  	}
   474  	_, err := f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(ctx, binding, metav1.CreateOptions{})
   475  	framework.ExpectNoError(err)
   476  }
   477  
   478  func bindClusterRBACRoleToServiceAccount(ctx context.Context, f *framework.Framework, serviceAccountName, rbacRoleName string) {
   479  	binding := &rbacv1.ClusterRoleBinding{
   480  		ObjectMeta: metav1.ObjectMeta{
   481  			Name:      f.Namespace.Name + "-rbac-binding",
   482  			Namespace: f.Namespace.Name,
   483  		},
   484  		Subjects: []rbacv1.Subject{
   485  			{
   486  				Kind:      "ServiceAccount",
   487  				Name:      serviceAccountName,
   488  				Namespace: f.Namespace.Name,
   489  			},
   490  		},
   491  		RoleRef: rbacv1.RoleRef{
   492  			APIGroup: "rbac.authorization.k8s.io",
   493  			Kind:     "ClusterRole",
   494  			Name:     rbacRoleName,
   495  		},
   496  	}
   497  	_, err := f.ClientSet.RbacV1().ClusterRoleBindings().Create(ctx, binding, metav1.CreateOptions{})
   498  	framework.ExpectNoError(err)
   499  }
   500  
   501  // createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
   502  func createPodWithGmsa(ctx context.Context, f *framework.Framework, serviceAccountName string) string {
   503  	podName := "pod-with-gmsa"
   504  	credSpecName := gmsaCustomResourceName
   505  
   506  	pod := &v1.Pod{
   507  		ObjectMeta: metav1.ObjectMeta{
   508  			Name:      podName,
   509  			Namespace: f.Namespace.Name,
   510  		},
   511  		Spec: v1.PodSpec{
   512  			ServiceAccountName: serviceAccountName,
   513  			Containers: []v1.Container{
   514  				{
   515  					Name:  podName,
   516  					Image: imageutils.GetE2EImage(imageutils.BusyBox),
   517  					Command: []string{
   518  						"powershell.exe",
   519  						"-Command",
   520  						"sleep -Seconds 600",
   521  					},
   522  				},
   523  			},
   524  			SecurityContext: &v1.PodSecurityContext{
   525  				WindowsOptions: &v1.WindowsSecurityContextOptions{
   526  					GMSACredentialSpecName: &credSpecName,
   527  				},
   528  			},
   529  		},
   530  	}
   531  	e2epod.NewPodClient(f).CreateSync(ctx, pod)
   532  
   533  	return podName
   534  }
   535  
   536  func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
   537  	namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
   538  	return e2ekubectl.RunKubectl(namespace, append([]string{"exec", namespaceOption}, args...)...)
   539  }
   540  
   541  func getGmsaDomainIP(f *framework.Framework, podName string) string {
   542  	output, _ := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell.exe", "--", "nslookup", gmsaDomain)
   543  	re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
   544  	idx := strings.Index(output, gmsaDomain)
   545  
   546  	submatchall := re.FindAllString(output[idx:], -1)
   547  	if len(submatchall) < 1 {
   548  		framework.Logf("fail to get the ip of the gmsa domain")
   549  		return ""
   550  	}
   551  	return submatchall[0]
   552  }
   553
View as plain text