Source file src/k8s.io/kubernetes/test/e2e/auth/service_accounts.go

Documentation: k8s.io/kubernetes/test/e2e/auth

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package auth
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"path"
    24  	"regexp"
    25  	"strings"
    26  	"time"
    27  
    28  	authenticationv1 "k8s.io/api/authentication/v1"
    29  	v1 "k8s.io/api/core/v1"
    30  	rbacv1 "k8s.io/api/rbac/v1"
    31  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/types"
    34  	utilrand "k8s.io/apimachinery/pkg/util/rand"
    35  	"k8s.io/apimachinery/pkg/util/sets"
    36  	"k8s.io/apimachinery/pkg/util/uuid"
    37  	"k8s.io/apimachinery/pkg/util/wait"
    38  	watch "k8s.io/apimachinery/pkg/watch"
    39  	"k8s.io/client-go/util/retry"
    40  	"k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
    41  	"k8s.io/kubernetes/test/e2e/framework"
    42  	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
    43  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    44  	e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
    45  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    46  	"k8s.io/kubernetes/test/e2e/nodefeature"
    47  	imageutils "k8s.io/kubernetes/test/utils/image"
    48  	admissionapi "k8s.io/pod-security-admission/api"
    49  	utilptr "k8s.io/utils/pointer"
    50  
    51  	"github.com/onsi/ginkgo/v2"
    52  	"github.com/onsi/gomega"
    53  )
    54  
    55  const rootCAConfigMapName = "kube-root-ca.crt"
    56  
    57  var _ = SIGDescribe("ServiceAccounts", func() {
    58  	f := framework.NewDefaultFramework("svcaccounts")
    59  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    60  
    61  	ginkgo.It("no secret-based service account token should be auto-generated", func(ctx context.Context) {
    62  		{
    63  			ginkgo.By("ensuring no secret-based service account token exists")
    64  			time.Sleep(10 * time.Second)
    65  			sa, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Get(ctx, "default", metav1.GetOptions{})
    66  			framework.ExpectNoError(err)
    67  			gomega.Expect(sa.Secrets).To(gomega.BeEmpty())
    68  		}
    69  	})
    70  
    71  	/*
    72  	   Release: v1.9
    73  	   Testname: Service Account Tokens Must AutoMount
    74  	   Description: Ensure that Service Account keys are mounted into the Container. Pod
    75  	                contains three containers each will read Service Account token,
    76  	                root CA and default namespace respectively from the default API
    77  	                Token Mount path. All these three files MUST exist and the Service
    78  	                Account mount path MUST be auto mounted to the Container.
    79  	*/
    80  	framework.ConformanceIt("should mount an API token into pods", func(ctx context.Context) {
    81  		sa, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "mount-test"}}, metav1.CreateOptions{})
    82  		framework.ExpectNoError(err)
    83  
    84  		zero := int64(0)
    85  		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, &v1.Pod{
    86  			ObjectMeta: metav1.ObjectMeta{
    87  				Name: "pod-service-account-" + string(uuid.NewUUID()),
    88  			},
    89  			Spec: v1.PodSpec{
    90  				ServiceAccountName: sa.Name,
    91  				Containers: []v1.Container{{
    92  					Name:    "test",
    93  					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
    94  					Command: []string{"sleep", "100000"},
    95  				}},
    96  				TerminationGracePeriodSeconds: &zero,
    97  				RestartPolicy:                 v1.RestartPolicyNever,
    98  			},
    99  		}, metav1.CreateOptions{})
   100  		framework.ExpectNoError(err)
   101  		framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
   102  
   103  		tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, f.Namespace.Name)
   104  		mountedToken, err := tk.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, path.Join(serviceaccount.DefaultAPITokenMountPath, v1.ServiceAccountTokenKey))
   105  		framework.ExpectNoError(err)
   106  		mountedCA, err := tk.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, path.Join(serviceaccount.DefaultAPITokenMountPath, v1.ServiceAccountRootCAKey))
   107  		framework.ExpectNoError(err)
   108  		mountedNamespace, err := tk.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, path.Join(serviceaccount.DefaultAPITokenMountPath, v1.ServiceAccountNamespaceKey))
   109  		framework.ExpectNoError(err)
   110  
   111  		// CA and namespace should be identical
   112  		rootCA, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(ctx, rootCAConfigMapName, metav1.GetOptions{})
   113  		framework.ExpectNoError(err)
   114  		framework.Logf("Got root ca configmap in namespace %q", f.Namespace.Name)
   115  		gomega.Expect(mountedCA).To(gomega.Equal(rootCA.Data["ca.crt"]))
   116  		gomega.Expect(mountedNamespace).To(gomega.Equal(f.Namespace.Name))
   117  		// Token should be a valid credential that identifies the pod's service account
   118  		tokenReview := &authenticationv1.TokenReview{Spec: authenticationv1.TokenReviewSpec{Token: mountedToken}}
   119  		tokenReview, err = f.ClientSet.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{})
   120  		framework.ExpectNoError(err)
   121  		if !tokenReview.Status.Authenticated {
   122  			framework.Fail("tokenReview is not authenticated")
   123  		}
   124  		gomega.Expect(tokenReview.Status.Error).To(gomega.BeEmpty())
   125  		gomega.Expect(tokenReview.Status.User.Username).To(gomega.Equal("system:serviceaccount:" + f.Namespace.Name + ":" + sa.Name))
   126  		groups := sets.NewString(tokenReview.Status.User.Groups...)
   127  		if !groups.Has("system:authenticated") {
   128  			framework.Failf("expected system:authenticated group, had %v", groups.List())
   129  		}
   130  		if !groups.Has("system:serviceaccounts") {
   131  			framework.Failf("expected system:serviceaccounts group, had %v", groups.List())
   132  		}
   133  		if !groups.Has("system:serviceaccounts:" + f.Namespace.Name) {
   134  			framework.Failf("expected system:serviceaccounts:%s group, had %v", f.Namespace.Name, groups.List())
   135  		}
   136  	})
   137  
   138  	/*
   139  	   Release: v1.9
   140  	   Testname: Service account tokens auto mount optionally
   141  	   Description: Ensure that Service Account keys are mounted into the Pod only
   142  	                when AutoMountServiceToken is not set to false. We test the
   143  	                following scenarios here.
   144  	   1. Create Pod, Pod Spec has AutomountServiceAccountToken set to nil
   145  	      a) Service Account with default value,
   146  	      b) Service Account is an configured AutomountServiceAccountToken set to true,
   147  	      c) Service Account is an configured AutomountServiceAccountToken set to false
   148  	   2. Create Pod, Pod Spec has AutomountServiceAccountToken set to true
   149  	      a) Service Account with default value,
   150  	      b) Service Account is configured with AutomountServiceAccountToken set to true,
   151  	      c) Service Account is configured with AutomountServiceAccountToken set to false
   152  	   3. Create Pod, Pod Spec has AutomountServiceAccountToken set to false
   153  	      a) Service Account with default value,
   154  	      b) Service Account is configured with AutomountServiceAccountToken set to true,
   155  	      c) Service Account is configured with AutomountServiceAccountToken set to false
   156  
   157  	   The Containers running in these pods MUST verify that the ServiceTokenVolume path is
   158  	   auto mounted only when Pod Spec has AutomountServiceAccountToken not set to false
   159  	   and ServiceAccount object has AutomountServiceAccountToken not set to false, this
   160  	   include test cases 1a,1b,2a,2b and 2c.
   161  	   In the test cases 1c,3a,3b and 3c the ServiceTokenVolume MUST not be auto mounted.
   162  	*/
   163  	framework.ConformanceIt("should allow opting out of API token automount", func(ctx context.Context) {
   164  
   165  		var err error
   166  		trueValue := true
   167  		falseValue := false
   168  		mountSA := &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "mount"}, AutomountServiceAccountToken: &trueValue}
   169  		nomountSA := &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "nomount"}, AutomountServiceAccountToken: &falseValue}
   170  		mountSA, err = f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, mountSA, metav1.CreateOptions{})
   171  		framework.ExpectNoError(err)
   172  		nomountSA, err = f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, nomountSA, metav1.CreateOptions{})
   173  		framework.ExpectNoError(err)
   174  
   175  		testcases := []struct {
   176  			PodName            string
   177  			ServiceAccountName string
   178  			AutomountPodSpec   *bool
   179  			ExpectTokenVolume  bool
   180  		}{
   181  			{
   182  				PodName:            "pod-service-account-defaultsa",
   183  				ServiceAccountName: "default",
   184  				AutomountPodSpec:   nil,
   185  				ExpectTokenVolume:  true, // default is true
   186  			},
   187  			{
   188  				PodName:            "pod-service-account-mountsa",
   189  				ServiceAccountName: mountSA.Name,
   190  				AutomountPodSpec:   nil,
   191  				ExpectTokenVolume:  true,
   192  			},
   193  			{
   194  				PodName:            "pod-service-account-nomountsa",
   195  				ServiceAccountName: nomountSA.Name,
   196  				AutomountPodSpec:   nil,
   197  				ExpectTokenVolume:  false,
   198  			},
   199  
   200  			// Make sure pod spec trumps when opting in
   201  			{
   202  				PodName:            "pod-service-account-defaultsa-mountspec",
   203  				ServiceAccountName: "default",
   204  				AutomountPodSpec:   &trueValue,
   205  				ExpectTokenVolume:  true,
   206  			},
   207  			{
   208  				PodName:            "pod-service-account-mountsa-mountspec",
   209  				ServiceAccountName: mountSA.Name,
   210  				AutomountPodSpec:   &trueValue,
   211  				ExpectTokenVolume:  true,
   212  			},
   213  			{
   214  				PodName:            "pod-service-account-nomountsa-mountspec",
   215  				ServiceAccountName: nomountSA.Name,
   216  				AutomountPodSpec:   &trueValue,
   217  				ExpectTokenVolume:  true, // pod spec trumps
   218  			},
   219  
   220  			// Make sure pod spec trumps when opting out
   221  			{
   222  				PodName:            "pod-service-account-defaultsa-nomountspec",
   223  				ServiceAccountName: "default",
   224  				AutomountPodSpec:   &falseValue,
   225  				ExpectTokenVolume:  false, // pod spec trumps
   226  			},
   227  			{
   228  				PodName:            "pod-service-account-mountsa-nomountspec",
   229  				ServiceAccountName: mountSA.Name,
   230  				AutomountPodSpec:   &falseValue,
   231  				ExpectTokenVolume:  false, // pod spec trumps
   232  			},
   233  			{
   234  				PodName:            "pod-service-account-nomountsa-nomountspec",
   235  				ServiceAccountName: nomountSA.Name,
   236  				AutomountPodSpec:   &falseValue,
   237  				ExpectTokenVolume:  false, // pod spec trumps
   238  			},
   239  		}
   240  
   241  		for _, tc := range testcases {
   242  			pod := &v1.Pod{
   243  				ObjectMeta: metav1.ObjectMeta{Name: tc.PodName},
   244  				Spec: v1.PodSpec{
   245  					Containers:                   []v1.Container{{Name: "token-test", Image: imageutils.GetE2EImage(imageutils.Agnhost)}},
   246  					RestartPolicy:                v1.RestartPolicyNever,
   247  					ServiceAccountName:           tc.ServiceAccountName,
   248  					AutomountServiceAccountToken: tc.AutomountPodSpec,
   249  				},
   250  			}
   251  			createdPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
   252  			framework.ExpectNoError(err)
   253  			framework.Logf("created pod %s", tc.PodName)
   254  
   255  			hasServiceAccountTokenVolume := false
   256  			for _, c := range createdPod.Spec.Containers {
   257  				for _, vm := range c.VolumeMounts {
   258  					if vm.MountPath == serviceaccount.DefaultAPITokenMountPath {
   259  						hasServiceAccountTokenVolume = true
   260  					}
   261  				}
   262  			}
   263  
   264  			if hasServiceAccountTokenVolume != tc.ExpectTokenVolume {
   265  				framework.Failf("%s: expected volume=%v, got %v (%#v)", tc.PodName, tc.ExpectTokenVolume, hasServiceAccountTokenVolume, createdPod)
   266  			} else {
   267  				framework.Logf("pod %s service account token volume mount: %v", tc.PodName, hasServiceAccountTokenVolume)
   268  			}
   269  		}
   270  	})
   271  
   272  	/*
   273  	  Release : v1.20
   274  	  Testname: TokenRequestProjection should mount a projected volume with token using TokenRequest API.
   275  	  Description: Ensure that projected service account token is mounted.
   276  	*/
   277  	framework.ConformanceIt("should mount projected service account token", func(ctx context.Context) {
   278  
   279  		var (
   280  			podName         = "test-pod-" + string(uuid.NewUUID())
   281  			volumeName      = "test-volume"
   282  			volumeMountPath = "/test-volume"
   283  			tokenVolumePath = "/test-volume/token"
   284  		)
   285  
   286  		volumes := []v1.Volume{
   287  			{
   288  				Name: volumeName,
   289  				VolumeSource: v1.VolumeSource{
   290  					Projected: &v1.ProjectedVolumeSource{
   291  						Sources: []v1.VolumeProjection{
   292  							{
   293  								ServiceAccountToken: &v1.ServiceAccountTokenProjection{
   294  									Path:              "token",
   295  									ExpirationSeconds: utilptr.Int64Ptr(60 * 60),
   296  								},
   297  							},
   298  						},
   299  					},
   300  				},
   301  			},
   302  		}
   303  		volumeMounts := []v1.VolumeMount{
   304  			{
   305  				Name:      volumeName,
   306  				MountPath: volumeMountPath,
   307  				ReadOnly:  true,
   308  			},
   309  		}
   310  		mounttestArgs := []string{
   311  			"mounttest",
   312  			fmt.Sprintf("--file_content=%v", tokenVolumePath),
   313  		}
   314  
   315  		pod := e2epod.NewAgnhostPod(f.Namespace.Name, podName, volumes, volumeMounts, nil, mounttestArgs...)
   316  		pod.Spec.RestartPolicy = v1.RestartPolicyNever
   317  
   318  		output := []string{
   319  			fmt.Sprintf("content of file \"%v\": %s", tokenVolumePath, `[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*`),
   320  		}
   321  
   322  		e2eoutput.TestContainerOutputRegexp(ctx, f, "service account token: ", pod, 0, output)
   323  	})
   324  
   325  	/*
   326  	   Testname: Projected service account token file ownership and permission.
   327  	   Description: Ensure that Projected Service Account Token is mounted with
   328  	               correct file ownership and permission mounted. We test the
   329  	               following scenarios here.
   330  	   1. RunAsUser is set,
   331  	   2. FsGroup is set,
   332  	   3. RunAsUser and FsGroup are set,
   333  	   4. Default, neither RunAsUser nor FsGroup is set,
   334  
   335  	   Containers MUST verify that the projected service account token can be
   336  	   read and has correct file mode set including ownership and permission.
   337  	*/
   338  	f.It("should set ownership and permission when RunAsUser or FsGroup is present [LinuxOnly]", nodefeature.FSGroup, func(ctx context.Context) {
   339  		e2eskipper.SkipIfNodeOSDistroIs("windows")
   340  
   341  		var (
   342  			podName         = "test-pod-" + string(uuid.NewUUID())
   343  			volumeName      = "test-volume"
   344  			volumeMountPath = "/test-volume"
   345  			tokenVolumePath = "/test-volume/token"
   346  		)
   347  
   348  		volumes := []v1.Volume{
   349  			{
   350  				Name: volumeName,
   351  				VolumeSource: v1.VolumeSource{
   352  					Projected: &v1.ProjectedVolumeSource{
   353  						Sources: []v1.VolumeProjection{
   354  							{
   355  								ServiceAccountToken: &v1.ServiceAccountTokenProjection{
   356  									Path:              "token",
   357  									ExpirationSeconds: utilptr.Int64Ptr(60 * 60),
   358  								},
   359  							},
   360  						},
   361  					},
   362  				},
   363  			},
   364  		}
   365  		volumeMounts := []v1.VolumeMount{
   366  			{
   367  				Name:      volumeName,
   368  				MountPath: volumeMountPath,
   369  				ReadOnly:  true,
   370  			},
   371  		}
   372  		mounttestArgs := []string{
   373  			"mounttest",
   374  			fmt.Sprintf("--file_perm=%v", tokenVolumePath),
   375  			fmt.Sprintf("--file_owner=%v", tokenVolumePath),
   376  			fmt.Sprintf("--file_content=%v", tokenVolumePath),
   377  		}
   378  
   379  		pod := e2epod.NewAgnhostPod(f.Namespace.Name, podName, volumes, volumeMounts, nil, mounttestArgs...)
   380  		pod.Spec.RestartPolicy = v1.RestartPolicyNever
   381  
   382  		testcases := []struct {
   383  			runAsUser bool
   384  			fsGroup   bool
   385  			wantPerm  string
   386  			wantUID   int64
   387  			wantGID   int64
   388  		}{
   389  			{
   390  				runAsUser: true,
   391  				wantPerm:  "-rw-------",
   392  				wantUID:   1000,
   393  				wantGID:   0,
   394  			},
   395  			{
   396  				fsGroup:  true,
   397  				wantPerm: "-rw-r-----",
   398  				wantUID:  0,
   399  				wantGID:  10000,
   400  			},
   401  			{
   402  				runAsUser: true,
   403  				fsGroup:   true,
   404  				wantPerm:  "-rw-r-----",
   405  				wantUID:   1000,
   406  				wantGID:   10000,
   407  			},
   408  			{
   409  				wantPerm: "-rw-r--r--",
   410  				wantUID:  0,
   411  				wantGID:  0,
   412  			},
   413  		}
   414  
   415  		for _, tc := range testcases {
   416  			pod.Spec.SecurityContext = &v1.PodSecurityContext{}
   417  			if tc.runAsUser {
   418  				pod.Spec.SecurityContext.RunAsUser = &tc.wantUID
   419  			}
   420  			if tc.fsGroup {
   421  				pod.Spec.SecurityContext.FSGroup = &tc.wantGID
   422  			}
   423  
   424  			output := []string{
   425  				fmt.Sprintf("perms of file \"%v\": %s", tokenVolumePath, tc.wantPerm),
   426  				fmt.Sprintf("content of file \"%v\": %s", tokenVolumePath, `[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*`),
   427  				fmt.Sprintf("owner UID of \"%v\": %d", tokenVolumePath, tc.wantUID),
   428  				fmt.Sprintf("owner GID of \"%v\": %d", tokenVolumePath, tc.wantGID),
   429  			}
   430  			e2eoutput.TestContainerOutputRegexp(ctx, f, "service account token: ", pod, 0, output)
   431  		}
   432  	})
   433  
   434  	f.It("should support InClusterConfig with token rotation", f.WithSlow(), func(ctx context.Context) {
   435  		tenMin := int64(10 * 60)
   436  		pod := &v1.Pod{
   437  			ObjectMeta: metav1.ObjectMeta{Name: "inclusterclient"},
   438  			Spec: v1.PodSpec{
   439  				Containers: []v1.Container{{
   440  					Name:  "inclusterclient",
   441  					Image: imageutils.GetE2EImage(imageutils.Agnhost),
   442  					Args:  []string{"inclusterclient"},
   443  					VolumeMounts: []v1.VolumeMount{{
   444  						MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
   445  						Name:      "kube-api-access-e2e",
   446  						ReadOnly:  true,
   447  					}},
   448  				}},
   449  				RestartPolicy:      v1.RestartPolicyNever,
   450  				ServiceAccountName: "default",
   451  				Volumes: []v1.Volume{{
   452  					Name: "kube-api-access-e2e",
   453  					VolumeSource: v1.VolumeSource{
   454  						Projected: &v1.ProjectedVolumeSource{
   455  							Sources: []v1.VolumeProjection{
   456  								{
   457  									ServiceAccountToken: &v1.ServiceAccountTokenProjection{
   458  										Path:              "token",
   459  										ExpirationSeconds: &tenMin,
   460  									},
   461  								},
   462  								{
   463  									ConfigMap: &v1.ConfigMapProjection{
   464  										LocalObjectReference: v1.LocalObjectReference{
   465  											Name: "kube-root-ca.crt",
   466  										},
   467  										Items: []v1.KeyToPath{
   468  											{
   469  												Key:  "ca.crt",
   470  												Path: "ca.crt",
   471  											},
   472  										},
   473  									},
   474  								},
   475  								{
   476  									DownwardAPI: &v1.DownwardAPIProjection{
   477  										Items: []v1.DownwardAPIVolumeFile{
   478  											{
   479  												Path: "namespace",
   480  												FieldRef: &v1.ObjectFieldSelector{
   481  													APIVersion: "v1",
   482  													FieldPath:  "metadata.namespace",
   483  												},
   484  											},
   485  										},
   486  									},
   487  								},
   488  							},
   489  						},
   490  					},
   491  				}},
   492  			},
   493  		}
   494  		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
   495  		framework.ExpectNoError(err)
   496  
   497  		framework.Logf("created pod")
   498  		framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name, time.Minute))
   499  
   500  		framework.Logf("pod is ready")
   501  
   502  		var logs string
   503  		if err := wait.Poll(1*time.Minute, 20*time.Minute, func() (done bool, err error) {
   504  			framework.Logf("polling logs")
   505  			logs, err = e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, "inclusterclient", "inclusterclient")
   506  			if err != nil {
   507  				framework.Logf("Error pulling logs: %v", err)
   508  				return false, nil
   509  			}
   510  			tokenCount, err := ParseInClusterClientLogs(logs)
   511  			if err != nil {
   512  				return false, fmt.Errorf("inclusterclient reported an error: %w", err)
   513  			}
   514  			if tokenCount < 2 {
   515  				framework.Logf("Retrying. Still waiting to see more unique tokens: got=%d, want=2", tokenCount)
   516  				return false, nil
   517  			}
   518  			return true, nil
   519  		}); err != nil {
   520  			framework.Failf("Unexpected error: %v\n%s", err, logs)
   521  		}
   522  	})
   523  
   524  	/*
   525  	   Release: v1.21
   526  	   Testname: OIDC Discovery (ServiceAccountIssuerDiscovery)
   527  	   Description: Ensure kube-apiserver serves correct OIDC discovery
   528  	   endpoints by deploying a Pod that verifies its own
   529  	   token against these endpoints.
   530  	*/
   531  	framework.ConformanceIt("ServiceAccountIssuerDiscovery should support OIDC discovery of service account issuer", func(ctx context.Context) {
   532  
   533  		// Allow the test pod access to the OIDC discovery non-resource URLs.
   534  		// The role should have already been automatically created as part of the
   535  		// RBAC bootstrap policy, but not the role binding. If RBAC is disabled,
   536  		// we skip creating the binding. We also make sure we clean up the
   537  		// binding after the test.
   538  		const clusterRoleName = "system:service-account-issuer-discovery"
   539  		crbName := fmt.Sprintf("%s-%s", f.Namespace.Name, clusterRoleName)
   540  		if crb, err := f.ClientSet.RbacV1().ClusterRoleBindings().Create(
   541  			ctx,
   542  			&rbacv1.ClusterRoleBinding{
   543  				ObjectMeta: metav1.ObjectMeta{
   544  					Name: crbName,
   545  				},
   546  				Subjects: []rbacv1.Subject{
   547  					{
   548  						Kind:      rbacv1.ServiceAccountKind,
   549  						APIGroup:  "",
   550  						Name:      "default",
   551  						Namespace: f.Namespace.Name,
   552  					},
   553  				},
   554  				RoleRef: rbacv1.RoleRef{
   555  					Name:     clusterRoleName,
   556  					APIGroup: rbacv1.GroupName,
   557  					Kind:     "ClusterRole",
   558  				},
   559  			},
   560  			metav1.CreateOptions{}); err != nil {
   561  			// Tolerate RBAC not being enabled
   562  			framework.Logf("error granting ClusterRoleBinding %s: %v", crbName, err)
   563  		} else {
   564  			defer func() {
   565  				framework.ExpectNoError(
   566  					f.ClientSet.RbacV1().ClusterRoleBindings().Delete(
   567  						ctx,
   568  						crb.Name, metav1.DeleteOptions{}))
   569  			}()
   570  		}
   571  
   572  		// Create the pod with tokens.
   573  		tokenPath := "/var/run/secrets/tokens"
   574  		tokenName := "sa-token"
   575  		audience := "oidc-discovery-test"
   576  		tenMin := int64(10 * 60)
   577  
   578  		pod := &v1.Pod{
   579  			ObjectMeta: metav1.ObjectMeta{Name: "oidc-discovery-validator"},
   580  			Spec: v1.PodSpec{
   581  				Containers: []v1.Container{{
   582  					Name:  "oidc-discovery-validator",
   583  					Image: imageutils.GetE2EImage(imageutils.Agnhost),
   584  					Args: []string{
   585  						"test-service-account-issuer-discovery",
   586  						"--token-path", path.Join(tokenPath, tokenName),
   587  						"--audience", audience,
   588  					},
   589  					VolumeMounts: []v1.VolumeMount{{
   590  						MountPath: tokenPath,
   591  						Name:      tokenName,
   592  						ReadOnly:  true,
   593  					}},
   594  				}},
   595  				RestartPolicy:      v1.RestartPolicyNever,
   596  				ServiceAccountName: "default",
   597  				Volumes: []v1.Volume{{
   598  					Name: tokenName,
   599  					VolumeSource: v1.VolumeSource{
   600  						Projected: &v1.ProjectedVolumeSource{
   601  							Sources: []v1.VolumeProjection{
   602  								{
   603  									ServiceAccountToken: &v1.ServiceAccountTokenProjection{
   604  										Path:              tokenName,
   605  										ExpirationSeconds: &tenMin,
   606  										Audience:          audience,
   607  									},
   608  								},
   609  							},
   610  						},
   611  					},
   612  				}},
   613  			},
   614  		}
   615  		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
   616  		framework.ExpectNoError(err)
   617  
   618  		framework.Logf("created pod")
   619  		podErr := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)
   620  
   621  		// Get the logs before calling ExpectNoError, so we can debug any errors.
   622  		var logs string
   623  		if err := wait.Poll(30*time.Second, 2*time.Minute, func() (done bool, err error) {
   624  			framework.Logf("polling logs")
   625  			logs, err = e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
   626  			if err != nil {
   627  				framework.Logf("Error pulling logs: %v", err)
   628  				return false, nil
   629  			}
   630  			return true, nil
   631  		}); err != nil {
   632  			framework.Failf("Unexpected error getting pod logs: %v\n%s", err, logs)
   633  		} else {
   634  			framework.Logf("Pod logs: \n%v", logs)
   635  		}
   636  
   637  		framework.ExpectNoError(podErr)
   638  		framework.Logf("completed pod")
   639  	})
   640  
   641  	/*
   642  			   Release: v1.19
   643  			   Testname: ServiceAccount lifecycle test
   644  			   Description: Creates a ServiceAccount with a static Label MUST be added as shown in watch event.
   645  		                        Patching the ServiceAccount MUST return it's new property.
   646  		                        Listing the ServiceAccounts MUST return the test ServiceAccount with it's patched values.
   647  		                        ServiceAccount will be deleted and MUST find a deleted watch event.
   648  	*/
   649  	framework.ConformanceIt("should run through the lifecycle of a ServiceAccount", func(ctx context.Context) {
   650  		testNamespaceName := f.Namespace.Name
   651  		testServiceAccountName := "testserviceaccount"
   652  		testServiceAccountStaticLabels := map[string]string{"test-serviceaccount-static": "true"}
   653  		testServiceAccountStaticLabelsFlat := "test-serviceaccount-static=true"
   654  
   655  		ginkgo.By("creating a ServiceAccount")
   656  		testServiceAccount := v1.ServiceAccount{
   657  			ObjectMeta: metav1.ObjectMeta{
   658  				Name:   testServiceAccountName,
   659  				Labels: testServiceAccountStaticLabels,
   660  			},
   661  		}
   662  		createdServiceAccount, err := f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Create(ctx, &testServiceAccount, metav1.CreateOptions{})
   663  		framework.ExpectNoError(err, "failed to create a ServiceAccount")
   664  
   665  		getServiceAccount, err := f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Get(ctx, testServiceAccountName, metav1.GetOptions{})
   666  		framework.ExpectNoError(err, "failed to fetch the created ServiceAccount")
   667  		gomega.Expect(createdServiceAccount.UID).To(gomega.Equal(getServiceAccount.UID))
   668  
   669  		ginkgo.By("watching for the ServiceAccount to be added")
   670  		resourceWatchTimeoutSeconds := int64(180)
   671  		resourceWatch, err := f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Watch(ctx, metav1.ListOptions{LabelSelector: testServiceAccountStaticLabelsFlat, TimeoutSeconds: &resourceWatchTimeoutSeconds})
   672  		if err != nil {
   673  			fmt.Println(err, "failed to setup watch on newly created ServiceAccount")
   674  			return
   675  		}
   676  
   677  		resourceWatchChan := resourceWatch.ResultChan()
   678  		eventFound := false
   679  		for watchEvent := range resourceWatchChan {
   680  			if watchEvent.Type == watch.Added {
   681  				eventFound = true
   682  				break
   683  			}
   684  		}
   685  		if !eventFound {
   686  			framework.Failf("failed to find %v event", watch.Added)
   687  		}
   688  		ginkgo.By("patching the ServiceAccount")
   689  		boolFalse := false
   690  		testServiceAccountPatchData, err := json.Marshal(v1.ServiceAccount{
   691  			AutomountServiceAccountToken: &boolFalse,
   692  		})
   693  		framework.ExpectNoError(err, "failed to marshal JSON patch for the ServiceAccount")
   694  		_, err = f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).Patch(ctx, testServiceAccountName, types.StrategicMergePatchType, []byte(testServiceAccountPatchData), metav1.PatchOptions{})
   695  		framework.ExpectNoError(err, "failed to patch the ServiceAccount")
   696  		eventFound = false
   697  		for watchEvent := range resourceWatchChan {
   698  			if watchEvent.Type == watch.Modified {
   699  				eventFound = true
   700  				break
   701  			}
   702  		}
   703  		if !eventFound {
   704  			framework.Failf("failed to find %v event", watch.Modified)
   705  		}
   706  		ginkgo.By("finding ServiceAccount in list of all ServiceAccounts (by LabelSelector)")
   707  		serviceAccountList, err := f.ClientSet.CoreV1().ServiceAccounts("").List(ctx, metav1.ListOptions{LabelSelector: testServiceAccountStaticLabelsFlat})
   708  		framework.ExpectNoError(err, "failed to list ServiceAccounts by LabelSelector")
   709  		foundServiceAccount := false
   710  		for _, serviceAccountItem := range serviceAccountList.Items {
   711  			if serviceAccountItem.ObjectMeta.Name == testServiceAccountName && serviceAccountItem.ObjectMeta.Namespace == testNamespaceName && *serviceAccountItem.AutomountServiceAccountToken == boolFalse {
   712  				foundServiceAccount = true
   713  				break
   714  			}
   715  		}
   716  		if !foundServiceAccount {
   717  			framework.Fail("failed to find the created ServiceAccount")
   718  		}
   719  		ginkgo.By("deleting the ServiceAccount")
   720  		err = f.ClientSet.CoreV1().ServiceAccounts(testNamespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
   721  		framework.ExpectNoError(err, "failed to delete the ServiceAccount by Collection")
   722  		eventFound = false
   723  		for watchEvent := range resourceWatchChan {
   724  			if watchEvent.Type == watch.Deleted {
   725  				eventFound = true
   726  				break
   727  			}
   728  		}
   729  		if !eventFound {
   730  			framework.Failf("failed to find %v event", watch.Deleted)
   731  		}
   732  	})
   733  
   734  	/*
   735  		Release: v1.21
   736  		Testname: RootCA ConfigMap test
   737  		Description: Ensure every namespace exist a ConfigMap for root ca cert.
   738  			1. Created automatically
   739  			2. Recreated if deleted
   740  			3. Reconciled if modified
   741  	*/
   742  	framework.ConformanceIt("should guarantee kube-root-ca.crt exist in any namespace", func(ctx context.Context) {
   743  		framework.ExpectNoError(wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   744  			_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(ctx, rootCAConfigMapName, metav1.GetOptions{})
   745  			if err == nil {
   746  				return true, nil
   747  			}
   748  			if apierrors.IsNotFound(err) {
   749  				ginkgo.By("root ca configmap not found, retrying")
   750  				return false, nil
   751  			}
   752  			return false, err
   753  		}))
   754  		framework.Logf("Got root ca configmap in namespace %q", f.Namespace.Name)
   755  
   756  		framework.ExpectNoError(f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, rootCAConfigMapName, metav1.DeleteOptions{GracePeriodSeconds: utilptr.Int64Ptr(0)}))
   757  		framework.Logf("Deleted root ca configmap in namespace %q", f.Namespace.Name)
   758  
   759  		framework.ExpectNoError(wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   760  			ginkgo.By("waiting for a new root ca configmap created")
   761  			_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(ctx, rootCAConfigMapName, metav1.GetOptions{})
   762  			if err == nil {
   763  				return true, nil
   764  			}
   765  			if apierrors.IsNotFound(err) {
   766  				ginkgo.By("root ca configmap not found, retrying")
   767  				return false, nil
   768  			}
   769  			return false, err
   770  		}))
   771  		framework.Logf("Recreated root ca configmap in namespace %q", f.Namespace.Name)
   772  
   773  		_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(ctx, &v1.ConfigMap{
   774  			ObjectMeta: metav1.ObjectMeta{
   775  				Name: rootCAConfigMapName,
   776  			},
   777  			Data: map[string]string{
   778  				"ca.crt": "",
   779  			},
   780  		}, metav1.UpdateOptions{})
   781  		framework.ExpectNoError(err)
   782  		framework.Logf("Updated root ca configmap in namespace %q", f.Namespace.Name)
   783  
   784  		framework.ExpectNoError(wait.Poll(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   785  			ginkgo.By("waiting for the root ca configmap reconciled")
   786  			cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(ctx, rootCAConfigMapName, metav1.GetOptions{})
   787  			if err != nil {
   788  				if apierrors.IsNotFound(err) {
   789  					ginkgo.By("root ca configmap not found, retrying")
   790  					return false, nil
   791  				}
   792  				return false, err
   793  			}
   794  			if value, ok := cm.Data["ca.crt"]; !ok || value == "" {
   795  				ginkgo.By("root ca configmap is not reconciled yet, retrying")
   796  				return false, nil
   797  			}
   798  			return true, nil
   799  		}))
   800  		framework.Logf("Reconciled root ca configmap in namespace %q", f.Namespace.Name)
   801  	})
   802  
   803  	/*
   804  		Release: v1.26
   805  		Testname: ServiceAccount, update a ServiceAccount
   806  		Description: A ServiceAccount is created which MUST succeed. When
   807  		updating the ServiceAccount it MUST succeed and the field MUST equal
   808  		the new value.
   809  	*/
   810  	framework.ConformanceIt("should update a ServiceAccount", func(ctx context.Context) {
   811  		saClient := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name)
   812  		saName := "e2e-sa-" + utilrand.String(5)
   813  
   814  		initialServiceAccount := &v1.ServiceAccount{
   815  			ObjectMeta: metav1.ObjectMeta{
   816  				Name: saName,
   817  			},
   818  			AutomountServiceAccountToken: utilptr.Bool(false),
   819  		}
   820  
   821  		ginkgo.By(fmt.Sprintf("Creating ServiceAccount %q ", saName))
   822  		createdServiceAccount, err := saClient.Create(ctx, initialServiceAccount, metav1.CreateOptions{})
   823  		framework.ExpectNoError(err)
   824  		gomega.Expect(createdServiceAccount.AutomountServiceAccountToken).To(gomega.Equal(utilptr.Bool(false)), "Failed to set AutomountServiceAccountToken")
   825  		framework.Logf("AutomountServiceAccountToken: %v", *createdServiceAccount.AutomountServiceAccountToken)
   826  
   827  		ginkgo.By(fmt.Sprintf("Updating ServiceAccount %q ", saName))
   828  		var updatedServiceAccount *v1.ServiceAccount
   829  
   830  		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   831  			updateServiceAccount, err := saClient.Get(ctx, saName, metav1.GetOptions{})
   832  			framework.ExpectNoError(err, "Unable to get ServiceAccount %q", saName)
   833  			updateServiceAccount.AutomountServiceAccountToken = utilptr.Bool(true)
   834  			updatedServiceAccount, err = saClient.Update(ctx, updateServiceAccount, metav1.UpdateOptions{})
   835  			return err
   836  		})
   837  		framework.ExpectNoError(err, "Failed to update ServiceAccount")
   838  		gomega.Expect(updatedServiceAccount.AutomountServiceAccountToken).To(gomega.Equal(utilptr.Bool(true)), "Failed to set AutomountServiceAccountToken")
   839  		framework.Logf("AutomountServiceAccountToken: %v", *updatedServiceAccount.AutomountServiceAccountToken)
   840  	})
   841  })
   842  
   843  var reportLogsParser = regexp.MustCompile("([a-zA-Z0-9-_]*)=([a-zA-Z0-9-_]*)$")
   844  
   845  // ParseInClusterClientLogs parses logs of pods using inclusterclient.
   846  func ParseInClusterClientLogs(logs string) (int, error) {
   847  	seenTokens := map[string]struct{}{}
   848  
   849  	lines := strings.Split(logs, "\n")
   850  	for _, line := range lines {
   851  		parts := reportLogsParser.FindStringSubmatch(line)
   852  		if len(parts) != 3 {
   853  			continue
   854  		}
   855  
   856  		key, value := parts[1], parts[2]
   857  		switch key {
   858  		case "authz_header":
   859  			if value == "<empty>" {
   860  				return 0, fmt.Errorf("saw empty Authorization header")
   861  			}
   862  			seenTokens[value] = struct{}{}
   863  		case "status":
   864  			if value == "failed" {
   865  				return 0, fmt.Errorf("saw status=failed")
   866  			}
   867  		}
   868  	}
   869  
   870  	return len(seenTokens), nil
   871  }
   872  

View as plain text