mount_rro_linux_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  
    22  	"github.com/onsi/ginkgo/v2"
    23  	"github.com/onsi/gomega"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/util/uuid"
    28  	"k8s.io/kubernetes/pkg/features"
    29  	"k8s.io/kubernetes/test/e2e/framework"
    30  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    31  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    32  	"k8s.io/kubernetes/test/e2e/nodefeature"
    33  	admissionapi "k8s.io/pod-security-admission/api"
    34  	"k8s.io/utils/ptr"
    35  )
    36  
    37  // Usage:
    38  // make test-e2e-node TEST_ARGS='--service-feature-gates=RecursiveReadOnlyMounts=true --kubelet-flags="--feature-gates=RecursiveReadOnlyMounts=true"' FOCUS="Mount recursive read-only" SKIP=""
    39  var _ = SIGDescribe("Mount recursive read-only [LinuxOnly]", framework.WithSerial(), nodefeature.RecursiveReadOnlyMounts, func() {
    40  	f := framework.NewDefaultFramework("mount-rro")
    41  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    42  	ginkgo.Describe("Mount recursive read-only", func() {
    43  		ginkgo.Context("when the runtime supports recursive read-only mounts", func() {
    44  			f.It("should accept recursive read-only mounts", func(ctx context.Context) {
    45  				ginkgo.By("waiting for the node to be ready", func() {
    46  					waitForNodeReady(ctx)
    47  					if !supportsRRO(ctx, f) {
    48  						e2eskipper.Skipf("runtime does not support recursive read-only mounts")
    49  					}
    50  				}) // By
    51  				var pod *v1.Pod
    52  				ginkgo.By("creating a pod", func() {
    53  					pod = e2epod.NewPodClient(f).Create(ctx,
    54  						podForRROSupported("mount-rro-"+string(uuid.NewUUID()), f.Namespace.Name))
    55  					framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
    56  					var err error
    57  					pod, err = f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
    58  					framework.ExpectNoError(err)
    59  				}) // By
    60  				ginkgo.By("checking containerStatuses.volumeMounts", func() {
    61  					gomega.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(3)) // "mount", "test", "unmount"
    62  					volMountStatuses := pod.Status.InitContainerStatuses[1].VolumeMounts
    63  					var verifiedVolMountStatuses int
    64  					for _, f := range volMountStatuses {
    65  						switch f.Name {
    66  						case "mnt":
    67  							switch f.MountPath {
    68  							case "/mnt-rro", "/mnt-rro-if-possible":
    69  								gomega.Expect(*f.RecursiveReadOnly).To(gomega.Equal(v1.RecursiveReadOnlyEnabled))
    70  								verifiedVolMountStatuses++
    71  							case "/mnt-rro-disabled", "/mnt-ro":
    72  								gomega.Expect(*f.RecursiveReadOnly).To(gomega.Equal(v1.RecursiveReadOnlyDisabled))
    73  								verifiedVolMountStatuses++
    74  							case "/mnt-rw":
    75  								gomega.Expect(f.RecursiveReadOnly).To(gomega.BeNil())
    76  								verifiedVolMountStatuses++
    77  							default:
    78  								framework.Failf("unexpected mount path: %q", f.MountPath)
    79  							}
    80  						default: // implicit secret volumes, etc.
    81  							// NOP
    82  						}
    83  					}
    84  					gomega.Expect(verifiedVolMountStatuses).To(gomega.Equal(5))
    85  				}) // By
    86  			}) // It
    87  			f.It("should reject invalid recursive read-only mounts", func(ctx context.Context) {
    88  				ginkgo.By("waiting for the node to be ready", func() {
    89  					waitForNodeReady(ctx)
    90  					if !supportsRRO(ctx, f) {
    91  						e2eskipper.Skipf("runtime does not support recursive read-only mounts")
    92  					}
    93  				}) // By
    94  				ginkgo.By("specifying RRO without RO", func() {
    95  					pod := &v1.Pod{
    96  						ObjectMeta: metav1.ObjectMeta{
    97  							Name:      "mount-rro-invalid-" + string(uuid.NewUUID()),
    98  							Namespace: f.Namespace.Name,
    99  						},
   100  						Spec: v1.PodSpec{
   101  							RestartPolicy: v1.RestartPolicyNever,
   102  							Containers: []v1.Container{
   103  								{
   104  									Image:   busyboxImage,
   105  									Name:    "busybox",
   106  									Command: []string{"echo", "this container should fail"},
   107  									VolumeMounts: []v1.VolumeMount{
   108  										{
   109  											Name:              "mnt",
   110  											MountPath:         "/mnt",
   111  											RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyEnabled),
   112  										},
   113  									},
   114  								},
   115  							},
   116  							Volumes: []v1.Volume{
   117  								{
   118  									Name: "mnt",
   119  									VolumeSource: v1.VolumeSource{
   120  										EmptyDir: &v1.EmptyDirVolumeSource{},
   121  									},
   122  								},
   123  							},
   124  						},
   125  					}
   126  					_, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
   127  					gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("spec.containers[0].volumeMounts.recursiveReadOnly: Forbidden: may only be specified when readOnly is true")))
   128  				}) // By
   129  				// See also the unit test [pkg/kubelet.TestResolveRecursiveReadOnly] for more invalid conditions (e.g., incompatible mount propagation)
   130  			}) // It
   131  		}) // Context
   132  		ginkgo.Context("when the runtime does not support recursive read-only mounts", func() {
   133  			f.It("should accept non-recursive read-only mounts", func(ctx context.Context) {
   134  				e2eskipper.SkipUnlessFeatureGateEnabled(features.RecursiveReadOnlyMounts)
   135  				ginkgo.By("waiting for the node to be ready", func() {
   136  					waitForNodeReady(ctx)
   137  					if supportsRRO(ctx, f) {
   138  						e2eskipper.Skipf("runtime supports recursive read-only mounts")
   139  					}
   140  				}) // By
   141  				var pod *v1.Pod
   142  				ginkgo.By("creating a pod", func() {
   143  					pod = e2epod.NewPodClient(f).Create(ctx,
   144  						podForRROUnsupported("mount-ro-"+string(uuid.NewUUID()), f.Namespace.Name))
   145  					framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace))
   146  					var err error
   147  					pod, err = f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
   148  					framework.ExpectNoError(err)
   149  				}) // By
   150  				ginkgo.By("checking containerStatuses.volumeMounts", func() {
   151  					gomega.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(3)) // "mount", "test", "unmount"
   152  					volMountStatuses := pod.Status.InitContainerStatuses[1].VolumeMounts
   153  					var verifiedVolMountStatuses int
   154  					for _, f := range volMountStatuses {
   155  						switch f.Name {
   156  						case "mnt":
   157  							switch f.MountPath {
   158  							case "/mnt-rro-if-possible", "/mnt-rro-disabled", "/mnt-ro":
   159  								gomega.Expect(*f.RecursiveReadOnly).To(gomega.Equal(v1.RecursiveReadOnlyDisabled))
   160  								verifiedVolMountStatuses++
   161  							case "/mnt-rw":
   162  								gomega.Expect(f.RecursiveReadOnly).To(gomega.BeNil())
   163  								verifiedVolMountStatuses++
   164  							default:
   165  								framework.Failf("unexpected mount path: %q", f.MountPath)
   166  							}
   167  						default: // implicit secret volumes, etc.
   168  							// NOP
   169  						}
   170  					}
   171  					gomega.Expect(verifiedVolMountStatuses).To(gomega.Equal(4))
   172  				}) // By
   173  			}) // It
   174  			f.It("should reject recursive read-only mounts", func(ctx context.Context) {
   175  				e2eskipper.SkipUnlessFeatureGateEnabled(features.RecursiveReadOnlyMounts)
   176  				ginkgo.By("waiting for the node to be ready", func() {
   177  					waitForNodeReady(ctx)
   178  					if supportsRRO(ctx, f) {
   179  						e2eskipper.Skipf("runtime supports recursive read-only mounts")
   180  					}
   181  				}) // By
   182  				ginkgo.By("specifying RRO explicitly", func() {
   183  					pod := &v1.Pod{
   184  						ObjectMeta: metav1.ObjectMeta{
   185  							Name:      "mount-rro-unsupported-" + string(uuid.NewUUID()),
   186  							Namespace: f.Namespace.Name,
   187  						},
   188  						Spec: v1.PodSpec{
   189  							RestartPolicy: v1.RestartPolicyNever,
   190  							Containers: []v1.Container{
   191  								{
   192  									Image:   busyboxImage,
   193  									Name:    "busybox",
   194  									Command: []string{"echo", "this container should fail"},
   195  									VolumeMounts: []v1.VolumeMount{
   196  										{
   197  											Name:              "mnt",
   198  											MountPath:         "/mnt",
   199  											ReadOnly:          true,
   200  											RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyEnabled),
   201  										},
   202  									},
   203  								},
   204  							},
   205  							Volumes: []v1.Volume{
   206  								{
   207  									Name: "mnt",
   208  									VolumeSource: v1.VolumeSource{
   209  										EmptyDir: &v1.EmptyDirVolumeSource{},
   210  									},
   211  								},
   212  							},
   213  						},
   214  					}
   215  					pod = e2epod.NewPodClient(f).Create(ctx, pod)
   216  					framework.ExpectNoError(e2epod.WaitForPodContainerToFail(ctx, f.ClientSet, pod.Namespace, pod.Name, 0, "CreateContainerConfigError", framework.PodStartShortTimeout))
   217  					var err error
   218  					pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
   219  					framework.ExpectNoError(err)
   220  					gomega.Expect(pod.Status.ContainerStatuses[0].State.Waiting.Message).To(
   221  						gomega.ContainSubstring("failed to resolve recursive read-only mode: volume \"mnt\" requested recursive read-only mode, but it is not supported by the runtime"))
   222  				}) // By
   223  			}) // It
   224  		}) // Context
   225  	}) // Describe
   226  }) // SIGDescribe
   227  
   228  func supportsRRO(ctx context.Context, f *framework.Framework) bool {
   229  	nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
   230  	framework.ExpectNoError(err)
   231  	// Assuming that there is only one node, because this is a node e2e test.
   232  	gomega.Expect(nodeList.Items).To(gomega.HaveLen(1))
   233  	node := nodeList.Items[0]
   234  	for _, f := range node.Status.RuntimeHandlers {
   235  		if f.Name == "" && f.Features != nil && *f.Features.RecursiveReadOnlyMounts {
   236  			return true
   237  		}
   238  	}
   239  	return false
   240  }
   241  
   242  func podForRROSupported(name, ns string) *v1.Pod {
   243  	return &v1.Pod{
   244  		ObjectMeta: metav1.ObjectMeta{
   245  			Name:      name,
   246  			Namespace: ns,
   247  		},
   248  		Spec: v1.PodSpec{
   249  			RestartPolicy: v1.RestartPolicyNever,
   250  			InitContainers: []v1.Container{
   251  				{
   252  					Image:   busyboxImage,
   253  					Name:    "mount",
   254  					Command: []string{"sh", "-euxc", "mkdir -p /mnt/tmpfs && mount -t tmpfs none /mnt/tmpfs"},
   255  					SecurityContext: &v1.SecurityContext{
   256  						Privileged: ptr.To(true),
   257  					},
   258  					VolumeMounts: []v1.VolumeMount{
   259  						{
   260  							Name:             "mnt",
   261  							MountPath:        "/mnt",
   262  							MountPropagation: ptr.To(v1.MountPropagationBidirectional),
   263  						},
   264  					},
   265  				},
   266  				{
   267  					Image: busyboxImage,
   268  					Name:  "test",
   269  					Command: []string{"sh", "-euxc", `
   270  for f in rro rro-if-possible; do touch /mnt-$f/tmpfs/foo 2>&1 | grep "Read-only"; done
   271  for f in rro-disabled ro rw; do touch /mnt-$f/tmpfs/foo; done
   272  `},
   273  					VolumeMounts: []v1.VolumeMount{
   274  						{
   275  							Name:              "mnt",
   276  							MountPath:         "/mnt-rro",
   277  							ReadOnly:          true,
   278  							MountPropagation:  ptr.To(v1.MountPropagationNone), // explicit
   279  							RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyEnabled),
   280  						},
   281  						{
   282  							Name:              "mnt",
   283  							MountPath:         "/mnt-rro-if-possible",
   284  							ReadOnly:          true,
   285  							RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyIfPossible),
   286  						},
   287  						{
   288  							Name:              "mnt",
   289  							MountPath:         "/mnt-rro-disabled",
   290  							ReadOnly:          true,
   291  							RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyDisabled), // explicit
   292  						},
   293  						{
   294  							Name:      "mnt",
   295  							MountPath: "/mnt-ro",
   296  							ReadOnly:  true,
   297  						},
   298  						{
   299  							Name:      "mnt",
   300  							MountPath: "/mnt-rw",
   301  						},
   302  					},
   303  				},
   304  				{
   305  					Image:   busyboxImage,
   306  					Name:    "unmount",
   307  					Command: []string{"umount", "/mnt/tmpfs"},
   308  					SecurityContext: &v1.SecurityContext{
   309  						Privileged: ptr.To(true),
   310  					},
   311  					VolumeMounts: []v1.VolumeMount{
   312  						{
   313  							Name:             "mnt",
   314  							MountPath:        "/mnt",
   315  							MountPropagation: ptr.To(v1.MountPropagationBidirectional),
   316  						},
   317  					},
   318  				},
   319  			},
   320  			Containers: []v1.Container{
   321  				{
   322  					Image:   busyboxImage,
   323  					Name:    "completion",
   324  					Command: []string{"echo", "OK"},
   325  				},
   326  			},
   327  			Volumes: []v1.Volume{
   328  				{
   329  					Name: "mnt",
   330  					VolumeSource: v1.VolumeSource{
   331  						EmptyDir: &v1.EmptyDirVolumeSource{},
   332  					},
   333  				},
   334  			},
   335  		},
   336  	}
   337  }
   338  
   339  func podForRROUnsupported(name, ns string) *v1.Pod {
   340  	return &v1.Pod{
   341  		ObjectMeta: metav1.ObjectMeta{
   342  			Name:      name,
   343  			Namespace: ns,
   344  		},
   345  		Spec: v1.PodSpec{
   346  			RestartPolicy: v1.RestartPolicyNever,
   347  			InitContainers: []v1.Container{
   348  				{
   349  					Image:   busyboxImage,
   350  					Name:    "mount",
   351  					Command: []string{"sh", "-euxc", "mkdir -p /mnt/tmpfs && mount -t tmpfs none /mnt/tmpfs"},
   352  					SecurityContext: &v1.SecurityContext{
   353  						Privileged: ptr.To(true),
   354  					},
   355  					VolumeMounts: []v1.VolumeMount{
   356  						{
   357  							Name:             "mnt",
   358  							MountPath:        "/mnt",
   359  							MountPropagation: ptr.To(v1.MountPropagationBidirectional),
   360  						},
   361  					},
   362  				},
   363  				{
   364  					Image: busyboxImage,
   365  					Name:  "test",
   366  					Command: []string{"sh", "-euxc", `
   367  for f in rro-if-possible rro-disabled ro rw; do touch /mnt-$f/tmpfs/foo; done
   368  `},
   369  					VolumeMounts: []v1.VolumeMount{
   370  						{
   371  							Name:              "mnt",
   372  							MountPath:         "/mnt-rro-if-possible",
   373  							ReadOnly:          true,
   374  							RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyIfPossible),
   375  						},
   376  						{
   377  							Name:              "mnt",
   378  							MountPath:         "/mnt-rro-disabled",
   379  							ReadOnly:          true,
   380  							RecursiveReadOnly: ptr.To(v1.RecursiveReadOnlyDisabled), // explicit
   381  						},
   382  						{
   383  							Name:      "mnt",
   384  							MountPath: "/mnt-ro",
   385  							ReadOnly:  true,
   386  						},
   387  						{
   388  							Name:      "mnt",
   389  							MountPath: "/mnt-rw",
   390  						},
   391  					},
   392  				},
   393  				{
   394  					Image:   busyboxImage,
   395  					Name:    "unmount",
   396  					Command: []string{"umount", "/mnt/tmpfs"},
   397  					SecurityContext: &v1.SecurityContext{
   398  						Privileged: ptr.To(true),
   399  					},
   400  					VolumeMounts: []v1.VolumeMount{
   401  						{
   402  							Name:             "mnt",
   403  							MountPath:        "/mnt",
   404  							MountPropagation: ptr.To(v1.MountPropagationBidirectional),
   405  						},
   406  					},
   407  				},
   408  			},
   409  			Containers: []v1.Container{
   410  				{
   411  					Image:   busyboxImage,
   412  					Name:    "completion",
   413  					Command: []string{"echo", "OK"},
   414  				},
   415  			},
   416  			Volumes: []v1.Volume{
   417  				{
   418  					Name: "mnt",
   419  					VolumeSource: v1.VolumeSource{
   420  						EmptyDir: &v1.EmptyDirVolumeSource{},
   421  					},
   422  				},
   423  			},
   424  		},
   425  	}
   426  }
   427
View as plain text