apparmor_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"fmt"
    23  	"os"
    24  	"os/exec"
    25  	"regexp"
    26  	"strconv"
    27  	"strings"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/fields"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"k8s.io/apimachinery/pkg/runtime/schema"
    35  	"k8s.io/apimachinery/pkg/util/dump"
    36  	"k8s.io/apimachinery/pkg/watch"
    37  	"k8s.io/client-go/tools/cache"
    38  	watchtools "k8s.io/client-go/tools/watch"
    39  	"k8s.io/klog/v2"
    40  	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
    41  	"k8s.io/kubernetes/test/e2e/feature"
    42  	"k8s.io/kubernetes/test/e2e/framework"
    43  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    44  	"k8s.io/kubernetes/test/e2e/nodefeature"
    45  	admissionapi "k8s.io/pod-security-admission/api"
    46  
    47  	"github.com/onsi/ginkgo/v2"
    48  	"github.com/onsi/gomega"
    49  	"github.com/opencontainers/runc/libcontainer/apparmor"
    50  )
    51  
    52  var _ = SIGDescribe("AppArmor", feature.AppArmor, nodefeature.AppArmor, func() {
    53  	if isAppArmorEnabled() {
    54  		ginkgo.BeforeEach(func() {
    55  			ginkgo.By("Loading AppArmor profiles for testing")
    56  			framework.ExpectNoError(loadTestProfiles(), "Could not load AppArmor test profiles")
    57  		})
    58  		ginkgo.Context("when running with AppArmor", func() {
    59  			f := framework.NewDefaultFramework("apparmor-test")
    60  			f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    61  
    62  			ginkgo.It("should reject an unloaded profile", func(ctx context.Context) {
    63  				status := runAppArmorTest(ctx, f, false, v1.DeprecatedAppArmorBetaProfileNamePrefix+"non-existent-profile")
    64  				gomega.Expect(status.ContainerStatuses[0].State.Waiting.Message).To(gomega.ContainSubstring("apparmor"))
    65  			})
    66  			ginkgo.It("should enforce a profile blocking writes", func(ctx context.Context) {
    67  				status := runAppArmorTest(ctx, f, true, v1.DeprecatedAppArmorBetaProfileNamePrefix+apparmorProfilePrefix+"deny-write")
    68  				if len(status.ContainerStatuses) == 0 {
    69  					framework.Failf("Unexpected pod status: %s", dump.Pretty(status))
    70  					return
    71  				}
    72  				state := status.ContainerStatuses[0].State.Terminated
    73  				gomega.Expect(state).ToNot(gomega.BeNil(), "ContainerState: %+v", status.ContainerStatuses[0].State)
    74  				gomega.Expect(state.ExitCode).To(gomega.Not(gomega.BeZero()), "ContainerStateTerminated: %+v", state)
    75  
    76  			})
    77  			ginkgo.It("should enforce a permissive profile", func(ctx context.Context) {
    78  				status := runAppArmorTest(ctx, f, true, v1.DeprecatedAppArmorBetaProfileNamePrefix+apparmorProfilePrefix+"audit-write")
    79  				if len(status.ContainerStatuses) == 0 {
    80  					framework.Failf("Unexpected pod status: %s", dump.Pretty(status))
    81  					return
    82  				}
    83  				state := status.ContainerStatuses[0].State.Terminated
    84  				gomega.Expect(state).ToNot(gomega.BeNil(), "ContainerState: %+v", status.ContainerStatuses[0].State)
    85  				gomega.Expect(state.ExitCode).To(gomega.BeZero(), "ContainerStateTerminated: %+v", state)
    86  			})
    87  		})
    88  	} else {
    89  		ginkgo.Context("when running without AppArmor", func() {
    90  			f := framework.NewDefaultFramework("apparmor-test")
    91  			f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    92  
    93  			ginkgo.It("should reject a pod with an AppArmor profile", func(ctx context.Context) {
    94  				status := runAppArmorTest(ctx, f, false, v1.DeprecatedAppArmorBetaProfileRuntimeDefault)
    95  				expectSoftRejection(status)
    96  			})
    97  		})
    98  	}
    99  })
   100  
   101  const apparmorProfilePrefix = "e2e-node-apparmor-test-"
   102  const testProfiles = `
   103  #include <tunables/global>
   104  
   105  profile e2e-node-apparmor-test-deny-write flags=(attach_disconnected) {
   106    #include <abstractions/base>
   107  
   108    file,
   109  
   110    # Deny all file writes.
   111    deny /** w,
   112  }
   113  
   114  profile e2e-node-apparmor-test-audit-write flags=(attach_disconnected) {
   115    #include <abstractions/base>
   116  
   117    file,
   118  
   119    # Only audit file writes.
   120    audit /** w,
   121  }
   122  `
   123  
   124  func loadTestProfiles() error {
   125  	f, err := os.CreateTemp("/tmp", "apparmor")
   126  	if err != nil {
   127  		return fmt.Errorf("failed to open temp file: %w", err)
   128  	}
   129  	defer os.Remove(f.Name())
   130  	defer f.Close()
   131  
   132  	if _, err := f.WriteString(testProfiles); err != nil {
   133  		return fmt.Errorf("failed to write profiles to file: %w", err)
   134  	}
   135  
   136  	cmd := exec.Command("apparmor_parser", "-r", "-W", f.Name())
   137  	stderr := &bytes.Buffer{}
   138  	cmd.Stderr = stderr
   139  	out, err := cmd.Output()
   140  	// apparmor_parser does not always return an error code, so consider any stderr output an error.
   141  	if err != nil || stderr.Len() > 0 {
   142  		if stderr.Len() > 0 {
   143  			klog.Warning(stderr.String())
   144  		}
   145  		if len(out) > 0 {
   146  			klog.Infof("apparmor_parser: %s", out)
   147  		}
   148  		return fmt.Errorf("failed to load profiles: %w", err)
   149  	}
   150  	klog.V(2).Infof("Loaded profiles: %v", out)
   151  	return nil
   152  }
   153  
   154  func runAppArmorTest(ctx context.Context, f *framework.Framework, shouldRun bool, profile string) v1.PodStatus {
   155  	pod := createPodWithAppArmor(ctx, f, profile)
   156  	if shouldRun {
   157  		// The pod needs to start before it stops, so wait for the longer start timeout.
   158  		framework.ExpectNoError(e2epod.WaitTimeoutForPodNoLongerRunningInNamespace(ctx,
   159  			f.ClientSet, pod.Name, f.Namespace.Name, framework.PodStartTimeout))
   160  	} else {
   161  		// Pod should remain in the pending state. Wait for the Reason to be set to "AppArmor".
   162  		fieldSelector := fields.OneTermEqualSelector("metadata.name", pod.Name).String()
   163  		w := &cache.ListWatch{
   164  			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
   165  				options.FieldSelector = fieldSelector
   166  				return e2epod.NewPodClient(f).List(ctx, options)
   167  			},
   168  			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
   169  				options.FieldSelector = fieldSelector
   170  				return e2epod.NewPodClient(f).Watch(ctx, options)
   171  			},
   172  		}
   173  		preconditionFunc := func(store cache.Store) (bool, error) {
   174  			_, exists, err := store.Get(&metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name})
   175  			if err != nil {
   176  				return true, err
   177  			}
   178  			if !exists {
   179  				// We need to make sure we see the object in the cache before we start waiting for events
   180  				// or we would be waiting for the timeout if such object didn't exist.
   181  				return true, apierrors.NewNotFound(v1.Resource("pods"), pod.Name)
   182  			}
   183  
   184  			return false, nil
   185  		}
   186  		ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, framework.PodStartTimeout)
   187  		defer cancel()
   188  		_, err := watchtools.UntilWithSync(ctx, w, &v1.Pod{}, preconditionFunc, func(e watch.Event) (bool, error) {
   189  			switch e.Type {
   190  			case watch.Deleted:
   191  				return false, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, pod.Name)
   192  			}
   193  			switch t := e.Object.(type) {
   194  			case *v1.Pod:
   195  				if t.Status.Reason == "AppArmor" {
   196  					return true, nil
   197  				}
   198  				// Loading a profile not available on disk should return a container creation error
   199  				if len(t.Status.ContainerStatuses) > 0 && t.Status.ContainerStatuses[0].State.Waiting.Reason == kuberuntime.ErrCreateContainer.Error() {
   200  					return true, nil
   201  				}
   202  			}
   203  			return false, nil
   204  		})
   205  		framework.ExpectNoError(err)
   206  	}
   207  	p, err := e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{})
   208  	framework.ExpectNoError(err)
   209  	return p.Status
   210  }
   211  
   212  func createPodWithAppArmor(ctx context.Context, f *framework.Framework, profile string) *v1.Pod {
   213  	pod := &v1.Pod{
   214  		ObjectMeta: metav1.ObjectMeta{
   215  			Name: fmt.Sprintf("test-apparmor-%s", strings.Replace(profile, "/", "-", -1)),
   216  			Annotations: map[string]string{
   217  				v1.DeprecatedAppArmorBetaContainerAnnotationKeyPrefix + "test": profile,
   218  			},
   219  		},
   220  		Spec: v1.PodSpec{
   221  			Containers: []v1.Container{{
   222  				Name:    "test",
   223  				Image:   busyboxImage,
   224  				Command: []string{"touch", "foo"},
   225  			}},
   226  			RestartPolicy: v1.RestartPolicyNever,
   227  		},
   228  	}
   229  	return e2epod.NewPodClient(f).Create(ctx, pod)
   230  }
   231  
   232  func expectSoftRejection(status v1.PodStatus) {
   233  	args := []interface{}{"PodStatus: %+v", status}
   234  	gomega.Expect(status.Phase).To(gomega.Equal(v1.PodPending), args...)
   235  	gomega.Expect(status.Reason).To(gomega.Equal("AppArmor"), args...)
   236  	gomega.Expect(status.Message).To(gomega.ContainSubstring("AppArmor"), args...)
   237  	gomega.Expect(status.ContainerStatuses[0].State.Waiting.Reason).To(gomega.Equal("Blocked"), args...)
   238  }
   239  
   240  func isAppArmorEnabled() bool {
   241  	// TODO(tallclair): Pass this through the image setup rather than hardcoding.
   242  	if strings.Contains(framework.TestContext.NodeName, "-gci-dev-") {
   243  		gciVersionRe := regexp.MustCompile("-gci-dev-([0-9]+)-")
   244  		matches := gciVersionRe.FindStringSubmatch(framework.TestContext.NodeName)
   245  		if len(matches) == 2 {
   246  			version, err := strconv.Atoi(matches[1])
   247  			if err != nil {
   248  				klog.Errorf("Error parsing GCI version from NodeName %q: %v", framework.TestContext.NodeName, err)
   249  				return false
   250  			}
   251  			return version >= 54
   252  		}
   253  		return false
   254  	}
   255  	if strings.Contains(framework.TestContext.NodeName, "-ubuntu-") {
   256  		return true
   257  	}
   258  	return apparmor.IsEnabled()
   259  }
   260
View as plain text