runtimeclass.go

Documentation: k8s.io/kubernetes/test/e2e/common/node

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package node
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	v1 "k8s.io/api/core/v1"
    25  	nodev1 "k8s.io/api/node/v1"
    26  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    27  	"k8s.io/apimachinery/pkg/api/resource"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/fields"
    30  	"k8s.io/apimachinery/pkg/labels"
    31  	types "k8s.io/apimachinery/pkg/types"
    32  	"k8s.io/apimachinery/pkg/util/wait"
    33  	"k8s.io/apimachinery/pkg/watch"
    34  	"k8s.io/kubernetes/pkg/kubelet/events"
    35  	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
    36  	"k8s.io/kubernetes/test/e2e/framework"
    37  	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
    38  	e2eruntimeclass "k8s.io/kubernetes/test/e2e/framework/node/runtimeclass"
    39  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    40  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    41  	"k8s.io/kubernetes/test/e2e/nodefeature"
    42  	admissionapi "k8s.io/pod-security-admission/api"
    43  
    44  	"github.com/onsi/ginkgo/v2"
    45  	"github.com/onsi/gomega"
    46  )
    47  
    48  var _ = SIGDescribe("RuntimeClass", func() {
    49  	f := framework.NewDefaultFramework("runtimeclass")
    50  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    51  
    52  	/*
    53  		Release: v1.20
    54  		Testname: Pod with the non-existing RuntimeClass is rejected.
    55  		Description: The Pod requesting the non-existing RuntimeClass must be rejected.
    56  	*/
    57  	framework.ConformanceIt("should reject a Pod requesting a non-existent RuntimeClass", f.WithNodeConformance(), func(ctx context.Context) {
    58  		rcName := f.Namespace.Name + "-nonexistent"
    59  		expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
    60  	})
    61  
    62  	// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler not being installed.
    63  	f.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", nodefeature.RuntimeHandler, func(ctx context.Context) {
    64  		handler := f.Namespace.Name + "-handler"
    65  		rcName := createRuntimeClass(ctx, f, "unconfigured-handler", handler, nil)
    66  		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
    67  		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
    68  		eventSelector := fields.Set{
    69  			"involvedObject.kind":      "Pod",
    70  			"involvedObject.name":      pod.Name,
    71  			"involvedObject.namespace": f.Namespace.Name,
    72  			"reason":                   events.FailedCreatePodSandBox,
    73  		}.AsSelector().String()
    74  		// Events are unreliable, don't depend on the event. It's used only to speed up the test.
    75  		err := e2eevents.WaitTimeoutForEvent(ctx, f.ClientSet, f.Namespace.Name, eventSelector, handler, framework.PodEventTimeout)
    76  		if err != nil {
    77  			framework.Logf("Warning: did not get event about FailedCreatePodSandBox. Err: %v", err)
    78  		}
    79  		// Check the pod is still not running
    80  		p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
    81  		framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)")
    82  		gomega.Expect(p.Status.Phase).To(gomega.Equal(v1.PodPending), "Pod phase isn't pending")
    83  	})
    84  
    85  	// This test requires that the PreconfiguredRuntimeClassHandler has already been set up on nodes.
    86  	// The test CANNOT be made a Conformance as it depends on a container runtime to have a specific handler installed and working.
    87  	f.It("should run a Pod requesting a RuntimeClass with a configured handler", nodefeature.RuntimeHandler, func(ctx context.Context) {
    88  		if err := e2eruntimeclass.NodeSupportsPreconfiguredRuntimeClassHandler(ctx, f); err != nil {
    89  			e2eskipper.Skipf("Skipping test as node does not have E2E runtime class handler preconfigured in container runtime config: %v", err)
    90  		}
    91  
    92  		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
    93  		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
    94  		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
    95  		expectPodSuccess(ctx, f, pod)
    96  	})
    97  
    98  	/*
    99  		Release: v1.20
   100  		Testname: Can schedule a pod requesting existing RuntimeClass.
   101  		Description: The Pod requesting the existing RuntimeClass must be scheduled.
   102  		This test doesn't validate that the Pod will actually start because this functionality
   103  		depends on container runtime and preconfigured handler. Runtime-specific functionality
   104  		is not being tested here.
   105  	*/
   106  	framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass without PodOverhead", f.WithNodeConformance(), func(ctx context.Context) {
   107  		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, nil)
   108  		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
   109  		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
   110  		// there is only one pod in the namespace
   111  		label := labels.SelectorFromSet(labels.Set(map[string]string{}))
   112  		pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
   113  		framework.ExpectNoError(err, "Failed to schedule Pod with the RuntimeClass")
   114  
   115  		gomega.Expect(pods.Items).To(gomega.HaveLen(1))
   116  		scheduledPod := &pods.Items[0]
   117  		gomega.Expect(scheduledPod.Name).To(gomega.Equal(pod.Name))
   118  
   119  		// Overhead should not be set
   120  		gomega.Expect(scheduledPod.Spec.Overhead).To(gomega.BeEmpty())
   121  	})
   122  
   123  	/*
   124  		Release: v1.24
   125  		Testname: RuntimeClass Overhead field must be respected.
   126  		Description: The Pod requesting the existing RuntimeClass must be scheduled.
   127  		This test doesn't validate that the Pod will actually start because this functionality
   128  		depends on container runtime and preconfigured handler. Runtime-specific functionality
   129  		is not being tested here.
   130  	*/
   131  	framework.ConformanceIt("should schedule a Pod requesting a RuntimeClass and initialize its Overhead", f.WithNodeConformance(), func(ctx context.Context) {
   132  		rcName := createRuntimeClass(ctx, f, "preconfigured-handler", e2eruntimeclass.PreconfiguredRuntimeClassHandler, &nodev1.Overhead{
   133  			PodFixed: v1.ResourceList{
   134  				v1.ResourceName(v1.ResourceCPU):    resource.MustParse("10m"),
   135  				v1.ResourceName(v1.ResourceMemory): resource.MustParse("1Mi"),
   136  			},
   137  		})
   138  		ginkgo.DeferCleanup(deleteRuntimeClass, f, rcName)
   139  		pod := e2epod.NewPodClient(f).Create(ctx, e2eruntimeclass.NewRuntimeClassPod(rcName))
   140  		// there is only one pod in the namespace
   141  		label := labels.SelectorFromSet(labels.Set(map[string]string{}))
   142  		pods, err := e2epod.WaitForPodsWithLabelScheduled(ctx, f.ClientSet, f.Namespace.Name, label)
   143  		framework.ExpectNoError(err, "Failed to schedule Pod with the RuntimeClass")
   144  
   145  		gomega.Expect(pods.Items).To(gomega.HaveLen(1))
   146  		scheduledPod := &pods.Items[0]
   147  		gomega.Expect(scheduledPod.Name).To(gomega.Equal(pod.Name))
   148  
   149  		gomega.Expect(scheduledPod.Spec.Overhead[v1.ResourceCPU]).To(gomega.Equal(resource.MustParse("10m")))
   150  		gomega.Expect(scheduledPod.Spec.Overhead[v1.ResourceMemory]).To(gomega.Equal(resource.MustParse("1Mi")))
   151  	})
   152  
   153  	/*
   154  		Release: v1.20
   155  		Testname: Pod with the deleted RuntimeClass is rejected.
   156  		Description: Pod requesting the deleted RuntimeClass must be rejected.
   157  	*/
   158  	framework.ConformanceIt("should reject a Pod requesting a deleted RuntimeClass", f.WithNodeConformance(), func(ctx context.Context) {
   159  		rcName := createRuntimeClass(ctx, f, "delete-me", "runc", nil)
   160  		rcClient := f.ClientSet.NodeV1().RuntimeClasses()
   161  
   162  		ginkgo.By("Deleting RuntimeClass "+rcName, func() {
   163  			err := rcClient.Delete(ctx, rcName, metav1.DeleteOptions{})
   164  			framework.ExpectNoError(err, "failed to delete RuntimeClass %s", rcName)
   165  
   166  			ginkgo.By("Waiting for the RuntimeClass to disappear")
   167  			framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) {
   168  				_, err := rcClient.Get(ctx, rcName, metav1.GetOptions{})
   169  				if apierrors.IsNotFound(err) {
   170  					return true, nil // done
   171  				}
   172  				if err != nil {
   173  					return true, err // stop wait with error
   174  				}
   175  				return false, nil
   176  			}))
   177  		})
   178  
   179  		expectPodRejection(ctx, f, e2eruntimeclass.NewRuntimeClassPod(rcName))
   180  	})
   181  
   182  	/*
   183  		Release: v1.20
   184  		Testname: RuntimeClass API
   185  		Description:
   186  		The node.k8s.io API group MUST exist in the /apis discovery document.
   187  		The node.k8s.io/v1 API group/version MUST exist in the /apis/mode.k8s.io discovery document.
   188  		The runtimeclasses resource MUST exist in the /apis/node.k8s.io/v1 discovery document.
   189  		The runtimeclasses resource must support create, get, list, watch, update, patch, delete, and deletecollection.
   190  	*/
   191  	framework.ConformanceIt("should support RuntimeClasses API operations", func(ctx context.Context) {
   192  		// Setup
   193  		rcVersion := "v1"
   194  		rcClient := f.ClientSet.NodeV1().RuntimeClasses()
   195  
   196  		// This is a conformance test that must configure opaque handlers to validate CRUD operations.
   197  		// Test should not use any existing handler like gVisor or runc
   198  		//
   199  		// All CRUD operations in this test are limited to the objects with the label test=f.UniqueName
   200  		rc := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler", f.UniqueName+"-conformance-runtime-class")
   201  		rc.SetLabels(map[string]string{"test": f.UniqueName})
   202  		rc2 := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler2", f.UniqueName+"-conformance-runtime-class2")
   203  		rc2.SetLabels(map[string]string{"test": f.UniqueName})
   204  		rc3 := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler3", f.UniqueName+"-conformance-runtime-class3")
   205  		rc3.SetLabels(map[string]string{"test": f.UniqueName})
   206  
   207  		// Discovery
   208  
   209  		ginkgo.By("getting /apis")
   210  		{
   211  			discoveryGroups, err := f.ClientSet.Discovery().ServerGroups()
   212  			framework.ExpectNoError(err)
   213  			found := false
   214  			for _, group := range discoveryGroups.Groups {
   215  				if group.Name == nodev1.GroupName {
   216  					for _, version := range group.Versions {
   217  						if version.Version == rcVersion {
   218  							found = true
   219  							break
   220  						}
   221  					}
   222  				}
   223  			}
   224  			if !found {
   225  				framework.Failf("expected RuntimeClass API group/version, got %#v", discoveryGroups.Groups)
   226  			}
   227  		}
   228  
   229  		ginkgo.By("getting /apis/node.k8s.io")
   230  		{
   231  			group := &metav1.APIGroup{}
   232  			err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/node.k8s.io").Do(ctx).Into(group)
   233  			framework.ExpectNoError(err)
   234  			found := false
   235  			for _, version := range group.Versions {
   236  				if version.Version == rcVersion {
   237  					found = true
   238  					break
   239  				}
   240  			}
   241  			if !found {
   242  				framework.Failf("expected RuntimeClass API version, got %#v", group.Versions)
   243  			}
   244  		}
   245  
   246  		ginkgo.By("getting /apis/node.k8s.io/" + rcVersion)
   247  		{
   248  			resources, err := f.ClientSet.Discovery().ServerResourcesForGroupVersion(nodev1.SchemeGroupVersion.String())
   249  			framework.ExpectNoError(err)
   250  			found := false
   251  			for _, resource := range resources.APIResources {
   252  				switch resource.Name {
   253  				case "runtimeclasses":
   254  					found = true
   255  				}
   256  			}
   257  			if !found {
   258  				framework.Failf("expected runtimeclasses, got %#v", resources.APIResources)
   259  			}
   260  		}
   261  
   262  		// Main resource create/read/update/watch operations
   263  
   264  		ginkgo.By("creating")
   265  		createdRC, err := rcClient.Create(ctx, rc, metav1.CreateOptions{})
   266  		framework.ExpectNoError(err)
   267  		_, err = rcClient.Create(ctx, rc, metav1.CreateOptions{})
   268  		if !apierrors.IsAlreadyExists(err) {
   269  			framework.Failf("expected 409, got %#v", err)
   270  		}
   271  		_, err = rcClient.Create(ctx, rc2, metav1.CreateOptions{})
   272  		framework.ExpectNoError(err)
   273  
   274  		ginkgo.By("watching")
   275  		framework.Logf("starting watch")
   276  		rcWatch, err := rcClient.Watch(ctx, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName})
   277  		framework.ExpectNoError(err)
   278  
   279  		// added for a watch
   280  		_, err = rcClient.Create(ctx, rc3, metav1.CreateOptions{})
   281  		framework.ExpectNoError(err)
   282  
   283  		ginkgo.By("getting")
   284  		gottenRC, err := rcClient.Get(ctx, rc.Name, metav1.GetOptions{})
   285  		framework.ExpectNoError(err)
   286  		gomega.Expect(gottenRC.UID).To(gomega.Equal(createdRC.UID))
   287  
   288  		ginkgo.By("listing")
   289  		rcs, err := rcClient.List(ctx, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName})
   290  		framework.ExpectNoError(err)
   291  		gomega.Expect(rcs.Items).To(gomega.HaveLen(3), "filtered list should have 3 items")
   292  
   293  		ginkgo.By("patching")
   294  		patchedRC, err := rcClient.Patch(ctx, createdRC.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patched":"true"}}}`), metav1.PatchOptions{})
   295  		framework.ExpectNoError(err)
   296  		gomega.Expect(patchedRC.Annotations).To(gomega.HaveKeyWithValue("patched", "true"), "patched object should have the applied annotation")
   297  
   298  		ginkgo.By("updating")
   299  		csrToUpdate := patchedRC.DeepCopy()
   300  		csrToUpdate.Annotations["updated"] = "true"
   301  		updatedRC, err := rcClient.Update(ctx, csrToUpdate, metav1.UpdateOptions{})
   302  		framework.ExpectNoError(err)
   303  		gomega.Expect(updatedRC.Annotations).To(gomega.HaveKeyWithValue("updated", "true"), "updated object should have the applied annotation")
   304  
   305  		framework.Logf("waiting for watch events with expected annotations")
   306  		for sawAdded, sawPatched, sawUpdated := false, false, false; !sawAdded && !sawPatched && !sawUpdated; {
   307  			select {
   308  			case evt, ok := <-rcWatch.ResultChan():
   309  				if !ok {
   310  					framework.Fail("watch channel should not close")
   311  				}
   312  				if evt.Type == watch.Modified {
   313  					watchedRC, isRC := evt.Object.(*nodev1.RuntimeClass)
   314  					if !isRC {
   315  						framework.Failf("expected RC, got %T", evt.Object)
   316  					}
   317  					if watchedRC.Annotations["patched"] == "true" {
   318  						framework.Logf("saw patched annotations")
   319  						sawPatched = true
   320  					} else if watchedRC.Annotations["updated"] == "true" {
   321  						framework.Logf("saw updated annotations")
   322  						sawUpdated = true
   323  					} else {
   324  						framework.Logf("missing expected annotations, waiting: %#v", watchedRC.Annotations)
   325  					}
   326  				} else if evt.Type == watch.Added {
   327  					_, isRC := evt.Object.(*nodev1.RuntimeClass)
   328  					if !isRC {
   329  						framework.Failf("expected RC, got %T", evt.Object)
   330  					}
   331  					sawAdded = true
   332  				}
   333  
   334  			case <-time.After(wait.ForeverTestTimeout):
   335  				framework.Fail("timed out waiting for watch event")
   336  			}
   337  		}
   338  		rcWatch.Stop()
   339  
   340  		// main resource delete operations
   341  
   342  		ginkgo.By("deleting")
   343  		err = rcClient.Delete(ctx, createdRC.Name, metav1.DeleteOptions{})
   344  		framework.ExpectNoError(err)
   345  		_, err = rcClient.Get(ctx, createdRC.Name, metav1.GetOptions{})
   346  		if !apierrors.IsNotFound(err) {
   347  			framework.Failf("expected 404, got %#v", err)
   348  		}
   349  		rcs, err = rcClient.List(ctx, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName})
   350  		framework.ExpectNoError(err)
   351  		gomega.Expect(rcs.Items).To(gomega.HaveLen(2), "filtered list should have 2 items")
   352  
   353  		ginkgo.By("deleting a collection")
   354  		err = rcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName})
   355  		framework.ExpectNoError(err)
   356  		rcs, err = rcClient.List(ctx, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName})
   357  		framework.ExpectNoError(err)
   358  		gomega.Expect(rcs.Items).To(gomega.BeEmpty(), "filtered list should have 0 items")
   359  	})
   360  })
   361  
   362  func deleteRuntimeClass(ctx context.Context, f *framework.Framework, name string) {
   363  	err := f.ClientSet.NodeV1().RuntimeClasses().Delete(ctx, name, metav1.DeleteOptions{})
   364  	framework.ExpectNoError(err, "failed to delete RuntimeClass resource")
   365  }
   366  
   367  // createRuntimeClass generates a RuntimeClass with the desired handler and a "namespaced" name,
   368  // synchronously creates it, and returns the generated name.
   369  func createRuntimeClass(ctx context.Context, f *framework.Framework, name, handler string, overhead *nodev1.Overhead) string {
   370  	uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name)
   371  	rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler)
   372  	rc.Overhead = overhead
   373  	rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(ctx, rc, metav1.CreateOptions{})
   374  	framework.ExpectNoError(err, "failed to create RuntimeClass resource")
   375  	return rc.GetName()
   376  }
   377  
   378  func expectPodRejection(ctx context.Context, f *framework.Framework, pod *v1.Pod) {
   379  	_, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
   380  	gomega.Expect(err).To(gomega.HaveOccurred(), "should be forbidden")
   381  	if !apierrors.IsForbidden(err) {
   382  		framework.Failf("expected forbidden error, got %#v", err)
   383  	}
   384  }
   385  
   386  // expectPodSuccess waits for the given pod to terminate successfully.
   387  func expectPodSuccess(ctx context.Context, f *framework.Framework, pod *v1.Pod) {
   388  	framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(
   389  		ctx, f.ClientSet, pod.Name, f.Namespace.Name))
   390  }
   391
View as plain text