dra.go

Documentation: k8s.io/kubernetes/test/e2e/dra

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package dra
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"errors"
    23  	"fmt"
    24  	"strings"
    25  	"sync"
    26  	"time"
    27  
    28  	"github.com/onsi/ginkgo/v2"
    29  	"github.com/onsi/gomega"
    30  	"github.com/onsi/gomega/gcustom"
    31  	"github.com/onsi/gomega/gstruct"
    32  
    33  	v1 "k8s.io/api/core/v1"
    34  	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
    35  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	"k8s.io/apimachinery/pkg/labels"
    38  	"k8s.io/apimachinery/pkg/runtime"
    39  	"k8s.io/client-go/kubernetes"
    40  	"k8s.io/dynamic-resource-allocation/controller"
    41  	"k8s.io/klog/v2"
    42  	"k8s.io/kubernetes/test/e2e/dra/test-driver/app"
    43  	"k8s.io/kubernetes/test/e2e/feature"
    44  	"k8s.io/kubernetes/test/e2e/framework"
    45  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    46  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    47  	admissionapi "k8s.io/pod-security-admission/api"
    48  	utilpointer "k8s.io/utils/pointer"
    49  	"k8s.io/utils/ptr"
    50  )
    51  
    52  const (
    53  	// podStartTimeout is how long to wait for the pod to be started.
    54  	podStartTimeout = 5 * time.Minute
    55  )
    56  
    57  // networkResources can be passed to NewDriver directly.
    58  func networkResources() app.Resources {
    59  	return app.Resources{
    60  		Shareable: true,
    61  	}
    62  }
    63  
    64  // perNode returns a function which can be passed to NewDriver. The nodes
    65  // parameter has be instantiated, but not initialized yet, so the returned
    66  // function has to capture it and use it when being called.
    67  func perNode(maxAllocations int, nodes *Nodes) func() app.Resources {
    68  	return func() app.Resources {
    69  		return app.Resources{
    70  			NodeLocal:      true,
    71  			MaxAllocations: maxAllocations,
    72  			Nodes:          nodes.NodeNames,
    73  		}
    74  	}
    75  }
    76  
    77  var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, func() {
    78  	f := framework.NewDefaultFramework("dra")
    79  
    80  	// The driver containers have to run with sufficient privileges to
    81  	// modify /var/lib/kubelet/plugins.
    82  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    83  
    84  	ginkgo.Context("kubelet", func() {
    85  		nodes := NewNodes(f, 1, 1)
    86  
    87  		ginkgo.Context("with ConfigMap parameters", func() {
    88  			driver := NewDriver(f, nodes, networkResources)
    89  			b := newBuilder(f, driver)
    90  
    91  			ginkgo.It("registers plugin", func() {
    92  				ginkgo.By("the driver is running")
    93  			})
    94  
    95  			ginkgo.It("must retry NodePrepareResources", func(ctx context.Context) {
    96  				// We have exactly one host.
    97  				m := MethodInstance{driver.Nodenames()[0], NodePrepareResourcesMethod}
    98  
    99  				driver.Fail(m, true)
   100  
   101  				ginkgo.By("waiting for container startup to fail")
   102  				parameters := b.parameters()
   103  				pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   104  
   105  				b.create(ctx, parameters, pod, template)
   106  
   107  				ginkgo.By("wait for NodePrepareResources call")
   108  				gomega.Eventually(ctx, func(ctx context.Context) error {
   109  					if driver.CallCount(m) == 0 {
   110  						return errors.New("NodePrepareResources not called yet")
   111  					}
   112  					return nil
   113  				}).WithTimeout(podStartTimeout).Should(gomega.Succeed())
   114  
   115  				ginkgo.By("allowing container startup to succeed")
   116  				callCount := driver.CallCount(m)
   117  				driver.Fail(m, false)
   118  				err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)
   119  				framework.ExpectNoError(err, "start pod with inline resource claim")
   120  				if driver.CallCount(m) == callCount {
   121  					framework.Fail("NodePrepareResources should have been called again")
   122  				}
   123  			})
   124  
   125  			ginkgo.It("must not run a pod if a claim is not reserved for it", func(ctx context.Context) {
   126  				// Pretend that the resource is allocated and reserved for some other entity.
   127  				// Until the resourceclaim controller learns to remove reservations for
   128  				// arbitrary types we can simply fake somthing here.
   129  				claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   130  				b.create(ctx, claim)
   131  				claim, err := f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
   132  				framework.ExpectNoError(err, "get claim")
   133  				claim.Status.Allocation = &resourcev1alpha2.AllocationResult{}
   134  				claim.Status.DriverName = driver.Name
   135  				claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourcev1alpha2.ResourceClaimConsumerReference{
   136  					APIGroup: "example.com",
   137  					Resource: "some",
   138  					Name:     "thing",
   139  					UID:      "12345",
   140  				})
   141  				_, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
   142  				framework.ExpectNoError(err, "update claim")
   143  
   144  				pod := b.podExternal()
   145  
   146  				// This bypasses scheduling and therefore the pod gets
   147  				// to run on the node although it never gets added to
   148  				// the `ReservedFor` field of the claim.
   149  				pod.Spec.NodeName = nodes.NodeNames[0]
   150  				b.create(ctx, pod)
   151  
   152  				gomega.Consistently(ctx, func(ctx context.Context) error {
   153  					testPod, err := b.f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
   154  					if err != nil {
   155  						return fmt.Errorf("expected the test pod %s to exist: %w", pod.Name, err)
   156  					}
   157  					if testPod.Status.Phase != v1.PodPending {
   158  						return fmt.Errorf("pod %s: unexpected status %s, expected status: %s", pod.Name, testPod.Status.Phase, v1.PodPending)
   159  					}
   160  					return nil
   161  				}, 20*time.Second, 200*time.Millisecond).Should(gomega.BeNil())
   162  			})
   163  
   164  			ginkgo.It("must unprepare resources for force-deleted pod", func(ctx context.Context) {
   165  				parameters := b.parameters()
   166  				claim := b.externalClaim(resourcev1alpha2.AllocationModeImmediate)
   167  				pod := b.podExternal()
   168  				zero := int64(0)
   169  				pod.Spec.TerminationGracePeriodSeconds = &zero
   170  
   171  				b.create(ctx, parameters, claim, pod)
   172  
   173  				b.testPod(ctx, f.ClientSet, pod)
   174  
   175  				ginkgo.By(fmt.Sprintf("force delete test pod %s", pod.Name))
   176  				err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero})
   177  				if !apierrors.IsNotFound(err) {
   178  					framework.ExpectNoError(err, "force delete test pod")
   179  				}
   180  
   181  				for host, plugin := range b.driver.Nodes {
   182  					ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host))
   183  					gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host)
   184  				}
   185  			})
   186  
   187  			ginkgo.It("must skip NodePrepareResource if not used by any container", func(ctx context.Context) {
   188  				parameters := b.parameters()
   189  				pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   190  				for i := range pod.Spec.Containers {
   191  					pod.Spec.Containers[i].Resources.Claims = nil
   192  				}
   193  				b.create(ctx, parameters, pod, template)
   194  				framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod), "start pod")
   195  				for host, plugin := range b.driver.Nodes {
   196  					gomega.Expect(plugin.GetPreparedResources()).Should(gomega.BeEmpty(), "not claims should be prepared on host %s while pod is running", host)
   197  				}
   198  			})
   199  
   200  		})
   201  	})
   202  
   203  	// claimTests tries out several different combinations of pods with
   204  	// claims, both inline and external.
   205  	claimTests := func(b *builder, driver *Driver, allocationMode resourcev1alpha2.AllocationMode) {
   206  		ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) {
   207  			objects, expectedEnv := b.flexibleParameters()
   208  			pod, template := b.podInline(allocationMode)
   209  			objects = append(objects, pod, template)
   210  			b.create(ctx, objects...)
   211  
   212  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   213  		})
   214  
   215  		ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) {
   216  			objects, expectedEnv := b.flexibleParameters()
   217  			pod, template := b.podInlineMultiple(allocationMode)
   218  			objects = append(objects, pod, template)
   219  			b.create(ctx, objects...)
   220  
   221  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   222  		})
   223  
   224  		ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) {
   225  			objects, expectedEnv := b.flexibleParameters()
   226  			pod := b.podExternal()
   227  			claim := b.externalClaim(allocationMode)
   228  			objects = append(objects, claim, pod)
   229  			b.create(ctx, objects...)
   230  
   231  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   232  		})
   233  
   234  		ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) {
   235  			objects, expectedEnv := b.flexibleParameters()
   236  			pod1 := b.podExternal()
   237  			pod2 := b.podExternal()
   238  			pod3 := b.podExternal()
   239  			claim := b.externalClaim(allocationMode)
   240  			objects = append(objects, claim, pod1, pod2, pod3)
   241  			b.create(ctx, objects...)
   242  
   243  			for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
   244  				b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   245  			}
   246  		})
   247  
   248  		ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) {
   249  			objects, expectedEnv := b.flexibleParameters()
   250  			pod1 := b.podExternalMultiple()
   251  			pod2 := b.podExternalMultiple()
   252  			pod3 := b.podExternalMultiple()
   253  			claim := b.externalClaim(allocationMode)
   254  			objects = append(objects, claim, pod1, pod2, pod3)
   255  			b.create(ctx, objects...)
   256  
   257  			for _, pod := range []*v1.Pod{pod1, pod2, pod3} {
   258  				b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   259  			}
   260  		})
   261  
   262  		ginkgo.It("supports init containers", func(ctx context.Context) {
   263  			objects, expectedEnv := b.flexibleParameters()
   264  			pod, template := b.podInline(allocationMode)
   265  			pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]}
   266  			pod.Spec.InitContainers[0].Name += "-init"
   267  			// This must succeed for the pod to start.
   268  			pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"}
   269  			objects = append(objects, pod, template)
   270  			b.create(ctx, objects...)
   271  
   272  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   273  		})
   274  
   275  		ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) {
   276  			objects, _ := b.flexibleParameters()
   277  			pod := b.podExternal()
   278  			claim := b.externalClaim(allocationMode)
   279  			pod.Spec.Containers[0].Command = []string{"true"}
   280  			objects = append(objects, claim, pod)
   281  			b.create(ctx, objects...)
   282  
   283  			ginkgo.By("waiting for pod to finish")
   284  			framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
   285  			ginkgo.By("waiting for claim to be unreserved")
   286  			gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1alpha2.ResourceClaim, error) {
   287  				return f.ClientSet.ResourceV1alpha2().ResourceClaims(pod.Namespace).Get(ctx, claim.Name, metav1.GetOptions{})
   288  			}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.ReservedFor", gomega.BeEmpty()), "reservation should have been removed")
   289  		})
   290  
   291  		ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) {
   292  			objects, _ := b.flexibleParameters()
   293  			pod, template := b.podInline(allocationMode)
   294  			pod.Spec.Containers[0].Command = []string{"true"}
   295  			objects = append(objects, template, pod)
   296  			b.create(ctx, objects...)
   297  
   298  			ginkgo.By("waiting for pod to finish")
   299  			framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish")
   300  			ginkgo.By("waiting for claim to be deleted")
   301  			gomega.Eventually(ctx, func(ctx context.Context) ([]resourcev1alpha2.ResourceClaim, error) {
   302  				claims, err := f.ClientSet.ResourceV1alpha2().ResourceClaims(pod.Namespace).List(ctx, metav1.ListOptions{})
   303  				if err != nil {
   304  					return nil, err
   305  				}
   306  				return claims.Items, nil
   307  			}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeEmpty(), "claim should have been deleted")
   308  		})
   309  
   310  		ginkgo.It("does not delete generated claims when pod is restarting", func(ctx context.Context) {
   311  			objects, _ := b.flexibleParameters()
   312  			pod, template := b.podInline(allocationMode)
   313  			pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"}
   314  			pod.Spec.RestartPolicy = v1.RestartPolicyAlways
   315  			objects = append(objects, template, pod)
   316  			b.create(ctx, objects...)
   317  
   318  			ginkgo.By("waiting for pod to restart twice")
   319  			gomega.Eventually(ctx, func(ctx context.Context) (*v1.Pod, error) {
   320  				return f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
   321  			}).WithTimeout(f.Timeouts.PodStartSlow).Should(gomega.HaveField("Status.ContainerStatuses", gomega.ContainElements(gomega.HaveField("RestartCount", gomega.BeNumerically(">=", 2)))))
   322  			if driver.Controller != nil {
   323  				gomega.Expect(driver.Controller.GetNumAllocations()).To(gomega.Equal(int64(1)), "number of allocations")
   324  			}
   325  		})
   326  
   327  		ginkgo.It("must deallocate after use when using delayed allocation", func(ctx context.Context) {
   328  			objects, expectedEnv := b.flexibleParameters()
   329  			pod := b.podExternal()
   330  			claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   331  			objects = append(objects, claim, pod)
   332  			b.create(ctx, objects...)
   333  
   334  			gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1alpha2.ResourceClaim, error) {
   335  				return b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
   336  			}).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourcev1alpha2.AllocationResult)(nil)))
   337  
   338  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   339  
   340  			ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod)))
   341  			framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{}))
   342  
   343  			ginkgo.By("waiting for claim to get deallocated")
   344  			gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1alpha2.ResourceClaim, error) {
   345  				return b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
   346  			}).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", (*resourcev1alpha2.AllocationResult)(nil)))
   347  		})
   348  	}
   349  
   350  	singleNodeTests := func(parameterMode parameterMode) {
   351  		nodes := NewNodes(f, 1, 1)
   352  		maxAllocations := 1
   353  		numPods := 10
   354  		generateResources := func() app.Resources {
   355  			resources := perNode(maxAllocations, nodes)()
   356  			resources.Shareable = true
   357  			return resources
   358  		}
   359  		driver := NewDriver(f, nodes, generateResources) // All tests get their own driver instance.
   360  		driver.parameterMode = parameterMode
   361  		b := newBuilder(f, driver)
   362  		// We need the parameters name *before* creating it.
   363  		b.parametersCounter = 1
   364  		b.classParametersName = b.parametersName()
   365  
   366  		ginkgo.It("supports claim and class parameters", func(ctx context.Context) {
   367  			objects, expectedEnv := b.flexibleParameters()
   368  
   369  			pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   370  			objects = append(objects, pod, template)
   371  
   372  			b.create(ctx, objects...)
   373  
   374  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   375  		})
   376  
   377  		ginkgo.It("supports reusing resources", func(ctx context.Context) {
   378  			objects, expectedEnv := b.flexibleParameters()
   379  			pods := make([]*v1.Pod, numPods)
   380  			for i := 0; i < numPods; i++ {
   381  				pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   382  				pods[i] = pod
   383  				objects = append(objects, pod, template)
   384  			}
   385  
   386  			b.create(ctx, objects...)
   387  
   388  			// We don't know the order. All that matters is that all of them get scheduled eventually.
   389  			var wg sync.WaitGroup
   390  			wg.Add(numPods)
   391  			for i := 0; i < numPods; i++ {
   392  				pod := pods[i]
   393  				go func() {
   394  					defer ginkgo.GinkgoRecover()
   395  					defer wg.Done()
   396  					b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   397  					err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
   398  					framework.ExpectNoError(err, "delete pod")
   399  					framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow))
   400  				}()
   401  			}
   402  			wg.Wait()
   403  		})
   404  
   405  		ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) {
   406  			objects, expectedEnv := b.flexibleParameters()
   407  			objects = append(objects, b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer))
   408  
   409  			pods := make([]*v1.Pod, numPods)
   410  			for i := 0; i < numPods; i++ {
   411  				pod := b.podExternal()
   412  				pods[i] = pod
   413  				objects = append(objects, pod)
   414  			}
   415  
   416  			b.create(ctx, objects...)
   417  
   418  			// We don't know the order. All that matters is that all of them get scheduled eventually.
   419  			f.Timeouts.PodStartSlow *= time.Duration(numPods)
   420  			var wg sync.WaitGroup
   421  			wg.Add(numPods)
   422  			for i := 0; i < numPods; i++ {
   423  				pod := pods[i]
   424  				go func() {
   425  					defer ginkgo.GinkgoRecover()
   426  					defer wg.Done()
   427  					b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   428  				}()
   429  			}
   430  			wg.Wait()
   431  		})
   432  
   433  		f.It("supports sharing a claim sequentially", f.WithSlow(), func(ctx context.Context) {
   434  			objects, expectedEnv := b.flexibleParameters()
   435  			numPods := numPods / 2
   436  
   437  			// Change from "shareable" to "not shareable", if possible.
   438  			switch parameterMode {
   439  			case parameterModeConfigMap:
   440  				ginkgo.Skip("cannot change the driver's controller behavior on-the-fly")
   441  			case parameterModeTranslated, parameterModeStructured:
   442  				objects[len(objects)-1].(*resourcev1alpha2.ResourceClaimParameters).Shareable = false
   443  			}
   444  
   445  			objects = append(objects, b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer))
   446  
   447  			pods := make([]*v1.Pod, numPods)
   448  			for i := 0; i < numPods; i++ {
   449  				pod := b.podExternal()
   450  				pods[i] = pod
   451  				objects = append(objects, pod)
   452  			}
   453  
   454  			b.create(ctx, objects...)
   455  
   456  			// We don't know the order. All that matters is that all of them get scheduled eventually.
   457  			f.Timeouts.PodStartSlow *= time.Duration(numPods)
   458  			var wg sync.WaitGroup
   459  			wg.Add(numPods)
   460  			for i := 0; i < numPods; i++ {
   461  				pod := pods[i]
   462  				go func() {
   463  					defer ginkgo.GinkgoRecover()
   464  					defer wg.Done()
   465  					b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   466  					// We need to delete each running pod, otherwise the others cannot use the claim.
   467  					err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{})
   468  					framework.ExpectNoError(err, "delete pod")
   469  					framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow))
   470  				}()
   471  			}
   472  			wg.Wait()
   473  		})
   474  
   475  		ginkgo.It("retries pod scheduling after creating resource class", func(ctx context.Context) {
   476  			objects, expectedEnv := b.flexibleParameters()
   477  			pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   478  			class, err := f.ClientSet.ResourceV1alpha2().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{})
   479  			framework.ExpectNoError(err)
   480  			template.Spec.Spec.ResourceClassName += "-b"
   481  			objects = append(objects, template, pod)
   482  			b.create(ctx, objects...)
   483  
   484  			// There's no way to be sure that the scheduler has checked the pod.
   485  			// But if we sleep for a short while, it's likely and if there are any
   486  			// bugs that prevent the scheduler from handling creation of the class,
   487  			// those bugs should show up as test flakes.
   488  			//
   489  			// TODO (https://github.com/kubernetes/kubernetes/issues/123805): check the Schedulable condition instead of
   490  			// sleeping.
   491  			time.Sleep(time.Second)
   492  
   493  			class.UID = ""
   494  			class.ResourceVersion = ""
   495  			class.Name = template.Spec.Spec.ResourceClassName
   496  			b.create(ctx, class)
   497  
   498  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   499  		})
   500  
   501  		ginkgo.It("retries pod scheduling after updating resource class", func(ctx context.Context) {
   502  			objects, expectedEnv := b.flexibleParameters()
   503  			pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   504  
   505  			// First modify the class so that it matches no nodes.
   506  			class, err := f.ClientSet.ResourceV1alpha2().ResourceClasses().Get(ctx, template.Spec.Spec.ResourceClassName, metav1.GetOptions{})
   507  			framework.ExpectNoError(err)
   508  			class.SuitableNodes = &v1.NodeSelector{
   509  				NodeSelectorTerms: []v1.NodeSelectorTerm{
   510  					{
   511  						MatchExpressions: []v1.NodeSelectorRequirement{
   512  							{
   513  								Key:      "no-such-label",
   514  								Operator: v1.NodeSelectorOpIn,
   515  								Values:   []string{"no-such-value"},
   516  							},
   517  						},
   518  					},
   519  				},
   520  			}
   521  			class, err = f.ClientSet.ResourceV1alpha2().ResourceClasses().Update(ctx, class, metav1.UpdateOptions{})
   522  			framework.ExpectNoError(err)
   523  
   524  			// Now create the pod.
   525  			objects = append(objects, template, pod)
   526  			b.create(ctx, objects...)
   527  
   528  			// There's no way to be sure that the scheduler has checked the pod.
   529  			// But if we sleep for a short while, it's likely and if there are any
   530  			// bugs that prevent the scheduler from handling updates of the class,
   531  			// those bugs should show up as test flakes.
   532  			time.Sleep(time.Second)
   533  
   534  			// Unblock the pod.
   535  			class.SuitableNodes = nil
   536  			_, err = f.ClientSet.ResourceV1alpha2().ResourceClasses().Update(ctx, class, metav1.UpdateOptions{})
   537  			framework.ExpectNoError(err)
   538  
   539  			b.testPod(ctx, f.ClientSet, pod, expectedEnv...)
   540  		})
   541  
   542  		ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) {
   543  			pod, _ /* template */ := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   544  			created := b.create(ctx, pod)
   545  			pod = created[0].(*v1.Pod)
   546  
   547  			// Normally, this pod would be stuck because the
   548  			// ResourceClaim cannot be created without the
   549  			// template. We allow it to run by communicating
   550  			// through the status that the ResourceClaim is not
   551  			// needed.
   552  			pod.Status.ResourceClaimStatuses = []v1.PodResourceClaimStatus{
   553  				{Name: pod.Spec.ResourceClaims[0].Name, ResourceClaimName: nil},
   554  			}
   555  			_, err := f.ClientSet.CoreV1().Pods(pod.Namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{})
   556  			framework.ExpectNoError(err)
   557  			framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
   558  		})
   559  
   560  		ginkgo.Context("with delayed allocation", func() {
   561  			claimTests(b, driver, resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   562  		})
   563  
   564  		ginkgo.Context("with immediate allocation", func() {
   565  			claimTests(b, driver, resourcev1alpha2.AllocationModeImmediate)
   566  		})
   567  	}
   568  
   569  	// These tests depend on having more than one node and a DRA driver controller.
   570  	multiNodeDRAControllerTests := func(nodes *Nodes) {
   571  		driver := NewDriver(f, nodes, networkResources)
   572  		b := newBuilder(f, driver)
   573  
   574  		ginkgo.It("schedules onto different nodes", func(ctx context.Context) {
   575  			parameters := b.parameters()
   576  			label := "app.kubernetes.io/instance"
   577  			instance := f.UniqueName + "-test-app"
   578  			antiAffinity := &v1.Affinity{
   579  				PodAntiAffinity: &v1.PodAntiAffinity{
   580  					RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
   581  						{
   582  							TopologyKey: "kubernetes.io/hostname",
   583  							LabelSelector: &metav1.LabelSelector{
   584  								MatchLabels: map[string]string{
   585  									label: instance,
   586  								},
   587  							},
   588  						},
   589  					},
   590  				},
   591  			}
   592  			createPod := func() *v1.Pod {
   593  				pod := b.podExternal()
   594  				pod.Labels[label] = instance
   595  				pod.Spec.Affinity = antiAffinity
   596  				return pod
   597  			}
   598  			pod1 := createPod()
   599  			pod2 := createPod()
   600  			claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   601  			b.create(ctx, parameters, claim, pod1, pod2)
   602  
   603  			for _, pod := range []*v1.Pod{pod1, pod2} {
   604  				err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
   605  				framework.ExpectNoError(err, "start pod")
   606  			}
   607  		})
   608  
   609  		// This test covers aspects of non graceful node shutdown by DRA controller
   610  		// More details about this can be found in the KEP:
   611  		// https://github.com/kubernetes/enhancements/tree/master/keps/sig-storage/2268-non-graceful-shutdown
   612  		// NOTE: this test depends on kind. It will only work with kind cluster as it shuts down one of the
   613  		// nodes by running `docker stop <node name>`, which is very kind-specific.
   614  		f.It(f.WithSerial(), f.WithDisruptive(), f.WithSlow(), "must deallocate on non graceful node shutdown", func(ctx context.Context) {
   615  			ginkgo.By("create test pod")
   616  			parameters := b.parameters()
   617  			label := "app.kubernetes.io/instance"
   618  			instance := f.UniqueName + "-test-app"
   619  			pod := b.podExternal()
   620  			pod.Labels[label] = instance
   621  			claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   622  			b.create(ctx, parameters, claim, pod)
   623  
   624  			ginkgo.By("wait for test pod " + pod.Name + " to run")
   625  			labelSelector := labels.SelectorFromSet(labels.Set(pod.Labels))
   626  			pods, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, pod.Namespace, labelSelector, 1, framework.PodStartTimeout)
   627  			framework.ExpectNoError(err, "start pod")
   628  			runningPod := &pods.Items[0]
   629  
   630  			nodeName := runningPod.Spec.NodeName
   631  			// Prevent builder tearDown to fail waiting for unprepared resources
   632  			delete(b.driver.Nodes, nodeName)
   633  			ginkgo.By("stop node " + nodeName + " non gracefully")
   634  			_, stderr, err := framework.RunCmd("docker", "stop", nodeName)
   635  			gomega.Expect(stderr).To(gomega.BeEmpty())
   636  			framework.ExpectNoError(err)
   637  			ginkgo.DeferCleanup(framework.RunCmd, "docker", "start", nodeName)
   638  			if ok := e2enode.WaitForNodeToBeNotReady(ctx, f.ClientSet, nodeName, f.Timeouts.NodeNotReady); !ok {
   639  				framework.Failf("Node %s failed to enter NotReady state", nodeName)
   640  			}
   641  
   642  			ginkgo.By("apply out-of-service taint on node " + nodeName)
   643  			taint := v1.Taint{
   644  				Key:    v1.TaintNodeOutOfService,
   645  				Effect: v1.TaintEffectNoExecute,
   646  			}
   647  			e2enode.AddOrUpdateTaintOnNode(ctx, f.ClientSet, nodeName, taint)
   648  			e2enode.ExpectNodeHasTaint(ctx, f.ClientSet, nodeName, &taint)
   649  			ginkgo.DeferCleanup(e2enode.RemoveTaintOffNode, f.ClientSet, nodeName, taint)
   650  
   651  			ginkgo.By("waiting for claim to get deallocated")
   652  			gomega.Eventually(ctx, framework.GetObject(b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get, claim.Name, metav1.GetOptions{})).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", gomega.BeNil()))
   653  		})
   654  	}
   655  
   656  	// The following tests only make sense when there is more than one node.
   657  	// They get skipped when there's only one node.
   658  	multiNodeTests := func(parameterMode parameterMode) {
   659  		nodes := NewNodes(f, 2, 8)
   660  
   661  		if parameterMode == parameterModeConfigMap {
   662  			ginkgo.Context("with network-attached resources", func() {
   663  				multiNodeDRAControllerTests(nodes)
   664  			})
   665  
   666  			ginkgo.Context("reallocation", func() {
   667  				var allocateWrapper2 app.AllocateWrapperType
   668  				driver := NewDriver(f, nodes, perNode(1, nodes))
   669  				driver2 := NewDriver(f, nodes, func() app.Resources {
   670  					return app.Resources{
   671  						NodeLocal:      true,
   672  						MaxAllocations: 1,
   673  						Nodes:          nodes.NodeNames,
   674  
   675  						AllocateWrapper: func(
   676  							ctx context.Context,
   677  							claimAllocations []*controller.ClaimAllocation,
   678  							selectedNode string,
   679  							handler func(
   680  								ctx context.Context,
   681  								claimAllocations []*controller.ClaimAllocation,
   682  								selectedNode string),
   683  						) {
   684  							allocateWrapper2(ctx, claimAllocations, selectedNode, handler)
   685  						},
   686  					}
   687  				})
   688  				driver2.NameSuffix = "-other"
   689  
   690  				b := newBuilder(f, driver)
   691  				b2 := newBuilder(f, driver2)
   692  
   693  				ginkgo.It("works", func(ctx context.Context) {
   694  					// A pod with multiple claims can run on a node, but
   695  					// only if allocation of all succeeds. This
   696  					// test simulates the scenario where one claim
   697  					// gets allocated from one driver, but the claims
   698  					// from second driver fail allocation because of a
   699  					// race with some other pod.
   700  					//
   701  					// To ensure the right timing, allocation of the
   702  					// claims from second driver are delayed while
   703  					// creating another pod that gets the remaining
   704  					// resource on the node from second driver.
   705  					ctx, cancel := context.WithCancel(ctx)
   706  					defer cancel()
   707  
   708  					parameters1 := b.parameters()
   709  					parameters2 := b2.parameters()
   710  					// Order is relevant here: each pod must be matched with its own claim.
   711  					pod1claim1 := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   712  					pod1 := b.podExternal()
   713  					pod2claim1 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   714  					pod2 := b2.podExternal()
   715  
   716  					// Add another claim to pod1.
   717  					pod1claim2 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   718  					pod1.Spec.ResourceClaims = append(pod1.Spec.ResourceClaims,
   719  						v1.PodResourceClaim{
   720  							Name: "claim-other",
   721  							Source: v1.ClaimSource{
   722  								ResourceClaimName: &pod1claim2.Name,
   723  							},
   724  						},
   725  					)
   726  
   727  					// Allocating the second claim in pod1 has to wait until pod2 has
   728  					// consumed the available resources on the node.
   729  					blockClaim, cancelBlockClaim := context.WithCancel(ctx)
   730  					defer cancelBlockClaim()
   731  					allocateWrapper2 = func(ctx context.Context,
   732  						claimAllocations []*controller.ClaimAllocation,
   733  						selectedNode string,
   734  						handler func(ctx context.Context,
   735  							claimAllocations []*controller.ClaimAllocation,
   736  							selectedNode string),
   737  					) {
   738  						if claimAllocations[0].Claim.Name == pod1claim2.Name {
   739  							<-blockClaim.Done()
   740  						}
   741  						handler(ctx, claimAllocations, selectedNode)
   742  					}
   743  
   744  					b.create(ctx, parameters1, parameters2, pod1claim1, pod1claim2, pod1)
   745  
   746  					ginkgo.By("waiting for one claim from driver1 to be allocated")
   747  					var nodeSelector *v1.NodeSelector
   748  					gomega.Eventually(ctx, func(ctx context.Context) (int, error) {
   749  						claims, err := f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).List(ctx, metav1.ListOptions{})
   750  						if err != nil {
   751  							return 0, err
   752  						}
   753  						allocated := 0
   754  						for _, claim := range claims.Items {
   755  							if claim.Status.Allocation != nil {
   756  								allocated++
   757  								nodeSelector = claim.Status.Allocation.AvailableOnNodes
   758  							}
   759  						}
   760  						return allocated, nil
   761  					}).WithTimeout(time.Minute).Should(gomega.Equal(1), "one claim allocated")
   762  
   763  					// Now create a second pod which we force to
   764  					// run on the same node that is currently being
   765  					// considered for the first one. We know what
   766  					// the node selector looks like and can
   767  					// directly access the key and value from it.
   768  					ginkgo.By(fmt.Sprintf("create second pod on the same node %s", nodeSelector))
   769  
   770  					req := nodeSelector.NodeSelectorTerms[0].MatchExpressions[0]
   771  					node := req.Values[0]
   772  					pod2.Spec.NodeSelector = map[string]string{req.Key: node}
   773  
   774  					b2.create(ctx, pod2claim1, pod2)
   775  					framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2), "start pod 2")
   776  
   777  					// Allow allocation of second claim in pod1 to proceed. It should fail now
   778  					// and the other node must be used instead, after deallocating
   779  					// the first claim.
   780  					ginkgo.By("move first pod to other node")
   781  					cancelBlockClaim()
   782  
   783  					framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1), "start pod 1")
   784  					pod1, err := f.ClientSet.CoreV1().Pods(pod1.Namespace).Get(ctx, pod1.Name, metav1.GetOptions{})
   785  					framework.ExpectNoError(err, "get first pod")
   786  					if pod1.Spec.NodeName == "" {
   787  						framework.Fail("first pod should be running on node, was not scheduled")
   788  					}
   789  					gomega.Expect(pod1.Spec.NodeName).ToNot(gomega.Equal(node), "first pod should run on different node than second one")
   790  					gomega.Expect(driver.Controller.GetNumDeallocations()).To(gomega.Equal(int64(1)), "number of deallocations")
   791  				})
   792  			})
   793  		}
   794  
   795  		ginkgo.Context("with node-local resources", func() {
   796  			driver := NewDriver(f, nodes, perNode(1, nodes))
   797  			driver.parameterMode = parameterMode
   798  			b := newBuilder(f, driver)
   799  
   800  			tests := func(allocationMode resourcev1alpha2.AllocationMode) {
   801  				ginkgo.It("uses all resources", func(ctx context.Context) {
   802  					objs, _ := b.flexibleParameters()
   803  					var pods []*v1.Pod
   804  					for i := 0; i < len(nodes.NodeNames); i++ {
   805  						pod, template := b.podInline(allocationMode)
   806  						pods = append(pods, pod)
   807  						objs = append(objs, pod, template)
   808  					}
   809  					b.create(ctx, objs...)
   810  
   811  					for _, pod := range pods {
   812  						err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
   813  						framework.ExpectNoError(err, "start pod")
   814  					}
   815  
   816  					// The pods all should run on different
   817  					// nodes because the maximum number of
   818  					// claims per node was limited to 1 for
   819  					// this test.
   820  					//
   821  					// We cannot know for sure why the pods
   822  					// ran on two different nodes (could
   823  					// also be a coincidence) but if they
   824  					// don't cover all nodes, then we have
   825  					// a problem.
   826  					used := make(map[string]*v1.Pod)
   827  					for _, pod := range pods {
   828  						pod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
   829  						framework.ExpectNoError(err, "get pod")
   830  						nodeName := pod.Spec.NodeName
   831  						if other, ok := used[nodeName]; ok {
   832  							framework.Failf("Pod %s got started on the same node %s as pod %s although claim allocation should have been limited to one claim per node.", pod.Name, nodeName, other.Name)
   833  						}
   834  						used[nodeName] = pod
   835  					}
   836  				})
   837  			}
   838  
   839  			ginkgo.Context("with delayed allocation", func() {
   840  				tests(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   841  			})
   842  
   843  			ginkgo.Context("with immediate allocation", func() {
   844  				tests(resourcev1alpha2.AllocationModeImmediate)
   845  			})
   846  		})
   847  	}
   848  
   849  	tests := func(parameterMode parameterMode) {
   850  		ginkgo.Context("on single node", func() {
   851  			singleNodeTests(parameterMode)
   852  		})
   853  		ginkgo.Context("on multiple nodes", func() {
   854  			multiNodeTests(parameterMode)
   855  		})
   856  	}
   857  
   858  	ginkgo.Context("with ConfigMap parameters", func() { tests(parameterModeConfigMap) })
   859  	ginkgo.Context("with translated parameters", func() { tests(parameterModeTranslated) })
   860  	ginkgo.Context("with structured parameters", func() { tests(parameterModeStructured) })
   861  
   862  	// TODO (https://github.com/kubernetes/kubernetes/issues/123699): move most of the test below into `testDriver` so that they get
   863  	// executed with different parameters.
   864  
   865  	ginkgo.Context("cluster", func() {
   866  		nodes := NewNodes(f, 1, 1)
   867  		driver := NewDriver(f, nodes, networkResources)
   868  		b := newBuilder(f, driver)
   869  
   870  		ginkgo.It("truncates the name of a generated resource claim", func(ctx context.Context) {
   871  			parameters := b.parameters()
   872  			pod, template := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   873  			pod.Name = strings.Repeat("p", 63)
   874  			pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63)
   875  			pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name
   876  			b.create(ctx, parameters, template, pod)
   877  
   878  			b.testPod(ctx, f.ClientSet, pod)
   879  		})
   880  	})
   881  
   882  	// The following tests are all about behavior in combination with a
   883  	// control-plane DRA driver controller.
   884  	ginkgo.Context("cluster with DRA driver controller", func() {
   885  		nodes := NewNodes(f, 1, 4)
   886  
   887  		ginkgo.Context("with structured parameters", func() {
   888  			driver := NewDriver(f, nodes, perNode(1, nodes))
   889  			driver.parameterMode = parameterModeStructured
   890  
   891  			f.It("must manage ResourceSlices", f.WithSlow(), func(ctx context.Context) {
   892  				nodeName := nodes.NodeNames[0]
   893  				driverName := driver.Name
   894  
   895  				// Check for gRPC call on one node. If that already fails, then
   896  				// we have a fundamental problem.
   897  				m := MethodInstance{nodeName, NodeListAndWatchResourcesMethod}
   898  				ginkgo.By("wait for NodeListAndWatchResources call")
   899  				gomega.Eventually(ctx, func() int64 {
   900  					return driver.CallCount(m)
   901  				}).WithTimeout(podStartTimeout).Should(gomega.BeNumerically(">", int64(0)), "NodeListAndWatchResources call count")
   902  
   903  				// Now check for exactly the right set of objects for all nodes.
   904  				ginkgo.By("check if ResourceSlice object(s) exist on the API server")
   905  				resourceClient := f.ClientSet.ResourceV1alpha2().ResourceSlices()
   906  				var expectedObjects []any
   907  				for _, nodeName := range nodes.NodeNames {
   908  					node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
   909  					framework.ExpectNoError(err, "get node")
   910  					expectedObjects = append(expectedObjects,
   911  						gstruct.MatchAllFields(gstruct.Fields{
   912  							"TypeMeta": gstruct.Ignore(),
   913  							"ObjectMeta": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
   914  								"OwnerReferences": gomega.ContainElements(
   915  									gstruct.MatchAllFields(gstruct.Fields{
   916  										"APIVersion":         gomega.Equal("v1"),
   917  										"Kind":               gomega.Equal("Node"),
   918  										"Name":               gomega.Equal(nodeName),
   919  										"UID":                gomega.Equal(node.UID),
   920  										"Controller":         gomega.Equal(ptr.To(true)),
   921  										"BlockOwnerDeletion": gomega.BeNil(),
   922  									}),
   923  								),
   924  							}),
   925  							"NodeName":   gomega.Equal(nodeName),
   926  							"DriverName": gomega.Equal(driver.Name),
   927  							"ResourceModel": gomega.Equal(resourcev1alpha2.ResourceModel{NamedResources: &resourcev1alpha2.NamedResourcesResources{
   928  								Instances: []resourcev1alpha2.NamedResourcesInstance{{Name: "instance-0"}},
   929  							}}),
   930  						}),
   931  					)
   932  				}
   933  				matchSlices := gomega.ContainElements(expectedObjects...)
   934  				getSlices := func(ctx context.Context) ([]resourcev1alpha2.ResourceSlice, error) {
   935  					slices, err := resourceClient.List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("driverName=%s", driverName)})
   936  					if err != nil {
   937  						return nil, err
   938  					}
   939  					return slices.Items, nil
   940  				}
   941  				gomega.Eventually(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices)
   942  				gomega.Consistently(ctx, getSlices).WithTimeout(20 * time.Second).Should(matchSlices)
   943  
   944  				// Removal of node resource slice is tested by the general driver removal code.
   945  			})
   946  
   947  			// TODO (https://github.com/kubernetes/kubernetes/issues/123699): more test scenarios:
   948  			// - driver returns "unimplemented" as method response
   949  			// - driver returns "Unimplemented" as part of stream
   950  			// - driver returns EOF
   951  			// - driver changes resources
   952  			//
   953  			// None of those matter if the publishing gets moved into the driver itself,
   954  			// which is the goal for 1.31 to support version skew for kubelet.
   955  		})
   956  
   957  		ginkgo.Context("with local unshared resources", func() {
   958  			driver := NewDriver(f, nodes, func() app.Resources {
   959  				return app.Resources{
   960  					NodeLocal:      true,
   961  					MaxAllocations: 10,
   962  					Nodes:          nodes.NodeNames,
   963  				}
   964  			})
   965  			b := newBuilder(f, driver)
   966  
   967  			// This test covers some special code paths in the scheduler:
   968  			// - Patching the ReservedFor during PreBind because in contrast
   969  			//   to claims specifically allocated for a pod, here the claim
   970  			//   gets allocated without reserving it.
   971  			// - Error handling when PreBind fails: multiple attempts to bind pods
   972  			//   are started concurrently, only one attempt succeeds.
   973  			// - Removing a ReservedFor entry because the first inline claim gets
   974  			//   reserved during allocation.
   975  			ginkgo.It("reuses an allocated immediate claim", func(ctx context.Context) {
   976  				objects := []klog.KMetadata{
   977  					b.parameters(),
   978  					b.externalClaim(resourcev1alpha2.AllocationModeImmediate),
   979  				}
   980  				podExternal := b.podExternal()
   981  
   982  				// Create many pods to increase the chance that the scheduler will
   983  				// try to bind two pods at the same time.
   984  				numPods := 5
   985  				for i := 0; i < numPods; i++ {
   986  					podInline, claimTemplate := b.podInline(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
   987  					podInline.Spec.Containers[0].Resources.Claims = append(podInline.Spec.Containers[0].Resources.Claims, podExternal.Spec.Containers[0].Resources.Claims[0])
   988  					podInline.Spec.ResourceClaims = append(podInline.Spec.ResourceClaims, podExternal.Spec.ResourceClaims[0])
   989  					objects = append(objects, claimTemplate, podInline)
   990  				}
   991  				b.create(ctx, objects...)
   992  
   993  				var runningPod *v1.Pod
   994  				haveRunningPod := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) {
   995  					numRunning := 0
   996  					runningPod = nil
   997  					for _, pod := range pods {
   998  						if pod.Status.Phase == v1.PodRunning {
   999  							pod := pod // Don't keep pointer to loop variable...
  1000  							runningPod = &pod
  1001  							numRunning++
  1002  						}
  1003  					}
  1004  					return numRunning == 1, nil
  1005  				}).WithTemplate("Expected one running Pod.\nGot instead:\n{{.FormattedActual}}")
  1006  
  1007  				for i := 0; i < numPods; i++ {
  1008  					ginkgo.By("waiting for exactly one pod to start")
  1009  					runningPod = nil
  1010  					gomega.Eventually(ctx, b.listTestPods).WithTimeout(f.Timeouts.PodStartSlow).Should(haveRunningPod)
  1011  
  1012  					ginkgo.By("checking that no other pod gets scheduled")
  1013  					havePendingPods := gcustom.MakeMatcher(func(pods []v1.Pod) (bool, error) {
  1014  						numPending := 0
  1015  						for _, pod := range pods {
  1016  							if pod.Status.Phase == v1.PodPending {
  1017  								numPending++
  1018  							}
  1019  						}
  1020  						return numPending == numPods-1-i, nil
  1021  					}).WithTemplate("Expected only one running Pod.\nGot instead:\n{{.FormattedActual}}")
  1022  					gomega.Consistently(ctx, b.listTestPods).WithTimeout(time.Second).Should(havePendingPods)
  1023  
  1024  					ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(runningPod)))
  1025  					framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, runningPod.Name, metav1.DeleteOptions{}))
  1026  
  1027  					ginkgo.By(fmt.Sprintf("waiting for pod %s to disappear", klog.KObj(runningPod)))
  1028  					framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, b.f.ClientSet, runningPod.Name, runningPod.Namespace, f.Timeouts.PodDelete))
  1029  				}
  1030  			})
  1031  		})
  1032  
  1033  		ginkgo.Context("with shared network resources", func() {
  1034  			driver := NewDriver(f, nodes, networkResources)
  1035  			b := newBuilder(f, driver)
  1036  
  1037  			// This test complements "reuses an allocated immediate claim" above:
  1038  			// because the claim can be shared, each PreBind attempt succeeds.
  1039  			ginkgo.It("shares an allocated immediate claim", func(ctx context.Context) {
  1040  				objects := []klog.KMetadata{
  1041  					b.parameters(),
  1042  					b.externalClaim(resourcev1alpha2.AllocationModeImmediate),
  1043  				}
  1044  				// Create many pods to increase the chance that the scheduler will
  1045  				// try to bind two pods at the same time.
  1046  				numPods := 5
  1047  				pods := make([]*v1.Pod, numPods)
  1048  				for i := 0; i < numPods; i++ {
  1049  					pods[i] = b.podExternal()
  1050  					objects = append(objects, pods[i])
  1051  				}
  1052  				b.create(ctx, objects...)
  1053  
  1054  				ginkgo.By("waiting all pods to start")
  1055  				framework.ExpectNoError(e2epod.WaitForPodsRunning(ctx, b.f.ClientSet, f.Namespace.Name, numPods+len(nodes.NodeNames) /* driver(s) */, f.Timeouts.PodStartSlow))
  1056  			})
  1057  		})
  1058  
  1059  		// kube-controller-manager can trigger delayed allocation for pods where the
  1060  		// node name was already selected when creating the pod. For immediate
  1061  		// allocation, the creator has to ensure that the node matches the claims.
  1062  		// This does not work for resource claim templates and only isn't
  1063  		// a problem here because the resource is network-attached and available
  1064  		// on all nodes.
  1065  		preScheduledTests := func(b *builder, driver *Driver, allocationMode resourcev1alpha2.AllocationMode) {
  1066  			ginkgo.It("supports scheduled pod referencing inline resource claim", func(ctx context.Context) {
  1067  				parameters := b.parameters()
  1068  				pod, template := b.podInline(allocationMode)
  1069  				pod.Spec.NodeName = nodes.NodeNames[0]
  1070  				b.create(ctx, parameters, pod, template)
  1071  
  1072  				b.testPod(ctx, f.ClientSet, pod)
  1073  			})
  1074  
  1075  			ginkgo.It("supports scheduled pod referencing external resource claim", func(ctx context.Context) {
  1076  				parameters := b.parameters()
  1077  				claim := b.externalClaim(allocationMode)
  1078  				pod := b.podExternal()
  1079  				pod.Spec.NodeName = nodes.NodeNames[0]
  1080  				b.create(ctx, parameters, claim, pod)
  1081  
  1082  				b.testPod(ctx, f.ClientSet, pod)
  1083  			})
  1084  		}
  1085  
  1086  		ginkgo.Context("with delayed allocation and setting ReservedFor", func() {
  1087  			driver := NewDriver(f, nodes, networkResources)
  1088  			b := newBuilder(f, driver)
  1089  			preScheduledTests(b, driver, resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1090  			claimTests(b, driver, resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1091  		})
  1092  
  1093  		ginkgo.Context("with delayed allocation and not setting ReservedFor", func() {
  1094  			driver := NewDriver(f, nodes, func() app.Resources {
  1095  				resources := networkResources()
  1096  				resources.DontSetReservedFor = true
  1097  				return resources
  1098  			})
  1099  			b := newBuilder(f, driver)
  1100  			preScheduledTests(b, driver, resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1101  			claimTests(b, driver, resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1102  		})
  1103  
  1104  		ginkgo.Context("with immediate allocation", func() {
  1105  			driver := NewDriver(f, nodes, networkResources)
  1106  			b := newBuilder(f, driver)
  1107  			preScheduledTests(b, driver, resourcev1alpha2.AllocationModeImmediate)
  1108  			claimTests(b, driver, resourcev1alpha2.AllocationModeImmediate)
  1109  		})
  1110  	})
  1111  
  1112  	multipleDrivers := func(nodeV1alpha2, nodeV1alpha3 bool) {
  1113  		nodes := NewNodes(f, 1, 4)
  1114  		driver1 := NewDriver(f, nodes, perNode(2, nodes))
  1115  		driver1.NodeV1alpha2 = nodeV1alpha2
  1116  		driver1.NodeV1alpha3 = nodeV1alpha3
  1117  		b1 := newBuilder(f, driver1)
  1118  
  1119  		driver2 := NewDriver(f, nodes, perNode(2, nodes))
  1120  		driver2.NameSuffix = "-other"
  1121  		driver2.NodeV1alpha2 = nodeV1alpha2
  1122  		driver2.NodeV1alpha3 = nodeV1alpha3
  1123  		b2 := newBuilder(f, driver2)
  1124  
  1125  		ginkgo.It("work", func(ctx context.Context) {
  1126  			parameters1 := b1.parameters()
  1127  			parameters2 := b2.parameters()
  1128  			claim1 := b1.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1129  			claim1b := b1.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1130  			claim2 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1131  			claim2b := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
  1132  			pod := b1.podExternal()
  1133  			for i, claim := range []*resourcev1alpha2.ResourceClaim{claim1b, claim2, claim2b} {
  1134  				claim := claim
  1135  				pod.Spec.ResourceClaims = append(pod.Spec.ResourceClaims,
  1136  					v1.PodResourceClaim{
  1137  						Name: fmt.Sprintf("claim%d", i+1),
  1138  						Source: v1.ClaimSource{
  1139  							ResourceClaimName: &claim.Name,
  1140  						},
  1141  					},
  1142  				)
  1143  			}
  1144  			b1.create(ctx, parameters1, parameters2, claim1, claim1b, claim2, claim2b, pod)
  1145  			b1.testPod(ctx, f.ClientSet, pod)
  1146  		})
  1147  	}
  1148  	multipleDriversContext := func(prefix string, nodeV1alpha2, nodeV1alpha3 bool) {
  1149  		ginkgo.Context(prefix, func() {
  1150  			multipleDrivers(nodeV1alpha2, nodeV1alpha3)
  1151  		})
  1152  	}
  1153  
  1154  	ginkgo.Context("multiple drivers", func() {
  1155  		multipleDriversContext("using only drapbv1alpha2", true, false)
  1156  		multipleDriversContext("using only drapbv1alpha3", false, true)
  1157  		multipleDriversContext("using both drapbv1alpha2 and drapbv1alpha3", true, true)
  1158  	})
  1159  })
  1160  
  1161  // builder contains a running counter to make objects unique within thir
  1162  // namespace.
  1163  type builder struct {
  1164  	f      *framework.Framework
  1165  	driver *Driver
  1166  
  1167  	podCounter        int
  1168  	parametersCounter int
  1169  	claimCounter      int
  1170  
  1171  	classParametersName string
  1172  }
  1173  
  1174  // className returns the default resource class name.
  1175  func (b *builder) className() string {
  1176  	return b.f.UniqueName + b.driver.NameSuffix + "-class"
  1177  }
  1178  
  1179  // class returns the resource class that the builder's other objects
  1180  // reference.
  1181  func (b *builder) class() *resourcev1alpha2.ResourceClass {
  1182  	class := &resourcev1alpha2.ResourceClass{
  1183  		ObjectMeta: metav1.ObjectMeta{
  1184  			Name: b.className(),
  1185  		},
  1186  		DriverName:           b.driver.Name,
  1187  		SuitableNodes:        b.nodeSelector(),
  1188  		StructuredParameters: ptr.To(b.driver.parameterMode != parameterModeConfigMap),
  1189  	}
  1190  	if b.classParametersName != "" {
  1191  		class.ParametersRef = &resourcev1alpha2.ResourceClassParametersReference{
  1192  			APIGroup:  b.driver.parameterAPIGroup,
  1193  			Kind:      b.driver.classParameterAPIKind,
  1194  			Name:      b.classParametersName,
  1195  			Namespace: b.f.Namespace.Name,
  1196  		}
  1197  	}
  1198  	return class
  1199  }
  1200  
  1201  // nodeSelector returns a node selector that matches all nodes on which the
  1202  // kubelet plugin was deployed.
  1203  func (b *builder) nodeSelector() *v1.NodeSelector {
  1204  	return &v1.NodeSelector{
  1205  		NodeSelectorTerms: []v1.NodeSelectorTerm{
  1206  			{
  1207  				MatchExpressions: []v1.NodeSelectorRequirement{
  1208  					{
  1209  						Key:      "kubernetes.io/hostname",
  1210  						Operator: v1.NodeSelectorOpIn,
  1211  						Values:   b.driver.Nodenames(),
  1212  					},
  1213  				},
  1214  			},
  1215  		},
  1216  	}
  1217  }
  1218  
  1219  // externalClaim returns external resource claim
  1220  // that test pods can reference
  1221  func (b *builder) externalClaim(allocationMode resourcev1alpha2.AllocationMode) *resourcev1alpha2.ResourceClaim {
  1222  	b.claimCounter++
  1223  	name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects.
  1224  	if b.claimCounter > 1 {
  1225  		name += fmt.Sprintf("-%d", b.claimCounter)
  1226  	}
  1227  	return &resourcev1alpha2.ResourceClaim{
  1228  		ObjectMeta: metav1.ObjectMeta{
  1229  			Name: name,
  1230  		},
  1231  		Spec: resourcev1alpha2.ResourceClaimSpec{
  1232  			ResourceClassName: b.className(),
  1233  			ParametersRef: &resourcev1alpha2.ResourceClaimParametersReference{
  1234  				APIGroup: b.driver.parameterAPIGroup,
  1235  				Kind:     b.driver.claimParameterAPIKind,
  1236  				Name:     b.parametersName(),
  1237  			},
  1238  			AllocationMode: allocationMode,
  1239  		},
  1240  	}
  1241  }
  1242  
  1243  // flexibleParameters returns parameter objects for claims and
  1244  // class with their type depending on the current parameter mode.
  1245  // It also returns the expected environment in a pod using
  1246  // the corresponding resource.
  1247  func (b *builder) flexibleParameters() ([]klog.KMetadata, []string) {
  1248  	var objects []klog.KMetadata
  1249  	switch b.driver.parameterMode {
  1250  	case parameterModeConfigMap:
  1251  		objects = append(objects,
  1252  			b.parameters("x", "y"),
  1253  			b.parameters("a", "b", "request_foo", "bar"),
  1254  		)
  1255  	case parameterModeTranslated:
  1256  		objects = append(objects,
  1257  			b.parameters("x", "y"),
  1258  			b.classParameters(b.parametersName(), "x", "y"),
  1259  			b.parameters("a", "b", "request_foo", "bar"),
  1260  			b.claimParameters(b.parametersName(), []string{"a", "b"}, []string{"request_foo", "bar"}),
  1261  		)
  1262  		// The parameters object is not the last one but the second-last.
  1263  		b.parametersCounter--
  1264  	case parameterModeStructured:
  1265  		objects = append(objects,
  1266  			b.classParameters("", "x", "y"),
  1267  			b.claimParameters("", []string{"a", "b"}, []string{"request_foo", "bar"}),
  1268  		)
  1269  	}
  1270  	env := []string{"user_a", "b", "user_request_foo", "bar"}
  1271  	if b.classParametersName != "" {
  1272  		env = append(env, "admin_x", "y")
  1273  	}
  1274  	return objects, env
  1275  }
  1276  
  1277  // parametersName returns the current ConfigMap name for resource
  1278  // claim or class parameters.
  1279  func (b *builder) parametersName() string {
  1280  	return fmt.Sprintf("parameters%s-%d", b.driver.NameSuffix, b.parametersCounter)
  1281  }
  1282  
  1283  // parametersEnv returns the default env variables.
  1284  func (b *builder) parametersEnv() map[string]string {
  1285  	return map[string]string{
  1286  		"a":           "b",
  1287  		"request_foo": "bar",
  1288  	}
  1289  }
  1290  
  1291  // parameters returns a config map with the default env variables.
  1292  func (b *builder) parameters(kv ...string) *v1.ConfigMap {
  1293  	data := b.parameterData(kv...)
  1294  	b.parametersCounter++
  1295  	return &v1.ConfigMap{
  1296  		ObjectMeta: metav1.ObjectMeta{
  1297  			Namespace: b.f.Namespace.Name,
  1298  			Name:      b.parametersName(),
  1299  		},
  1300  		Data: data,
  1301  	}
  1302  }
  1303  
  1304  func (b *builder) classParameters(generatedFrom string, kv ...string) *resourcev1alpha2.ResourceClassParameters {
  1305  	raw := b.rawParameterData(kv...)
  1306  	b.parametersCounter++
  1307  	parameters := &resourcev1alpha2.ResourceClassParameters{
  1308  		ObjectMeta: metav1.ObjectMeta{
  1309  			Namespace: b.f.Namespace.Name,
  1310  			Name:      b.parametersName(),
  1311  		},
  1312  
  1313  		VendorParameters: []resourcev1alpha2.VendorParameters{
  1314  			{DriverName: b.driver.Name, Parameters: runtime.RawExtension{Raw: raw}},
  1315  		},
  1316  	}
  1317  
  1318  	if generatedFrom != "" {
  1319  		parameters.GeneratedFrom = &resourcev1alpha2.ResourceClassParametersReference{
  1320  			Kind:      "ConfigMap",
  1321  			Namespace: b.f.Namespace.Name,
  1322  			Name:      generatedFrom,
  1323  		}
  1324  	}
  1325  
  1326  	return parameters
  1327  }
  1328  
  1329  func (b *builder) claimParameters(generatedFrom string, claimKV, requestKV []string) *resourcev1alpha2.ResourceClaimParameters {
  1330  	b.parametersCounter++
  1331  	parameters := &resourcev1alpha2.ResourceClaimParameters{
  1332  		ObjectMeta: metav1.ObjectMeta{
  1333  			Namespace: b.f.Namespace.Name,
  1334  			Name:      b.parametersName(),
  1335  		},
  1336  
  1337  		Shareable: true,
  1338  
  1339  		// Without any request, nothing gets allocated and vendor
  1340  		// parameters are also not passed down because they get
  1341  		// attached to the allocation result.
  1342  		DriverRequests: []resourcev1alpha2.DriverRequests{
  1343  			{
  1344  				DriverName:       b.driver.Name,
  1345  				VendorParameters: runtime.RawExtension{Raw: b.rawParameterData(claimKV...)},
  1346  				Requests: []resourcev1alpha2.ResourceRequest{
  1347  					{
  1348  						VendorParameters: runtime.RawExtension{Raw: b.rawParameterData(requestKV...)},
  1349  						ResourceRequestModel: resourcev1alpha2.ResourceRequestModel{
  1350  							NamedResources: &resourcev1alpha2.NamedResourcesRequest{
  1351  								Selector: "true",
  1352  							},
  1353  						},
  1354  					},
  1355  				},
  1356  			},
  1357  		},
  1358  	}
  1359  
  1360  	if generatedFrom != "" {
  1361  		parameters.GeneratedFrom = &resourcev1alpha2.ResourceClaimParametersReference{
  1362  			Kind: "ConfigMap",
  1363  			Name: generatedFrom,
  1364  		}
  1365  	}
  1366  
  1367  	return parameters
  1368  }
  1369  
  1370  func (b *builder) parameterData(kv ...string) map[string]string {
  1371  	data := map[string]string{}
  1372  	for i := 0; i < len(kv); i += 2 {
  1373  		data[kv[i]] = kv[i+1]
  1374  	}
  1375  	if len(data) == 0 {
  1376  		data = b.parametersEnv()
  1377  	}
  1378  	return data
  1379  }
  1380  
  1381  func (b *builder) rawParameterData(kv ...string) []byte {
  1382  	data := b.parameterData(kv...)
  1383  	raw, err := json.Marshal(data)
  1384  	framework.ExpectNoError(err, "JSON encoding of parameter data")
  1385  	return raw
  1386  }
  1387  
  1388  // makePod returns a simple pod with no resource claims.
  1389  // The pod prints its env and waits.
  1390  func (b *builder) pod() *v1.Pod {
  1391  	pod := e2epod.MakePod(b.f.Namespace.Name, nil, nil, b.f.NamespacePodSecurityLevel, "env && sleep 100000")
  1392  	pod.Labels = make(map[string]string)
  1393  	pod.Spec.RestartPolicy = v1.RestartPolicyNever
  1394  	// Let kubelet kill the pods quickly. Setting
  1395  	// TerminationGracePeriodSeconds to zero would bypass kubelet
  1396  	// completely because then the apiserver enables a force-delete even
  1397  	// when DeleteOptions for the pod don't ask for it (see
  1398  	// https://github.com/kubernetes/kubernetes/blob/0f582f7c3f504e807550310d00f130cb5c18c0c3/pkg/registry/core/pod/strategy.go#L151-L171).
  1399  	//
  1400  	// We don't do that because it breaks tracking of claim usage: the
  1401  	// kube-controller-manager assumes that kubelet is done with the pod
  1402  	// once it got removed or has a grace period of 0. Setting the grace
  1403  	// period to zero directly in DeletionOptions or indirectly through
  1404  	// TerminationGracePeriodSeconds causes the controller to remove
  1405  	// the pod from ReservedFor before it actually has stopped on
  1406  	// the node.
  1407  	one := int64(1)
  1408  	pod.Spec.TerminationGracePeriodSeconds = &one
  1409  	pod.ObjectMeta.GenerateName = ""
  1410  	b.podCounter++
  1411  	pod.ObjectMeta.Name = fmt.Sprintf("tester%s-%d", b.driver.NameSuffix, b.podCounter)
  1412  	return pod
  1413  }
  1414  
  1415  // makePodInline adds an inline resource claim with default class name and parameters.
  1416  func (b *builder) podInline(allocationMode resourcev1alpha2.AllocationMode) (*v1.Pod, *resourcev1alpha2.ResourceClaimTemplate) {
  1417  	pod := b.pod()
  1418  	pod.Spec.Containers[0].Name = "with-resource"
  1419  	podClaimName := "my-inline-claim"
  1420  	pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
  1421  	pod.Spec.ResourceClaims = []v1.PodResourceClaim{
  1422  		{
  1423  			Name: podClaimName,
  1424  			Source: v1.ClaimSource{
  1425  				ResourceClaimTemplateName: utilpointer.String(pod.Name),
  1426  			},
  1427  		},
  1428  	}
  1429  	template := &resourcev1alpha2.ResourceClaimTemplate{
  1430  		ObjectMeta: metav1.ObjectMeta{
  1431  			Name:      pod.Name,
  1432  			Namespace: pod.Namespace,
  1433  		},
  1434  		Spec: resourcev1alpha2.ResourceClaimTemplateSpec{
  1435  			Spec: resourcev1alpha2.ResourceClaimSpec{
  1436  				ResourceClassName: b.className(),
  1437  				ParametersRef: &resourcev1alpha2.ResourceClaimParametersReference{
  1438  					APIGroup: b.driver.parameterAPIGroup,
  1439  					Kind:     b.driver.claimParameterAPIKind,
  1440  					Name:     b.parametersName(),
  1441  				},
  1442  				AllocationMode: allocationMode,
  1443  			},
  1444  		},
  1445  	}
  1446  	return pod, template
  1447  }
  1448  
  1449  // podInlineMultiple returns a pod with inline resource claim referenced by 3 containers
  1450  func (b *builder) podInlineMultiple(allocationMode resourcev1alpha2.AllocationMode) (*v1.Pod, *resourcev1alpha2.ResourceClaimTemplate) {
  1451  	pod, template := b.podInline(allocationMode)
  1452  	pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy())
  1453  	pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1"
  1454  	pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2"
  1455  	return pod, template
  1456  }
  1457  
  1458  // podExternal adds a pod that references external resource claim with default class name and parameters.
  1459  func (b *builder) podExternal() *v1.Pod {
  1460  	pod := b.pod()
  1461  	pod.Spec.Containers[0].Name = "with-resource"
  1462  	podClaimName := "resource-claim"
  1463  	externalClaimName := "external-claim" + b.driver.NameSuffix
  1464  	pod.Spec.ResourceClaims = []v1.PodResourceClaim{
  1465  		{
  1466  			Name: podClaimName,
  1467  			Source: v1.ClaimSource{
  1468  				ResourceClaimName: &externalClaimName,
  1469  			},
  1470  		},
  1471  	}
  1472  	pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
  1473  	return pod
  1474  }
  1475  
  1476  // podShared returns a pod with 3 containers that reference external resource claim with default class name and parameters.
  1477  func (b *builder) podExternalMultiple() *v1.Pod {
  1478  	pod := b.podExternal()
  1479  	pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy(), *pod.Spec.Containers[0].DeepCopy())
  1480  	pod.Spec.Containers[1].Name = pod.Spec.Containers[1].Name + "-1"
  1481  	pod.Spec.Containers[2].Name = pod.Spec.Containers[1].Name + "-2"
  1482  	return pod
  1483  }
  1484  
  1485  // create takes a bunch of objects and calls their Create function.
  1486  func (b *builder) create(ctx context.Context, objs ...klog.KMetadata) []klog.KMetadata {
  1487  	var createdObjs []klog.KMetadata
  1488  	for _, obj := range objs {
  1489  		ginkgo.By(fmt.Sprintf("creating %T %s", obj, obj.GetName()))
  1490  		var err error
  1491  		var createdObj klog.KMetadata
  1492  		switch obj := obj.(type) {
  1493  		case *resourcev1alpha2.ResourceClass:
  1494  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceClasses().Create(ctx, obj, metav1.CreateOptions{})
  1495  			ginkgo.DeferCleanup(func(ctx context.Context) {
  1496  				err := b.f.ClientSet.ResourceV1alpha2().ResourceClasses().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{})
  1497  				framework.ExpectNoError(err, "delete resource class")
  1498  			})
  1499  		case *v1.Pod:
  1500  			createdObj, err = b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1501  		case *v1.ConfigMap:
  1502  			createdObj, err = b.f.ClientSet.CoreV1().ConfigMaps(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1503  		case *resourcev1alpha2.ResourceClaim:
  1504  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1505  		case *resourcev1alpha2.ResourceClaimTemplate:
  1506  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1507  		case *resourcev1alpha2.ResourceClassParameters:
  1508  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceClassParameters(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1509  		case *resourcev1alpha2.ResourceClaimParameters:
  1510  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceClaimParameters(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{})
  1511  		case *resourcev1alpha2.ResourceSlice:
  1512  			createdObj, err = b.f.ClientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, obj, metav1.CreateOptions{})
  1513  			ginkgo.DeferCleanup(func(ctx context.Context) {
  1514  				err := b.f.ClientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{})
  1515  				framework.ExpectNoError(err, "delete node resource slice")
  1516  			})
  1517  		default:
  1518  			framework.Fail(fmt.Sprintf("internal error, unsupported type %T", obj), 1)
  1519  		}
  1520  		framework.ExpectNoErrorWithOffset(1, err, "create %T", obj)
  1521  		createdObjs = append(createdObjs, createdObj)
  1522  	}
  1523  	return createdObjs
  1524  }
  1525  
  1526  // testPod runs pod and checks if container logs contain expected environment variables
  1527  func (b *builder) testPod(ctx context.Context, clientSet kubernetes.Interface, pod *v1.Pod, env ...string) {
  1528  	err := e2epod.WaitForPodRunningInNamespace(ctx, clientSet, pod)
  1529  	framework.ExpectNoError(err, "start pod")
  1530  
  1531  	for _, container := range pod.Spec.Containers {
  1532  		log, err := e2epod.GetPodLogs(ctx, clientSet, pod.Namespace, pod.Name, container.Name)
  1533  		framework.ExpectNoError(err, "get logs")
  1534  		if len(env) == 0 {
  1535  			for key, value := range b.parametersEnv() {
  1536  				envStr := fmt.Sprintf("\nuser_%s=%s\n", key, value)
  1537  				gomega.Expect(log).To(gomega.ContainSubstring(envStr), "container env variables")
  1538  			}
  1539  		} else {
  1540  			for i := 0; i < len(env); i += 2 {
  1541  				envStr := fmt.Sprintf("\n%s=%s\n", env[i], env[i+1])
  1542  				gomega.Expect(log).To(gomega.ContainSubstring(envStr), "container env variables")
  1543  			}
  1544  		}
  1545  	}
  1546  }
  1547  
  1548  func newBuilder(f *framework.Framework, driver *Driver) *builder {
  1549  	b := &builder{f: f, driver: driver}
  1550  
  1551  	ginkgo.BeforeEach(b.setUp)
  1552  
  1553  	return b
  1554  }
  1555  
  1556  func (b *builder) setUp() {
  1557  	b.podCounter = 0
  1558  	b.parametersCounter = 0
  1559  	b.claimCounter = 0
  1560  	b.create(context.Background(), b.class())
  1561  	ginkgo.DeferCleanup(b.tearDown)
  1562  }
  1563  
  1564  func (b *builder) tearDown(ctx context.Context) {
  1565  	// Before we allow the namespace and all objects in it do be deleted by
  1566  	// the framework, we must ensure that test pods and the claims that
  1567  	// they use are deleted. Otherwise the driver might get deleted first,
  1568  	// in which case deleting the claims won't work anymore.
  1569  	ginkgo.By("delete pods and claims")
  1570  	pods, err := b.listTestPods(ctx)
  1571  	framework.ExpectNoError(err, "list pods")
  1572  	for _, pod := range pods {
  1573  		if pod.DeletionTimestamp != nil {
  1574  			continue
  1575  		}
  1576  		ginkgo.By(fmt.Sprintf("deleting %T %s", &pod, klog.KObj(&pod)))
  1577  		err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{})
  1578  		if !apierrors.IsNotFound(err) {
  1579  			framework.ExpectNoError(err, "delete pod")
  1580  		}
  1581  	}
  1582  	gomega.Eventually(func() ([]v1.Pod, error) {
  1583  		return b.listTestPods(ctx)
  1584  	}).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "remaining pods despite deletion")
  1585  
  1586  	claims, err := b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
  1587  	framework.ExpectNoError(err, "get resource claims")
  1588  	for _, claim := range claims.Items {
  1589  		if claim.DeletionTimestamp != nil {
  1590  			continue
  1591  		}
  1592  		ginkgo.By(fmt.Sprintf("deleting %T %s", &claim, klog.KObj(&claim)))
  1593  		err := b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Delete(ctx, claim.Name, metav1.DeleteOptions{})
  1594  		if !apierrors.IsNotFound(err) {
  1595  			framework.ExpectNoError(err, "delete claim")
  1596  		}
  1597  	}
  1598  
  1599  	for host, plugin := range b.driver.Nodes {
  1600  		ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host))
  1601  		gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host)
  1602  	}
  1603  
  1604  	ginkgo.By("waiting for claims to be deallocated and deleted")
  1605  	gomega.Eventually(func() ([]resourcev1alpha2.ResourceClaim, error) {
  1606  		claims, err := b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
  1607  		if err != nil {
  1608  			return nil, err
  1609  		}
  1610  		return claims.Items, nil
  1611  	}).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "claims in the namespaces")
  1612  }
  1613  
  1614  func (b *builder) listTestPods(ctx context.Context) ([]v1.Pod, error) {
  1615  	pods, err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).List(ctx, metav1.ListOptions{})
  1616  	if err != nil {
  1617  		return nil, err
  1618  	}
  1619  
  1620  	var testPods []v1.Pod
  1621  	for _, pod := range pods.Items {
  1622  		if pod.Labels["app.kubernetes.io/part-of"] == "dra-test-driver" {
  1623  			continue
  1624  		}
  1625  		testPods = append(testPods, pod)
  1626  	}
  1627  	return testPods, nil
  1628  }
  1629
View as plain text