nvidia-gpus.go

Documentation: k8s.io/kubernetes/test/e2e/scheduling

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2017 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package scheduling
    21  
    22  import (
    23  	"context"
    24  	"os"
    25  	"regexp"
    26  	"time"
    27  
    28  	appsv1 "k8s.io/api/apps/v1"
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/apimachinery/pkg/api/resource"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/util/uuid"
    33  	extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions"
    34  	"k8s.io/kubernetes/test/e2e/feature"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
    37  	e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu"
    38  	e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
    39  	e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest"
    40  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    41  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    42  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    43  	e2eresource "k8s.io/kubernetes/test/e2e/framework/resource"
    44  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    45  	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
    46  	imageutils "k8s.io/kubernetes/test/utils/image"
    47  	admissionapi "k8s.io/pod-security-admission/api"
    48  
    49  	"github.com/onsi/ginkgo/v2"
    50  	"github.com/onsi/gomega"
    51  )
    52  
    53  const (
    54  	testPodNamePrefix = "nvidia-gpu-"
    55  	// Nvidia driver installation can take upwards of 5 minutes.
    56  	driverInstallTimeout = 10 * time.Minute
    57  )
    58  
    59  var (
    60  	gpuResourceName v1.ResourceName
    61  )
    62  
    63  func makeCudaAdditionDevicePluginTestPod() *v1.Pod {
    64  	podName := testPodNamePrefix + string(uuid.NewUUID())
    65  	testContainers := []v1.Container{
    66  		{
    67  			Name:  "vector-addition-cuda8",
    68  			Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd),
    69  			Resources: v1.ResourceRequirements{
    70  				Limits: v1.ResourceList{
    71  					gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
    72  				},
    73  			},
    74  		},
    75  		{
    76  			Name:  "vector-addition-cuda10",
    77  			Image: imageutils.GetE2EImage(imageutils.CudaVectorAdd2),
    78  			Resources: v1.ResourceRequirements{
    79  				Limits: v1.ResourceList{
    80  					gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
    81  				},
    82  			},
    83  		},
    84  	}
    85  	testPod := &v1.Pod{
    86  		ObjectMeta: metav1.ObjectMeta{
    87  			Name: podName,
    88  		},
    89  		Spec: v1.PodSpec{
    90  			RestartPolicy: v1.RestartPolicyNever,
    91  		},
    92  	}
    93  
    94  	testPod.Spec.Containers = testContainers
    95  	if os.Getenv("TEST_MAX_GPU_COUNT") == "1" {
    96  		testPod.Spec.Containers = []v1.Container{testContainers[0]}
    97  	}
    98  	framework.Logf("testPod.Spec.Containers {%#v}", testPod.Spec.Containers)
    99  	return testPod
   100  }
   101  
   102  func logOSImages(ctx context.Context, f *framework.Framework) {
   103  	nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
   104  	framework.ExpectNoError(err, "getting node list")
   105  	for _, node := range nodeList.Items {
   106  		framework.Logf("Nodename: %v, OS Image: %v", node.Name, node.Status.NodeInfo.OSImage)
   107  	}
   108  }
   109  
   110  func isControlPlaneNode(node v1.Node) bool {
   111  	_, isControlPlane := node.Labels["node-role.kubernetes.io/control-plane"]
   112  	if isControlPlane {
   113  		framework.Logf("Node: %q is a control-plane node (label)", node.Name)
   114  		return true
   115  	}
   116  
   117  	for _, taint := range node.Spec.Taints {
   118  		if taint.Key == "node-role.kubernetes.io/control-plane" {
   119  			framework.Logf("Node: %q is a control-plane node (taint)", node.Name)
   120  			return true
   121  		}
   122  	}
   123  	framework.Logf("Node: %q is NOT a control-plane node", node.Name)
   124  	return false
   125  }
   126  
   127  func areGPUsAvailableOnAllSchedulableNodes(ctx context.Context, f *framework.Framework) bool {
   128  	framework.Logf("Getting list of Nodes from API server")
   129  	nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
   130  	framework.ExpectNoError(err, "getting node list")
   131  	for _, node := range nodeList.Items {
   132  		if node.Spec.Unschedulable || isControlPlaneNode(node) {
   133  			continue
   134  		}
   135  
   136  		if val, ok := node.Status.Capacity[gpuResourceName]; !ok || val.Value() == 0 {
   137  			framework.Logf("Nvidia GPUs not available on Node: %q", node.Name)
   138  			return false
   139  		}
   140  	}
   141  	framework.Logf("Nvidia GPUs exist on all schedulable worker nodes")
   142  	return true
   143  }
   144  
   145  func getGPUsAvailable(ctx context.Context, f *framework.Framework) int64 {
   146  	nodeList, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
   147  	framework.ExpectNoError(err, "getting node list")
   148  	var gpusAvailable int64
   149  	for _, node := range nodeList.Items {
   150  		if val, ok := node.Status.Allocatable[gpuResourceName]; ok {
   151  			gpusAvailable += (&val).Value()
   152  		}
   153  	}
   154  	return gpusAvailable
   155  }
   156  
   157  // SetupNVIDIAGPUNode install Nvidia Drivers and wait for Nvidia GPUs to be available on nodes
   158  func SetupNVIDIAGPUNode(ctx context.Context, f *framework.Framework, setupResourceGatherer bool) *e2edebug.ContainerResourceGatherer {
   159  	logOSImages(ctx, f)
   160  
   161  	var err error
   162  	var ds *appsv1.DaemonSet
   163  	dsYamlURLFromEnv := os.Getenv("NVIDIA_DRIVER_INSTALLER_DAEMONSET")
   164  	if dsYamlURLFromEnv != "" {
   165  		// Using DaemonSet from remote URL
   166  		framework.Logf("Using remote nvidia-driver-installer daemonset manifest from %v", dsYamlURLFromEnv)
   167  		ds, err = e2emanifest.DaemonSetFromURL(ctx, dsYamlURLFromEnv)
   168  		framework.ExpectNoError(err, "failed get remote")
   169  	} else {
   170  		// Using default local DaemonSet
   171  		framework.Logf("Using default local nvidia-driver-installer daemonset manifest.")
   172  		data, err := e2etestfiles.Read("test/e2e/testing-manifests/scheduling/nvidia-driver-installer.yaml")
   173  		framework.ExpectNoError(err, "failed to read local manifest for nvidia-driver-installer daemonset")
   174  		ds, err = e2emanifest.DaemonSetFromData(data)
   175  		framework.ExpectNoError(err, "failed to parse local manifest for nvidia-driver-installer daemonset")
   176  	}
   177  	gpuResourceName = e2egpu.NVIDIAGPUResourceName
   178  	ds.Namespace = f.Namespace.Name
   179  
   180  	_, err = f.ClientSet.AppsV1().DaemonSets(ds.Namespace).Create(ctx, ds, metav1.CreateOptions{})
   181  	framework.ExpectNoError(err, "failed to create nvidia-driver-installer daemonset")
   182  	framework.Logf("Successfully created daemonset to install Nvidia drivers.")
   183  
   184  	pods, err := e2eresource.WaitForControlledPods(ctx, f.ClientSet, ds.Namespace, ds.Name, extensionsinternal.Kind("DaemonSet"))
   185  	framework.ExpectNoError(err, "failed to get pods controlled by the nvidia-driver-installer daemonset")
   186  
   187  	devicepluginPods, err := e2eresource.WaitForControlledPods(ctx, f.ClientSet, "kube-system", "nvidia-gpu-device-plugin", extensionsinternal.Kind("DaemonSet"))
   188  	if err == nil {
   189  		framework.Logf("Adding deviceplugin addon pod.")
   190  		pods.Items = append(pods.Items, devicepluginPods.Items...)
   191  	}
   192  
   193  	var rsgather *e2edebug.ContainerResourceGatherer
   194  	if setupResourceGatherer {
   195  		framework.Logf("Starting ResourceUsageGather for the created DaemonSet pods.")
   196  		rsgather, err = e2edebug.NewResourceUsageGatherer(ctx, f.ClientSet, e2edebug.ResourceGathererOptions{InKubemark: false, Nodes: e2edebug.AllNodes, ResourceDataGatheringPeriod: 2 * time.Second, ProbeDuration: 2 * time.Second, PrintVerboseLogs: true}, pods)
   197  		framework.ExpectNoError(err, "creating ResourceUsageGather for the daemonset pods")
   198  		go rsgather.StartGatheringData(ctx)
   199  	}
   200  
   201  	// Wait for Nvidia GPUs to be available on nodes
   202  	framework.Logf("Waiting for drivers to be installed and GPUs to be available in Node Capacity...")
   203  	gomega.Eventually(ctx, func(ctx context.Context) bool {
   204  		return areGPUsAvailableOnAllSchedulableNodes(ctx, f)
   205  	}, driverInstallTimeout, time.Second).Should(gomega.BeTrue())
   206  
   207  	return rsgather
   208  }
   209  
   210  func getGPUsPerPod() int64 {
   211  	var gpusPerPod int64
   212  	gpuPod := makeCudaAdditionDevicePluginTestPod()
   213  	for _, container := range gpuPod.Spec.Containers {
   214  		if val, ok := container.Resources.Limits[gpuResourceName]; ok {
   215  			gpusPerPod += (&val).Value()
   216  		}
   217  	}
   218  	return gpusPerPod
   219  }
   220  
   221  func testNvidiaGPUs(ctx context.Context, f *framework.Framework) {
   222  	rsgather := SetupNVIDIAGPUNode(ctx, f, true)
   223  	gpuPodNum := getGPUsAvailable(ctx, f) / getGPUsPerPod()
   224  	framework.Logf("Creating %d pods and have the pods run a CUDA app", gpuPodNum)
   225  	podList := []*v1.Pod{}
   226  	for i := int64(0); i < gpuPodNum; i++ {
   227  		podList = append(podList, e2epod.NewPodClient(f).Create(ctx, makeCudaAdditionDevicePluginTestPod()))
   228  	}
   229  	framework.Logf("Wait for all test pods to succeed")
   230  	// Wait for all pods to succeed
   231  	for _, pod := range podList {
   232  		e2epod.NewPodClient(f).WaitForSuccess(ctx, pod.Name, 5*time.Minute)
   233  		logContainers(ctx, f, pod)
   234  	}
   235  
   236  	framework.Logf("Stopping ResourceUsageGather")
   237  	constraints := make(map[string]e2edebug.ResourceConstraint)
   238  	// For now, just gets summary. Can pass valid constraints in the future.
   239  	summary, err := rsgather.StopAndSummarize([]int{50, 90, 100}, constraints)
   240  	f.TestSummaries = append(f.TestSummaries, summary)
   241  	framework.ExpectNoError(err, "getting resource usage summary")
   242  }
   243  
   244  func logContainers(ctx context.Context, f *framework.Framework, pod *v1.Pod) {
   245  	for _, container := range pod.Spec.Containers {
   246  		logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, pod.Name, container.Name)
   247  		framework.ExpectNoError(err, "Should be able to get container logs for container: %s", container.Name)
   248  		framework.Logf("Got container logs for %s:\n%v", container.Name, logs)
   249  	}
   250  }
   251  
   252  var _ = SIGDescribe(feature.GPUDevicePlugin, func() {
   253  	f := framework.NewDefaultFramework("device-plugin-gpus")
   254  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
   255  	ginkgo.It("run Nvidia GPU Device Plugin tests", func(ctx context.Context) {
   256  		testNvidiaGPUs(ctx, f)
   257  	})
   258  })
   259  
   260  func testNvidiaGPUsJob(ctx context.Context, f *framework.Framework) {
   261  	_ = SetupNVIDIAGPUNode(ctx, f, false)
   262  	// Job set to have 5 completions with parallelism of 1 to ensure that it lasts long enough to experience the node recreation
   263  	completions := int32(5)
   264  	ginkgo.By("Starting GPU job")
   265  	StartJob(ctx, f, completions)
   266  
   267  	job, err := e2ejob.GetJob(ctx, f.ClientSet, f.Namespace.Name, "cuda-add")
   268  	framework.ExpectNoError(err)
   269  
   270  	// make sure job is running by waiting for its first pod to start running
   271  	err = e2ejob.WaitForJobPodsRunning(ctx, f.ClientSet, f.Namespace.Name, job.Name, 1)
   272  	framework.ExpectNoError(err)
   273  
   274  	numNodes, err := e2enode.TotalRegistered(ctx, f.ClientSet)
   275  	framework.ExpectNoError(err)
   276  	nodes, err := e2enode.CheckReady(ctx, f.ClientSet, numNodes, framework.NodeReadyInitialTimeout)
   277  	framework.ExpectNoError(err)
   278  
   279  	ginkgo.By("Recreating nodes")
   280  	err = gce.RecreateNodes(f.ClientSet, nodes)
   281  	framework.ExpectNoError(err)
   282  	ginkgo.By("Done recreating nodes")
   283  
   284  	ginkgo.By("Waiting for gpu job to finish")
   285  	err = e2ejob.WaitForJobFinish(ctx, f.ClientSet, f.Namespace.Name, job.Name)
   286  	framework.ExpectNoError(err)
   287  	ginkgo.By("Done with gpu job")
   288  
   289  	gomega.Expect(job.Status.Failed).To(gomega.BeZero(), "Job pods failed during node recreation: %v", job.Status.Failed)
   290  
   291  	VerifyJobNCompletions(ctx, f, completions)
   292  }
   293  
   294  // StartJob starts a simple CUDA job that requests gpu and the specified number of completions
   295  func StartJob(ctx context.Context, f *framework.Framework, completions int32) {
   296  	var activeSeconds int64 = 3600
   297  	testJob := e2ejob.NewTestJob("succeed", "cuda-add", v1.RestartPolicyAlways, 1, completions, &activeSeconds, 6)
   298  	testJob.Spec.Template.Spec = v1.PodSpec{
   299  		RestartPolicy: v1.RestartPolicyOnFailure,
   300  		Containers: []v1.Container{
   301  			{
   302  				Name:    "vector-addition",
   303  				Image:   imageutils.GetE2EImage(imageutils.CudaVectorAdd),
   304  				Command: []string{"/bin/sh", "-c", "./vectorAdd && sleep 60"},
   305  				Resources: v1.ResourceRequirements{
   306  					Limits: v1.ResourceList{
   307  						gpuResourceName: *resource.NewQuantity(1, resource.DecimalSI),
   308  					},
   309  				},
   310  			},
   311  		},
   312  	}
   313  	ns := f.Namespace.Name
   314  	_, err := e2ejob.CreateJob(ctx, f.ClientSet, ns, testJob)
   315  	framework.ExpectNoError(err)
   316  	framework.Logf("Created job %v", testJob)
   317  }
   318  
   319  // VerifyJobNCompletions verifies that the job has completions number of successful pods
   320  func VerifyJobNCompletions(ctx context.Context, f *framework.Framework, completions int32) {
   321  	ns := f.Namespace.Name
   322  	pods, err := e2ejob.GetJobPods(ctx, f.ClientSet, f.Namespace.Name, "cuda-add")
   323  	framework.ExpectNoError(err)
   324  	createdPods := pods.Items
   325  	createdPodNames := podNames(createdPods)
   326  	framework.Logf("Got the following pods for job cuda-add: %v", createdPodNames)
   327  
   328  	successes := int32(0)
   329  	regex := regexp.MustCompile("PASSED")
   330  	for _, podName := range createdPodNames {
   331  		e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 5*time.Minute)
   332  		logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, ns, podName, "vector-addition")
   333  		framework.ExpectNoError(err, "Should be able to get logs for pod %v", podName)
   334  		if regex.MatchString(logs) {
   335  			successes++
   336  		}
   337  	}
   338  	if successes != completions {
   339  		framework.Failf("Only got %v completions. Expected %v completions.", successes, completions)
   340  	}
   341  }
   342  
   343  func podNames(pods []v1.Pod) []string {
   344  	originalPodNames := make([]string, len(pods))
   345  	for i, p := range pods {
   346  		originalPodNames[i] = p.ObjectMeta.Name
   347  	}
   348  	return originalPodNames
   349  }
   350  
   351  var _ = SIGDescribe("GPUDevicePluginAcrossRecreate", feature.Recreate, func() {
   352  	ginkgo.BeforeEach(func() {
   353  		e2eskipper.SkipUnlessProviderIs("gce", "gke")
   354  	})
   355  	f := framework.NewDefaultFramework("device-plugin-gpus-recreate")
   356  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
   357  	ginkgo.It("run Nvidia GPU Device Plugin tests with a recreation", func(ctx context.Context) {
   358  		testNvidiaGPUsJob(ctx, f)
   359  	})
   360  })
   361
View as plain text