pod_devices_test.go

Documentation: k8s.io/kubernetes/pkg/kubelet/cm/devicemanager

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package devicemanager
    18  
    19  import (
    20  	"encoding/json"
    21  	"testing"
    22  
    23  	"github.com/stretchr/testify/assert"
    24  	"github.com/stretchr/testify/require"
    25  
    26  	"k8s.io/apimachinery/pkg/util/sets"
    27  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    28  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    29  	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
    30  	"k8s.io/kubernetes/pkg/features"
    31  	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
    32  	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
    33  )
    34  
    35  func TestGetContainerDevices(t *testing.T) {
    36  	podDevices := newPodDevices()
    37  	resourceName1 := "domain1.com/resource1"
    38  	podID := "pod1"
    39  	contID := "con1"
    40  	devices := checkpoint.DevicesPerNUMA{0: []string{"dev1"}, 1: []string{"dev1"}}
    41  
    42  	podDevices.insert(podID, contID, resourceName1,
    43  		devices,
    44  		newContainerAllocateResponse(
    45  			withDevices(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"}),
    46  			withMounts(map[string]string{"/home/r1lib1": "/usr/r1lib1"}),
    47  		),
    48  	)
    49  
    50  	resContDevices := podDevices.getContainerDevices(podID, contID)
    51  	contDevices, ok := resContDevices[resourceName1]
    52  	require.True(t, ok, "resource %q not present", resourceName1)
    53  
    54  	for devID, plugInfo := range contDevices {
    55  		nodes := plugInfo.GetTopology().GetNodes()
    56  		require.Equal(t, len(nodes), len(devices), "Incorrect container devices: %v - %v (nodes %v)", devices, contDevices, nodes)
    57  
    58  		for _, node := range plugInfo.GetTopology().GetNodes() {
    59  			dev, ok := devices[node.ID]
    60  			require.True(t, ok, "NUMA id %v doesn't exist in result", node.ID)
    61  			require.Equal(t, devID, dev[0], "Can't find device %s in result", dev[0])
    62  		}
    63  	}
    64  }
    65  
    66  func TestResourceDeviceInstanceFilter(t *testing.T) {
    67  	var expected string
    68  	var cond map[string]sets.Set[string]
    69  	var resp ResourceDeviceInstances
    70  	devs := ResourceDeviceInstances{
    71  		"foo": DeviceInstances{
    72  			"dev-foo1": pluginapi.Device{
    73  				ID: "foo1",
    74  			},
    75  			"dev-foo2": pluginapi.Device{
    76  				ID: "foo2",
    77  			},
    78  			"dev-foo3": pluginapi.Device{
    79  				ID: "foo3",
    80  			},
    81  		},
    82  		"bar": DeviceInstances{
    83  			"dev-bar1": pluginapi.Device{
    84  				ID: "bar1",
    85  			},
    86  			"dev-bar2": pluginapi.Device{
    87  				ID: "bar2",
    88  			},
    89  			"dev-bar3": pluginapi.Device{
    90  				ID: "bar3",
    91  			},
    92  		},
    93  		"baz": DeviceInstances{
    94  			"dev-baz1": pluginapi.Device{
    95  				ID: "baz1",
    96  			},
    97  			"dev-baz2": pluginapi.Device{
    98  				ID: "baz2",
    99  			},
   100  			"dev-baz3": pluginapi.Device{
   101  				ID: "baz3",
   102  			},
   103  		},
   104  	}
   105  
   106  	resp = devs.Filter(map[string]sets.Set[string]{})
   107  	expected = `{}`
   108  	expectResourceDeviceInstances(t, resp, expected)
   109  
   110  	cond = map[string]sets.Set[string]{
   111  		"foo": sets.New[string]("dev-foo1", "dev-foo2"),
   112  		"bar": sets.New[string]("dev-bar1"),
   113  	}
   114  	resp = devs.Filter(cond)
   115  	expected = `{"bar":{"dev-bar1":{"ID":"bar1"}},"foo":{"dev-foo1":{"ID":"foo1"},"dev-foo2":{"ID":"foo2"}}}`
   116  	expectResourceDeviceInstances(t, resp, expected)
   117  
   118  	cond = map[string]sets.Set[string]{
   119  		"foo": sets.New[string]("dev-foo1", "dev-foo2", "dev-foo3"),
   120  		"bar": sets.New[string]("dev-bar1", "dev-bar2", "dev-bar3"),
   121  		"baz": sets.New[string]("dev-baz1", "dev-baz2", "dev-baz3"),
   122  	}
   123  	resp = devs.Filter(cond)
   124  	expected = `{"bar":{"dev-bar1":{"ID":"bar1"},"dev-bar2":{"ID":"bar2"},"dev-bar3":{"ID":"bar3"}},"baz":{"dev-baz1":{"ID":"baz1"},"dev-baz2":{"ID":"baz2"},"dev-baz3":{"ID":"baz3"}},"foo":{"dev-foo1":{"ID":"foo1"},"dev-foo2":{"ID":"foo2"},"dev-foo3":{"ID":"foo3"}}}`
   125  	expectResourceDeviceInstances(t, resp, expected)
   126  
   127  	cond = map[string]sets.Set[string]{
   128  		"foo": sets.New[string]("dev-foo1", "dev-foo2", "dev-foo3", "dev-foo4"),
   129  		"bar": sets.New[string]("dev-bar1", "dev-bar2", "dev-bar3", "dev-bar4"),
   130  		"baz": sets.New[string]("dev-baz1", "dev-baz2", "dev-baz3", "dev-bar4"),
   131  	}
   132  	resp = devs.Filter(cond)
   133  	expected = `{"bar":{"dev-bar1":{"ID":"bar1"},"dev-bar2":{"ID":"bar2"},"dev-bar3":{"ID":"bar3"}},"baz":{"dev-baz1":{"ID":"baz1"},"dev-baz2":{"ID":"baz2"},"dev-baz3":{"ID":"baz3"}},"foo":{"dev-foo1":{"ID":"foo1"},"dev-foo2":{"ID":"foo2"},"dev-foo3":{"ID":"foo3"}}}`
   134  	expectResourceDeviceInstances(t, resp, expected)
   135  
   136  	cond = map[string]sets.Set[string]{
   137  		"foo": sets.New[string]("dev-foo1", "dev-foo4", "dev-foo7"),
   138  		"bar": sets.New[string]("dev-bar1", "dev-bar4", "dev-bar7"),
   139  		"baz": sets.New[string]("dev-baz1", "dev-baz4", "dev-baz7"),
   140  	}
   141  	resp = devs.Filter(cond)
   142  	expected = `{"bar":{"dev-bar1":{"ID":"bar1"}},"baz":{"dev-baz1":{"ID":"baz1"}},"foo":{"dev-foo1":{"ID":"foo1"}}}`
   143  	expectResourceDeviceInstances(t, resp, expected)
   144  
   145  }
   146  
   147  func expectResourceDeviceInstances(t *testing.T, resp ResourceDeviceInstances, expected string) {
   148  	// per docs in https://pkg.go.dev/encoding/json#Marshal
   149  	// "Map values encode as JSON objects. The map's key type must either be a string, an integer type, or
   150  	// implement encoding.TextMarshaler. The map keys are sorted [...]"
   151  	// so this check is expected to be stable and not flaky
   152  	data, err := json.Marshal(resp)
   153  	if err != nil {
   154  		t.Fatalf("unexpected JSON marshalling error: %v", err)
   155  	}
   156  	got := string(data)
   157  	if got != expected {
   158  		t.Errorf("expected %q got %q", expected, got)
   159  	}
   160  }
   161  
   162  func TestDeviceRunContainerOptions(t *testing.T) {
   163  	const (
   164  		podUID        = "pod"
   165  		containerName = "container"
   166  		resource1     = "example1.com/resource1"
   167  		resource2     = "example2.com/resource2"
   168  	)
   169  	testCases := []struct {
   170  		description          string
   171  		gate                 bool
   172  		responsesPerResource map[string]*pluginapi.ContainerAllocateResponse
   173  		expected             *DeviceRunContainerOptions
   174  	}{
   175  		{
   176  			description: "empty response",
   177  			gate:        false,
   178  			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
   179  				resource1: newContainerAllocateResponse(),
   180  			},
   181  			expected: &DeviceRunContainerOptions{},
   182  		},
   183  		{
   184  			description: "cdi devices are ingored when feature gate is disabled",
   185  			gate:        false,
   186  			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
   187  				resource1: newContainerAllocateResponse(
   188  					withDevices(map[string]string{"/dev/r1": "/dev/r1"}),
   189  					withMounts(map[string]string{"/home/lib1": "/home/lib1"}),
   190  					withEnvs(map[string]string{"ENV1": "VALUE1"}),
   191  					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
   192  				),
   193  			},
   194  			expected: &DeviceRunContainerOptions{
   195  				Devices: []kubecontainer.DeviceInfo{
   196  					{PathOnHost: "/dev/r1", PathInContainer: "/dev/r1", Permissions: "mrw"},
   197  				},
   198  				Mounts: []kubecontainer.Mount{
   199  					{Name: "/home/lib1", HostPath: "/home/lib1", ContainerPath: "/home/lib1", ReadOnly: true},
   200  				},
   201  				Envs: []kubecontainer.EnvVar{
   202  					{Name: "ENV1", Value: "VALUE1"},
   203  				},
   204  			},
   205  		},
   206  		{
   207  			description: "cdi devices are handled when feature gate is enabled",
   208  			gate:        true,
   209  			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
   210  				resource1: newContainerAllocateResponse(
   211  					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
   212  				),
   213  			},
   214  			expected: &DeviceRunContainerOptions{
   215  				Annotations: []kubecontainer.Annotation{
   216  					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2"},
   217  				},
   218  				CDIDevices: []kubecontainer.CDIDevice{
   219  					{Name: "vendor1.com/class1=device1"},
   220  					{Name: "vendor2.com/class2=device2"},
   221  				},
   222  			},
   223  		},
   224  		{
   225  			description: "cdi devices from multiple resources are handled when feature gate is enabled",
   226  			gate:        true,
   227  			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
   228  				resource1: newContainerAllocateResponse(
   229  					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
   230  				),
   231  				resource2: newContainerAllocateResponse(
   232  					withCDIDevices("vendor3.com/class3=device3", "vendor4.com/class4=device4"),
   233  				),
   234  			},
   235  			expected: &DeviceRunContainerOptions{
   236  				Annotations: []kubecontainer.Annotation{
   237  					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3,vendor4.com/class4=device4"},
   238  				},
   239  				CDIDevices: []kubecontainer.CDIDevice{
   240  					{Name: "vendor1.com/class1=device1"},
   241  					{Name: "vendor2.com/class2=device2"},
   242  					{Name: "vendor3.com/class3=device3"},
   243  					{Name: "vendor4.com/class4=device4"},
   244  				},
   245  			},
   246  		},
   247  		{
   248  			description: "duplicate cdi devices are skipped",
   249  			gate:        true,
   250  			responsesPerResource: map[string]*pluginapi.ContainerAllocateResponse{
   251  				resource1: newContainerAllocateResponse(
   252  					withCDIDevices("vendor1.com/class1=device1", "vendor2.com/class2=device2"),
   253  				),
   254  				resource2: newContainerAllocateResponse(
   255  					withCDIDevices("vendor2.com/class2=device2", "vendor3.com/class3=device3"),
   256  				),
   257  			},
   258  			expected: &DeviceRunContainerOptions{
   259  				Annotations: []kubecontainer.Annotation{
   260  					{Name: "cdi.k8s.io/devicemanager_pod-container", Value: "vendor1.com/class1=device1,vendor2.com/class2=device2,vendor3.com/class3=device3"},
   261  				},
   262  				CDIDevices: []kubecontainer.CDIDevice{
   263  					{Name: "vendor1.com/class1=device1"},
   264  					{Name: "vendor2.com/class2=device2"},
   265  					{Name: "vendor3.com/class3=device3"},
   266  				},
   267  			},
   268  		},
   269  	}
   270  
   271  	for _, tc := range testCases {
   272  		t.Run(tc.description, func(t *testing.T) {
   273  			as := assert.New(t)
   274  
   275  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DevicePluginCDIDevices, tc.gate)()
   276  			podDevices := newPodDevices()
   277  			for resourceName, response := range tc.responsesPerResource {
   278  				podDevices.insert("pod", "container", resourceName,
   279  					nil,
   280  					response,
   281  				)
   282  			}
   283  			opts := podDevices.deviceRunContainerOptions(podUID, containerName)
   284  
   285  			// The exact ordering of the options depends on the order of the resources in the map.
   286  			// We therefore use `ElementsMatch` instead of `Equal` on the member slices.
   287  			as.ElementsMatch(tc.expected.Annotations, opts.Annotations)
   288  			as.ElementsMatch(tc.expected.CDIDevices, opts.CDIDevices)
   289  			as.ElementsMatch(tc.expected.Devices, opts.Devices)
   290  			as.ElementsMatch(tc.expected.Envs, opts.Envs)
   291  			as.ElementsMatch(tc.expected.Mounts, opts.Mounts)
   292  		})
   293  	}
   294  }
   295
View as plain text