1
16
17 package nodevolumelimits
18
19 import (
20 "context"
21 "errors"
22 "fmt"
23 "reflect"
24 "strings"
25 "testing"
26
27 "github.com/google/go-cmp/cmp"
28 v1 "k8s.io/api/core/v1"
29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30 csilibplugins "k8s.io/csi-translation-lib/plugins"
31 "k8s.io/klog/v2/ktesting"
32 "k8s.io/kubernetes/pkg/scheduler/framework"
33 "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
34 st "k8s.io/kubernetes/pkg/scheduler/testing"
35 tf "k8s.io/kubernetes/pkg/scheduler/testing/framework"
36 "k8s.io/utils/ptr"
37 )
38
39 var (
40 nonApplicablePod = st.MakePod().Volume(v1.Volume{
41 VolumeSource: v1.VolumeSource{
42 HostPath: &v1.HostPathVolumeSource{},
43 },
44 }).Obj()
45 onlyConfigmapAndSecretPod = st.MakePod().Volume(v1.Volume{
46 VolumeSource: v1.VolumeSource{
47 ConfigMap: &v1.ConfigMapVolumeSource{},
48 },
49 }).Volume(v1.Volume{
50 VolumeSource: v1.VolumeSource{
51 Secret: &v1.SecretVolumeSource{},
52 },
53 }).Obj()
54 pvcPodWithConfigmapAndSecret = st.MakePod().PVC("pvcWithDeletedPV").Volume(v1.Volume{
55 VolumeSource: v1.VolumeSource{
56 ConfigMap: &v1.ConfigMapVolumeSource{},
57 },
58 }).Volume(v1.Volume{
59 VolumeSource: v1.VolumeSource{
60 Secret: &v1.SecretVolumeSource{},
61 },
62 }).Obj()
63
64 deletedPVCPod = st.MakePod().PVC("deletedPVC").Obj()
65 twoDeletedPVCPod = st.MakePod().PVC("deletedPVC").PVC("anotherDeletedPVC").Obj()
66 deletedPVPod = st.MakePod().PVC("pvcWithDeletedPV").Obj()
67
68 deletedPVPod2 = st.MakePod().PVC("pvcWithDeletedPV").Obj()
69 anotherDeletedPVPod = st.MakePod().PVC("anotherPVCWithDeletedPV").Obj()
70 emptyPod = st.MakePod().Obj()
71 unboundPVCPod = st.MakePod().PVC("unboundPVC").Obj()
72
73 unboundPVCPod2 = st.MakePod().PVC("unboundPVC").Obj()
74
75 anotherUnboundPVCPod = st.MakePod().PVC("anotherUnboundPVC").Obj()
76 )
77
78 func TestEphemeralLimits(t *testing.T) {
79
80
81 filterName := gcePDVolumeFilterType
82 driverName := csilibplugins.GCEPDInTreePluginName
83
84 ephemeralVolumePod := st.MakePod().Name("abc").Namespace("test").UID("12345").Volume(v1.Volume{
85 Name: "xyz",
86 VolumeSource: v1.VolumeSource{
87 Ephemeral: &v1.EphemeralVolumeSource{},
88 },
89 }).Obj()
90
91 controller := true
92 ephemeralClaim := &v1.PersistentVolumeClaim{
93 ObjectMeta: metav1.ObjectMeta{
94 Namespace: ephemeralVolumePod.Namespace,
95 Name: ephemeralVolumePod.Name + "-" + ephemeralVolumePod.Spec.Volumes[0].Name,
96 OwnerReferences: []metav1.OwnerReference{
97 {
98 Kind: "Pod",
99 Name: ephemeralVolumePod.Name,
100 UID: ephemeralVolumePod.UID,
101 Controller: &controller,
102 },
103 },
104 },
105 Spec: v1.PersistentVolumeClaimSpec{
106 VolumeName: "missing",
107 StorageClassName: &filterName,
108 },
109 }
110 conflictingClaim := ephemeralClaim.DeepCopy()
111 conflictingClaim.OwnerReferences = nil
112
113 ephemeralPodWithConfigmapAndSecret := st.MakePod().Name("abc").Namespace("test").UID("12345").Volume(v1.Volume{
114 Name: "xyz",
115 VolumeSource: v1.VolumeSource{
116 Ephemeral: &v1.EphemeralVolumeSource{},
117 },
118 }).Volume(v1.Volume{
119 VolumeSource: v1.VolumeSource{
120 ConfigMap: &v1.ConfigMapVolumeSource{},
121 },
122 }).Volume(v1.Volume{
123 VolumeSource: v1.VolumeSource{
124 Secret: &v1.SecretVolumeSource{},
125 },
126 }).Obj()
127
128 tests := []struct {
129 newPod *v1.Pod
130 existingPods []*v1.Pod
131 extraClaims []v1.PersistentVolumeClaim
132 ephemeralEnabled bool
133 maxVols int32
134 test string
135 wantStatus *framework.Status
136 wantPreFilterStatus *framework.Status
137 }{
138 {
139 newPod: ephemeralVolumePod,
140 ephemeralEnabled: true,
141 test: "volume missing",
142 wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, `looking up PVC test/abc-xyz: persistentvolumeclaims "abc-xyz" not found`),
143 },
144 {
145 newPod: ephemeralVolumePod,
146 ephemeralEnabled: true,
147 extraClaims: []v1.PersistentVolumeClaim{*conflictingClaim},
148 test: "volume not owned",
149 wantStatus: framework.AsStatus(errors.New("PVC test/abc-xyz was not created for pod test/abc (pod is not owner)")),
150 },
151 {
152 newPod: ephemeralVolumePod,
153 ephemeralEnabled: true,
154 extraClaims: []v1.PersistentVolumeClaim{*ephemeralClaim},
155 maxVols: 1,
156 test: "volume unbound, allowed",
157 },
158 {
159 newPod: ephemeralVolumePod,
160 ephemeralEnabled: true,
161 extraClaims: []v1.PersistentVolumeClaim{*ephemeralClaim},
162 maxVols: 0,
163 test: "volume unbound, exceeds limit",
164 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
165 },
166 {
167 newPod: onlyConfigmapAndSecretPod,
168 ephemeralEnabled: true,
169 extraClaims: []v1.PersistentVolumeClaim{*ephemeralClaim},
170 maxVols: 1,
171 test: "skip Filter when the pod only uses secrets and configmaps",
172 wantPreFilterStatus: framework.NewStatus(framework.Skip),
173 },
174 {
175 newPod: ephemeralPodWithConfigmapAndSecret,
176 ephemeralEnabled: true,
177 extraClaims: []v1.PersistentVolumeClaim{*ephemeralClaim},
178 maxVols: 1,
179 test: "don't skip Filter when the pods has ephemeral volumes",
180 },
181 }
182
183 for _, test := range tests {
184 t.Run(test.test, func(t *testing.T) {
185 _, ctx := ktesting.NewTestContext(t)
186 fts := feature.Features{}
187 node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, test.maxVols, filterName)
188 p := newNonCSILimits(ctx, filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(filterName, driverName), getFakePVLister(filterName), append(getFakePVCLister(filterName), test.extraClaims...), fts).(framework.FilterPlugin)
189 _, gotPreFilterStatus := p.(*nonCSILimits).PreFilter(ctx, nil, test.newPod)
190 if diff := cmp.Diff(test.wantPreFilterStatus, gotPreFilterStatus); diff != "" {
191 t.Errorf("PreFilter status does not match (-want, +got): %s", diff)
192 }
193
194 if gotPreFilterStatus.Code() != framework.Skip {
195 gotStatus := p.Filter(ctx, nil, test.newPod, node)
196 if !reflect.DeepEqual(gotStatus, test.wantStatus) {
197 t.Errorf("Filter status does not match: %v, want: %v", gotStatus, test.wantStatus)
198 }
199 }
200 })
201 }
202 }
203
204 func TestAzureDiskLimits(t *testing.T) {
205 oneAzureDiskPod := st.MakePod().Volume(v1.Volume{
206 VolumeSource: v1.VolumeSource{
207 AzureDisk: &v1.AzureDiskVolumeSource{},
208 },
209 }).Obj()
210 twoAzureDiskPod := st.MakePod().Volume(v1.Volume{
211 VolumeSource: v1.VolumeSource{
212 AzureDisk: &v1.AzureDiskVolumeSource{},
213 },
214 }).Volume(v1.Volume{
215 VolumeSource: v1.VolumeSource{
216 AzureDisk: &v1.AzureDiskVolumeSource{},
217 },
218 }).Obj()
219 splitAzureDiskPod := st.MakePod().Volume(v1.Volume{
220 VolumeSource: v1.VolumeSource{
221 HostPath: &v1.HostPathVolumeSource{},
222 },
223 }).Volume(v1.Volume{
224 VolumeSource: v1.VolumeSource{
225 AzureDisk: &v1.AzureDiskVolumeSource{},
226 },
227 }).Obj()
228 AzureDiskPodWithConfigmapAndSecret := st.MakePod().Volume(v1.Volume{
229 VolumeSource: v1.VolumeSource{
230 AzureDisk: &v1.AzureDiskVolumeSource{},
231 },
232 }).Volume(v1.Volume{
233 VolumeSource: v1.VolumeSource{
234 ConfigMap: &v1.ConfigMapVolumeSource{},
235 },
236 }).Volume(v1.Volume{
237 VolumeSource: v1.VolumeSource{
238 Secret: &v1.SecretVolumeSource{},
239 },
240 }).Obj()
241 tests := []struct {
242 newPod *v1.Pod
243 existingPods []*v1.Pod
244 filterName string
245 driverName string
246 maxVols int32
247 test string
248 wantStatus *framework.Status
249 wantPreFilterStatus *framework.Status
250 }{
251 {
252 newPod: oneAzureDiskPod,
253 existingPods: []*v1.Pod{twoAzureDiskPod, oneAzureDiskPod},
254 filterName: azureDiskVolumeFilterType,
255 maxVols: 4,
256 test: "fits when node capacity >= new pod's AzureDisk volumes",
257 },
258 {
259 newPod: twoAzureDiskPod,
260 existingPods: []*v1.Pod{oneAzureDiskPod},
261 filterName: azureDiskVolumeFilterType,
262 maxVols: 2,
263 test: "fit when node capacity < new pod's AzureDisk volumes",
264 },
265 {
266 newPod: splitAzureDiskPod,
267 existingPods: []*v1.Pod{twoAzureDiskPod},
268 filterName: azureDiskVolumeFilterType,
269 maxVols: 3,
270 test: "new pod's count ignores non-AzureDisk volumes",
271 },
272 {
273 newPod: twoAzureDiskPod,
274 existingPods: []*v1.Pod{splitAzureDiskPod, nonApplicablePod, emptyPod},
275 filterName: azureDiskVolumeFilterType,
276 maxVols: 3,
277 test: "existing pods' counts ignore non-AzureDisk volumes",
278 },
279 {
280 newPod: onePVCPod(azureDiskVolumeFilterType),
281 existingPods: []*v1.Pod{splitAzureDiskPod, nonApplicablePod, emptyPod},
282 filterName: azureDiskVolumeFilterType,
283 maxVols: 3,
284 test: "new pod's count considers PVCs backed by AzureDisk volumes",
285 },
286 {
287 newPod: splitPVCPod(azureDiskVolumeFilterType),
288 existingPods: []*v1.Pod{splitAzureDiskPod, oneAzureDiskPod},
289 filterName: azureDiskVolumeFilterType,
290 maxVols: 3,
291 test: "new pod's count ignores PVCs not backed by AzureDisk volumes",
292 },
293 {
294 newPod: twoAzureDiskPod,
295 existingPods: []*v1.Pod{oneAzureDiskPod, onePVCPod(azureDiskVolumeFilterType)},
296 filterName: azureDiskVolumeFilterType,
297 maxVols: 3,
298 test: "existing pods' counts considers PVCs backed by AzureDisk volumes",
299 },
300 {
301 newPod: twoAzureDiskPod,
302 existingPods: []*v1.Pod{oneAzureDiskPod, twoAzureDiskPod, onePVCPod(azureDiskVolumeFilterType)},
303 filterName: azureDiskVolumeFilterType,
304 maxVols: 4,
305 test: "already-mounted AzureDisk volumes are always ok to allow",
306 },
307 {
308 newPod: splitAzureDiskPod,
309 existingPods: []*v1.Pod{oneAzureDiskPod, oneAzureDiskPod, onePVCPod(azureDiskVolumeFilterType)},
310 filterName: azureDiskVolumeFilterType,
311 maxVols: 3,
312 test: "the same AzureDisk volumes are not counted multiple times",
313 },
314 {
315 newPod: onePVCPod(azureDiskVolumeFilterType),
316 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVCPod},
317 filterName: azureDiskVolumeFilterType,
318 maxVols: 2,
319 test: "pod with missing PVC is counted towards the PV limit",
320 },
321 {
322 newPod: onePVCPod(azureDiskVolumeFilterType),
323 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVCPod},
324 filterName: azureDiskVolumeFilterType,
325 maxVols: 3,
326 test: "pod with missing PVC is counted towards the PV limit",
327 },
328 {
329 newPod: onePVCPod(azureDiskVolumeFilterType),
330 existingPods: []*v1.Pod{oneAzureDiskPod, twoDeletedPVCPod},
331 filterName: azureDiskVolumeFilterType,
332 maxVols: 3,
333 test: "pod with missing two PVCs is counted towards the PV limit twice",
334 },
335 {
336 newPod: onePVCPod(azureDiskVolumeFilterType),
337 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVPod},
338 filterName: azureDiskVolumeFilterType,
339 maxVols: 2,
340 test: "pod with missing PV is counted towards the PV limit",
341 },
342 {
343 newPod: onePVCPod(azureDiskVolumeFilterType),
344 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVPod},
345 filterName: azureDiskVolumeFilterType,
346 maxVols: 3,
347 test: "pod with missing PV is counted towards the PV limit",
348 },
349 {
350 newPod: deletedPVPod2,
351 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVPod},
352 filterName: azureDiskVolumeFilterType,
353 maxVols: 2,
354 test: "two pods missing the same PV are counted towards the PV limit only once",
355 },
356 {
357 newPod: anotherDeletedPVPod,
358 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVPod},
359 filterName: azureDiskVolumeFilterType,
360 maxVols: 2,
361 test: "two pods missing different PVs are counted towards the PV limit twice",
362 },
363 {
364 newPod: onePVCPod(azureDiskVolumeFilterType),
365 existingPods: []*v1.Pod{oneAzureDiskPod, unboundPVCPod},
366 filterName: azureDiskVolumeFilterType,
367 maxVols: 2,
368 test: "pod with unbound PVC is counted towards the PV limit",
369 },
370 {
371 newPod: onePVCPod(azureDiskVolumeFilterType),
372 existingPods: []*v1.Pod{oneAzureDiskPod, unboundPVCPod},
373 filterName: azureDiskVolumeFilterType,
374 maxVols: 3,
375 test: "pod with unbound PVC is counted towards the PV limit",
376 },
377 {
378 newPod: unboundPVCPod2,
379 existingPods: []*v1.Pod{oneAzureDiskPod, unboundPVCPod},
380 filterName: azureDiskVolumeFilterType,
381 maxVols: 2,
382 test: "the same unbound PVC in multiple pods is counted towards the PV limit only once",
383 },
384 {
385 newPod: anotherUnboundPVCPod,
386 existingPods: []*v1.Pod{oneAzureDiskPod, unboundPVCPod},
387 filterName: azureDiskVolumeFilterType,
388 maxVols: 2,
389 test: "two different unbound PVCs are counted towards the PV limit as two volumes",
390 },
391 {
392 newPod: onlyConfigmapAndSecretPod,
393 existingPods: []*v1.Pod{twoAzureDiskPod, oneAzureDiskPod},
394 filterName: azureDiskVolumeFilterType,
395 maxVols: 4,
396 test: "skip Filter when the pod only uses secrets and configmaps",
397 wantPreFilterStatus: framework.NewStatus(framework.Skip),
398 },
399 {
400 newPod: pvcPodWithConfigmapAndSecret,
401 existingPods: []*v1.Pod{oneAzureDiskPod, deletedPVPod},
402 filterName: azureDiskVolumeFilterType,
403 maxVols: 2,
404 test: "don't skip Filter when the pod has pvcs",
405 },
406 {
407 newPod: AzureDiskPodWithConfigmapAndSecret,
408 existingPods: []*v1.Pod{twoAzureDiskPod, oneAzureDiskPod},
409 filterName: azureDiskVolumeFilterType,
410 maxVols: 4,
411 test: "don't skip Filter when the pod has AzureDisk volumes",
412 },
413 }
414
415 for _, test := range tests {
416 t.Run(test.test, func(t *testing.T) {
417 _, ctx := ktesting.NewTestContext(t)
418 node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, test.maxVols, test.filterName)
419 p := newNonCSILimits(ctx, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin)
420 _, gotPreFilterStatus := p.(*nonCSILimits).PreFilter(context.Background(), nil, test.newPod)
421 if diff := cmp.Diff(test.wantPreFilterStatus, gotPreFilterStatus); diff != "" {
422 t.Errorf("PreFilter status does not match (-want, +got): %s", diff)
423 }
424
425 if gotPreFilterStatus.Code() != framework.Skip {
426 gotStatus := p.Filter(context.Background(), nil, test.newPod, node)
427 if !reflect.DeepEqual(gotStatus, test.wantStatus) {
428 t.Errorf("Filter status does not match: %v, want: %v", gotStatus, test.wantStatus)
429 }
430 }
431 })
432 }
433 }
434
435 func TestEBSLimits(t *testing.T) {
436 oneVolPod := st.MakePod().Volume(v1.Volume{
437 VolumeSource: v1.VolumeSource{
438 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"},
439 },
440 }).Obj()
441 twoVolPod := st.MakePod().Volume(v1.Volume{
442 VolumeSource: v1.VolumeSource{
443 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp1"},
444 },
445 }).Volume(v1.Volume{
446 VolumeSource: v1.VolumeSource{
447 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "tvp2"},
448 },
449 }).Obj()
450 splitVolsPod := st.MakePod().Volume(v1.Volume{
451 VolumeSource: v1.VolumeSource{
452 HostPath: &v1.HostPathVolumeSource{},
453 },
454 }).Volume(v1.Volume{
455 VolumeSource: v1.VolumeSource{
456 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "svp"},
457 },
458 }).Obj()
459
460 unboundPVCWithInvalidSCPod := st.MakePod().PVC("unboundPVCWithInvalidSCPod").Obj()
461 unboundPVCWithDefaultSCPod := st.MakePod().PVC("unboundPVCWithDefaultSCPod").Obj()
462
463 EBSPodWithConfigmapAndSecret := st.MakePod().Volume(v1.Volume{
464 VolumeSource: v1.VolumeSource{
465 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: "ovp"},
466 },
467 }).Volume(v1.Volume{
468 VolumeSource: v1.VolumeSource{
469 ConfigMap: &v1.ConfigMapVolumeSource{},
470 },
471 }).Volume(v1.Volume{
472 VolumeSource: v1.VolumeSource{
473 Secret: &v1.SecretVolumeSource{},
474 },
475 }).Obj()
476
477 tests := []struct {
478 newPod *v1.Pod
479 existingPods []*v1.Pod
480 filterName string
481 driverName string
482 maxVols int32
483 test string
484 wantStatus *framework.Status
485 wantPreFilterStatus *framework.Status
486 }{
487 {
488 newPod: oneVolPod,
489 existingPods: []*v1.Pod{twoVolPod, oneVolPod},
490 filterName: ebsVolumeFilterType,
491 driverName: csilibplugins.AWSEBSInTreePluginName,
492 maxVols: 4,
493 test: "fits when node capacity >= new pod's EBS volumes",
494 },
495 {
496 newPod: twoVolPod,
497 existingPods: []*v1.Pod{oneVolPod},
498 filterName: ebsVolumeFilterType,
499 driverName: csilibplugins.AWSEBSInTreePluginName,
500 maxVols: 2,
501 test: "doesn't fit when node capacity < new pod's EBS volumes",
502 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
503 },
504 {
505 newPod: splitVolsPod,
506 existingPods: []*v1.Pod{twoVolPod},
507 filterName: ebsVolumeFilterType,
508 driverName: csilibplugins.AWSEBSInTreePluginName,
509 maxVols: 3,
510 test: "new pod's count ignores non-EBS volumes",
511 },
512 {
513 newPod: twoVolPod,
514 existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod},
515 filterName: ebsVolumeFilterType,
516 driverName: csilibplugins.AWSEBSInTreePluginName,
517 maxVols: 3,
518 test: "existing pods' counts ignore non-EBS volumes",
519 },
520 {
521 newPod: onePVCPod(ebsVolumeFilterType),
522 existingPods: []*v1.Pod{splitVolsPod, nonApplicablePod, emptyPod},
523 filterName: ebsVolumeFilterType,
524 driverName: csilibplugins.AWSEBSInTreePluginName,
525 maxVols: 3,
526 test: "new pod's count considers PVCs backed by EBS volumes",
527 },
528 {
529 newPod: splitPVCPod(ebsVolumeFilterType),
530 existingPods: []*v1.Pod{splitVolsPod, oneVolPod},
531 filterName: ebsVolumeFilterType,
532 driverName: csilibplugins.AWSEBSInTreePluginName,
533 maxVols: 3,
534 test: "new pod's count ignores PVCs not backed by EBS volumes",
535 },
536 {
537 newPod: twoVolPod,
538 existingPods: []*v1.Pod{oneVolPod, onePVCPod(ebsVolumeFilterType)},
539 filterName: ebsVolumeFilterType,
540 driverName: csilibplugins.AWSEBSInTreePluginName,
541 maxVols: 3,
542 test: "existing pods' counts considers PVCs backed by EBS volumes",
543 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
544 },
545 {
546 newPod: twoVolPod,
547 existingPods: []*v1.Pod{oneVolPod, twoVolPod, onePVCPod(ebsVolumeFilterType)},
548 filterName: ebsVolumeFilterType,
549 driverName: csilibplugins.AWSEBSInTreePluginName,
550 maxVols: 4,
551 test: "already-mounted EBS volumes are always ok to allow",
552 },
553 {
554 newPod: splitVolsPod,
555 existingPods: []*v1.Pod{oneVolPod, oneVolPod, onePVCPod(ebsVolumeFilterType)},
556 filterName: ebsVolumeFilterType,
557 driverName: csilibplugins.AWSEBSInTreePluginName,
558 maxVols: 3,
559 test: "the same EBS volumes are not counted multiple times",
560 },
561 {
562 newPod: onePVCPod(ebsVolumeFilterType),
563 existingPods: []*v1.Pod{oneVolPod, deletedPVCPod},
564 filterName: ebsVolumeFilterType,
565 driverName: csilibplugins.AWSEBSInTreePluginName,
566 maxVols: 1,
567 test: "missing PVC is not counted towards the PV limit",
568 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
569 },
570 {
571 newPod: onePVCPod(ebsVolumeFilterType),
572 existingPods: []*v1.Pod{oneVolPod, deletedPVCPod},
573 filterName: ebsVolumeFilterType,
574 driverName: csilibplugins.AWSEBSInTreePluginName,
575 maxVols: 2,
576 test: "missing PVC is not counted towards the PV limit",
577 },
578 {
579 newPod: onePVCPod(ebsVolumeFilterType),
580 existingPods: []*v1.Pod{oneVolPod, twoDeletedPVCPod},
581 filterName: ebsVolumeFilterType,
582 driverName: csilibplugins.AWSEBSInTreePluginName,
583 maxVols: 2,
584 test: "two missing PVCs are not counted towards the PV limit twice",
585 },
586 {
587 newPod: unboundPVCWithInvalidSCPod,
588 existingPods: []*v1.Pod{oneVolPod},
589 filterName: ebsVolumeFilterType,
590 driverName: csilibplugins.AWSEBSInTreePluginName,
591 maxVols: 1,
592 test: "unbound PVC with invalid SC is not counted towards the PV limit",
593 },
594 {
595 newPod: unboundPVCWithDefaultSCPod,
596 existingPods: []*v1.Pod{oneVolPod},
597 filterName: ebsVolumeFilterType,
598 driverName: csilibplugins.AWSEBSInTreePluginName,
599 maxVols: 1,
600 test: "unbound PVC from different provisioner is not counted towards the PV limit",
601 },
602 {
603 newPod: onePVCPod(ebsVolumeFilterType),
604 existingPods: []*v1.Pod{oneVolPod, deletedPVPod},
605 filterName: ebsVolumeFilterType,
606 driverName: csilibplugins.AWSEBSInTreePluginName,
607 maxVols: 2,
608 test: "pod with missing PV is counted towards the PV limit",
609 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
610 },
611 {
612 newPod: onePVCPod(ebsVolumeFilterType),
613 existingPods: []*v1.Pod{oneVolPod, deletedPVPod},
614 filterName: ebsVolumeFilterType,
615 driverName: csilibplugins.AWSEBSInTreePluginName,
616 maxVols: 3,
617 test: "pod with missing PV is counted towards the PV limit",
618 },
619 {
620 newPod: deletedPVPod2,
621 existingPods: []*v1.Pod{oneVolPod, deletedPVPod},
622 filterName: ebsVolumeFilterType,
623 driverName: csilibplugins.AWSEBSInTreePluginName,
624 maxVols: 2,
625 test: "two pods missing the same PV are counted towards the PV limit only once",
626 },
627 {
628 newPod: anotherDeletedPVPod,
629 existingPods: []*v1.Pod{oneVolPod, deletedPVPod},
630 filterName: ebsVolumeFilterType,
631 driverName: csilibplugins.AWSEBSInTreePluginName,
632 maxVols: 2,
633 test: "two pods missing different PVs are counted towards the PV limit twice",
634 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
635 },
636 {
637 newPod: onePVCPod(ebsVolumeFilterType),
638 existingPods: []*v1.Pod{oneVolPod, unboundPVCPod},
639 filterName: ebsVolumeFilterType,
640 driverName: csilibplugins.AWSEBSInTreePluginName,
641 maxVols: 2,
642 test: "pod with unbound PVC is counted towards the PV limit",
643 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
644 },
645 {
646 newPod: onePVCPod(ebsVolumeFilterType),
647 existingPods: []*v1.Pod{oneVolPod, unboundPVCPod},
648 filterName: ebsVolumeFilterType,
649 driverName: csilibplugins.AWSEBSInTreePluginName,
650 maxVols: 3,
651 test: "pod with unbound PVC is counted towards the PV limit",
652 },
653 {
654 newPod: unboundPVCPod2,
655 existingPods: []*v1.Pod{oneVolPod, unboundPVCPod},
656 filterName: ebsVolumeFilterType,
657 driverName: csilibplugins.AWSEBSInTreePluginName,
658 maxVols: 2,
659 test: "the same unbound PVC in multiple pods is counted towards the PV limit only once",
660 },
661 {
662 newPod: anotherUnboundPVCPod,
663 existingPods: []*v1.Pod{oneVolPod, unboundPVCPod},
664 filterName: ebsVolumeFilterType,
665 driverName: csilibplugins.AWSEBSInTreePluginName,
666 maxVols: 2,
667 test: "two different unbound PVCs are counted towards the PV limit as two volumes",
668 wantStatus: framework.NewStatus(framework.Unschedulable, ErrReasonMaxVolumeCountExceeded),
669 },
670 {
671 newPod: onlyConfigmapAndSecretPod,
672 existingPods: []*v1.Pod{twoVolPod, oneVolPod},
673 filterName: ebsVolumeFilterType,
674 driverName: csilibplugins.AWSEBSInTreePluginName,
675 maxVols: 4,
676 test: "skip Filter when the pod only uses secrets and configmaps",
677 wantPreFilterStatus: framework.NewStatus(framework.Skip),
678 },
679 {
680 newPod: pvcPodWithConfigmapAndSecret,
681 existingPods: []*v1.Pod{oneVolPod, deletedPVPod},
682 filterName: ebsVolumeFilterType,
683 driverName: csilibplugins.AWSEBSInTreePluginName,
684 maxVols: 2,
685 test: "don't skip Filter when the pod has pvcs",
686 },
687 {
688 newPod: EBSPodWithConfigmapAndSecret,
689 existingPods: []*v1.Pod{twoVolPod, oneVolPod},
690 filterName: ebsVolumeFilterType,
691 driverName: csilibplugins.AWSEBSInTreePluginName,
692 maxVols: 4,
693 test: "don't skip Filter when the pod has EBS volumes",
694 },
695 }
696
697 for _, test := range tests {
698 t.Run(test.test, func(t *testing.T) {
699 _, ctx := ktesting.NewTestContext(t)
700 node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, test.maxVols, test.filterName)
701 p := newNonCSILimits(ctx, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin)
702 _, gotPreFilterStatus := p.(*nonCSILimits).PreFilter(ctx, nil, test.newPod)
703 if diff := cmp.Diff(test.wantPreFilterStatus, gotPreFilterStatus); diff != "" {
704 t.Errorf("PreFilter status does not match (-want, +got): %s", diff)
705 }
706
707 if gotPreFilterStatus.Code() != framework.Skip {
708 gotStatus := p.Filter(ctx, nil, test.newPod, node)
709 if !reflect.DeepEqual(gotStatus, test.wantStatus) {
710 t.Errorf("Filter status does not match: %v, want: %v", gotStatus, test.wantStatus)
711 }
712 }
713 })
714 }
715 }
716
717 func TestGCEPDLimits(t *testing.T) {
718 oneGCEPDPod := st.MakePod().Volume(v1.Volume{
719 VolumeSource: v1.VolumeSource{
720 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
721 },
722 }).Obj()
723 twoGCEPDPod := st.MakePod().Volume(v1.Volume{
724 VolumeSource: v1.VolumeSource{
725 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
726 },
727 }).Volume(v1.Volume{
728 VolumeSource: v1.VolumeSource{
729 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
730 },
731 }).Obj()
732 splitGCEPDPod := st.MakePod().Volume(v1.Volume{
733 VolumeSource: v1.VolumeSource{
734 HostPath: &v1.HostPathVolumeSource{},
735 },
736 }).Volume(v1.Volume{
737 VolumeSource: v1.VolumeSource{
738 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
739 },
740 }).Obj()
741 GCEPDPodWithConfigmapAndSecret := st.MakePod().Volume(v1.Volume{
742 VolumeSource: v1.VolumeSource{
743 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
744 },
745 }).Volume(v1.Volume{
746 VolumeSource: v1.VolumeSource{
747 ConfigMap: &v1.ConfigMapVolumeSource{},
748 },
749 }).Volume(v1.Volume{
750 VolumeSource: v1.VolumeSource{
751 Secret: &v1.SecretVolumeSource{},
752 },
753 }).Obj()
754 tests := []struct {
755 newPod *v1.Pod
756 existingPods []*v1.Pod
757 filterName string
758 driverName string
759 maxVols int32
760 test string
761 wantStatus *framework.Status
762 wantPreFilterStatus *framework.Status
763 }{
764 {
765 newPod: oneGCEPDPod,
766 existingPods: []*v1.Pod{twoGCEPDPod, oneGCEPDPod},
767 filterName: gcePDVolumeFilterType,
768 maxVols: 4,
769 test: "fits when node capacity >= new pod's GCE volumes",
770 },
771 {
772 newPod: twoGCEPDPod,
773 existingPods: []*v1.Pod{oneGCEPDPod},
774 filterName: gcePDVolumeFilterType,
775 maxVols: 2,
776 test: "fit when node capacity < new pod's GCE volumes",
777 },
778 {
779 newPod: splitGCEPDPod,
780 existingPods: []*v1.Pod{twoGCEPDPod},
781 filterName: gcePDVolumeFilterType,
782 maxVols: 3,
783 test: "new pod's count ignores non-GCE volumes",
784 },
785 {
786 newPod: twoGCEPDPod,
787 existingPods: []*v1.Pod{splitGCEPDPod, nonApplicablePod, emptyPod},
788 filterName: gcePDVolumeFilterType,
789 maxVols: 3,
790 test: "existing pods' counts ignore non-GCE volumes",
791 },
792 {
793 newPod: onePVCPod(gcePDVolumeFilterType),
794 existingPods: []*v1.Pod{splitGCEPDPod, nonApplicablePod, emptyPod},
795 filterName: gcePDVolumeFilterType,
796 maxVols: 3,
797 test: "new pod's count considers PVCs backed by GCE volumes",
798 },
799 {
800 newPod: splitPVCPod(gcePDVolumeFilterType),
801 existingPods: []*v1.Pod{splitGCEPDPod, oneGCEPDPod},
802 filterName: gcePDVolumeFilterType,
803 maxVols: 3,
804 test: "new pod's count ignores PVCs not backed by GCE volumes",
805 },
806 {
807 newPod: twoGCEPDPod,
808 existingPods: []*v1.Pod{oneGCEPDPod, onePVCPod(gcePDVolumeFilterType)},
809 filterName: gcePDVolumeFilterType,
810 maxVols: 3,
811 test: "existing pods' counts considers PVCs backed by GCE volumes",
812 },
813 {
814 newPod: twoGCEPDPod,
815 existingPods: []*v1.Pod{oneGCEPDPod, twoGCEPDPod, onePVCPod(gcePDVolumeFilterType)},
816 filterName: gcePDVolumeFilterType,
817 maxVols: 4,
818 test: "already-mounted EBS volumes are always ok to allow",
819 },
820 {
821 newPod: splitGCEPDPod,
822 existingPods: []*v1.Pod{oneGCEPDPod, oneGCEPDPod, onePVCPod(gcePDVolumeFilterType)},
823 filterName: gcePDVolumeFilterType,
824 maxVols: 3,
825 test: "the same GCE volumes are not counted multiple times",
826 },
827 {
828 newPod: onePVCPod(gcePDVolumeFilterType),
829 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVCPod},
830 filterName: gcePDVolumeFilterType,
831 maxVols: 2,
832 test: "pod with missing PVC is counted towards the PV limit",
833 },
834 {
835 newPod: onePVCPod(gcePDVolumeFilterType),
836 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVCPod},
837 filterName: gcePDVolumeFilterType,
838 maxVols: 3,
839 test: "pod with missing PVC is counted towards the PV limit",
840 },
841 {
842 newPod: onePVCPod(gcePDVolumeFilterType),
843 existingPods: []*v1.Pod{oneGCEPDPod, twoDeletedPVCPod},
844 filterName: gcePDVolumeFilterType,
845 maxVols: 3,
846 test: "pod with missing two PVCs is counted towards the PV limit twice",
847 },
848 {
849 newPod: onePVCPod(gcePDVolumeFilterType),
850 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVPod},
851 filterName: gcePDVolumeFilterType,
852 maxVols: 2,
853 test: "pod with missing PV is counted towards the PV limit",
854 },
855 {
856 newPod: onePVCPod(gcePDVolumeFilterType),
857 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVPod},
858 filterName: gcePDVolumeFilterType,
859 maxVols: 3,
860 test: "pod with missing PV is counted towards the PV limit",
861 },
862 {
863 newPod: deletedPVPod2,
864 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVPod},
865 filterName: gcePDVolumeFilterType,
866 maxVols: 2,
867 test: "two pods missing the same PV are counted towards the PV limit only once",
868 },
869 {
870 newPod: anotherDeletedPVPod,
871 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVPod},
872 filterName: gcePDVolumeFilterType,
873 maxVols: 2,
874 test: "two pods missing different PVs are counted towards the PV limit twice",
875 },
876 {
877 newPod: onePVCPod(gcePDVolumeFilterType),
878 existingPods: []*v1.Pod{oneGCEPDPod, unboundPVCPod},
879 filterName: gcePDVolumeFilterType,
880 maxVols: 2,
881 test: "pod with unbound PVC is counted towards the PV limit",
882 },
883 {
884 newPod: onePVCPod(gcePDVolumeFilterType),
885 existingPods: []*v1.Pod{oneGCEPDPod, unboundPVCPod},
886 filterName: gcePDVolumeFilterType,
887 maxVols: 3,
888 test: "pod with unbound PVC is counted towards the PV limit",
889 },
890 {
891 newPod: unboundPVCPod2,
892 existingPods: []*v1.Pod{oneGCEPDPod, unboundPVCPod},
893 filterName: gcePDVolumeFilterType,
894 maxVols: 2,
895 test: "the same unbound PVC in multiple pods is counted towards the PV limit only once",
896 },
897 {
898 newPod: anotherUnboundPVCPod,
899 existingPods: []*v1.Pod{oneGCEPDPod, unboundPVCPod},
900 filterName: gcePDVolumeFilterType,
901 maxVols: 2,
902 test: "two different unbound PVCs are counted towards the PV limit as two volumes",
903 },
904 {
905 newPod: onlyConfigmapAndSecretPod,
906 existingPods: []*v1.Pod{twoGCEPDPod, oneGCEPDPod},
907 filterName: gcePDVolumeFilterType,
908 maxVols: 4,
909 test: "skip Filter when the pod only uses secrets and configmaps",
910 wantPreFilterStatus: framework.NewStatus(framework.Skip),
911 },
912 {
913 newPod: pvcPodWithConfigmapAndSecret,
914 existingPods: []*v1.Pod{oneGCEPDPod, deletedPVPod},
915 filterName: gcePDVolumeFilterType,
916 maxVols: 2,
917 test: "don't skip Filter when the pods has pvcs",
918 },
919 {
920 newPod: GCEPDPodWithConfigmapAndSecret,
921 existingPods: []*v1.Pod{twoGCEPDPod, oneGCEPDPod},
922 filterName: gcePDVolumeFilterType,
923 maxVols: 4,
924 test: "don't skip Filter when the pods has GCE volumes",
925 },
926 }
927
928 for _, test := range tests {
929 t.Run(test.test, func(t *testing.T) {
930 _, ctx := ktesting.NewTestContext(t)
931 node, csiNode := getNodeWithPodAndVolumeLimits("node", test.existingPods, test.maxVols, test.filterName)
932 p := newNonCSILimits(ctx, test.filterName, getFakeCSINodeLister(csiNode), getFakeCSIStorageClassLister(test.filterName, test.driverName), getFakePVLister(test.filterName), getFakePVCLister(test.filterName), feature.Features{}).(framework.FilterPlugin)
933 _, gotPreFilterStatus := p.(*nonCSILimits).PreFilter(context.Background(), nil, test.newPod)
934 if diff := cmp.Diff(test.wantPreFilterStatus, gotPreFilterStatus); diff != "" {
935 t.Errorf("PreFilter status does not match (-want, +got): %s", diff)
936 }
937
938 if gotPreFilterStatus.Code() != framework.Skip {
939 gotStatus := p.Filter(context.Background(), nil, test.newPod, node)
940 if !reflect.DeepEqual(gotStatus, test.wantStatus) {
941 t.Errorf("Filter status does not match: %v, want: %v", gotStatus, test.wantStatus)
942 }
943 }
944 })
945 }
946 }
947
948 func TestGetMaxVols(t *testing.T) {
949 tests := []struct {
950 rawMaxVols string
951 expected int
952 name string
953 }{
954 {
955 rawMaxVols: "invalid",
956 expected: -1,
957 name: "Unable to parse maximum PD volumes value, using default value",
958 },
959 {
960 rawMaxVols: "-2",
961 expected: -1,
962 name: "Maximum PD volumes must be a positive value, using default value",
963 },
964 {
965 rawMaxVols: "40",
966 expected: 40,
967 name: "Parse maximum PD volumes value from env",
968 },
969 }
970
971 for _, test := range tests {
972 t.Run(test.name, func(t *testing.T) {
973 logger, _ := ktesting.NewTestContext(t)
974 t.Setenv(KubeMaxPDVols, test.rawMaxVols)
975 result := getMaxVolLimitFromEnv(logger)
976 if result != test.expected {
977 t.Errorf("expected %v got %v", test.expected, result)
978 }
979 })
980 }
981 }
982
983 func getFakePVCLister(filterName string) tf.PersistentVolumeClaimLister {
984 return tf.PersistentVolumeClaimLister{
985 {
986 ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"},
987 Spec: v1.PersistentVolumeClaimSpec{
988 VolumeName: "some" + filterName + "Vol",
989 StorageClassName: &filterName,
990 },
991 },
992 {
993 ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"},
994 Spec: v1.PersistentVolumeClaimSpec{
995 VolumeName: "someNon" + filterName + "Vol",
996 StorageClassName: &filterName,
997 },
998 },
999 {
1000 ObjectMeta: metav1.ObjectMeta{Name: "pvcWithDeletedPV"},
1001 Spec: v1.PersistentVolumeClaimSpec{
1002 VolumeName: "pvcWithDeletedPV",
1003 StorageClassName: &filterName,
1004 },
1005 },
1006 {
1007 ObjectMeta: metav1.ObjectMeta{Name: "anotherPVCWithDeletedPV"},
1008 Spec: v1.PersistentVolumeClaimSpec{
1009 VolumeName: "anotherPVCWithDeletedPV",
1010 StorageClassName: &filterName,
1011 },
1012 },
1013 {
1014 ObjectMeta: metav1.ObjectMeta{Name: "unboundPVC"},
1015 Spec: v1.PersistentVolumeClaimSpec{
1016 VolumeName: "",
1017 StorageClassName: &filterName,
1018 },
1019 },
1020 {
1021 ObjectMeta: metav1.ObjectMeta{Name: "anotherUnboundPVC"},
1022 Spec: v1.PersistentVolumeClaimSpec{
1023 VolumeName: "",
1024 StorageClassName: &filterName,
1025 },
1026 },
1027 {
1028 ObjectMeta: metav1.ObjectMeta{Name: "unboundPVCWithDefaultSCPod"},
1029 Spec: v1.PersistentVolumeClaimSpec{
1030 VolumeName: "",
1031 StorageClassName: ptr.To("standard-sc"),
1032 },
1033 },
1034 {
1035 ObjectMeta: metav1.ObjectMeta{Name: "unboundPVCWithInvalidSCPod"},
1036 Spec: v1.PersistentVolumeClaimSpec{
1037 VolumeName: "",
1038 StorageClassName: ptr.To("invalid-sc"),
1039 },
1040 },
1041 }
1042 }
1043
1044 func getFakePVLister(filterName string) tf.PersistentVolumeLister {
1045 return tf.PersistentVolumeLister{
1046 {
1047 ObjectMeta: metav1.ObjectMeta{Name: "some" + filterName + "Vol"},
1048 Spec: v1.PersistentVolumeSpec{
1049 PersistentVolumeSource: v1.PersistentVolumeSource{
1050 AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{VolumeID: strings.ToLower(filterName) + "Vol"},
1051 },
1052 },
1053 },
1054 {
1055 ObjectMeta: metav1.ObjectMeta{Name: "someNon" + filterName + "Vol"},
1056 Spec: v1.PersistentVolumeSpec{
1057 PersistentVolumeSource: v1.PersistentVolumeSource{},
1058 },
1059 },
1060 }
1061 }
1062
1063 func onePVCPod(filterName string) *v1.Pod {
1064 return st.MakePod().PVC(fmt.Sprintf("some%sVol", filterName)).Obj()
1065 }
1066
1067 func splitPVCPod(filterName string) *v1.Pod {
1068 return st.MakePod().PVC(fmt.Sprintf("someNon%sVol", filterName)).PVC(fmt.Sprintf("some%sVol", filterName)).Obj()
1069 }
1070
View as plain text