1
16
17 package operationexecutor
18
19 import (
20 "fmt"
21 "os"
22 "testing"
23
24 "k8s.io/apimachinery/pkg/api/resource"
25 "k8s.io/apimachinery/pkg/runtime"
26 utilfeature "k8s.io/apiserver/pkg/util/feature"
27 core "k8s.io/client-go/testing"
28 "k8s.io/client-go/tools/record"
29 featuregatetesting "k8s.io/component-base/featuregate/testing"
30 "k8s.io/kubernetes/pkg/features"
31
32 "github.com/stretchr/testify/assert"
33 v1 "k8s.io/api/core/v1"
34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35 "k8s.io/apimachinery/pkg/types"
36 "k8s.io/apimachinery/pkg/util/uuid"
37 fakeclient "k8s.io/client-go/kubernetes/fake"
38 "k8s.io/component-base/metrics/testutil"
39 "k8s.io/kubernetes/pkg/volume"
40 csitesting "k8s.io/kubernetes/pkg/volume/csi/testing"
41 volumetesting "k8s.io/kubernetes/pkg/volume/testing"
42 "k8s.io/kubernetes/pkg/volume/util"
43 volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
44 )
45
46
47
48 func TestOperationGenerator_GenerateUnmapVolumeFunc_PluginName(t *testing.T) {
49 type testcase struct {
50 name string
51 pluginName string
52 pvSpec v1.PersistentVolumeSpec
53 probVolumePlugins []volume.VolumePlugin
54 }
55
56 testcases := []testcase{
57 {
58 name: "gce pd plugin: csi migration disabled",
59 pluginName: "fake-plugin",
60 pvSpec: v1.PersistentVolumeSpec{
61 PersistentVolumeSource: v1.PersistentVolumeSource{
62 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
63 }},
64 probVolumePlugins: volumetesting.ProbeVolumePlugins(volume.VolumeConfig{}),
65 },
66 }
67
68 for _, tc := range testcases {
69 expectedPluginName := tc.pluginName
70 volumePluginMgr, tmpDir := initTestPlugins(t, tc.probVolumePlugins, tc.pluginName)
71 defer os.RemoveAll(tmpDir)
72
73 operationGenerator := getTestOperationGenerator(volumePluginMgr)
74
75 pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID(string(uuid.NewUUID()))}}
76 volumeToUnmount := getTestVolumeToUnmount(pod, tc.pvSpec, tc.pluginName)
77
78 unmapVolumeFunc, e := operationGenerator.GenerateUnmapVolumeFunc(volumeToUnmount, nil)
79 if e != nil {
80 t.Fatalf("Error occurred while generating unmapVolumeFunc: %v", e)
81 }
82
83 m := util.StorageOperationMetric.WithLabelValues(expectedPluginName, "unmap_volume", "success", "false")
84 storageOperationDurationSecondsMetricBefore, _ := testutil.GetHistogramMetricCount(m)
85
86 var ee error
87 unmapVolumeFunc.CompleteFunc(volumetypes.CompleteFuncParam{Err: &ee})
88
89 storageOperationDurationSecondsMetricAfter, _ := testutil.GetHistogramMetricCount(m)
90 metricValueDiff := storageOperationDurationSecondsMetricAfter - storageOperationDurationSecondsMetricBefore
91 assert.Equal(t, uint64(1), metricValueDiff, tc.name)
92 }
93 }
94
95 func TestOperationGenerator_GenerateExpandAndRecoverVolumeFunc(t *testing.T) {
96 nodeResizePending := v1.PersistentVolumeClaimNodeResizePending
97 nodeResizeFailed := v1.PersistentVolumeClaimNodeResizeFailed
98 var tests = []struct {
99 name string
100 pvc *v1.PersistentVolumeClaim
101 pv *v1.PersistentVolume
102 recoverFeatureGate bool
103 disableNodeExpansion bool
104
105 expectedResizeStatus v1.ClaimResourceStatus
106 expectedAllocatedSize resource.Quantity
107 expectResizeCall bool
108 }{
109 {
110 name: "pvc.spec.size > pv.spec.size, recover_expansion=on",
111 pvc: getTestPVC("test-vol0", "2G", "1G", "", nil),
112 pv: getTestPV("test-vol0", "1G"),
113 recoverFeatureGate: true,
114 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
115 expectedAllocatedSize: resource.MustParse("2G"),
116 expectResizeCall: true,
117 },
118 {
119 name: "pvc.spec.size = pv.spec.size, recover_expansion=on",
120 pvc: getTestPVC("test-vol0", "1G", "1G", "", nil),
121 pv: getTestPV("test-vol0", "1G"),
122 recoverFeatureGate: true,
123 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
124 expectedAllocatedSize: resource.MustParse("1G"),
125 expectResizeCall: true,
126 },
127 {
128 name: "pvc.spec.size = pv.spec.size, recover_expansion=on",
129 pvc: getTestPVC("test-vol0", "1G", "1G", "1G", &nodeResizePending),
130 pv: getTestPV("test-vol0", "1G"),
131 recoverFeatureGate: true,
132 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
133 expectedAllocatedSize: resource.MustParse("1G"),
134 expectResizeCall: false,
135 },
136 {
137 name: "pvc.spec.size > pv.spec.size, recover_expansion=on, disable_node_expansion=true",
138 pvc: getTestPVC("test-vol0", "2G", "1G", "", nil),
139 pv: getTestPV("test-vol0", "1G"),
140 disableNodeExpansion: true,
141 recoverFeatureGate: true,
142 expectedResizeStatus: "",
143 expectedAllocatedSize: resource.MustParse("2G"),
144 expectResizeCall: true,
145 },
146 {
147 name: "pv.spec.size >= pvc.spec.size, recover_expansion=on, resize_status=node_expansion_failed",
148 pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizeFailed),
149 pv: getTestPV("test-vol0", "2G"),
150 recoverFeatureGate: true,
151 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizePending,
152 expectedAllocatedSize: resource.MustParse("2G"),
153 expectResizeCall: false,
154 },
155 }
156 for i := range tests {
157 test := tests[i]
158 t.Run(test.name, func(t *testing.T) {
159 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, test.recoverFeatureGate)()
160 volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t)
161 fakePlugin.DisableNodeExpansion = test.disableNodeExpansion
162 pvc := test.pvc
163 pv := test.pv
164 og := getTestOperationGenerator(volumePluginMgr, pvc, pv)
165 rsOpts := inTreeResizeOpts{
166 pvc: pvc,
167 pv: pv,
168 resizerName: fakePlugin.GetPluginName(),
169 volumePlugin: fakePlugin,
170 }
171 ogInstance, _ := og.(*operationGenerator)
172
173 expansionResponse := ogInstance.expandAndRecoverFunction(rsOpts)
174 if expansionResponse.err != nil {
175 t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: %v", expansionResponse.err)
176 }
177 updatedPVC := expansionResponse.pvc
178 actualResizeStatus := updatedPVC.Status.AllocatedResourceStatuses[v1.ResourceStorage]
179 assert.Equal(t, actualResizeStatus, test.expectedResizeStatus)
180 actualAllocatedSize := updatedPVC.Status.AllocatedResources.Storage()
181 if test.expectedAllocatedSize.Cmp(*actualAllocatedSize) != 0 {
182 t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: expected allocated size %s, got %s", test.expectedAllocatedSize.String(), actualAllocatedSize.String())
183 }
184 if test.expectResizeCall != expansionResponse.resizeCalled {
185 t.Fatalf("GenerateExpandAndRecoverVolumeFunc failed: expected resize called %t, got %t", test.expectResizeCall, expansionResponse.resizeCalled)
186 }
187 })
188 }
189 }
190
191 func TestOperationGenerator_nodeExpandVolume(t *testing.T) {
192 getSizeFunc := func(size string) *resource.Quantity {
193 x := resource.MustParse(size)
194 return &x
195 }
196
197 nodeResizeFailed := v1.PersistentVolumeClaimNodeResizeFailed
198 nodeResizePending := v1.PersistentVolumeClaimNodeResizePending
199 var tests = []struct {
200 name string
201 pvc *v1.PersistentVolumeClaim
202 pv *v1.PersistentVolume
203
204
205 desiredSize *resource.Quantity
206
207 actualSize *resource.Quantity
208
209
210 expectedResizeStatus v1.ClaimResourceStatus
211 expectedStatusSize resource.Quantity
212 resizeCallCount int
213 expectError bool
214 }{
215 {
216 name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_failed",
217 pvc: getTestPVC("test-vol0", "2G", "1G", "", &nodeResizeFailed),
218 pv: getTestPV("test-vol0", "2G"),
219
220 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
221 resizeCallCount: 0,
222 expectedStatusSize: resource.MustParse("1G"),
223 },
224 {
225 name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending",
226 pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending),
227 pv: getTestPV("test-vol0", "2G"),
228 expectedResizeStatus: "",
229 resizeCallCount: 1,
230 expectedStatusSize: resource.MustParse("2G"),
231 },
232 {
233 name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=failing",
234 pvc: getTestPVC(volumetesting.AlwaysFailNodeExpansion, "2G", "1G", "2G", &nodeResizePending),
235 pv: getTestPV(volumetesting.AlwaysFailNodeExpansion, "2G"),
236 expectError: true,
237 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
238 resizeCallCount: 1,
239 expectedStatusSize: resource.MustParse("1G"),
240 },
241 {
242 name: "pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize = actualSize",
243 pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil),
244 pv: getTestPV("test-vol0", "2G"),
245
246 expectedResizeStatus: "",
247 resizeCallCount: 0,
248 expectedStatusSize: resource.MustParse("2G"),
249 },
250 {
251 name: "pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize",
252 pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil),
253 pv: getTestPV("test-vol0", "2G"),
254 desiredSize: getSizeFunc("2G"),
255 actualSize: getSizeFunc("1G"),
256
257 expectedResizeStatus: "",
258 resizeCallCount: 1,
259 expectedStatusSize: resource.MustParse("2G"),
260 },
261 {
262 name: "pv.spec.cap = pvc.status.cap, resizeStatus=node-expansion-failed, desiredSize > actualSize",
263 pvc: getTestPVC("test-vol0", "2G", "2G", "2G", &nodeResizeFailed),
264 pv: getTestPV("test-vol0", "2G"),
265 desiredSize: getSizeFunc("2G"),
266 actualSize: getSizeFunc("1G"),
267
268 expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeFailed,
269 resizeCallCount: 0,
270 expectedStatusSize: resource.MustParse("2G"),
271 },
272 }
273 for i := range tests {
274 test := tests[i]
275 t.Run(test.name, func(t *testing.T) {
276 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, true)()
277 volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t)
278 test.pv.Spec.ClaimRef = &v1.ObjectReference{
279 Namespace: test.pvc.Namespace,
280 Name: test.pvc.Name,
281 }
282
283 pvc := test.pvc
284 pv := test.pv
285 pod := getTestPod("test-pod", pvc.Name)
286 og := getTestOperatorGeneratorWithPVPVC(volumePluginMgr, pvc, pv)
287 vmt := VolumeToMount{
288 Pod: pod,
289 VolumeName: v1.UniqueVolumeName(pv.Name),
290 VolumeSpec: volume.NewSpecFromPersistentVolume(pv, false),
291 }
292 desiredSize := test.desiredSize
293 if desiredSize == nil {
294 desiredSize = pv.Spec.Capacity.Storage()
295 }
296 actualSize := test.actualSize
297 if actualSize == nil {
298 actualSize = pvc.Status.Capacity.Storage()
299 }
300 pluginResizeOpts := volume.NodeResizeOptions{
301 VolumeSpec: vmt.VolumeSpec,
302 NewSize: *desiredSize,
303 OldSize: *actualSize,
304 }
305
306 ogInstance, _ := og.(*operationGenerator)
307 _, err := ogInstance.nodeExpandVolume(vmt, nil, pluginResizeOpts)
308
309 if !test.expectError && err != nil {
310 t.Errorf("For test %s, expected no error got: %v", test.name, err)
311 }
312 if test.expectError && err == nil {
313 t.Errorf("For test %s, expected error but got none", test.name)
314 }
315 if test.resizeCallCount != fakePlugin.NodeExpandCallCount {
316 t.Errorf("for test %s, expected node-expand call count to be %d, got %d", test.name, test.resizeCallCount, fakePlugin.NodeExpandCallCount)
317 }
318 })
319 }
320 }
321
322 func getTestPod(podName, pvcName string) *v1.Pod {
323 return &v1.Pod{
324 ObjectMeta: metav1.ObjectMeta{
325 Name: podName,
326 UID: "test-pod-uid",
327 Namespace: "ns",
328 },
329 Spec: v1.PodSpec{
330 Volumes: []v1.Volume{
331 {
332 Name: pvcName,
333 VolumeSource: v1.VolumeSource{
334 PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
335 ClaimName: pvcName,
336 },
337 },
338 },
339 },
340 },
341 }
342 }
343
344 func getTestPVC(volumeName string, specSize, statusSize, allocatedSize string, resizeStatus *v1.ClaimResourceStatus) *v1.PersistentVolumeClaim {
345 pvc := &v1.PersistentVolumeClaim{
346 ObjectMeta: metav1.ObjectMeta{
347 Name: "claim01",
348 Namespace: "ns",
349 UID: "test-uid",
350 },
351 Spec: v1.PersistentVolumeClaimSpec{
352 AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
353 Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse(specSize)}},
354 VolumeName: volumeName,
355 },
356 Status: v1.PersistentVolumeClaimStatus{
357 Phase: v1.ClaimBound,
358 },
359 }
360 if len(statusSize) > 0 {
361 pvc.Status.Capacity = v1.ResourceList{v1.ResourceStorage: resource.MustParse(statusSize)}
362 }
363 if len(allocatedSize) > 0 {
364 pvc.Status.AllocatedResources = v1.ResourceList{v1.ResourceStorage: resource.MustParse(allocatedSize)}
365 }
366 if resizeStatus != nil {
367 pvc.Status.AllocatedResourceStatuses = map[v1.ResourceName]v1.ClaimResourceStatus{
368 v1.ResourceStorage: *resizeStatus,
369 }
370 }
371 return pvc
372 }
373
374 func getTestPV(volumeName string, specSize string) *v1.PersistentVolume {
375 return &v1.PersistentVolume{
376 ObjectMeta: metav1.ObjectMeta{
377 Name: volumeName,
378 UID: "test-uid",
379 },
380 Spec: v1.PersistentVolumeSpec{
381 AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
382 Capacity: v1.ResourceList{
383 v1.ResourceStorage: resource.MustParse(specSize),
384 },
385 },
386 Status: v1.PersistentVolumeStatus{
387 Phase: v1.VolumeBound,
388 },
389 }
390 }
391
392 func getTestOperationGenerator(volumePluginMgr *volume.VolumePluginMgr, objects ...runtime.Object) OperationGenerator {
393 fakeKubeClient := fakeclient.NewSimpleClientset(objects...)
394 fakeRecorder := &record.FakeRecorder{}
395 fakeHandler := volumetesting.NewBlockVolumePathHandler()
396 operationGenerator := NewOperationGenerator(
397 fakeKubeClient,
398 volumePluginMgr,
399 fakeRecorder,
400 fakeHandler)
401 return operationGenerator
402 }
403
404 func getTestOperatorGeneratorWithPVPVC(volumePluginMgr *volume.VolumePluginMgr, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) OperationGenerator {
405 fakeKubeClient := fakeclient.NewSimpleClientset(pvc, pv)
406 fakeKubeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
407 return true, pvc, nil
408 })
409 fakeKubeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) {
410 return true, pv, nil
411 })
412 fakeKubeClient.AddReactor("patch", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
413 if action.GetSubresource() == "status" {
414 return true, pvc, nil
415 }
416 return true, nil, fmt.Errorf("no reaction implemented for %s", action)
417 })
418
419 fakeRecorder := &record.FakeRecorder{}
420 fakeHandler := volumetesting.NewBlockVolumePathHandler()
421 operationGenerator := NewOperationGenerator(
422 fakeKubeClient,
423 volumePluginMgr,
424 fakeRecorder,
425 fakeHandler)
426 return operationGenerator
427 }
428
429 func getTestVolumeToUnmount(pod *v1.Pod, pvSpec v1.PersistentVolumeSpec, pluginName string) MountedVolume {
430 volumeSpec := &volume.Spec{
431 PersistentVolume: &v1.PersistentVolume{
432 Spec: pvSpec,
433 },
434 }
435 volumeToUnmount := MountedVolume{
436 VolumeName: v1.UniqueVolumeName("pd-volume"),
437 PodUID: pod.UID,
438 PluginName: pluginName,
439 VolumeSpec: volumeSpec,
440 }
441 return volumeToUnmount
442 }
443
444 func initTestPlugins(t *testing.T, plugs []volume.VolumePlugin, pluginName string) (*volume.VolumePluginMgr, string) {
445 client := fakeclient.NewSimpleClientset()
446 pluginMgr, _, tmpDir := csitesting.NewTestPlugin(t, client)
447
448 err := pluginMgr.InitPlugins(plugs, nil, pluginMgr.Host)
449 if err != nil {
450 t.Fatalf("Can't init volume plugins: %v", err)
451 }
452
453 _, e := pluginMgr.FindPluginByName(pluginName)
454 if e != nil {
455 t.Fatalf("Can't find the plugin by name: %s", pluginName)
456 }
457
458 return pluginMgr, tmpDir
459 }
460
View as plain text