Source file src/k8s.io/kubernetes/pkg/volume/csi/csi_attacher_test.go

Documentation: k8s.io/kubernetes/pkg/volume/csi

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package csi
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"fmt"
    23  	"os"
    24  	"os/user"
    25  	"path/filepath"
    26  	"reflect"
    27  	goruntime "runtime"
    28  	"sync"
    29  	"testing"
    30  	"time"
    31  
    32  	v1 "k8s.io/api/core/v1"
    33  	storage "k8s.io/api/storage/v1"
    34  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    35  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	"k8s.io/apimachinery/pkg/runtime"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/watch"
    39  	clientset "k8s.io/client-go/kubernetes"
    40  	fakeclient "k8s.io/client-go/kubernetes/fake"
    41  	core "k8s.io/client-go/testing"
    42  	"k8s.io/kubernetes/pkg/volume"
    43  	fakecsi "k8s.io/kubernetes/pkg/volume/csi/fake"
    44  	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
    45  )
    46  
    47  const (
    48  	testWatchTimeout     = 10 * time.Second
    49  	testWatchFailTimeout = 2 * time.Second
    50  )
    51  
    52  var (
    53  	bFalse = false
    54  	bTrue  = true
    55  )
    56  
    57  func makeTestAttachment(attachID, nodeName, pvName string) *storage.VolumeAttachment {
    58  	return &storage.VolumeAttachment{
    59  		ObjectMeta: metav1.ObjectMeta{
    60  			Name: attachID,
    61  		},
    62  		Spec: storage.VolumeAttachmentSpec{
    63  			NodeName: nodeName,
    64  			Attacher: "mock",
    65  			Source: storage.VolumeAttachmentSource{
    66  				PersistentVolumeName: &pvName,
    67  			},
    68  		},
    69  		Status: storage.VolumeAttachmentStatus{
    70  			Attached:    false,
    71  			AttachError: nil,
    72  			DetachError: nil,
    73  		},
    74  	}
    75  }
    76  
    77  func markVolumeAttached(t *testing.T, client clientset.Interface, watch *watch.RaceFreeFakeWatcher, attachID string, status storage.VolumeAttachmentStatus) {
    78  	ticker := time.NewTicker(10 * time.Millisecond)
    79  	var attach *storage.VolumeAttachment
    80  	var err error
    81  	defer ticker.Stop()
    82  	// wait for attachment to be saved
    83  	for i := 0; i < 100; i++ {
    84  		attach, err = client.StorageV1().VolumeAttachments().Get(context.TODO(), attachID, metav1.GetOptions{})
    85  		if err != nil {
    86  			if apierrors.IsNotFound(err) {
    87  				<-ticker.C
    88  				continue
    89  			}
    90  			t.Error(err)
    91  		}
    92  		if attach != nil {
    93  			t.Logf("attachment found on try %d, stopping wait...", i)
    94  			break
    95  		}
    96  	}
    97  	t.Logf("stopped waiting for attachment")
    98  
    99  	if attach == nil {
   100  		t.Logf("attachment not found for id:%v", attachID)
   101  	} else {
   102  		attach.Status = status
   103  		t.Logf("updating attachment %s with attach status %v", attachID, status)
   104  		_, err := client.StorageV1().VolumeAttachments().Update(context.TODO(), attach, metav1.UpdateOptions{})
   105  		if err != nil {
   106  			t.Error(err)
   107  		}
   108  		if watch != nil {
   109  			watch.Modify(attach)
   110  		}
   111  	}
   112  }
   113  
   114  func TestAttacherAttach(t *testing.T) {
   115  	testCases := []struct {
   116  		name                string
   117  		nodeName            string
   118  		driverName          string
   119  		volumeName          string
   120  		attachID            string
   121  		spec                *volume.Spec
   122  		injectAttacherError bool
   123  		shouldFail          bool
   124  		watchTimeout        time.Duration
   125  	}{
   126  		{
   127  			name:       "test ok 1",
   128  			nodeName:   "testnode-01",
   129  			driverName: "testdriver-01",
   130  			volumeName: "testvol-01",
   131  			attachID:   getAttachmentName("testvol-01", "testdriver-01", "testnode-01"),
   132  			spec:       volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "testdriver-01", "testvol-01"), false),
   133  		},
   134  		{
   135  			name:       "test ok 2",
   136  			nodeName:   "node02",
   137  			driverName: "driver02",
   138  			volumeName: "vol02",
   139  			attachID:   getAttachmentName("vol02", "driver02", "node02"),
   140  			spec:       volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
   141  		},
   142  		{
   143  			name:         "mismatch vol",
   144  			nodeName:     "node02",
   145  			driverName:   "driver02",
   146  			volumeName:   "vol01",
   147  			attachID:     getAttachmentName("vol02", "driver02", "node02"),
   148  			spec:         volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol01"), false),
   149  			shouldFail:   true,
   150  			watchTimeout: testWatchFailTimeout,
   151  		},
   152  		{
   153  			name:         "mismatch driver",
   154  			nodeName:     "node02",
   155  			driverName:   "driver000",
   156  			volumeName:   "vol02",
   157  			attachID:     getAttachmentName("vol02", "driver02", "node02"),
   158  			spec:         volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver01", "vol02"), false),
   159  			shouldFail:   true,
   160  			watchTimeout: testWatchFailTimeout,
   161  		},
   162  		{
   163  			name:         "mismatch node",
   164  			nodeName:     "node000",
   165  			driverName:   "driver000",
   166  			volumeName:   "vol02",
   167  			attachID:     getAttachmentName("vol02", "driver02", "node02"),
   168  			spec:         volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
   169  			shouldFail:   true,
   170  			watchTimeout: testWatchFailTimeout,
   171  		},
   172  		{
   173  			name:                "attacher error",
   174  			nodeName:            "node02",
   175  			driverName:          "driver02",
   176  			volumeName:          "vol02",
   177  			attachID:            getAttachmentName("vol02", "driver02", "node02"),
   178  			spec:                volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
   179  			injectAttacherError: true,
   180  			shouldFail:          true,
   181  		},
   182  		{
   183  			name:       "test with volume source",
   184  			nodeName:   "node000",
   185  			driverName: "driver000",
   186  			volumeName: "vol02",
   187  			attachID:   getAttachmentName("vol02", "driver02", "node02"),
   188  			spec:       volume.NewSpecFromVolume(makeTestVol("pv01", "driver02")),
   189  			shouldFail: true, // csi not enabled
   190  		},
   191  		{
   192  			name:       "missing spec",
   193  			nodeName:   "node000",
   194  			driverName: "driver000",
   195  			volumeName: "vol02",
   196  			attachID:   getAttachmentName("vol02", "driver02", "node02"),
   197  			shouldFail: true, // csi not enabled
   198  		},
   199  	}
   200  
   201  	// attacher loop
   202  	for _, tc := range testCases {
   203  		t.Run(tc.name, func(t *testing.T) {
   204  			t.Logf("test case: %s", tc.name)
   205  			fakeClient := fakeclient.NewSimpleClientset()
   206  			plug, tmpDir := newTestPluginWithAttachDetachVolumeHost(t, fakeClient)
   207  			defer os.RemoveAll(tmpDir)
   208  
   209  			attacher, err := plug.NewAttacher()
   210  			if err != nil {
   211  				t.Fatalf("failed to create new attacher: %v", err)
   212  			}
   213  
   214  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   215  
   216  			var wg sync.WaitGroup
   217  			wg.Add(1)
   218  			go func(spec *volume.Spec, nodename string, fail bool) {
   219  				defer wg.Done()
   220  				attachID, err := csiAttacher.Attach(spec, types.NodeName(nodename))
   221  				if !fail && err != nil {
   222  					t.Errorf("expecting no failure, but got err: %v", err)
   223  				}
   224  				if fail && err == nil {
   225  					t.Errorf("expecting failure, but got no err")
   226  				}
   227  				if attachID != "" {
   228  					t.Errorf("expecting empty attachID, got %v", attachID)
   229  				}
   230  			}(tc.spec, tc.nodeName, tc.shouldFail)
   231  
   232  			var status storage.VolumeAttachmentStatus
   233  			if tc.injectAttacherError {
   234  				status.Attached = false
   235  				status.AttachError = &storage.VolumeError{
   236  					Message: "attacher error",
   237  				}
   238  			} else {
   239  				status.Attached = true
   240  			}
   241  			markVolumeAttached(t, csiAttacher.k8s, nil, tc.attachID, status)
   242  			wg.Wait()
   243  		})
   244  	}
   245  }
   246  
   247  func TestAttacherAttachWithInline(t *testing.T) {
   248  	testCases := []struct {
   249  		name                string
   250  		nodeName            string
   251  		driverName          string
   252  		volumeName          string
   253  		attachID            string
   254  		spec                *volume.Spec
   255  		injectAttacherError bool
   256  		shouldFail          bool
   257  		watchTimeout        time.Duration
   258  	}{
   259  		{
   260  			name:     "test ok 1 with PV",
   261  			nodeName: "node01",
   262  			attachID: getAttachmentName("vol01", "driver01", "node01"),
   263  			spec:     volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver01", "vol01"), false),
   264  		},
   265  		{
   266  			name:       "test failure, attach with volSrc",
   267  			nodeName:   "node01",
   268  			attachID:   getAttachmentName("vol01", "driver01", "node01"),
   269  			spec:       volume.NewSpecFromVolume(makeTestVol("vol01", "driver01")),
   270  			shouldFail: true,
   271  		},
   272  		{
   273  			name:                "attacher error",
   274  			nodeName:            "node02",
   275  			attachID:            getAttachmentName("vol02", "driver02", "node02"),
   276  			spec:                volume.NewSpecFromPersistentVolume(makeTestPV("pv02", 10, "driver02", "vol02"), false),
   277  			injectAttacherError: true,
   278  			shouldFail:          true,
   279  		},
   280  		{
   281  			name:       "missing spec",
   282  			nodeName:   "node02",
   283  			attachID:   getAttachmentName("vol02", "driver02", "node02"),
   284  			shouldFail: true,
   285  		},
   286  	}
   287  
   288  	// attacher loop
   289  	for _, tc := range testCases {
   290  		t.Run(tc.name, func(t *testing.T) {
   291  			t.Logf("test case: %s", tc.name)
   292  			fakeClient := fakeclient.NewSimpleClientset()
   293  			plug, tmpDir := newTestPluginWithAttachDetachVolumeHost(t, fakeClient)
   294  			defer os.RemoveAll(tmpDir)
   295  
   296  			attacher, err := plug.NewAttacher()
   297  			if err != nil {
   298  				t.Fatalf("failed to create new attacher: %v", err)
   299  			}
   300  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   301  
   302  			var wg sync.WaitGroup
   303  			wg.Add(1)
   304  			go func(spec *volume.Spec, nodename string, fail bool) {
   305  				defer wg.Done()
   306  				attachID, err := csiAttacher.Attach(spec, types.NodeName(nodename))
   307  				if fail != (err != nil) {
   308  					t.Errorf("expecting no failure, but got err: %v", err)
   309  				}
   310  				if attachID != "" {
   311  					t.Errorf("expecting empty attachID, got %v", attachID)
   312  				}
   313  			}(tc.spec, tc.nodeName, tc.shouldFail)
   314  
   315  			var status storage.VolumeAttachmentStatus
   316  			if tc.injectAttacherError {
   317  				status.Attached = false
   318  				status.AttachError = &storage.VolumeError{
   319  					Message: "attacher error",
   320  				}
   321  			} else {
   322  				status.Attached = true
   323  			}
   324  			markVolumeAttached(t, csiAttacher.k8s, nil, tc.attachID, status)
   325  			wg.Wait()
   326  		})
   327  	}
   328  }
   329  
   330  func TestAttacherWithCSIDriver(t *testing.T) {
   331  	tests := []struct {
   332  		name                   string
   333  		driver                 string
   334  		expectVolumeAttachment bool
   335  		watchTimeout           time.Duration
   336  	}{
   337  		{
   338  			name:                   "CSIDriver not attachable",
   339  			driver:                 "not-attachable",
   340  			expectVolumeAttachment: false,
   341  		},
   342  		{
   343  			name:                   "CSIDriver is attachable",
   344  			driver:                 "attachable",
   345  			expectVolumeAttachment: true,
   346  		},
   347  		{
   348  			name:                   "CSIDriver.AttachRequired not set  -> failure",
   349  			driver:                 "nil",
   350  			expectVolumeAttachment: true,
   351  		},
   352  		{
   353  			name:                   "CSIDriver does not exist not set  -> failure",
   354  			driver:                 "unknown",
   355  			expectVolumeAttachment: true,
   356  		},
   357  	}
   358  
   359  	for _, test := range tests {
   360  		t.Run(test.name, func(t *testing.T) {
   361  			fakeClient := fakeclient.NewSimpleClientset(
   362  				getTestCSIDriver("not-attachable", nil, &bFalse, nil),
   363  				getTestCSIDriver("attachable", nil, &bTrue, nil),
   364  				getTestCSIDriver("nil", nil, nil, nil),
   365  			)
   366  			plug, tmpDir := newTestPluginWithAttachDetachVolumeHost(t, fakeClient)
   367  			defer os.RemoveAll(tmpDir)
   368  
   369  			attacher, err := plug.NewAttacher()
   370  			if err != nil {
   371  				t.Fatalf("failed to create new attacher: %v", err)
   372  			}
   373  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, test.watchTimeout)
   374  			spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, test.driver, "test-vol"), false)
   375  
   376  			pluginCanAttach, err := plug.CanAttach(spec)
   377  			if err != nil {
   378  				t.Fatalf("attacher.CanAttach failed: %s", err)
   379  			}
   380  			if pluginCanAttach != test.expectVolumeAttachment {
   381  				t.Errorf("attacher.CanAttach does not match expected attachment status %t", test.expectVolumeAttachment)
   382  			}
   383  
   384  			if !pluginCanAttach {
   385  				t.Log("plugin is not attachable")
   386  				return
   387  			}
   388  			var wg sync.WaitGroup
   389  			wg.Add(1)
   390  			go func(volSpec *volume.Spec) {
   391  				attachID, err := csiAttacher.Attach(volSpec, "fakeNode")
   392  				defer wg.Done()
   393  
   394  				if err != nil {
   395  					t.Errorf("Attach() failed: %s", err)
   396  				}
   397  				if attachID != "" {
   398  					t.Errorf("Expected empty attachID, got %q", attachID)
   399  				}
   400  			}(spec)
   401  
   402  			if test.expectVolumeAttachment {
   403  				expectedAttachID := getAttachmentName("test-vol", test.driver, "fakeNode")
   404  				status := storage.VolumeAttachmentStatus{
   405  					Attached: true,
   406  				}
   407  				markVolumeAttached(t, csiAttacher.k8s, nil, expectedAttachID, status)
   408  			}
   409  			wg.Wait()
   410  		})
   411  	}
   412  }
   413  
   414  func TestAttacherWaitForVolumeAttachmentWithCSIDriver(t *testing.T) {
   415  	// In order to detect if the volume plugin would skip WaitForAttach for non-attachable drivers,
   416  	// we do not instantiate any VolumeAttachment. So if the plugin does not skip attach,  WaitForVolumeAttachment
   417  	// will return an error that volume attachment was not found.
   418  	tests := []struct {
   419  		name         string
   420  		driver       string
   421  		expectError  bool
   422  		watchTimeout time.Duration
   423  	}{
   424  		{
   425  			name:        "CSIDriver not attachable -> success",
   426  			driver:      "not-attachable",
   427  			expectError: false,
   428  		},
   429  		{
   430  			name:        "CSIDriver is attachable -> failure",
   431  			driver:      "attachable",
   432  			expectError: true,
   433  		},
   434  		{
   435  			name:        "CSIDriver.AttachRequired not set  -> failure",
   436  			driver:      "nil",
   437  			expectError: true,
   438  		},
   439  		{
   440  			name:        "CSIDriver does not exist not set  -> failure",
   441  			driver:      "unknown",
   442  			expectError: true,
   443  		},
   444  	}
   445  
   446  	for _, test := range tests {
   447  		t.Run(test.name, func(t *testing.T) {
   448  			fakeClient := fakeclient.NewSimpleClientset(
   449  				getTestCSIDriver("not-attachable", nil, &bFalse, nil),
   450  				getTestCSIDriver("attachable", nil, &bTrue, nil),
   451  				getTestCSIDriver("nil", nil, nil, nil),
   452  				&v1.Node{
   453  					ObjectMeta: metav1.ObjectMeta{
   454  						Name: "fakeNode",
   455  					},
   456  					Spec: v1.NodeSpec{},
   457  				},
   458  			)
   459  			plug, tmpDir := newTestPlugin(t, fakeClient)
   460  			defer os.RemoveAll(tmpDir)
   461  
   462  			attacher, err := plug.NewAttacher()
   463  			if err != nil {
   464  				t.Fatalf("failed to create new attacher: %v", err)
   465  			}
   466  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, test.watchTimeout)
   467  			spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, test.driver, "test-vol"), false)
   468  
   469  			pluginCanAttach, err := plug.CanAttach(spec)
   470  			if err != nil {
   471  				t.Fatalf("plugin.CanAttach test failed: %s", err)
   472  			}
   473  			if !pluginCanAttach {
   474  				t.Log("plugin is not attachable")
   475  				return
   476  			}
   477  
   478  			_, err = csiAttacher.WaitForAttach(spec, "", nil, time.Second)
   479  			if err != nil && !test.expectError {
   480  				t.Errorf("Unexpected error: %s", err)
   481  			}
   482  			if err == nil && test.expectError {
   483  				t.Errorf("Expected error, got none")
   484  			}
   485  		})
   486  	}
   487  }
   488  
   489  func TestAttacherWaitForAttach(t *testing.T) {
   490  	tests := []struct {
   491  		name             string
   492  		driver           string
   493  		makeAttachment   func() *storage.VolumeAttachment
   494  		spec             *volume.Spec
   495  		expectedAttachID string
   496  		expectError      bool
   497  		watchTimeout     time.Duration
   498  	}{
   499  		{
   500  			name:   "successful attach",
   501  			driver: "attachable",
   502  			makeAttachment: func() *storage.VolumeAttachment {
   503  
   504  				testAttachID := getAttachmentName("test-vol", "attachable", "fakeNode")
   505  				successfulAttachment := makeTestAttachment(testAttachID, "fakeNode", "test-pv")
   506  				successfulAttachment.Status.Attached = true
   507  				return successfulAttachment
   508  			},
   509  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "attachable", "test-vol"), false),
   510  			expectedAttachID: getAttachmentName("test-vol", "attachable", "fakeNode"),
   511  			expectError:      false,
   512  		},
   513  		{
   514  			name: "failed attach with vol source",
   515  			makeAttachment: func() *storage.VolumeAttachment {
   516  
   517  				testAttachID := getAttachmentName("test-vol", "attachable", "fakeNode")
   518  				successfulAttachment := makeTestAttachment(testAttachID, "fakeNode", "volSrc01")
   519  				successfulAttachment.Status.Attached = true
   520  				return successfulAttachment
   521  			},
   522  			spec:        volume.NewSpecFromVolume(makeTestVol("volSrc01", "attachable")),
   523  			expectError: true,
   524  		},
   525  		{
   526  			name:        "failed attach",
   527  			driver:      "attachable",
   528  			expectError: true,
   529  		},
   530  	}
   531  
   532  	for _, test := range tests {
   533  		t.Run(test.name, func(t *testing.T) {
   534  			fakeClient := fakeclient.NewSimpleClientset()
   535  			plug, tmpDir := newTestPlugin(t, fakeClient)
   536  			defer os.RemoveAll(tmpDir)
   537  
   538  			attacher, err := plug.NewAttacher()
   539  			if err != nil {
   540  				t.Fatalf("failed to create new attacher: %v", err)
   541  			}
   542  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, test.watchTimeout)
   543  
   544  			if test.makeAttachment != nil {
   545  				attachment := test.makeAttachment()
   546  				_, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   547  				if err != nil {
   548  					t.Fatalf("failed to create VolumeAttachment: %v", err)
   549  				}
   550  				gotAttachment, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(context.TODO(), attachment.Name, metav1.GetOptions{})
   551  				if err != nil {
   552  					t.Fatalf("failed to get created VolumeAttachment: %v", err)
   553  				}
   554  				t.Logf("created test VolumeAttachment %+v", gotAttachment)
   555  			}
   556  
   557  			attachID, err := csiAttacher.WaitForAttach(test.spec, "", nil, time.Second)
   558  			if err != nil && !test.expectError {
   559  				t.Errorf("Unexpected error: %s", err)
   560  			}
   561  			if err == nil && test.expectError {
   562  				t.Errorf("Expected error, got none")
   563  			}
   564  			if attachID != test.expectedAttachID {
   565  				t.Errorf("Expected attachID %q, got %q", test.expectedAttachID, attachID)
   566  			}
   567  		})
   568  	}
   569  }
   570  
   571  func TestAttacherWaitForAttachWithInline(t *testing.T) {
   572  	tests := []struct {
   573  		name             string
   574  		driver           string
   575  		makeAttachment   func() *storage.VolumeAttachment
   576  		spec             *volume.Spec
   577  		expectedAttachID string
   578  		expectError      bool
   579  		watchTimeout     time.Duration
   580  	}{
   581  		{
   582  			name: "successful attach with PV",
   583  			makeAttachment: func() *storage.VolumeAttachment {
   584  
   585  				testAttachID := getAttachmentName("test-vol", "attachable", "fakeNode")
   586  				successfulAttachment := makeTestAttachment(testAttachID, "fakeNode", "test-pv")
   587  				successfulAttachment.Status.Attached = true
   588  				return successfulAttachment
   589  			},
   590  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "attachable", "test-vol"), false),
   591  			expectedAttachID: getAttachmentName("test-vol", "attachable", "fakeNode"),
   592  			expectError:      false,
   593  		},
   594  		{
   595  			name: "failed attach with volSrc",
   596  			makeAttachment: func() *storage.VolumeAttachment {
   597  
   598  				testAttachID := getAttachmentName("test-vol", "attachable", "fakeNode")
   599  				successfulAttachment := makeTestAttachment(testAttachID, "fakeNode", "volSrc01")
   600  				successfulAttachment.Status.Attached = true
   601  				return successfulAttachment
   602  			},
   603  			spec:        volume.NewSpecFromVolume(makeTestVol("volSrc01", "attachable")),
   604  			expectError: true,
   605  		},
   606  		{
   607  			name:        "failed attach",
   608  			driver:      "non-attachable",
   609  			spec:        volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "non-attachable", "test-vol"), false),
   610  			expectError: true,
   611  		},
   612  	}
   613  
   614  	for _, test := range tests {
   615  		t.Run(test.name, func(t *testing.T) {
   616  			fakeClient := fakeclient.NewSimpleClientset()
   617  			plug, tmpDir := newTestPlugin(t, fakeClient)
   618  			defer os.RemoveAll(tmpDir)
   619  
   620  			attacher, err := plug.NewAttacher()
   621  			if err != nil {
   622  				t.Fatalf("failed to create new attacher: %v", err)
   623  			}
   624  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, test.watchTimeout)
   625  
   626  			if test.makeAttachment != nil {
   627  				attachment := test.makeAttachment()
   628  				_, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   629  				if err != nil {
   630  					t.Fatalf("failed to create VolumeAttachment: %v", err)
   631  				}
   632  				gotAttachment, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(context.TODO(), attachment.Name, metav1.GetOptions{})
   633  				if err != nil {
   634  					t.Fatalf("failed to get created VolumeAttachment: %v", err)
   635  				}
   636  				t.Logf("created test VolumeAttachment %+v", gotAttachment)
   637  			}
   638  
   639  			attachID, err := csiAttacher.WaitForAttach(test.spec, "", nil, time.Second)
   640  			if test.expectError != (err != nil) {
   641  				t.Errorf("Unexpected error: %s", err)
   642  				return
   643  			}
   644  			if attachID != test.expectedAttachID {
   645  				t.Errorf("Expected attachID %q, got %q", test.expectedAttachID, attachID)
   646  			}
   647  		})
   648  	}
   649  }
   650  
   651  func TestAttacherWaitForVolumeAttachment(t *testing.T) {
   652  	nodeName := "fakeNode"
   653  	testCases := []struct {
   654  		name                 string
   655  		initAttached         bool
   656  		finalAttached        bool
   657  		trigerWatchEventTime time.Duration
   658  		initAttachErr        *storage.VolumeError
   659  		finalAttachErr       *storage.VolumeError
   660  		timeout              time.Duration
   661  		shouldFail           bool
   662  		watchTimeout         time.Duration
   663  	}{
   664  		{
   665  			name:         "attach success at get",
   666  			initAttached: true,
   667  			timeout:      50 * time.Millisecond,
   668  			shouldFail:   false,
   669  		},
   670  		{
   671  			name:          "attachment error ant get",
   672  			initAttachErr: &storage.VolumeError{Message: "missing volume"},
   673  			timeout:       30 * time.Millisecond,
   674  			shouldFail:    true,
   675  		},
   676  		{
   677  			name:                 "attach success at watch",
   678  			initAttached:         false,
   679  			finalAttached:        true,
   680  			trigerWatchEventTime: 5 * time.Millisecond,
   681  			timeout:              50 * time.Millisecond,
   682  			shouldFail:           false,
   683  		},
   684  		{
   685  			name:                 "attachment error ant watch",
   686  			initAttached:         false,
   687  			finalAttached:        false,
   688  			finalAttachErr:       &storage.VolumeError{Message: "missing volume"},
   689  			trigerWatchEventTime: 5 * time.Millisecond,
   690  			timeout:              30 * time.Millisecond,
   691  			shouldFail:           true,
   692  		},
   693  		{
   694  			name:                 "time ran out",
   695  			initAttached:         false,
   696  			finalAttached:        true,
   697  			trigerWatchEventTime: 100 * time.Millisecond,
   698  			timeout:              50 * time.Millisecond,
   699  			shouldFail:           true,
   700  		},
   701  	}
   702  
   703  	for i, tc := range testCases {
   704  		t.Run(tc.name, func(t *testing.T) {
   705  			fakeClient := fakeclient.NewSimpleClientset()
   706  			plug, tmpDir := newTestPlugin(t, fakeClient)
   707  			defer os.RemoveAll(tmpDir)
   708  
   709  			fakeWatcher := watch.NewRaceFreeFake()
   710  			fakeClient.Fake.PrependWatchReactor("volumeattachments", core.DefaultWatchReactor(fakeWatcher, nil))
   711  
   712  			attacher, err := plug.NewAttacher()
   713  			if err != nil {
   714  				t.Fatalf("failed to create new attacher: %v", err)
   715  			}
   716  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   717  
   718  			t.Logf("running test: %v", tc.name)
   719  			pvName := fmt.Sprintf("test-pv-%d", i)
   720  			volID := fmt.Sprintf("test-vol-%d", i)
   721  			attachID := getAttachmentName(volID, testDriver, nodeName)
   722  			attachment := makeTestAttachment(attachID, nodeName, pvName)
   723  			attachment.Status.Attached = tc.initAttached
   724  			attachment.Status.AttachError = tc.initAttachErr
   725  			_, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   726  			if err != nil {
   727  				t.Fatalf("failed to attach: %v", err)
   728  			}
   729  
   730  			trigerWatchEventTime := tc.trigerWatchEventTime
   731  			finalAttached := tc.finalAttached
   732  			finalAttachErr := tc.finalAttachErr
   733  			var wg sync.WaitGroup
   734  			// after timeout, fakeWatcher will be closed by csiAttacher.waitForVolumeAttachment
   735  			if tc.trigerWatchEventTime > 0 && tc.trigerWatchEventTime < tc.timeout {
   736  				wg.Add(1)
   737  				go func() {
   738  					defer wg.Done()
   739  					time.Sleep(trigerWatchEventTime)
   740  					attachment := makeTestAttachment(attachID, nodeName, pvName)
   741  					attachment.Status.Attached = finalAttached
   742  					attachment.Status.AttachError = finalAttachErr
   743  					fakeWatcher.Modify(attachment)
   744  				}()
   745  			}
   746  
   747  			retID, err := csiAttacher.waitForVolumeAttachment(volID, attachID, tc.timeout)
   748  			if tc.shouldFail && err == nil {
   749  				t.Error("expecting failure, but err is nil")
   750  			}
   751  			if tc.initAttachErr != nil && err != nil {
   752  				if tc.initAttachErr.Message != err.Error() {
   753  					t.Errorf("expecting error [%v], got [%v]", tc.initAttachErr.Message, err.Error())
   754  				}
   755  			}
   756  			if err == nil && retID != attachID {
   757  				t.Errorf("attacher.WaitForAttach not returning attachment ID")
   758  			}
   759  			wg.Wait()
   760  		})
   761  	}
   762  }
   763  
   764  func TestAttacherVolumesAreAttached(t *testing.T) {
   765  	type attachedSpec struct {
   766  		volName  string
   767  		spec     *volume.Spec
   768  		attached bool
   769  	}
   770  	testCases := []struct {
   771  		name          string
   772  		attachedSpecs []attachedSpec
   773  		watchTimeout  time.Duration
   774  	}{
   775  		{
   776  			name: "attach and detach",
   777  			attachedSpecs: []attachedSpec{
   778  				{"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
   779  				{"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), true},
   780  				{"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), false},
   781  				{"vol3", volume.NewSpecFromPersistentVolume(makeTestPV("pv3", 10, testDriver, "vol3"), false), false},
   782  				{"vol4", volume.NewSpecFromPersistentVolume(makeTestPV("pv4", 20, testDriver, "vol4"), false), true},
   783  			},
   784  		},
   785  		{
   786  			name: "all detached",
   787  			attachedSpecs: []attachedSpec{
   788  				{"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), false},
   789  				{"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), false},
   790  				{"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), false},
   791  			},
   792  		},
   793  		{
   794  			name: "all attached",
   795  			attachedSpecs: []attachedSpec{
   796  				{"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
   797  				{"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), true},
   798  			},
   799  		},
   800  		{
   801  			name: "include non-attable",
   802  			attachedSpecs: []attachedSpec{
   803  				{"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
   804  				{"vol1", volume.NewSpecFromVolume(makeTestVol("pv1", testDriver)), false},
   805  			},
   806  		},
   807  	}
   808  
   809  	for _, tc := range testCases {
   810  		t.Run(tc.name, func(t *testing.T) {
   811  			plug, tmpDir := newTestPluginWithAttachDetachVolumeHost(t, nil)
   812  			defer os.RemoveAll(tmpDir)
   813  
   814  			attacher, err := plug.NewAttacher()
   815  			if err != nil {
   816  				t.Fatalf("failed to create new attacher: %v", err)
   817  			}
   818  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   819  			nodeName := "fakeNode"
   820  
   821  			var specs []*volume.Spec
   822  			// create and save volume attchments
   823  			for _, attachedSpec := range tc.attachedSpecs {
   824  				specs = append(specs, attachedSpec.spec)
   825  				attachID := getAttachmentName(attachedSpec.volName, testDriver, nodeName)
   826  				attachment := makeTestAttachment(attachID, nodeName, attachedSpec.spec.Name())
   827  				attachment.Status.Attached = attachedSpec.attached
   828  				_, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   829  				if err != nil {
   830  					t.Fatalf("failed to attach: %v", err)
   831  				}
   832  			}
   833  
   834  			// retrieve attached status
   835  			stats, err := csiAttacher.VolumesAreAttached(specs, types.NodeName(nodeName))
   836  			if err != nil {
   837  				t.Fatal(err)
   838  			}
   839  			if len(tc.attachedSpecs) != len(stats) {
   840  				t.Errorf("expecting %d attachment status, got %d", len(tc.attachedSpecs), len(stats))
   841  			}
   842  
   843  			// compare attachment status for each spec
   844  			for _, attached := range tc.attachedSpecs {
   845  				stat, ok := stats[attached.spec]
   846  				if attached.attached && !ok {
   847  					t.Error("failed to retrieve attached status for:", attached.spec)
   848  				}
   849  				if attached.attached != stat {
   850  					t.Errorf("expecting volume attachment %t, got %t", attached.attached, stat)
   851  				}
   852  			}
   853  		})
   854  	}
   855  }
   856  
   857  func TestAttacherVolumesAreAttachedWithInline(t *testing.T) {
   858  	type attachedSpec struct {
   859  		volName  string
   860  		spec     *volume.Spec
   861  		attached bool
   862  	}
   863  	testCases := []struct {
   864  		name          string
   865  		attachedSpecs []attachedSpec
   866  		watchTimeout  time.Duration
   867  	}{
   868  		{
   869  			name: "attach and detach with volume sources",
   870  			attachedSpecs: []attachedSpec{
   871  				{"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
   872  				{"vol1", volume.NewSpecFromVolume(makeTestVol("pv1", testDriver)), false},
   873  				{"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), true},
   874  				{"vol3", volume.NewSpecFromVolume(makeTestVol("pv3", testDriver)), false},
   875  				{"vol4", volume.NewSpecFromPersistentVolume(makeTestPV("pv4", 20, testDriver, "vol4"), false), true},
   876  			},
   877  		},
   878  	}
   879  
   880  	for _, tc := range testCases {
   881  		t.Run(tc.name, func(t *testing.T) {
   882  			plug, tmpDir := newTestPlugin(t, nil)
   883  			defer os.RemoveAll(tmpDir)
   884  
   885  			attacher, err := plug.NewAttacher()
   886  			if err != nil {
   887  				t.Fatalf("failed to create new attacher: %v", err)
   888  			}
   889  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   890  			nodeName := "fakeNode"
   891  
   892  			var specs []*volume.Spec
   893  			// create and save volume attchments
   894  			for _, attachedSpec := range tc.attachedSpecs {
   895  				specs = append(specs, attachedSpec.spec)
   896  				attachID := getAttachmentName(attachedSpec.volName, testDriver, nodeName)
   897  				attachment := makeTestAttachment(attachID, nodeName, attachedSpec.spec.Name())
   898  				attachment.Status.Attached = attachedSpec.attached
   899  				_, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   900  				if err != nil {
   901  					t.Fatalf("failed to attach: %v", err)
   902  				}
   903  			}
   904  
   905  			// retrieve attached status
   906  			stats, err := csiAttacher.VolumesAreAttached(specs, types.NodeName(nodeName))
   907  			if err != nil {
   908  				t.Fatal(err)
   909  			}
   910  			if len(tc.attachedSpecs) != len(stats) {
   911  				t.Errorf("expecting %d attachment status, got %d", len(tc.attachedSpecs), len(stats))
   912  			}
   913  
   914  			// compare attachment status for each spec
   915  			for _, attached := range tc.attachedSpecs {
   916  				stat, ok := stats[attached.spec]
   917  				if attached.attached && !ok {
   918  					t.Error("failed to retrieve attached status for:", attached.spec)
   919  				}
   920  				if attached.attached != stat {
   921  					t.Errorf("expecting volume attachment %t, got %t", attached.attached, stat)
   922  				}
   923  			}
   924  		})
   925  	}
   926  }
   927  
   928  func TestAttacherDetach(t *testing.T) {
   929  	nodeName := "fakeNode"
   930  	testCases := []struct {
   931  		name         string
   932  		volID        string
   933  		attachID     string
   934  		shouldFail   bool
   935  		reactor      func(action core.Action) (handled bool, ret runtime.Object, err error)
   936  		watchTimeout time.Duration
   937  	}{
   938  		{name: "normal test", volID: "vol-001", attachID: getAttachmentName("vol-001", testDriver, nodeName)},
   939  		{name: "normal test 2", volID: "vol-002", attachID: getAttachmentName("vol-002", testDriver, nodeName)},
   940  		{name: "object not found", volID: "vol-non-existing", attachID: getAttachmentName("vol-003", testDriver, nodeName)},
   941  		{
   942  			name:       "API error",
   943  			volID:      "vol-004",
   944  			attachID:   getAttachmentName("vol-004", testDriver, nodeName),
   945  			shouldFail: true, // All other API errors should be propagated to caller
   946  			reactor: func(action core.Action) (handled bool, ret runtime.Object, err error) {
   947  				// return Forbidden to all DELETE requests
   948  				if action.Matches("delete", "volumeattachments") {
   949  					return true, nil, apierrors.NewForbidden(action.GetResource().GroupResource(), action.GetNamespace(), fmt.Errorf("mock error"))
   950  				}
   951  				return false, nil, nil
   952  			},
   953  		},
   954  	}
   955  
   956  	for _, tc := range testCases {
   957  		t.Run(tc.name, func(t *testing.T) {
   958  			t.Logf("running test: %v", tc.name)
   959  			fakeClient := fakeclient.NewSimpleClientset()
   960  			plug, tmpDir := newTestPluginWithAttachDetachVolumeHost(t, fakeClient)
   961  			defer os.RemoveAll(tmpDir)
   962  
   963  			if tc.reactor != nil {
   964  				fakeClient.PrependReactor("*", "*", tc.reactor)
   965  			}
   966  
   967  			attacher, err0 := plug.NewAttacher()
   968  			if err0 != nil {
   969  				t.Fatalf("failed to create new attacher: %v", err0)
   970  			}
   971  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
   972  
   973  			pv := makeTestPV("test-pv", 10, testDriver, tc.volID)
   974  			spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly)
   975  			attachment := makeTestAttachment(tc.attachID, nodeName, "test-pv")
   976  			_, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
   977  			if err != nil {
   978  				t.Fatalf("failed to attach: %v", err)
   979  			}
   980  			volumeName, err := plug.GetVolumeName(spec)
   981  			if err != nil {
   982  				t.Errorf("test case %s failed: %v", tc.name, err)
   983  			}
   984  
   985  			err = csiAttacher.Detach(volumeName, types.NodeName(nodeName))
   986  			if tc.shouldFail && err == nil {
   987  				t.Fatal("expecting failure, but err = nil")
   988  			}
   989  			if !tc.shouldFail && err != nil {
   990  				t.Fatalf("unexpected err: %v", err)
   991  			}
   992  			attach, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(context.TODO(), tc.attachID, metav1.GetOptions{})
   993  			if err != nil {
   994  				if !apierrors.IsNotFound(err) {
   995  					t.Fatalf("unexpected err: %v", err)
   996  				}
   997  			} else {
   998  				if attach == nil {
   999  					t.Errorf("expecting attachment not to be nil, but it is")
  1000  				}
  1001  			}
  1002  		})
  1003  	}
  1004  }
  1005  
  1006  func TestAttacherGetDeviceMountPath(t *testing.T) {
  1007  	// Setup
  1008  	// Create a new attacher
  1009  	fakeClient := fakeclient.NewSimpleClientset()
  1010  	plug, tmpDir := newTestPlugin(t, fakeClient)
  1011  	defer os.RemoveAll(tmpDir)
  1012  	attacher, err0 := plug.NewAttacher()
  1013  	if err0 != nil {
  1014  		t.Fatalf("failed to create new attacher: %v", err0)
  1015  	}
  1016  	csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, testWatchTimeout)
  1017  
  1018  	pluginDir := csiAttacher.plugin.host.GetPluginDir(plug.GetPluginName())
  1019  	testCases := []struct {
  1020  		testName           string
  1021  		pvName             string
  1022  		volumeId           string
  1023  		skipPVCSISource    bool // The test clears PV.Spec.CSI
  1024  		shouldFail         bool
  1025  		addVolSource       bool // The test adds a Volume.VolumeSource.CSI.
  1026  		removeVolumeHandle bool // The test force removes CSI volume handle.
  1027  	}{
  1028  		{
  1029  			testName: "success test",
  1030  			pvName:   "test-pv1",
  1031  			volumeId: "test-vol1",
  1032  		},
  1033  		{
  1034  			testName:        "fail test, failed to create device mount path due to missing CSI source",
  1035  			pvName:          "test-pv1",
  1036  			volumeId:        "test-vol1",
  1037  			skipPVCSISource: true,
  1038  			shouldFail:      true,
  1039  		},
  1040  		{
  1041  			testName:     "fail test, failed to create device mount path, CSIVolumeSource found",
  1042  			pvName:       "test-pv1",
  1043  			volumeId:     "test-vol1",
  1044  			addVolSource: true,
  1045  			shouldFail:   true,
  1046  		},
  1047  		{
  1048  			testName:           "fail test, failed to create device mount path, missing CSI volume handle",
  1049  			pvName:             "test-pv1",
  1050  			volumeId:           "test-vol1",
  1051  			shouldFail:         true,
  1052  			removeVolumeHandle: true,
  1053  		},
  1054  	}
  1055  
  1056  	for _, tc := range testCases {
  1057  		t.Logf("Running test case: %s", tc.testName)
  1058  		var spec *volume.Spec
  1059  
  1060  		// Create spec
  1061  		pv := makeTestPV(tc.pvName, 10, testDriver, tc.volumeId)
  1062  		if tc.removeVolumeHandle {
  1063  			pv.Spec.PersistentVolumeSource.CSI.VolumeHandle = ""
  1064  		}
  1065  		if tc.addVolSource {
  1066  			spec = volume.NewSpecFromVolume(makeTestVol(tc.pvName, testDriver))
  1067  		} else {
  1068  			spec = volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly)
  1069  			if tc.skipPVCSISource {
  1070  				spec.PersistentVolume.Spec.CSI = nil
  1071  			}
  1072  		}
  1073  		// Run
  1074  		mountPath, err := csiAttacher.GetDeviceMountPath(spec)
  1075  
  1076  		// Verify
  1077  		if err != nil && !tc.shouldFail {
  1078  			t.Errorf("test should not fail, but error occurred: %v", err)
  1079  		} else if err == nil {
  1080  			expectedMountPath := filepath.Join(pluginDir, testDriver, generateSha(tc.volumeId), globalMountInGlobalPath)
  1081  			if tc.shouldFail {
  1082  				t.Errorf("test should fail, but no error occurred")
  1083  			} else if mountPath != expectedMountPath {
  1084  				t.Errorf("mountPath does not equal expectedMountPath. Got: %s. Expected: %s", mountPath, expectedMountPath)
  1085  			}
  1086  		}
  1087  	}
  1088  }
  1089  
  1090  func TestAttacherMountDevice(t *testing.T) {
  1091  	pvName := "test-pv"
  1092  	var testFSGroup int64 = 3000
  1093  	nonFinalError := volumetypes.NewUncertainProgressError("")
  1094  	transientError := volumetypes.NewTransientOperationFailure("")
  1095  
  1096  	testCases := []struct {
  1097  		testName                       string
  1098  		volName                        string
  1099  		devicePath                     string
  1100  		deviceMountPath                string
  1101  		stageUnstageSet                bool
  1102  		fsGroup                        *int64
  1103  		expectedVolumeMountGroup       string
  1104  		driverSupportsVolumeMountGroup bool
  1105  		shouldFail                     bool
  1106  		skipOnWindows                  bool
  1107  		createAttachment               bool
  1108  		populateDeviceMountPath        bool
  1109  		exitError                      error
  1110  		spec                           *volume.Spec
  1111  		watchTimeout                   time.Duration
  1112  		skipClientSetup                bool
  1113  	}{
  1114  		{
  1115  			testName:         "normal PV",
  1116  			volName:          "test-vol1",
  1117  			devicePath:       "path1",
  1118  			deviceMountPath:  "path2",
  1119  			stageUnstageSet:  true,
  1120  			createAttachment: true,
  1121  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1122  		},
  1123  		{
  1124  			testName:         "normal PV with mount options",
  1125  			volName:          "test-vol1",
  1126  			devicePath:       "path1",
  1127  			deviceMountPath:  "path2",
  1128  			stageUnstageSet:  true,
  1129  			createAttachment: true,
  1130  			spec:             volume.NewSpecFromPersistentVolume(makeTestPVWithMountOptions(pvName, 10, testDriver, "test-vol1", []string{"test-op"}), false),
  1131  		},
  1132  		{
  1133  			testName:         "normal PV but with missing attachment should result in no-change",
  1134  			volName:          "test-vol1",
  1135  			devicePath:       "path1",
  1136  			deviceMountPath:  "path2",
  1137  			stageUnstageSet:  true,
  1138  			createAttachment: false,
  1139  			shouldFail:       true,
  1140  			exitError:        transientError,
  1141  			spec:             volume.NewSpecFromPersistentVolume(makeTestPVWithMountOptions(pvName, 10, testDriver, "test-vol1", []string{"test-op"}), false),
  1142  		},
  1143  		{
  1144  			testName:         "no vol name",
  1145  			volName:          "",
  1146  			devicePath:       "path1",
  1147  			deviceMountPath:  "path2",
  1148  			stageUnstageSet:  true,
  1149  			shouldFail:       true,
  1150  			createAttachment: true,
  1151  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, ""), false),
  1152  		},
  1153  		{
  1154  			testName:         "no device path",
  1155  			volName:          "test-vol1",
  1156  			devicePath:       "",
  1157  			deviceMountPath:  "path2",
  1158  			stageUnstageSet:  true,
  1159  			shouldFail:       false,
  1160  			createAttachment: true,
  1161  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1162  		},
  1163  		{
  1164  			testName:         "no device mount path",
  1165  			volName:          "test-vol1",
  1166  			devicePath:       "path1",
  1167  			deviceMountPath:  "",
  1168  			stageUnstageSet:  true,
  1169  			shouldFail:       true,
  1170  			createAttachment: true,
  1171  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1172  		},
  1173  		{
  1174  			testName:         "stage_unstage cap not set",
  1175  			volName:          "test-vol1",
  1176  			devicePath:       "path1",
  1177  			deviceMountPath:  "path2",
  1178  			stageUnstageSet:  false,
  1179  			createAttachment: true,
  1180  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1181  		},
  1182  		{
  1183  			testName:         "failure with volume source",
  1184  			volName:          "test-vol1",
  1185  			devicePath:       "path1",
  1186  			deviceMountPath:  "path2",
  1187  			shouldFail:       true,
  1188  			createAttachment: true,
  1189  			spec:             volume.NewSpecFromVolume(makeTestVol(pvName, testDriver)),
  1190  		},
  1191  		{
  1192  			testName:         "pv with nodestage timeout should result in in-progress device",
  1193  			volName:          fakecsi.NodeStageTimeOut_VolumeID,
  1194  			devicePath:       "path1",
  1195  			deviceMountPath:  "path2",
  1196  			stageUnstageSet:  true,
  1197  			createAttachment: true,
  1198  			spec:             volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, fakecsi.NodeStageTimeOut_VolumeID), false),
  1199  			exitError:        nonFinalError,
  1200  			shouldFail:       true,
  1201  		},
  1202  		{
  1203  			testName:                "failure PV with existing data",
  1204  			volName:                 "test-vol1",
  1205  			devicePath:              "path1",
  1206  			deviceMountPath:         "path2",
  1207  			stageUnstageSet:         true,
  1208  			createAttachment:        true,
  1209  			populateDeviceMountPath: true,
  1210  			shouldFail:              true,
  1211  			// NOTE: We're skipping this test on Windows because os.Chmod is not working as intended, which means that
  1212  			// this test won't fail on Windows due to permission denied errors.
  1213  			// TODO: Remove the skip once Windows file permissions support is added.
  1214  			// https://github.com/kubernetes/kubernetes/pull/110921
  1215  			skipOnWindows: true,
  1216  			spec:          volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), true),
  1217  		},
  1218  		{
  1219  			testName:                       "fsgroup provided, driver supports volume mount group; expect fsgroup to be passed to NodeStageVolume",
  1220  			volName:                        "test-vol1",
  1221  			devicePath:                     "path1",
  1222  			deviceMountPath:                "path2",
  1223  			fsGroup:                        &testFSGroup,
  1224  			driverSupportsVolumeMountGroup: true,
  1225  			expectedVolumeMountGroup:       "3000",
  1226  			stageUnstageSet:                true,
  1227  			createAttachment:               true,
  1228  			spec:                           volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1229  		},
  1230  		{
  1231  			testName:                       "fsgroup not provided, driver supports volume mount group; expect fsgroup not to be passed to NodeStageVolume",
  1232  			volName:                        "test-vol1",
  1233  			devicePath:                     "path1",
  1234  			deviceMountPath:                "path2",
  1235  			driverSupportsVolumeMountGroup: true,
  1236  			expectedVolumeMountGroup:       "",
  1237  			stageUnstageSet:                true,
  1238  			createAttachment:               true,
  1239  			spec:                           volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1240  		},
  1241  		{
  1242  			testName:                       "fsgroup provided, driver does not support volume mount group; expect fsgroup not to be passed to NodeStageVolume",
  1243  			volName:                        "test-vol1",
  1244  			devicePath:                     "path1",
  1245  			deviceMountPath:                "path2",
  1246  			fsGroup:                        &testFSGroup,
  1247  			driverSupportsVolumeMountGroup: false,
  1248  			expectedVolumeMountGroup:       "",
  1249  			stageUnstageSet:                true,
  1250  			createAttachment:               true,
  1251  			spec:                           volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1252  		},
  1253  		{
  1254  			testName:                "driver not specified",
  1255  			volName:                 "test-vol1",
  1256  			devicePath:              "path1",
  1257  			deviceMountPath:         "path2",
  1258  			fsGroup:                 &testFSGroup,
  1259  			stageUnstageSet:         true,
  1260  			createAttachment:        true,
  1261  			populateDeviceMountPath: false,
  1262  			spec:                    volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, "not-found", "test-vol1"), false),
  1263  			exitError:               transientError,
  1264  			shouldFail:              true,
  1265  			skipClientSetup:         true,
  1266  		},
  1267  	}
  1268  
  1269  	for _, tc := range testCases {
  1270  		user, err := user.Current()
  1271  		if err != nil {
  1272  			t.Logf("Current user could not be determined, assuming non-root: %v", err)
  1273  		} else {
  1274  			if tc.populateDeviceMountPath && user.Uid == "0" {
  1275  				t.Skipf("Skipping intentional failure on existing data when running as root.")
  1276  			}
  1277  		}
  1278  		t.Run(tc.testName, func(t *testing.T) {
  1279  			if tc.skipOnWindows && goruntime.GOOS == "windows" {
  1280  				t.Skipf("Skipping test case on Windows: %s", tc.testName)
  1281  			}
  1282  			t.Logf("Running test case: %s", tc.testName)
  1283  
  1284  			// Setup
  1285  			// Create a new attacher
  1286  			fakeClient := fakeclient.NewSimpleClientset()
  1287  			plug, tmpDir := newTestPlugin(t, fakeClient)
  1288  			defer os.RemoveAll(tmpDir)
  1289  
  1290  			attacher, err0 := plug.NewAttacher()
  1291  			if err0 != nil {
  1292  				t.Fatalf("failed to create new attacher: %v", err0)
  1293  			}
  1294  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
  1295  			if !tc.skipClientSetup {
  1296  				csiAttacher.csiClient = setupClientWithVolumeMountGroup(t, tc.stageUnstageSet, tc.driverSupportsVolumeMountGroup)
  1297  			}
  1298  
  1299  			if tc.deviceMountPath != "" {
  1300  				tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1301  			}
  1302  
  1303  			nodeName := string(csiAttacher.plugin.host.GetNodeName())
  1304  			attachID := getAttachmentName(tc.volName, testDriver, nodeName)
  1305  
  1306  			if tc.createAttachment {
  1307  				// Set up volume attachment
  1308  				attachment := makeTestAttachment(attachID, nodeName, pvName)
  1309  				_, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
  1310  				if err != nil {
  1311  					t.Fatalf("failed to attach: %v", err)
  1312  				}
  1313  			}
  1314  
  1315  			parent := filepath.Dir(tc.deviceMountPath)
  1316  			filePath := filepath.Join(parent, "newfile")
  1317  			if tc.populateDeviceMountPath {
  1318  				// We need to create the deviceMountPath before we Mount,
  1319  				// so that we can correctly create the file without errors.
  1320  				err := os.MkdirAll(tc.deviceMountPath, 0750)
  1321  				if err != nil {
  1322  					t.Errorf("error attempting to create the directory")
  1323  				}
  1324  				_, err = os.Create(filePath)
  1325  				if err != nil {
  1326  					t.Errorf("error attempting to populate file on parent path: %v", err)
  1327  				}
  1328  				err = os.Chmod(parent, 0555)
  1329  				if err != nil {
  1330  					t.Errorf("error attempting to modify directory permissions: %v", err)
  1331  				}
  1332  			}
  1333  
  1334  			// Run
  1335  			err := csiAttacher.MountDevice(
  1336  				tc.spec,
  1337  				tc.devicePath,
  1338  				tc.deviceMountPath,
  1339  				volume.DeviceMounterArgs{FsGroup: tc.fsGroup})
  1340  
  1341  			// Verify
  1342  			if err != nil {
  1343  				if !tc.shouldFail {
  1344  					t.Errorf("test should not fail, but error occurred: %v", err)
  1345  				}
  1346  				if tc.populateDeviceMountPath {
  1347  					// We're expecting saveVolumeData to fail, which is responsible
  1348  					// for creating this file. It shouldn't exist.
  1349  					_, err := os.Stat(filepath.Join(parent, volDataFileName))
  1350  					if !os.IsNotExist(err) {
  1351  						t.Errorf("vol_data.json should not exist: %v", err)
  1352  					}
  1353  					_, err = os.Stat(filePath)
  1354  					if os.IsNotExist(err) {
  1355  						t.Errorf("expecting file to exist after err received: %v", err)
  1356  					}
  1357  					err = os.Chmod(parent, 0777)
  1358  					if err != nil {
  1359  						t.Errorf("failed to modify permissions after test: %v", err)
  1360  					}
  1361  				}
  1362  				if tc.exitError != nil && reflect.TypeOf(tc.exitError) != reflect.TypeOf(err) {
  1363  					t.Fatalf("expected exitError type: %v got: %v (%v)", reflect.TypeOf(tc.exitError), reflect.TypeOf(err), err)
  1364  				}
  1365  				return
  1366  			}
  1367  			if tc.shouldFail {
  1368  				t.Errorf("test should fail, but no error occurred")
  1369  			}
  1370  
  1371  			// Verify call goes through all the way
  1372  			numStaged := 1
  1373  			if !tc.stageUnstageSet {
  1374  				numStaged = 0
  1375  			}
  1376  
  1377  			cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1378  			staged := cdc.nodeClient.GetNodeStagedVolumes()
  1379  			if len(staged) != numStaged {
  1380  				t.Errorf("got wrong number of staged volumes, expecting %v got: %v", numStaged, len(staged))
  1381  			}
  1382  			if tc.stageUnstageSet {
  1383  				vol, ok := staged[tc.volName]
  1384  				if !ok {
  1385  					t.Errorf("could not find staged volume: %s", tc.volName)
  1386  				}
  1387  				if vol.Path != tc.deviceMountPath {
  1388  					t.Errorf("expected mount path: %s. got: %s", tc.deviceMountPath, vol.Path)
  1389  				}
  1390  				if !reflect.DeepEqual(vol.MountFlags, tc.spec.PersistentVolume.Spec.MountOptions) {
  1391  					t.Errorf("expected mount options: %v, got: %v", tc.spec.PersistentVolume.Spec.MountOptions, vol.MountFlags)
  1392  				}
  1393  				if vol.VolumeMountGroup != tc.expectedVolumeMountGroup {
  1394  					t.Errorf("expected volume mount group %q, got: %q", tc.expectedVolumeMountGroup, vol.VolumeMountGroup)
  1395  				}
  1396  			}
  1397  
  1398  			// Verify the deviceMountPath was created by the plugin
  1399  			if tc.stageUnstageSet {
  1400  				s, err := os.Stat(tc.deviceMountPath)
  1401  				if err != nil {
  1402  					t.Errorf("expected staging directory %s to be created and be a directory, got error: %s", tc.deviceMountPath, err)
  1403  				} else {
  1404  					if !s.IsDir() {
  1405  						t.Errorf("expected staging directory %s to be directory, got something else", tc.deviceMountPath)
  1406  					}
  1407  				}
  1408  			}
  1409  		})
  1410  	}
  1411  }
  1412  
  1413  func TestAttacherMountDeviceWithInline(t *testing.T) {
  1414  	pvName := "test-pv"
  1415  	var testFSGroup int64 = 3000
  1416  	testCases := []struct {
  1417  		testName                 string
  1418  		volName                  string
  1419  		devicePath               string
  1420  		deviceMountPath          string
  1421  		fsGroup                  *int64
  1422  		expectedVolumeMountGroup string
  1423  		stageUnstageSet          bool
  1424  		shouldFail               bool
  1425  		spec                     *volume.Spec
  1426  		watchTimeout             time.Duration
  1427  	}{
  1428  		{
  1429  			testName:        "normal PV",
  1430  			volName:         "test-vol1",
  1431  			devicePath:      "path1",
  1432  			deviceMountPath: "path2",
  1433  			stageUnstageSet: true,
  1434  			spec:            volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1435  		},
  1436  		{
  1437  			testName:        "failure with volSrc",
  1438  			volName:         "test-vol1",
  1439  			devicePath:      "path1",
  1440  			deviceMountPath: "path2",
  1441  			shouldFail:      true,
  1442  			spec:            volume.NewSpecFromVolume(makeTestVol(pvName, testDriver)),
  1443  		},
  1444  		{
  1445  			testName:        "no vol name",
  1446  			volName:         "",
  1447  			devicePath:      "path1",
  1448  			deviceMountPath: "path2",
  1449  			stageUnstageSet: true,
  1450  			shouldFail:      true,
  1451  			spec:            volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, ""), false),
  1452  		},
  1453  		{
  1454  			testName:        "no device path",
  1455  			volName:         "test-vol1",
  1456  			devicePath:      "",
  1457  			deviceMountPath: "path2",
  1458  			stageUnstageSet: true,
  1459  			shouldFail:      false,
  1460  			spec:            volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1461  		},
  1462  		{
  1463  			testName:        "no device mount path",
  1464  			volName:         "test-vol1",
  1465  			devicePath:      "path1",
  1466  			deviceMountPath: "",
  1467  			stageUnstageSet: true,
  1468  			shouldFail:      true,
  1469  			spec:            volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1470  		},
  1471  		{
  1472  			testName:        "stage_unstage cap not set",
  1473  			volName:         "test-vol1",
  1474  			devicePath:      "path1",
  1475  			deviceMountPath: "path2",
  1476  			stageUnstageSet: false,
  1477  			spec:            volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1478  		},
  1479  		{
  1480  			testName:        "missing spec",
  1481  			volName:         "test-vol1",
  1482  			devicePath:      "path1",
  1483  			deviceMountPath: "path2",
  1484  			shouldFail:      true,
  1485  		},
  1486  		{
  1487  			testName:                 "fsgroup set",
  1488  			volName:                  "test-vol1",
  1489  			devicePath:               "path1",
  1490  			deviceMountPath:          "path2",
  1491  			fsGroup:                  &testFSGroup,
  1492  			expectedVolumeMountGroup: "3000",
  1493  			stageUnstageSet:          true,
  1494  			spec:                     volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1495  		},
  1496  	}
  1497  
  1498  	for _, tc := range testCases {
  1499  		t.Run(tc.testName, func(t *testing.T) {
  1500  			t.Logf("Running test case: %s", tc.testName)
  1501  
  1502  			// Setup
  1503  			// Create a new attacher
  1504  			fakeClient := fakeclient.NewSimpleClientset()
  1505  			plug, tmpDir := newTestPlugin(t, fakeClient)
  1506  			defer os.RemoveAll(tmpDir)
  1507  
  1508  			fakeWatcher := watch.NewRaceFreeFake()
  1509  			fakeClient.Fake.PrependWatchReactor("volumeattachments", core.DefaultWatchReactor(fakeWatcher, nil))
  1510  
  1511  			attacher, err0 := plug.NewAttacher()
  1512  			if err0 != nil {
  1513  				t.Fatalf("failed to create new attacher: %v", err0)
  1514  			}
  1515  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
  1516  			csiAttacher.csiClient = setupClientWithVolumeMountGroup(t, tc.stageUnstageSet, true /* volumeMountGroup */)
  1517  
  1518  			if tc.deviceMountPath != "" {
  1519  				tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1520  			}
  1521  
  1522  			nodeName := string(csiAttacher.plugin.host.GetNodeName())
  1523  			attachID := getAttachmentName(tc.volName, testDriver, nodeName)
  1524  
  1525  			// Set up volume attachment
  1526  			attachment := makeTestAttachment(attachID, nodeName, pvName)
  1527  			_, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(context.TODO(), attachment, metav1.CreateOptions{})
  1528  			if err != nil {
  1529  				t.Fatalf("failed to attach: %v", err)
  1530  			}
  1531  
  1532  			var wg sync.WaitGroup
  1533  			wg.Add(1)
  1534  
  1535  			go func() {
  1536  				defer wg.Done()
  1537  				fakeWatcher.Delete(attachment)
  1538  			}()
  1539  
  1540  			// Run
  1541  			err = csiAttacher.MountDevice(
  1542  				tc.spec,
  1543  				tc.devicePath,
  1544  				tc.deviceMountPath,
  1545  				volume.DeviceMounterArgs{FsGroup: tc.fsGroup})
  1546  
  1547  			// Verify
  1548  			if err != nil {
  1549  				if !tc.shouldFail {
  1550  					t.Errorf("test should not fail, but error occurred: %v", err)
  1551  				}
  1552  				return
  1553  			}
  1554  			if tc.shouldFail {
  1555  				t.Errorf("test should fail, but no error occurred")
  1556  			}
  1557  
  1558  			// Verify call goes through all the way
  1559  			numStaged := 1
  1560  			if !tc.stageUnstageSet {
  1561  				numStaged = 0
  1562  			}
  1563  
  1564  			cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1565  			staged := cdc.nodeClient.GetNodeStagedVolumes()
  1566  			if len(staged) != numStaged {
  1567  				t.Errorf("got wrong number of staged volumes, expecting %v got: %v", numStaged, len(staged))
  1568  			}
  1569  			if tc.stageUnstageSet {
  1570  				vol, ok := staged[tc.volName]
  1571  				if !ok {
  1572  					t.Errorf("could not find staged volume: %s", tc.volName)
  1573  				}
  1574  				if vol.Path != tc.deviceMountPath {
  1575  					t.Errorf("expected mount path: %s. got: %s", tc.deviceMountPath, vol.Path)
  1576  				}
  1577  				if vol.VolumeMountGroup != tc.expectedVolumeMountGroup {
  1578  					t.Errorf("expected volume mount group %q, got: %q", tc.expectedVolumeMountGroup, vol.VolumeMountGroup)
  1579  				}
  1580  			}
  1581  
  1582  			wg.Wait()
  1583  		})
  1584  	}
  1585  }
  1586  
  1587  func TestAttacherUnmountDevice(t *testing.T) {
  1588  	transientError := volumetypes.NewTransientOperationFailure("")
  1589  	testCases := []struct {
  1590  		testName        string
  1591  		volID           string
  1592  		deviceMountPath string
  1593  		jsonFile        string
  1594  		createPV        bool
  1595  		stageUnstageSet bool
  1596  		shouldFail      bool
  1597  		watchTimeout    time.Duration
  1598  		exitError       error
  1599  		unsetClient     bool
  1600  	}{
  1601  		// PV agnostic path positive test cases
  1602  		{
  1603  			testName:        "success, json file exists",
  1604  			volID:           "project/zone/test-vol1",
  1605  			deviceMountPath: "plugins/csi/" + generateSha("project/zone/test-vol1") + "/globalmount",
  1606  			jsonFile:        `{"driverName": "csi", "volumeHandle":"project/zone/test-vol1"}`,
  1607  			stageUnstageSet: true,
  1608  		},
  1609  		{
  1610  			testName:        "stage_unstage not set, PV agnostic path, unmount device is skipped",
  1611  			deviceMountPath: "plugins/csi/" + generateSha("project/zone/test-vol1") + "/globalmount",
  1612  			jsonFile:        `{"driverName":"test-driver","volumeHandle":"test-vol1"}`,
  1613  			stageUnstageSet: false,
  1614  		},
  1615  		// PV agnostic path negative test cases
  1616  		{
  1617  			testName:        "success: json file doesn't exist, unmount device is skipped",
  1618  			deviceMountPath: "plugins/csi/" + generateSha("project/zone/test-vol1") + "/globalmount",
  1619  			jsonFile:        "",
  1620  			stageUnstageSet: true,
  1621  			createPV:        true,
  1622  		},
  1623  		{
  1624  			testName:        "fail: invalid json, fail to retrieve driver and volumeID from globalpath",
  1625  			volID:           "project/zone/test-vol1",
  1626  			deviceMountPath: "plugins/csi/" + generateSha("project/zone/test-vol1") + "/globalmount",
  1627  			jsonFile:        `{"driverName"}}`,
  1628  			stageUnstageSet: true,
  1629  			shouldFail:      true,
  1630  		},
  1631  		// Ensure that a transient error is returned if the client is not established
  1632  		{
  1633  			testName:        "fail with transient error, json file exists but client not found",
  1634  			volID:           "project/zone/test-vol1",
  1635  			deviceMountPath: "plugins/csi/" + generateSha("project/zone/test-vol1") + "/globalmount",
  1636  			jsonFile:        `{"driverName": "unknown-driver", "volumeHandle":"project/zone/test-vol1"}`,
  1637  			stageUnstageSet: true,
  1638  			shouldFail:      true,
  1639  			exitError:       transientError,
  1640  			unsetClient:     true,
  1641  		},
  1642  	}
  1643  
  1644  	for _, tc := range testCases {
  1645  		t.Run(tc.testName, func(t *testing.T) {
  1646  			t.Logf("Running test case: %s", tc.testName)
  1647  			// Setup
  1648  			// Create a new attacher
  1649  			fakeClient := fakeclient.NewSimpleClientset()
  1650  			plug, tmpDir := newTestPlugin(t, fakeClient)
  1651  			defer os.RemoveAll(tmpDir)
  1652  			attacher, err0 := plug.NewAttacher()
  1653  			if err0 != nil {
  1654  				t.Fatalf("failed to create new attacher: %v", err0)
  1655  			}
  1656  			csiAttacher := getCsiAttacherFromVolumeAttacher(attacher, tc.watchTimeout)
  1657  			csiAttacher.csiClient = setupClient(t, tc.stageUnstageSet)
  1658  
  1659  			if tc.deviceMountPath != "" {
  1660  				tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1661  			}
  1662  			// Add the volume to NodeStagedVolumes
  1663  			cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1664  			cdc.nodeClient.AddNodeStagedVolume(tc.volID, tc.deviceMountPath, nil)
  1665  
  1666  			// Make the device staged path
  1667  			if tc.deviceMountPath != "" {
  1668  				if err := os.MkdirAll(tc.deviceMountPath, 0755); err != nil {
  1669  					t.Fatalf("error creating directory %s: %s", tc.deviceMountPath, err)
  1670  				}
  1671  			}
  1672  			dir := filepath.Dir(tc.deviceMountPath)
  1673  			// Make JSON for this object
  1674  			if tc.jsonFile != "" {
  1675  				dataPath := filepath.Join(dir, volDataFileName)
  1676  				if err := os.WriteFile(dataPath, []byte(tc.jsonFile), 0644); err != nil {
  1677  					t.Fatalf("error creating %s: %s", dataPath, err)
  1678  				}
  1679  			}
  1680  			if tc.createPV {
  1681  				// Make the PV for this object
  1682  				pvName := filepath.Base(dir)
  1683  				pv := makeTestPV(pvName, 5, "csi", tc.volID)
  1684  				_, err := csiAttacher.k8s.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
  1685  				if err != nil && !tc.shouldFail {
  1686  					t.Fatalf("Failed to create PV: %v", err)
  1687  				}
  1688  			}
  1689  			// Clear out the client if specified
  1690  			// The lookup to generate a new client will fail
  1691  			if tc.unsetClient {
  1692  				csiAttacher.csiClient = nil
  1693  			}
  1694  
  1695  			// Run
  1696  			err := csiAttacher.UnmountDevice(tc.deviceMountPath)
  1697  			// Verify
  1698  			if err != nil {
  1699  				if !tc.shouldFail {
  1700  					t.Errorf("test should not fail, but error occurred: %v", err)
  1701  				}
  1702  				if tc.exitError != nil && reflect.TypeOf(tc.exitError) != reflect.TypeOf(err) {
  1703  					t.Fatalf("expected exitError type: %v got: %v (%v)", reflect.TypeOf(tc.exitError), reflect.TypeOf(err), err)
  1704  				}
  1705  				return
  1706  			}
  1707  			if tc.shouldFail {
  1708  				t.Errorf("test should fail, but no error occurred")
  1709  			}
  1710  
  1711  			// Verify call goes through all the way
  1712  			expectedSet := 0
  1713  			if !tc.stageUnstageSet || tc.volID == "" {
  1714  				expectedSet = 1
  1715  			}
  1716  			staged := cdc.nodeClient.GetNodeStagedVolumes()
  1717  			if len(staged) != expectedSet {
  1718  				t.Errorf("got wrong number of staged volumes, expecting %v got: %v", expectedSet, len(staged))
  1719  			}
  1720  
  1721  			_, ok := staged[tc.volID]
  1722  			if ok && tc.stageUnstageSet && tc.volID != "" {
  1723  				t.Errorf("found unexpected staged volume: %s", tc.volID)
  1724  			} else if !ok && !tc.stageUnstageSet {
  1725  				t.Errorf("could not find expected staged volume: %s", tc.volID)
  1726  			}
  1727  
  1728  			if tc.jsonFile != "" && !tc.shouldFail {
  1729  				dataPath := filepath.Join(dir, volDataFileName)
  1730  				if _, err := os.Stat(dataPath); !os.IsNotExist(err) {
  1731  					if err != nil {
  1732  						t.Errorf("error checking file %s: %s", dataPath, err)
  1733  					} else {
  1734  						t.Errorf("json file %s should not exists, but it does", dataPath)
  1735  					}
  1736  				} else {
  1737  					t.Logf("json file %s was correctly removed", dataPath)
  1738  				}
  1739  			}
  1740  		})
  1741  	}
  1742  }
  1743  
  1744  func getCsiAttacherFromVolumeAttacher(attacher volume.Attacher, watchTimeout time.Duration) *csiAttacher {
  1745  	if watchTimeout == 0 {
  1746  		watchTimeout = testWatchTimeout
  1747  	}
  1748  	csiAttacher := attacher.(*csiAttacher)
  1749  	csiAttacher.watchTimeout = watchTimeout
  1750  	return csiAttacher
  1751  }
  1752  
  1753  func getCsiAttacherFromVolumeDetacher(detacher volume.Detacher, watchTimeout time.Duration) *csiAttacher {
  1754  	if watchTimeout == 0 {
  1755  		watchTimeout = testWatchTimeout
  1756  	}
  1757  	csiAttacher := detacher.(*csiAttacher)
  1758  	csiAttacher.watchTimeout = watchTimeout
  1759  	return csiAttacher
  1760  }
  1761  
  1762  func getCsiAttacherFromDeviceMounter(deviceMounter volume.DeviceMounter, watchTimeout time.Duration) *csiAttacher {
  1763  	if watchTimeout == 0 {
  1764  		watchTimeout = testWatchTimeout
  1765  	}
  1766  	csiAttacher := deviceMounter.(*csiAttacher)
  1767  	csiAttacher.watchTimeout = watchTimeout
  1768  	return csiAttacher
  1769  }
  1770  
  1771  func getCsiAttacherFromDeviceUnmounter(deviceUnmounter volume.DeviceUnmounter, watchTimeout time.Duration) *csiAttacher {
  1772  	if watchTimeout == 0 {
  1773  		watchTimeout = testWatchTimeout
  1774  	}
  1775  	csiAttacher := deviceUnmounter.(*csiAttacher)
  1776  	csiAttacher.watchTimeout = watchTimeout
  1777  	return csiAttacher
  1778  }
  1779  
  1780  func generateSha(handle string) string {
  1781  	result := sha256.Sum256([]byte(fmt.Sprintf("%s", handle)))
  1782  	return fmt.Sprintf("%x", result)
  1783  }
  1784  

View as plain text