in_tree.go

Documentation: k8s.io/kubernetes/test/e2e/storage/drivers

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18   * This file defines various in-tree volume test drivers for TestSuites.
    19   *
    20   * There are two ways, how to prepare test drivers:
    21   * 1) With containerized server (NFS, Ceph, iSCSI, ...)
    22   * It creates a server pod which defines one volume for the tests.
    23   * These tests work only when privileged containers are allowed, exporting
    24   * various filesystems (like NFS) usually needs some mounting or
    25   * other privileged magic in the server pod.
    26   *
    27   * Note that the server containers are for testing purposes only and should not
    28   * be used in production.
    29   *
    30   * 2) With server or cloud provider outside of Kubernetes (Cinder, GCE, AWS, Azure, ...)
    31   * Appropriate server or cloud provider must exist somewhere outside
    32   * the tested Kubernetes cluster. CreateVolume will create a new volume to be
    33   * used in the TestSuites for inlineVolume or DynamicPV tests.
    34   */
    35  
    36  package drivers
    37  
    38  import (
    39  	"context"
    40  	"fmt"
    41  	"strconv"
    42  	"strings"
    43  	"time"
    44  
    45  	"github.com/onsi/ginkgo/v2"
    46  	v1 "k8s.io/api/core/v1"
    47  	rbacv1 "k8s.io/api/rbac/v1"
    48  	storagev1 "k8s.io/api/storage/v1"
    49  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    50  	"k8s.io/apimachinery/pkg/runtime/schema"
    51  	"k8s.io/apimachinery/pkg/util/sets"
    52  	"k8s.io/apiserver/pkg/authentication/serviceaccount"
    53  	clientset "k8s.io/client-go/kubernetes"
    54  	"k8s.io/kubernetes/test/e2e/feature"
    55  	"k8s.io/kubernetes/test/e2e/framework"
    56  	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
    57  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    58  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    59  	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
    60  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    61  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    62  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    63  	"k8s.io/kubernetes/test/e2e/storage/utils"
    64  	imageutils "k8s.io/kubernetes/test/utils/image"
    65  )
    66  
    67  const (
    68  	// Template for iSCSI IQN.
    69  	iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s"
    70  )
    71  
    72  // NFS
    73  type nfsDriver struct {
    74  	externalProvisionerPod *v1.Pod
    75  	externalPluginName     string
    76  
    77  	driverInfo storageframework.DriverInfo
    78  }
    79  
    80  type nfsVolume struct {
    81  	serverHost string
    82  	serverPod  *v1.Pod
    83  	f          *framework.Framework
    84  }
    85  
    86  var _ storageframework.TestDriver = &nfsDriver{}
    87  var _ storageframework.PreprovisionedVolumeTestDriver = &nfsDriver{}
    88  var _ storageframework.InlineVolumeTestDriver = &nfsDriver{}
    89  var _ storageframework.PreprovisionedPVTestDriver = &nfsDriver{}
    90  var _ storageframework.DynamicPVTestDriver = &nfsDriver{}
    91  
    92  // InitNFSDriver returns nfsDriver that implements TestDriver interface
    93  func InitNFSDriver() storageframework.TestDriver {
    94  	return &nfsDriver{
    95  		driverInfo: storageframework.DriverInfo{
    96  			Name:             "nfs",
    97  			InTreePluginName: "kubernetes.io/nfs",
    98  			MaxFileSize:      storageframework.FileSizeLarge,
    99  			SupportedSizeRange: e2evolume.SizeRange{
   100  				Min: "1Gi",
   101  			},
   102  			SupportedFsType: sets.NewString(
   103  				"", // Default fsType
   104  			),
   105  			SupportedMountOption: sets.NewString("relatime"),
   106  			RequiredMountOption:  sets.NewString("vers=4.0"),
   107  			Capabilities: map[storageframework.Capability]bool{
   108  				storageframework.CapPersistence:       true,
   109  				storageframework.CapExec:              true,
   110  				storageframework.CapRWX:               true,
   111  				storageframework.CapMultiPODs:         true,
   112  				storageframework.CapMultiplePVsSameID: true,
   113  			},
   114  		},
   115  	}
   116  }
   117  
   118  func (n *nfsDriver) GetDriverInfo() *storageframework.DriverInfo {
   119  	return &n.driverInfo
   120  }
   121  
   122  func (n *nfsDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   123  }
   124  
   125  func (n *nfsDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   126  	nv, ok := e2evolume.(*nfsVolume)
   127  	if !ok {
   128  		framework.Failf("Failed to cast test volume of type %T to the NFS test volume", e2evolume)
   129  	}
   130  	return &v1.VolumeSource{
   131  		NFS: &v1.NFSVolumeSource{
   132  			Server:   nv.serverHost,
   133  			Path:     "/",
   134  			ReadOnly: readOnly,
   135  		},
   136  	}
   137  }
   138  
   139  func (n *nfsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
   140  	nv, ok := e2evolume.(*nfsVolume)
   141  	if !ok {
   142  		framework.Failf("Failed to cast test volume of type %T to the NFS test volume", e2evolume)
   143  	}
   144  	return &v1.PersistentVolumeSource{
   145  		NFS: &v1.NFSVolumeSource{
   146  			Server:   nv.serverHost,
   147  			Path:     "/",
   148  			ReadOnly: readOnly,
   149  		},
   150  	}, nil
   151  }
   152  
   153  func (n *nfsDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
   154  	provisioner := n.externalPluginName
   155  	parameters := map[string]string{"mountOptions": "vers=4.0"}
   156  	ns := config.Framework.Namespace.Name
   157  
   158  	return storageframework.GetStorageClass(provisioner, parameters, nil, ns)
   159  }
   160  
   161  func (n *nfsDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   162  	cs := f.ClientSet
   163  	ns := f.Namespace
   164  	n.externalPluginName = fmt.Sprintf("example.com/nfs-%s", ns.Name)
   165  
   166  	// TODO(mkimuram): cluster-admin gives too much right but system:persistent-volume-provisioner
   167  	// is not enough. We should create new clusterrole for testing.
   168  	err := e2eauth.BindClusterRole(ctx, cs.RbacV1(), "cluster-admin", ns.Name,
   169  		rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Namespace: ns.Name, Name: "default"})
   170  	framework.ExpectNoError(err)
   171  	ginkgo.DeferCleanup(cs.RbacV1().ClusterRoleBindings().Delete, ns.Name+"--"+"cluster-admin", *metav1.NewDeleteOptions(0))
   172  
   173  	err = e2eauth.WaitForAuthorizationUpdate(ctx, cs.AuthorizationV1(),
   174  		serviceaccount.MakeUsername(ns.Name, "default"),
   175  		"", "get", schema.GroupResource{Group: "storage.k8s.io", Resource: "storageclasses"}, true)
   176  	framework.ExpectNoError(err, "Failed to update authorization: %v", err)
   177  
   178  	ginkgo.By("creating an external dynamic provisioner pod")
   179  	n.externalProvisionerPod = utils.StartExternalProvisioner(ctx, cs, ns.Name, n.externalPluginName)
   180  	ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, n.externalProvisionerPod)
   181  
   182  	return &storageframework.PerTestConfig{
   183  		Driver:    n,
   184  		Prefix:    "nfs",
   185  		Framework: f,
   186  	}
   187  }
   188  
   189  func (n *nfsDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   190  	f := config.Framework
   191  	cs := f.ClientSet
   192  	ns := f.Namespace
   193  
   194  	// NewNFSServer creates a pod for InlineVolume and PreprovisionedPV,
   195  	// and startExternalProvisioner creates a pod for DynamicPV.
   196  	// Therefore, we need a different PrepareTest logic for volType.
   197  	switch volType {
   198  	case storageframework.InlineVolume:
   199  		fallthrough
   200  	case storageframework.PreprovisionedPV:
   201  		c, serverPod, serverHost := e2evolume.NewNFSServer(ctx, cs, ns.Name, []string{})
   202  		config.ServerConfig = &c
   203  		return &nfsVolume{
   204  			serverHost: serverHost,
   205  			serverPod:  serverPod,
   206  			f:          f,
   207  		}
   208  	case storageframework.DynamicPV:
   209  		// Do nothing
   210  	default:
   211  		framework.Failf("Unsupported volType:%v is specified", volType)
   212  	}
   213  	return nil
   214  }
   215  
   216  func (v *nfsVolume) DeleteVolume(ctx context.Context) {
   217  	cleanUpVolumeServer(ctx, v.f, v.serverPod)
   218  }
   219  
   220  // iSCSI
   221  // The iscsiadm utility and iscsi target kernel modules must be installed on all nodes.
   222  type iSCSIDriver struct {
   223  	driverInfo storageframework.DriverInfo
   224  }
   225  type iSCSIVolume struct {
   226  	serverPod *v1.Pod
   227  	serverIP  string
   228  	f         *framework.Framework
   229  	iqn       string
   230  }
   231  
   232  var _ storageframework.TestDriver = &iSCSIDriver{}
   233  var _ storageframework.PreprovisionedVolumeTestDriver = &iSCSIDriver{}
   234  var _ storageframework.InlineVolumeTestDriver = &iSCSIDriver{}
   235  var _ storageframework.PreprovisionedPVTestDriver = &iSCSIDriver{}
   236  
   237  // InitISCSIDriver returns iSCSIDriver that implements TestDriver interface
   238  func InitISCSIDriver() storageframework.TestDriver {
   239  	return &iSCSIDriver{
   240  		driverInfo: storageframework.DriverInfo{
   241  			Name:             "iscsi",
   242  			InTreePluginName: "kubernetes.io/iscsi",
   243  			TestTags:         []interface{}{feature.Volumes},
   244  			MaxFileSize:      storageframework.FileSizeMedium,
   245  			SupportedFsType: sets.NewString(
   246  				"", // Default fsType
   247  				"ext4",
   248  			),
   249  			TopologyKeys: []string{v1.LabelHostname},
   250  			Capabilities: map[storageframework.Capability]bool{
   251  				storageframework.CapPersistence:       true,
   252  				storageframework.CapFsGroup:           true,
   253  				storageframework.CapBlock:             true,
   254  				storageframework.CapExec:              true,
   255  				storageframework.CapMultiPODs:         true,
   256  				storageframework.CapTopology:          true,
   257  				storageframework.CapMultiplePVsSameID: true,
   258  			},
   259  		},
   260  	}
   261  }
   262  
   263  func (i *iSCSIDriver) GetDriverInfo() *storageframework.DriverInfo {
   264  	return &i.driverInfo
   265  }
   266  
   267  func (i *iSCSIDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   268  }
   269  
   270  func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   271  	iv, ok := e2evolume.(*iSCSIVolume)
   272  	if !ok {
   273  		framework.Failf("failed to cast test volume of type %T to the iSCSI test volume", e2evolume)
   274  	}
   275  
   276  	volSource := v1.VolumeSource{
   277  		ISCSI: &v1.ISCSIVolumeSource{
   278  			TargetPortal: "127.0.0.1:3260",
   279  			IQN:          iv.iqn,
   280  			Lun:          0,
   281  			ReadOnly:     readOnly,
   282  		},
   283  	}
   284  	if fsType != "" {
   285  		volSource.ISCSI.FSType = fsType
   286  	}
   287  	return &volSource
   288  }
   289  
   290  func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
   291  	iv, ok := e2evolume.(*iSCSIVolume)
   292  	if !ok {
   293  		framework.Failf("failed to cast test volume of type %T to the iSCSI test volume", e2evolume)
   294  	}
   295  
   296  	pvSource := v1.PersistentVolumeSource{
   297  		ISCSI: &v1.ISCSIPersistentVolumeSource{
   298  			TargetPortal: "127.0.0.1:3260",
   299  			IQN:          iv.iqn,
   300  			Lun:          0,
   301  			ReadOnly:     readOnly,
   302  		},
   303  	}
   304  	if fsType != "" {
   305  		pvSource.ISCSI.FSType = fsType
   306  	}
   307  	return &pvSource, nil
   308  }
   309  
   310  func (i *iSCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   311  	return &storageframework.PerTestConfig{
   312  		Driver:    i,
   313  		Prefix:    "iscsi",
   314  		Framework: f,
   315  	}
   316  }
   317  
   318  func (i *iSCSIDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   319  	f := config.Framework
   320  	cs := f.ClientSet
   321  	ns := f.Namespace
   322  
   323  	c, serverPod, serverIP, iqn := newISCSIServer(ctx, cs, ns.Name)
   324  	config.ServerConfig = &c
   325  	config.ClientNodeSelection = c.ClientNodeSelection
   326  	return &iSCSIVolume{
   327  		serverPod: serverPod,
   328  		serverIP:  serverIP,
   329  		iqn:       iqn,
   330  		f:         f,
   331  	}
   332  }
   333  
   334  // newISCSIServer is an iSCSI-specific wrapper for CreateStorageServer.
   335  func newISCSIServer(ctx context.Context, cs clientset.Interface, namespace string) (config e2evolume.TestConfig, pod *v1.Pod, ip, iqn string) {
   336  	// Generate cluster-wide unique IQN
   337  	iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace)
   338  	config = e2evolume.TestConfig{
   339  		Namespace:   namespace,
   340  		Prefix:      "iscsi",
   341  		ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
   342  		ServerArgs:  []string{iqn},
   343  		ServerVolumes: map[string]string{
   344  			// iSCSI container needs to insert modules from the host
   345  			"/lib/modules": "/lib/modules",
   346  			// iSCSI container needs to configure kernel
   347  			"/sys/kernel": "/sys/kernel",
   348  			// iSCSI source "block devices" must be available on the host
   349  			"/srv/iscsi": "/srv/iscsi",
   350  			// targetcli uses dbus
   351  			"/run/dbus": "/run/dbus",
   352  		},
   353  		ServerReadyMessage: "iscsi target started",
   354  		ServerHostNetwork:  true,
   355  	}
   356  	pod, ip = e2evolume.CreateStorageServer(ctx, cs, config)
   357  	// Make sure the client runs on the same node as server so we don't need to open any firewalls.
   358  	config.ClientNodeSelection = e2epod.NodeSelection{Name: pod.Spec.NodeName}
   359  	return config, pod, ip, iqn
   360  }
   361  
   362  // newRBDServer is a CephRBD-specific wrapper for CreateStorageServer.
   363  func newRBDServer(ctx context.Context, cs clientset.Interface, namespace string) (config e2evolume.TestConfig, pod *v1.Pod, secret *v1.Secret, ip string) {
   364  	config = e2evolume.TestConfig{
   365  		Namespace:   namespace,
   366  		Prefix:      "rbd",
   367  		ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer),
   368  		ServerPorts: []int{6789},
   369  		ServerVolumes: map[string]string{
   370  			"/lib/modules": "/lib/modules",
   371  		},
   372  		ServerReadyMessage: "Ceph is ready",
   373  	}
   374  	pod, ip = e2evolume.CreateStorageServer(ctx, cs, config)
   375  	// create secrets for the server
   376  	secret = &v1.Secret{
   377  		TypeMeta: metav1.TypeMeta{
   378  			Kind:       "Secret",
   379  			APIVersion: "v1",
   380  		},
   381  		ObjectMeta: metav1.ObjectMeta{
   382  			Name: config.Prefix + "-secret",
   383  		},
   384  		Data: map[string][]byte{
   385  			// from test/images/volumes-tester/rbd/keyring
   386  			"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
   387  		},
   388  		Type: "kubernetes.io/rbd",
   389  	}
   390  
   391  	secret, err := cs.CoreV1().Secrets(config.Namespace).Create(ctx, secret, metav1.CreateOptions{})
   392  	if err != nil {
   393  		framework.Failf("Failed to create secrets for Ceph RBD: %v", err)
   394  	}
   395  
   396  	return config, pod, secret, ip
   397  }
   398  
   399  func (v *iSCSIVolume) DeleteVolume(ctx context.Context) {
   400  	cleanUpVolumeServer(ctx, v.f, v.serverPod)
   401  }
   402  
   403  // Ceph RBD
   404  type rbdDriver struct {
   405  	driverInfo storageframework.DriverInfo
   406  }
   407  
   408  type rbdVolume struct {
   409  	serverPod *v1.Pod
   410  	serverIP  string
   411  	secret    *v1.Secret
   412  	f         *framework.Framework
   413  }
   414  
   415  var _ storageframework.TestDriver = &rbdDriver{}
   416  var _ storageframework.PreprovisionedVolumeTestDriver = &rbdDriver{}
   417  var _ storageframework.InlineVolumeTestDriver = &rbdDriver{}
   418  var _ storageframework.PreprovisionedPVTestDriver = &rbdDriver{}
   419  
   420  // InitRbdDriver returns rbdDriver that implements TestDriver interface
   421  func InitRbdDriver() storageframework.TestDriver {
   422  	return &rbdDriver{
   423  		driverInfo: storageframework.DriverInfo{
   424  			Name:             "rbd",
   425  			InTreePluginName: "kubernetes.io/rbd",
   426  			TestTags:         []interface{}{feature.Volumes, framework.WithSerial()},
   427  			MaxFileSize:      storageframework.FileSizeMedium,
   428  			SupportedSizeRange: e2evolume.SizeRange{
   429  				Min: "1Gi",
   430  			},
   431  			SupportedFsType: sets.NewString(
   432  				"", // Default fsType
   433  				"ext4",
   434  			),
   435  			Capabilities: map[storageframework.Capability]bool{
   436  				storageframework.CapPersistence:       true,
   437  				storageframework.CapFsGroup:           true,
   438  				storageframework.CapBlock:             true,
   439  				storageframework.CapExec:              true,
   440  				storageframework.CapMultiPODs:         true,
   441  				storageframework.CapMultiplePVsSameID: true,
   442  			},
   443  		},
   444  	}
   445  }
   446  
   447  func (r *rbdDriver) GetDriverInfo() *storageframework.DriverInfo {
   448  	return &r.driverInfo
   449  }
   450  
   451  func (r *rbdDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   452  }
   453  
   454  func (r *rbdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   455  	rv, ok := e2evolume.(*rbdVolume)
   456  	if !ok {
   457  		framework.Failf("failed to cast test volume of type %T to the RBD test volume", e2evolume)
   458  	}
   459  
   460  	volSource := v1.VolumeSource{
   461  		RBD: &v1.RBDVolumeSource{
   462  			CephMonitors: []string{rv.serverIP},
   463  			RBDPool:      "rbd",
   464  			RBDImage:     "foo",
   465  			RadosUser:    "admin",
   466  			SecretRef: &v1.LocalObjectReference{
   467  				Name: rv.secret.Name,
   468  			},
   469  			ReadOnly: readOnly,
   470  		},
   471  	}
   472  	if fsType != "" {
   473  		volSource.RBD.FSType = fsType
   474  	}
   475  	return &volSource
   476  }
   477  
   478  func (r *rbdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
   479  	rv, ok := e2evolume.(*rbdVolume)
   480  	if !ok {
   481  		framework.Failf("failed to cast test volume of type %T to the RBD test volume", e2evolume)
   482  	}
   483  
   484  	f := rv.f
   485  	ns := f.Namespace
   486  
   487  	pvSource := v1.PersistentVolumeSource{
   488  		RBD: &v1.RBDPersistentVolumeSource{
   489  			CephMonitors: []string{rv.serverIP},
   490  			RBDPool:      "rbd",
   491  			RBDImage:     "foo",
   492  			RadosUser:    "admin",
   493  			SecretRef: &v1.SecretReference{
   494  				Name:      rv.secret.Name,
   495  				Namespace: ns.Name,
   496  			},
   497  			ReadOnly: readOnly,
   498  		},
   499  	}
   500  	if fsType != "" {
   501  		pvSource.RBD.FSType = fsType
   502  	}
   503  	return &pvSource, nil
   504  }
   505  
   506  func (r *rbdDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   507  	return &storageframework.PerTestConfig{
   508  		Driver:    r,
   509  		Prefix:    "rbd",
   510  		Framework: f,
   511  	}
   512  }
   513  
   514  func (r *rbdDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   515  	f := config.Framework
   516  	cs := f.ClientSet
   517  	ns := f.Namespace
   518  
   519  	c, serverPod, secret, serverIP := newRBDServer(ctx, cs, ns.Name)
   520  	config.ServerConfig = &c
   521  	return &rbdVolume{
   522  		serverPod: serverPod,
   523  		serverIP:  serverIP,
   524  		secret:    secret,
   525  		f:         f,
   526  	}
   527  }
   528  
   529  func (v *rbdVolume) DeleteVolume(ctx context.Context) {
   530  	cleanUpVolumeServerWithSecret(ctx, v.f, v.serverPod, v.secret)
   531  }
   532  
   533  // Ceph
   534  type cephFSDriver struct {
   535  	driverInfo storageframework.DriverInfo
   536  }
   537  
   538  type cephVolume struct {
   539  	serverPod *v1.Pod
   540  	serverIP  string
   541  	secret    *v1.Secret
   542  	f         *framework.Framework
   543  }
   544  
   545  var _ storageframework.TestDriver = &cephFSDriver{}
   546  var _ storageframework.PreprovisionedVolumeTestDriver = &cephFSDriver{}
   547  var _ storageframework.InlineVolumeTestDriver = &cephFSDriver{}
   548  var _ storageframework.PreprovisionedPVTestDriver = &cephFSDriver{}
   549  
   550  // InitCephFSDriver returns cephFSDriver that implements TestDriver interface
   551  func InitCephFSDriver() storageframework.TestDriver {
   552  	return &cephFSDriver{
   553  		driverInfo: storageframework.DriverInfo{
   554  			Name:             "ceph",
   555  			InTreePluginName: "kubernetes.io/cephfs",
   556  			TestTags:         []interface{}{feature.Volumes, framework.WithSerial()},
   557  			MaxFileSize:      storageframework.FileSizeMedium,
   558  			SupportedSizeRange: e2evolume.SizeRange{
   559  				Min: "1Gi",
   560  			},
   561  			SupportedFsType: sets.NewString(
   562  				"", // Default fsType
   563  			),
   564  			Capabilities: map[storageframework.Capability]bool{
   565  				storageframework.CapPersistence:       true,
   566  				storageframework.CapExec:              true,
   567  				storageframework.CapRWX:               true,
   568  				storageframework.CapMultiPODs:         true,
   569  				storageframework.CapMultiplePVsSameID: true,
   570  			},
   571  		},
   572  	}
   573  }
   574  
   575  func (c *cephFSDriver) GetDriverInfo() *storageframework.DriverInfo {
   576  	return &c.driverInfo
   577  }
   578  
   579  func (c *cephFSDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   580  }
   581  
   582  func (c *cephFSDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   583  	cv, ok := e2evolume.(*cephVolume)
   584  	if !ok {
   585  		framework.Failf("Failed to cast test volume of type %T to the Ceph test volume", e2evolume)
   586  	}
   587  
   588  	return &v1.VolumeSource{
   589  		CephFS: &v1.CephFSVolumeSource{
   590  			Monitors: []string{cv.serverIP + ":6789"},
   591  			User:     "kube",
   592  			SecretRef: &v1.LocalObjectReference{
   593  				Name: cv.secret.Name,
   594  			},
   595  			ReadOnly: readOnly,
   596  		},
   597  	}
   598  }
   599  
   600  func (c *cephFSDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
   601  	cv, ok := e2evolume.(*cephVolume)
   602  	if !ok {
   603  		framework.Failf("Failed to cast test volume of type %T to the Ceph test volume", e2evolume)
   604  	}
   605  
   606  	ns := cv.f.Namespace
   607  
   608  	return &v1.PersistentVolumeSource{
   609  		CephFS: &v1.CephFSPersistentVolumeSource{
   610  			Monitors: []string{cv.serverIP + ":6789"},
   611  			User:     "kube",
   612  			SecretRef: &v1.SecretReference{
   613  				Name:      cv.secret.Name,
   614  				Namespace: ns.Name,
   615  			},
   616  			ReadOnly: readOnly,
   617  		},
   618  	}, nil
   619  }
   620  
   621  func (c *cephFSDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   622  	return &storageframework.PerTestConfig{
   623  		Driver:    c,
   624  		Prefix:    "cephfs",
   625  		Framework: f,
   626  	}
   627  }
   628  
   629  func (c *cephFSDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   630  	f := config.Framework
   631  	cs := f.ClientSet
   632  	ns := f.Namespace
   633  
   634  	cfg, serverPod, secret, serverIP := newRBDServer(ctx, cs, ns.Name)
   635  	config.ServerConfig = &cfg
   636  	return &cephVolume{
   637  		serverPod: serverPod,
   638  		serverIP:  serverIP,
   639  		secret:    secret,
   640  		f:         f,
   641  	}
   642  }
   643  
   644  func (v *cephVolume) DeleteVolume(ctx context.Context) {
   645  	cleanUpVolumeServerWithSecret(ctx, v.f, v.serverPod, v.secret)
   646  }
   647  
   648  // Hostpath
   649  type hostPathDriver struct {
   650  	driverInfo storageframework.DriverInfo
   651  }
   652  
   653  var _ storageframework.TestDriver = &hostPathDriver{}
   654  var _ storageframework.PreprovisionedVolumeTestDriver = &hostPathDriver{}
   655  var _ storageframework.InlineVolumeTestDriver = &hostPathDriver{}
   656  
   657  // InitHostPathDriver returns hostPathDriver that implements TestDriver interface
   658  func InitHostPathDriver() storageframework.TestDriver {
   659  	return &hostPathDriver{
   660  		driverInfo: storageframework.DriverInfo{
   661  			Name:             "hostPath",
   662  			InTreePluginName: "kubernetes.io/host-path",
   663  			MaxFileSize:      storageframework.FileSizeMedium,
   664  			SupportedFsType: sets.NewString(
   665  				"", // Default fsType
   666  			),
   667  			TopologyKeys: []string{v1.LabelHostname},
   668  			Capabilities: map[storageframework.Capability]bool{
   669  				storageframework.CapPersistence:       true,
   670  				storageframework.CapMultiPODs:         true,
   671  				storageframework.CapSingleNodeVolume:  true,
   672  				storageframework.CapTopology:          true,
   673  				storageframework.CapMultiplePVsSameID: true,
   674  			},
   675  		},
   676  	}
   677  }
   678  
   679  func (h *hostPathDriver) GetDriverInfo() *storageframework.DriverInfo {
   680  	return &h.driverInfo
   681  }
   682  
   683  func (h *hostPathDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   684  }
   685  
   686  func (h *hostPathDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   687  	// hostPath doesn't support readOnly volume
   688  	if readOnly {
   689  		return nil
   690  	}
   691  	return &v1.VolumeSource{
   692  		HostPath: &v1.HostPathVolumeSource{
   693  			Path: "/tmp",
   694  		},
   695  	}
   696  }
   697  
   698  func (h *hostPathDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   699  	return &storageframework.PerTestConfig{
   700  		Driver:    h,
   701  		Prefix:    "hostpath",
   702  		Framework: f,
   703  	}
   704  }
   705  
   706  func (h *hostPathDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   707  	f := config.Framework
   708  	cs := f.ClientSet
   709  
   710  	// pods should be scheduled on the node
   711  	node, err := e2enode.GetRandomReadySchedulableNode(ctx, cs)
   712  	framework.ExpectNoError(err)
   713  	config.ClientNodeSelection = e2epod.NodeSelection{Name: node.Name}
   714  	return nil
   715  }
   716  
   717  // HostPathSymlink
   718  type hostPathSymlinkDriver struct {
   719  	driverInfo storageframework.DriverInfo
   720  }
   721  
   722  type hostPathSymlinkVolume struct {
   723  	targetPath string
   724  	sourcePath string
   725  	prepPod    *v1.Pod
   726  	f          *framework.Framework
   727  }
   728  
   729  var _ storageframework.TestDriver = &hostPathSymlinkDriver{}
   730  var _ storageframework.PreprovisionedVolumeTestDriver = &hostPathSymlinkDriver{}
   731  var _ storageframework.InlineVolumeTestDriver = &hostPathSymlinkDriver{}
   732  
   733  // InitHostPathSymlinkDriver returns hostPathSymlinkDriver that implements TestDriver interface
   734  func InitHostPathSymlinkDriver() storageframework.TestDriver {
   735  	return &hostPathSymlinkDriver{
   736  		driverInfo: storageframework.DriverInfo{
   737  			Name:             "hostPathSymlink",
   738  			InTreePluginName: "kubernetes.io/host-path",
   739  			MaxFileSize:      storageframework.FileSizeMedium,
   740  			SupportedFsType: sets.NewString(
   741  				"", // Default fsType
   742  			),
   743  			TopologyKeys: []string{v1.LabelHostname},
   744  			Capabilities: map[storageframework.Capability]bool{
   745  				storageframework.CapPersistence:       true,
   746  				storageframework.CapMultiPODs:         true,
   747  				storageframework.CapSingleNodeVolume:  true,
   748  				storageframework.CapTopology:          true,
   749  				storageframework.CapMultiplePVsSameID: true,
   750  			},
   751  		},
   752  	}
   753  }
   754  
   755  func (h *hostPathSymlinkDriver) GetDriverInfo() *storageframework.DriverInfo {
   756  	return &h.driverInfo
   757  }
   758  
   759  func (h *hostPathSymlinkDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   760  }
   761  
   762  func (h *hostPathSymlinkDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   763  	hv, ok := e2evolume.(*hostPathSymlinkVolume)
   764  	if !ok {
   765  		framework.Failf("Failed to cast test volume of type %T to the Hostpath Symlink test volume", e2evolume)
   766  	}
   767  
   768  	// hostPathSymlink doesn't support readOnly volume
   769  	if readOnly {
   770  		return nil
   771  	}
   772  	return &v1.VolumeSource{
   773  		HostPath: &v1.HostPathVolumeSource{
   774  			Path: hv.targetPath,
   775  		},
   776  	}
   777  }
   778  
   779  func (h *hostPathSymlinkDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   780  	return &storageframework.PerTestConfig{
   781  		Driver:    h,
   782  		Prefix:    "hostpathsymlink",
   783  		Framework: f,
   784  	}
   785  }
   786  
   787  func (h *hostPathSymlinkDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   788  	f := config.Framework
   789  	cs := f.ClientSet
   790  
   791  	sourcePath := fmt.Sprintf("/tmp/%v", f.Namespace.Name)
   792  	targetPath := fmt.Sprintf("/tmp/%v-link", f.Namespace.Name)
   793  	volumeName := "test-volume"
   794  
   795  	// pods should be scheduled on the node
   796  	node, err := e2enode.GetRandomReadySchedulableNode(ctx, cs)
   797  	framework.ExpectNoError(err)
   798  	config.ClientNodeSelection = e2epod.NodeSelection{Name: node.Name}
   799  
   800  	cmd := fmt.Sprintf("mkdir %v -m 777 && ln -s %v %v", sourcePath, sourcePath, targetPath)
   801  	privileged := true
   802  
   803  	// Launch pod to initialize hostPath directory and symlink
   804  	prepPod := &v1.Pod{
   805  		ObjectMeta: metav1.ObjectMeta{
   806  			Name: fmt.Sprintf("hostpath-symlink-prep-%s", f.Namespace.Name),
   807  		},
   808  		Spec: v1.PodSpec{
   809  			Containers: []v1.Container{
   810  				{
   811  					Name:    fmt.Sprintf("init-volume-%s", f.Namespace.Name),
   812  					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
   813  					Command: []string{"/bin/sh", "-ec", cmd},
   814  					VolumeMounts: []v1.VolumeMount{
   815  						{
   816  							Name:      volumeName,
   817  							MountPath: "/tmp",
   818  						},
   819  					},
   820  					SecurityContext: &v1.SecurityContext{
   821  						Privileged: &privileged,
   822  					},
   823  				},
   824  			},
   825  			RestartPolicy: v1.RestartPolicyNever,
   826  			Volumes: []v1.Volume{
   827  				{
   828  					Name: volumeName,
   829  					VolumeSource: v1.VolumeSource{
   830  						HostPath: &v1.HostPathVolumeSource{
   831  							Path: "/tmp",
   832  						},
   833  					},
   834  				},
   835  			},
   836  			NodeName: node.Name,
   837  		},
   838  	}
   839  	// h.prepPod will be reused in cleanupDriver.
   840  	pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, prepPod, metav1.CreateOptions{})
   841  	framework.ExpectNoError(err, "while creating hostPath init pod")
   842  
   843  	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodStart)
   844  	framework.ExpectNoError(err, "while waiting for hostPath init pod to succeed")
   845  
   846  	err = e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
   847  	framework.ExpectNoError(err, "while deleting hostPath init pod")
   848  	return &hostPathSymlinkVolume{
   849  		sourcePath: sourcePath,
   850  		targetPath: targetPath,
   851  		prepPod:    prepPod,
   852  		f:          f,
   853  	}
   854  }
   855  
   856  func (v *hostPathSymlinkVolume) DeleteVolume(ctx context.Context) {
   857  	f := v.f
   858  
   859  	cmd := fmt.Sprintf("rm -rf %v&& rm -rf %v", v.targetPath, v.sourcePath)
   860  	v.prepPod.Spec.Containers[0].Command = []string{"/bin/sh", "-ec", cmd}
   861  
   862  	pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, v.prepPod, metav1.CreateOptions{})
   863  	framework.ExpectNoError(err, "while creating hostPath teardown pod")
   864  
   865  	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodStart)
   866  	framework.ExpectNoError(err, "while waiting for hostPath teardown pod to succeed")
   867  
   868  	err = e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
   869  	framework.ExpectNoError(err, "while deleting hostPath teardown pod")
   870  }
   871  
   872  // emptydir
   873  type emptydirDriver struct {
   874  	driverInfo storageframework.DriverInfo
   875  }
   876  
   877  var _ storageframework.TestDriver = &emptydirDriver{}
   878  var _ storageframework.PreprovisionedVolumeTestDriver = &emptydirDriver{}
   879  var _ storageframework.InlineVolumeTestDriver = &emptydirDriver{}
   880  
   881  // InitEmptydirDriver returns emptydirDriver that implements TestDriver interface
   882  func InitEmptydirDriver() storageframework.TestDriver {
   883  	return &emptydirDriver{
   884  		driverInfo: storageframework.DriverInfo{
   885  			Name:             "emptydir",
   886  			InTreePluginName: "kubernetes.io/empty-dir",
   887  			MaxFileSize:      storageframework.FileSizeMedium,
   888  			SupportedFsType: sets.NewString(
   889  				"", // Default fsType
   890  			),
   891  			Capabilities: map[storageframework.Capability]bool{
   892  				storageframework.CapExec:             true,
   893  				storageframework.CapSingleNodeVolume: true,
   894  			},
   895  		},
   896  	}
   897  }
   898  
   899  func (e *emptydirDriver) GetDriverInfo() *storageframework.DriverInfo {
   900  	return &e.driverInfo
   901  }
   902  
   903  func (e *emptydirDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   904  }
   905  
   906  func (e *emptydirDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
   907  	// emptydir doesn't support readOnly volume
   908  	if readOnly {
   909  		return nil
   910  	}
   911  	return &v1.VolumeSource{
   912  		EmptyDir: &v1.EmptyDirVolumeSource{},
   913  	}
   914  }
   915  
   916  func (e *emptydirDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
   917  	return nil
   918  }
   919  
   920  func (e *emptydirDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   921  	return &storageframework.PerTestConfig{
   922  		Driver:    e,
   923  		Prefix:    "emptydir",
   924  		Framework: f,
   925  	}
   926  }
   927  
   928  // Cinder
   929  // This tests only CSI migration with dynamically provisioned volumes.
   930  type cinderDriver struct {
   931  	driverInfo storageframework.DriverInfo
   932  }
   933  
   934  var _ storageframework.TestDriver = &cinderDriver{}
   935  var _ storageframework.DynamicPVTestDriver = &cinderDriver{}
   936  
   937  // InitCinderDriver returns cinderDriver that implements TestDriver interface
   938  func InitCinderDriver() storageframework.TestDriver {
   939  	return &cinderDriver{
   940  		driverInfo: storageframework.DriverInfo{
   941  			Name:             "cinder",
   942  			InTreePluginName: "kubernetes.io/cinder",
   943  			MaxFileSize:      storageframework.FileSizeMedium,
   944  			SupportedSizeRange: e2evolume.SizeRange{
   945  				Min: "1Gi",
   946  			},
   947  			SupportedFsType: sets.NewString(
   948  				"", // Default fsType
   949  			),
   950  			TopologyKeys: []string{v1.LabelFailureDomainBetaZone},
   951  			Capabilities: map[storageframework.Capability]bool{
   952  				storageframework.CapPersistence: true,
   953  				storageframework.CapFsGroup:     true,
   954  				storageframework.CapExec:        true,
   955  				storageframework.CapBlock:       true,
   956  				// Cinder supports volume limits, but the test creates large
   957  				// number of volumes and times out test suites.
   958  				storageframework.CapVolumeLimits: false,
   959  				storageframework.CapTopology:     true,
   960  			},
   961  		},
   962  	}
   963  }
   964  
   965  func (c *cinderDriver) GetDriverInfo() *storageframework.DriverInfo {
   966  	return &c.driverInfo
   967  }
   968  
   969  func (c *cinderDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   970  	e2eskipper.SkipUnlessProviderIs("openstack")
   971  }
   972  
   973  func (c *cinderDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
   974  	provisioner := "kubernetes.io/cinder"
   975  	parameters := map[string]string{}
   976  	if fsType != "" {
   977  		parameters["fsType"] = fsType
   978  	}
   979  	ns := config.Framework.Namespace.Name
   980  
   981  	return storageframework.GetStorageClass(provisioner, parameters, nil, ns)
   982  }
   983  
   984  func (c *cinderDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   985  	return &storageframework.PerTestConfig{
   986  		Driver:    c,
   987  		Prefix:    "cinder",
   988  		Framework: f,
   989  	}
   990  }
   991  
   992  // GCE
   993  type gcePdDriver struct {
   994  	driverInfo storageframework.DriverInfo
   995  }
   996  
   997  type gcePdVolume struct {
   998  	volumeName string
   999  }
  1000  
  1001  var _ storageframework.TestDriver = &gcePdDriver{}
  1002  var _ storageframework.PreprovisionedVolumeTestDriver = &gcePdDriver{}
  1003  var _ storageframework.InlineVolumeTestDriver = &gcePdDriver{}
  1004  var _ storageframework.PreprovisionedPVTestDriver = &gcePdDriver{}
  1005  var _ storageframework.DynamicPVTestDriver = &gcePdDriver{}
  1006  
  1007  // InitGcePdDriver returns gcePdDriver that implements TestDriver interface
  1008  func InitGcePdDriver() storageframework.TestDriver {
  1009  	supportedTypes := sets.NewString(
  1010  		"", // Default fsType
  1011  		"ext2",
  1012  		"ext3",
  1013  		"ext4",
  1014  		"xfs",
  1015  	)
  1016  	return &gcePdDriver{
  1017  		driverInfo: storageframework.DriverInfo{
  1018  			Name:             "gcepd",
  1019  			InTreePluginName: "kubernetes.io/gce-pd",
  1020  			MaxFileSize:      storageframework.FileSizeMedium,
  1021  			SupportedSizeRange: e2evolume.SizeRange{
  1022  				Min: "1Gi",
  1023  			},
  1024  			SupportedFsType:      supportedTypes,
  1025  			SupportedMountOption: sets.NewString("debug", "nouid32"),
  1026  			TopologyKeys:         []string{v1.LabelTopologyZone},
  1027  			Capabilities: map[storageframework.Capability]bool{
  1028  				storageframework.CapPersistence:         true,
  1029  				storageframework.CapFsGroup:             true,
  1030  				storageframework.CapBlock:               true,
  1031  				storageframework.CapExec:                true,
  1032  				storageframework.CapMultiPODs:           true,
  1033  				storageframework.CapControllerExpansion: true,
  1034  				storageframework.CapOfflineExpansion:    true,
  1035  				storageframework.CapOnlineExpansion:     true,
  1036  				storageframework.CapNodeExpansion:       true,
  1037  				// GCE supports volume limits, but the test creates large
  1038  				// number of volumes and times out test suites.
  1039  				storageframework.CapVolumeLimits:      false,
  1040  				storageframework.CapTopology:          true,
  1041  				storageframework.CapMultiplePVsSameID: true,
  1042  			},
  1043  		},
  1044  	}
  1045  }
  1046  
  1047  // InitWindowsGcePdDriver returns gcePdDriver running on Windows cluster that implements TestDriver interface
  1048  // In current test structure, it first initialize the driver and then set up
  1049  // the new framework, so we cannot get the correct OS here and select which file system is supported.
  1050  // So here uses a separate Windows in-tree gce pd driver
  1051  func InitWindowsGcePdDriver() storageframework.TestDriver {
  1052  	supportedTypes := sets.NewString(
  1053  		"ntfs",
  1054  	)
  1055  	return &gcePdDriver{
  1056  		driverInfo: storageframework.DriverInfo{
  1057  			Name:             "windows-gcepd",
  1058  			InTreePluginName: "kubernetes.io/gce-pd",
  1059  			MaxFileSize:      storageframework.FileSizeMedium,
  1060  			SupportedSizeRange: e2evolume.SizeRange{
  1061  				Min: "1Gi",
  1062  			},
  1063  			SupportedFsType: supportedTypes,
  1064  			TopologyKeys:    []string{v1.LabelZoneFailureDomain},
  1065  			Capabilities: map[storageframework.Capability]bool{
  1066  				storageframework.CapControllerExpansion: false,
  1067  				storageframework.CapPersistence:         true,
  1068  				storageframework.CapExec:                true,
  1069  				storageframework.CapMultiPODs:           true,
  1070  				// GCE supports volume limits, but the test creates large
  1071  				// number of volumes and times out test suites.
  1072  				storageframework.CapVolumeLimits:      false,
  1073  				storageframework.CapTopology:          true,
  1074  				storageframework.CapMultiplePVsSameID: true,
  1075  			},
  1076  		},
  1077  	}
  1078  }
  1079  
  1080  func (g *gcePdDriver) GetDriverInfo() *storageframework.DriverInfo {
  1081  	return &g.driverInfo
  1082  }
  1083  
  1084  func (g *gcePdDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1085  	e2eskipper.SkipUnlessProviderIs("gce", "gke")
  1086  	for _, tag := range pattern.TestTags {
  1087  		if tag == feature.Windows {
  1088  			e2eskipper.SkipUnlessNodeOSDistroIs("windows")
  1089  		}
  1090  	}
  1091  }
  1092  
  1093  func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) *v1.VolumeSource {
  1094  	gv, ok := e2evolume.(*gcePdVolume)
  1095  	if !ok {
  1096  		framework.Failf("Failed to cast test volume of type %T to the GCE PD test volume", e2evolume)
  1097  	}
  1098  	volSource := v1.VolumeSource{
  1099  		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
  1100  			PDName:   gv.volumeName,
  1101  			ReadOnly: readOnly,
  1102  		},
  1103  	}
  1104  	if fsType != "" {
  1105  		volSource.GCEPersistentDisk.FSType = fsType
  1106  	}
  1107  	return &volSource
  1108  }
  1109  
  1110  func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
  1111  	gv, ok := e2evolume.(*gcePdVolume)
  1112  	if !ok {
  1113  		framework.Failf("Failed to cast test volume of type %T to the GCE PD test volume", e2evolume)
  1114  	}
  1115  	pvSource := v1.PersistentVolumeSource{
  1116  		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
  1117  			PDName:   gv.volumeName,
  1118  			ReadOnly: readOnly,
  1119  		},
  1120  	}
  1121  	if fsType != "" {
  1122  		pvSource.GCEPersistentDisk.FSType = fsType
  1123  	}
  1124  	return &pvSource, nil
  1125  }
  1126  
  1127  func (g *gcePdDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
  1128  	provisioner := "kubernetes.io/gce-pd"
  1129  	parameters := map[string]string{}
  1130  	if fsType != "" {
  1131  		parameters["fsType"] = fsType
  1132  	}
  1133  	ns := config.Framework.Namespace.Name
  1134  	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
  1135  
  1136  	return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns)
  1137  }
  1138  
  1139  func (g *gcePdDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1140  	config := &storageframework.PerTestConfig{
  1141  		Driver:    g,
  1142  		Prefix:    "gcepd",
  1143  		Framework: f,
  1144  	}
  1145  
  1146  	if framework.NodeOSDistroIs("windows") {
  1147  		config.ClientNodeSelection = e2epod.NodeSelection{
  1148  			Selector: map[string]string{
  1149  				"kubernetes.io/os": "windows",
  1150  			},
  1151  		}
  1152  	}
  1153  	return config
  1154  
  1155  }
  1156  
  1157  func (g *gcePdDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
  1158  	zone := getInlineVolumeZone(ctx, config.Framework)
  1159  	if volType == storageframework.InlineVolume {
  1160  		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
  1161  		// so pods should be also scheduled there.
  1162  		config.ClientNodeSelection = e2epod.NodeSelection{
  1163  			Selector: map[string]string{
  1164  				v1.LabelTopologyZone: zone,
  1165  			},
  1166  		}
  1167  	}
  1168  	ginkgo.By("creating a test gce pd volume")
  1169  	vname, err := e2epv.CreatePDWithRetryAndZone(ctx, zone)
  1170  	framework.ExpectNoError(err)
  1171  	return &gcePdVolume{
  1172  		volumeName: vname,
  1173  	}
  1174  }
  1175  
  1176  func (v *gcePdVolume) DeleteVolume(ctx context.Context) {
  1177  	_ = e2epv.DeletePDWithRetry(ctx, v.volumeName)
  1178  }
  1179  
  1180  // vSphere
  1181  type vSphereDriver struct {
  1182  	driverInfo storageframework.DriverInfo
  1183  }
  1184  
  1185  var _ storageframework.TestDriver = &vSphereDriver{}
  1186  var _ storageframework.DynamicPVTestDriver = &vSphereDriver{}
  1187  
  1188  // InitVSphereDriver returns vSphereDriver that implements TestDriver interface
  1189  func InitVSphereDriver() storageframework.TestDriver {
  1190  	return &vSphereDriver{
  1191  		driverInfo: storageframework.DriverInfo{
  1192  			Name:             "vsphere",
  1193  			InTreePluginName: "kubernetes.io/vsphere-volume",
  1194  			MaxFileSize:      storageframework.FileSizeMedium,
  1195  			SupportedSizeRange: e2evolume.SizeRange{
  1196  				Min: "1Gi",
  1197  			},
  1198  			SupportedFsType: sets.NewString(
  1199  				"", // Default fsType
  1200  				"ext4",
  1201  				"ntfs",
  1202  			),
  1203  			TopologyKeys: []string{v1.LabelFailureDomainBetaZone},
  1204  			Capabilities: map[storageframework.Capability]bool{
  1205  				storageframework.CapPersistence:       true,
  1206  				storageframework.CapFsGroup:           true,
  1207  				storageframework.CapExec:              true,
  1208  				storageframework.CapMultiPODs:         true,
  1209  				storageframework.CapTopology:          true,
  1210  				storageframework.CapBlock:             true,
  1211  				storageframework.CapMultiplePVsSameID: false,
  1212  			},
  1213  		},
  1214  	}
  1215  }
  1216  
  1217  func (v *vSphereDriver) GetDriverInfo() *storageframework.DriverInfo {
  1218  	return &v.driverInfo
  1219  }
  1220  
  1221  func (v *vSphereDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1222  	e2eskipper.SkipUnlessProviderIs("vsphere")
  1223  }
  1224  
  1225  func (v *vSphereDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
  1226  	provisioner := "kubernetes.io/vsphere-volume"
  1227  	parameters := map[string]string{}
  1228  	if fsType != "" {
  1229  		parameters["fsType"] = fsType
  1230  	}
  1231  	ns := config.Framework.Namespace.Name
  1232  
  1233  	return storageframework.GetStorageClass(provisioner, parameters, nil, ns)
  1234  }
  1235  
  1236  func (v *vSphereDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1237  	return &storageframework.PerTestConfig{
  1238  		Driver:    v,
  1239  		Prefix:    "vsphere",
  1240  		Framework: f,
  1241  	}
  1242  }
  1243  
  1244  // Azure Disk
  1245  type azureDiskDriver struct {
  1246  	driverInfo storageframework.DriverInfo
  1247  }
  1248  
  1249  var _ storageframework.TestDriver = &azureDiskDriver{}
  1250  var _ storageframework.DynamicPVTestDriver = &azureDiskDriver{}
  1251  var _ storageframework.CustomTimeoutsTestDriver = &azureDiskDriver{}
  1252  
  1253  // InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
  1254  func InitAzureDiskDriver() storageframework.TestDriver {
  1255  	return &azureDiskDriver{
  1256  		driverInfo: storageframework.DriverInfo{
  1257  			Name:             "azure-disk",
  1258  			InTreePluginName: "kubernetes.io/azure-disk",
  1259  			MaxFileSize:      storageframework.FileSizeMedium,
  1260  			SupportedSizeRange: e2evolume.SizeRange{
  1261  				Min: "1Gi",
  1262  			},
  1263  			SupportedFsType: sets.NewString(
  1264  				"", // Default fsType
  1265  				"ext4",
  1266  				"xfs",
  1267  			),
  1268  			TopologyKeys: []string{v1.LabelFailureDomainBetaZone},
  1269  			Capabilities: map[storageframework.Capability]bool{
  1270  				storageframework.CapPersistence: true,
  1271  				storageframework.CapFsGroup:     true,
  1272  				storageframework.CapBlock:       true,
  1273  				storageframework.CapExec:        true,
  1274  				storageframework.CapMultiPODs:   true,
  1275  				// Azure supports volume limits, but the test creates large
  1276  				// number of volumes and times out test suites.
  1277  				storageframework.CapVolumeLimits:      false,
  1278  				storageframework.CapTopology:          true,
  1279  				storageframework.CapMultiplePVsSameID: true,
  1280  			},
  1281  		},
  1282  	}
  1283  }
  1284  
  1285  func (a *azureDiskDriver) GetDriverInfo() *storageframework.DriverInfo {
  1286  	return &a.driverInfo
  1287  }
  1288  
  1289  func (a *azureDiskDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1290  	e2eskipper.SkipUnlessProviderIs("azure")
  1291  }
  1292  
  1293  func (a *azureDiskDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
  1294  	provisioner := "kubernetes.io/azure-disk"
  1295  	parameters := map[string]string{}
  1296  	if fsType != "" {
  1297  		parameters["fsType"] = fsType
  1298  	}
  1299  	ns := config.Framework.Namespace.Name
  1300  	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
  1301  
  1302  	return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns)
  1303  }
  1304  
  1305  func (a *azureDiskDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1306  	return &storageframework.PerTestConfig{
  1307  		Driver:    a,
  1308  		Prefix:    "azure",
  1309  		Framework: f,
  1310  	}
  1311  }
  1312  
  1313  func (a *azureDiskDriver) GetTimeouts() *framework.TimeoutContext {
  1314  	timeouts := framework.NewTimeoutContext()
  1315  	timeouts.PodStart = time.Minute * 15
  1316  	timeouts.PodDelete = time.Minute * 15
  1317  	timeouts.PVDelete = time.Minute * 20
  1318  	return timeouts
  1319  }
  1320  
  1321  // AWS
  1322  type awsDriver struct {
  1323  	driverInfo storageframework.DriverInfo
  1324  }
  1325  
  1326  var _ storageframework.TestDriver = &awsDriver{}
  1327  var _ storageframework.DynamicPVTestDriver = &awsDriver{}
  1328  
  1329  // InitAwsDriver returns awsDriver that implements TestDriver interface
  1330  func InitAwsDriver() storageframework.TestDriver {
  1331  	return &awsDriver{
  1332  		driverInfo: storageframework.DriverInfo{
  1333  			Name:             "aws",
  1334  			InTreePluginName: "kubernetes.io/aws-ebs",
  1335  			MaxFileSize:      storageframework.FileSizeMedium,
  1336  			SupportedSizeRange: e2evolume.SizeRange{
  1337  				Min: "1Gi",
  1338  			},
  1339  			SupportedFsType: sets.NewString(
  1340  				"", // Default fsType
  1341  				"ext4",
  1342  				"xfs",
  1343  				"ntfs",
  1344  			),
  1345  			SupportedMountOption: sets.NewString("debug", "nouid32"),
  1346  			TopologyKeys:         []string{v1.LabelTopologyZone},
  1347  			Capabilities: map[storageframework.Capability]bool{
  1348  				storageframework.CapPersistence:         true,
  1349  				storageframework.CapFsGroup:             true,
  1350  				storageframework.CapBlock:               true,
  1351  				storageframework.CapExec:                true,
  1352  				storageframework.CapMultiPODs:           true,
  1353  				storageframework.CapControllerExpansion: true,
  1354  				storageframework.CapNodeExpansion:       true,
  1355  				storageframework.CapOfflineExpansion:    true,
  1356  				storageframework.CapOnlineExpansion:     true,
  1357  				// AWS supports volume limits, but the test creates large
  1358  				// number of volumes and times out test suites.
  1359  				storageframework.CapVolumeLimits:      false,
  1360  				storageframework.CapTopology:          true,
  1361  				storageframework.CapMultiplePVsSameID: true,
  1362  			},
  1363  		},
  1364  	}
  1365  }
  1366  
  1367  func (a *awsDriver) GetDriverInfo() *storageframework.DriverInfo {
  1368  	return &a.driverInfo
  1369  }
  1370  
  1371  func (a *awsDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1372  	e2eskipper.SkipUnlessProviderIs("aws")
  1373  }
  1374  
  1375  func (a *awsDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
  1376  	provisioner := "kubernetes.io/aws-ebs"
  1377  	parameters := map[string]string{}
  1378  	if fsType != "" {
  1379  		parameters["fsType"] = fsType
  1380  	}
  1381  	ns := config.Framework.Namespace.Name
  1382  	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
  1383  
  1384  	return storageframework.GetStorageClass(provisioner, parameters, &delayedBinding, ns)
  1385  }
  1386  
  1387  func (a *awsDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1388  	config := &storageframework.PerTestConfig{
  1389  		Driver:    a,
  1390  		Prefix:    "aws",
  1391  		Framework: f,
  1392  	}
  1393  
  1394  	if framework.NodeOSDistroIs("windows") {
  1395  		config.ClientNodeSelection = e2epod.NodeSelection{
  1396  			Selector: map[string]string{
  1397  				"kubernetes.io/os": "windows",
  1398  			},
  1399  		}
  1400  	}
  1401  	return config
  1402  }
  1403  
  1404  // local
  1405  type localDriver struct {
  1406  	driverInfo storageframework.DriverInfo
  1407  	node       *v1.Node
  1408  	hostExec   utils.HostExec
  1409  	// volumeType represents local volume type we are testing, e.g.  tmpfs,
  1410  	// directory, block device.
  1411  	volumeType utils.LocalVolumeType
  1412  	ltrMgr     utils.LocalTestResourceManager
  1413  }
  1414  
  1415  type localVolume struct {
  1416  	ltrMgr utils.LocalTestResourceManager
  1417  	ltr    *utils.LocalTestResource
  1418  }
  1419  
  1420  var (
  1421  	// capabilities
  1422  	defaultLocalVolumeCapabilities = map[storageframework.Capability]bool{
  1423  		storageframework.CapPersistence:       true,
  1424  		storageframework.CapFsGroup:           true,
  1425  		storageframework.CapBlock:             false,
  1426  		storageframework.CapExec:              true,
  1427  		storageframework.CapMultiPODs:         true,
  1428  		storageframework.CapSingleNodeVolume:  true,
  1429  		storageframework.CapMultiplePVsSameID: true,
  1430  	}
  1431  	localVolumeCapabitilies = map[utils.LocalVolumeType]map[storageframework.Capability]bool{
  1432  		utils.LocalVolumeBlock: {
  1433  			storageframework.CapPersistence:       true,
  1434  			storageframework.CapFsGroup:           true,
  1435  			storageframework.CapBlock:             true,
  1436  			storageframework.CapExec:              true,
  1437  			storageframework.CapMultiPODs:         true,
  1438  			storageframework.CapSingleNodeVolume:  true,
  1439  			storageframework.CapMultiplePVsSameID: true,
  1440  		},
  1441  	}
  1442  	// fstype
  1443  	defaultLocalVolumeSupportedFsTypes = sets.NewString("")
  1444  	localVolumeSupportedFsTypes        = map[utils.LocalVolumeType]sets.String{
  1445  		utils.LocalVolumeBlock: sets.NewString(
  1446  			"", // Default fsType
  1447  			"ext4",
  1448  			//"xfs", disabled see issue https://github.com/kubernetes/kubernetes/issues/74095
  1449  		),
  1450  	}
  1451  	// max file size
  1452  	defaultLocalVolumeMaxFileSize = storageframework.FileSizeSmall
  1453  	localVolumeMaxFileSizes       = map[utils.LocalVolumeType]int64{}
  1454  )
  1455  
  1456  var _ storageframework.TestDriver = &localDriver{}
  1457  var _ storageframework.PreprovisionedVolumeTestDriver = &localDriver{}
  1458  var _ storageframework.PreprovisionedPVTestDriver = &localDriver{}
  1459  
  1460  // InitLocalDriverWithVolumeType initializes the local driver based on the volume type.
  1461  func InitLocalDriverWithVolumeType(volumeType utils.LocalVolumeType) func() storageframework.TestDriver {
  1462  	maxFileSize := defaultLocalVolumeMaxFileSize
  1463  	if maxFileSizeByVolType, ok := localVolumeMaxFileSizes[volumeType]; ok {
  1464  		maxFileSize = maxFileSizeByVolType
  1465  	}
  1466  	supportedFsTypes := defaultLocalVolumeSupportedFsTypes
  1467  	if supportedFsTypesByType, ok := localVolumeSupportedFsTypes[volumeType]; ok {
  1468  		supportedFsTypes = supportedFsTypesByType
  1469  	}
  1470  	capabilities := defaultLocalVolumeCapabilities
  1471  	if capabilitiesByType, ok := localVolumeCapabitilies[volumeType]; ok {
  1472  		capabilities = capabilitiesByType
  1473  	}
  1474  	return func() storageframework.TestDriver {
  1475  		// custom tag to distinguish from tests of other volume types
  1476  		testTags := []interface{}{fmt.Sprintf("[LocalVolumeType: %s]", volumeType)}
  1477  		// For GCE Local SSD volumes, we must run serially
  1478  		if volumeType == utils.LocalVolumeGCELocalSSD {
  1479  			testTags = append(testTags, framework.WithSerial())
  1480  		}
  1481  		return &localDriver{
  1482  			driverInfo: storageframework.DriverInfo{
  1483  				Name:             "local",
  1484  				InTreePluginName: "kubernetes.io/local-volume",
  1485  				TestTags:         testTags,
  1486  				MaxFileSize:      maxFileSize,
  1487  				SupportedFsType:  supportedFsTypes,
  1488  				Capabilities:     capabilities,
  1489  			},
  1490  			volumeType: volumeType,
  1491  		}
  1492  	}
  1493  }
  1494  
  1495  func (l *localDriver) GetDriverInfo() *storageframework.DriverInfo {
  1496  	return &l.driverInfo
  1497  }
  1498  
  1499  func (l *localDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1500  }
  1501  
  1502  func (l *localDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1503  	var err error
  1504  	l.node, err = e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
  1505  	framework.ExpectNoError(err)
  1506  
  1507  	l.hostExec = utils.NewHostExec(f)
  1508  	l.ltrMgr = utils.NewLocalResourceManager("local-driver", l.hostExec, "/tmp")
  1509  
  1510  	// This can't be done in SkipUnsupportedTest because the test framework is not initialized yet
  1511  	if l.volumeType == utils.LocalVolumeGCELocalSSD {
  1512  		ssdInterface := "scsi"
  1513  		filesystemType := "fs"
  1514  		ssdCmd := fmt.Sprintf("ls -1 /mnt/disks/by-uuid/google-local-ssds-%s-%s/ | wc -l", ssdInterface, filesystemType)
  1515  		res, err := l.hostExec.IssueCommandWithResult(ctx, ssdCmd, l.node)
  1516  		framework.ExpectNoError(err)
  1517  		num, err := strconv.Atoi(strings.TrimSpace(res))
  1518  		framework.ExpectNoError(err)
  1519  		if num < 1 {
  1520  			e2eskipper.Skipf("Requires at least 1 %s %s localSSD ", ssdInterface, filesystemType)
  1521  		}
  1522  	}
  1523  
  1524  	ginkgo.DeferCleanup(l.hostExec.Cleanup)
  1525  	return &storageframework.PerTestConfig{
  1526  		Driver:              l,
  1527  		Prefix:              "local",
  1528  		Framework:           f,
  1529  		ClientNodeSelection: e2epod.NodeSelection{Name: l.node.Name},
  1530  	}
  1531  }
  1532  
  1533  func (l *localDriver) CreateVolume(ctx context.Context, config *storageframework.PerTestConfig, volType storageframework.TestVolType) storageframework.TestVolume {
  1534  	switch volType {
  1535  	case storageframework.PreprovisionedPV:
  1536  		node := l.node
  1537  		// assign this to schedule pod on this node
  1538  		config.ClientNodeSelection = e2epod.NodeSelection{Name: node.Name}
  1539  		return &localVolume{
  1540  			ltrMgr: l.ltrMgr,
  1541  			ltr:    l.ltrMgr.Create(ctx, node, l.volumeType, nil),
  1542  		}
  1543  	default:
  1544  		framework.Failf("Unsupported volType: %v is specified", volType)
  1545  	}
  1546  	return nil
  1547  }
  1548  
  1549  func (v *localVolume) DeleteVolume(ctx context.Context) {
  1550  	v.ltrMgr.Remove(ctx, v.ltr)
  1551  }
  1552  
  1553  func (l *localDriver) nodeAffinityForNode(node *v1.Node) *v1.VolumeNodeAffinity {
  1554  	nodeKey := "kubernetes.io/hostname"
  1555  	if node.Labels == nil {
  1556  		framework.Failf("Node does not have labels")
  1557  	}
  1558  	nodeValue, found := node.Labels[nodeKey]
  1559  	if !found {
  1560  		framework.Failf("Node does not have required label %q", nodeKey)
  1561  	}
  1562  	return &v1.VolumeNodeAffinity{
  1563  		Required: &v1.NodeSelector{
  1564  			NodeSelectorTerms: []v1.NodeSelectorTerm{
  1565  				{
  1566  					MatchExpressions: []v1.NodeSelectorRequirement{
  1567  						{
  1568  							Key:      nodeKey,
  1569  							Operator: v1.NodeSelectorOpIn,
  1570  							Values:   []string{nodeValue},
  1571  						},
  1572  					},
  1573  				},
  1574  			},
  1575  		},
  1576  	}
  1577  }
  1578  
  1579  func (l *localDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
  1580  	lv, ok := e2evolume.(*localVolume)
  1581  	if !ok {
  1582  		framework.Failf("Failed to cast test volume of type %T to the local test volume", e2evolume)
  1583  	}
  1584  	return &v1.PersistentVolumeSource{
  1585  		Local: &v1.LocalVolumeSource{
  1586  			Path:   lv.ltr.Path,
  1587  			FSType: &fsType,
  1588  		},
  1589  	}, l.nodeAffinityForNode(lv.ltr.Node)
  1590  }
  1591  
  1592  // cleanUpVolumeServer is a wrapper of cleanup function for volume server without secret created by specific CreateStorageServer function.
  1593  func cleanUpVolumeServer(ctx context.Context, f *framework.Framework, serverPod *v1.Pod) {
  1594  	cleanUpVolumeServerWithSecret(ctx, f, serverPod, nil)
  1595  }
  1596  
  1597  func getInlineVolumeZone(ctx context.Context, f *framework.Framework) string {
  1598  	if framework.TestContext.CloudConfig.Zone != "" {
  1599  		return framework.TestContext.CloudConfig.Zone
  1600  	}
  1601  	// if zone is not specified we will randomly pick a zone from schedulable nodes for inline tests
  1602  	node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
  1603  	framework.ExpectNoError(err)
  1604  	zone, ok := node.Labels[v1.LabelFailureDomainBetaZone]
  1605  	if ok {
  1606  		return zone
  1607  	}
  1608  	topologyZone, ok := node.Labels[v1.LabelTopologyZone]
  1609  	if ok {
  1610  		return topologyZone
  1611  	}
  1612  	return ""
  1613  }
  1614  
  1615  // cleanUpVolumeServerWithSecret is a wrapper of cleanup function for volume server with secret created by specific CreateStorageServer function.
  1616  func cleanUpVolumeServerWithSecret(ctx context.Context, f *framework.Framework, serverPod *v1.Pod, secret *v1.Secret) {
  1617  	cs := f.ClientSet
  1618  	ns := f.Namespace
  1619  
  1620  	if secret != nil {
  1621  		framework.Logf("Deleting server secret %q...", secret.Name)
  1622  		err := cs.CoreV1().Secrets(ns.Name).Delete(ctx, secret.Name, metav1.DeleteOptions{})
  1623  		if err != nil {
  1624  			framework.Logf("Delete secret failed: %v", err)
  1625  		}
  1626  	}
  1627  
  1628  	framework.Logf("Deleting server pod %q...", serverPod.Name)
  1629  	err := e2epod.DeletePodWithWait(ctx, cs, serverPod)
  1630  	if err != nil {
  1631  		framework.Logf("Server pod delete failed: %v", err)
  1632  	}
  1633  }
  1634  
  1635  // Azure File
  1636  type azureFileDriver struct {
  1637  	driverInfo storageframework.DriverInfo
  1638  }
  1639  
  1640  var _ storageframework.TestDriver = &azureFileDriver{}
  1641  var _ storageframework.DynamicPVTestDriver = &azureFileDriver{}
  1642  
  1643  // InitAzureFileDriver returns azureFileDriver that implements TestDriver interface
  1644  func InitAzureFileDriver() storageframework.TestDriver {
  1645  	return &azureFileDriver{
  1646  		driverInfo: storageframework.DriverInfo{
  1647  			Name:             "azure-file",
  1648  			InTreePluginName: "kubernetes.io/azure-file",
  1649  			MaxFileSize:      storageframework.FileSizeMedium,
  1650  			SupportedSizeRange: e2evolume.SizeRange{
  1651  				Min: "1Gi",
  1652  			},
  1653  			SupportedFsType: sets.NewString(
  1654  				"", // Default fsType
  1655  			),
  1656  			Capabilities: map[storageframework.Capability]bool{
  1657  				storageframework.CapPersistence:         true,
  1658  				storageframework.CapExec:                true,
  1659  				storageframework.CapRWX:                 true,
  1660  				storageframework.CapMultiPODs:           true,
  1661  				storageframework.CapControllerExpansion: true,
  1662  				storageframework.CapNodeExpansion:       true,
  1663  				storageframework.CapMultiplePVsSameID:   true,
  1664  			},
  1665  		},
  1666  	}
  1667  }
  1668  
  1669  func (a *azureFileDriver) GetDriverInfo() *storageframework.DriverInfo {
  1670  	return &a.driverInfo
  1671  }
  1672  
  1673  func (a *azureFileDriver) SkipUnsupportedTest(pattern storageframework.TestPattern) {
  1674  	e2eskipper.SkipUnlessProviderIs("azure")
  1675  }
  1676  
  1677  func (a *azureFileDriver) GetDynamicProvisionStorageClass(ctx context.Context, config *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
  1678  	provisioner := "kubernetes.io/azure-file"
  1679  	parameters := map[string]string{}
  1680  	ns := config.Framework.Namespace.Name
  1681  	immediateBinding := storagev1.VolumeBindingImmediate
  1682  	return storageframework.GetStorageClass(provisioner, parameters, &immediateBinding, ns)
  1683  }
  1684  
  1685  func (a *azureFileDriver) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
  1686  	return &storageframework.PerTestConfig{
  1687  		Driver:    a,
  1688  		Prefix:    "azure-file",
  1689  		Framework: f,
  1690  	}
  1691  }
  1692
View as plain text