volume_provisioning.go

Documentation: k8s.io/kubernetes/test/e2e/storage

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2016 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package storage
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"strings"
    26  	"time"
    27  
    28  	"github.com/onsi/ginkgo/v2"
    29  	"github.com/onsi/gomega"
    30  
    31  	v1 "k8s.io/api/core/v1"
    32  	rbacv1 "k8s.io/api/rbac/v1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/runtime/schema"
    36  	"k8s.io/apimachinery/pkg/types"
    37  	"k8s.io/apimachinery/pkg/util/rand"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/apiserver/pkg/authentication/serviceaccount"
    40  	clientset "k8s.io/client-go/kubernetes"
    41  	storageutil "k8s.io/kubernetes/pkg/apis/storage/util"
    42  	"k8s.io/kubernetes/test/e2e/feature"
    43  	"k8s.io/kubernetes/test/e2e/framework"
    44  	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
    45  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    46  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    47  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    48  	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
    49  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    50  	"k8s.io/kubernetes/test/e2e/storage/testsuites"
    51  	"k8s.io/kubernetes/test/e2e/storage/utils"
    52  	admissionapi "k8s.io/pod-security-admission/api"
    53  )
    54  
    55  const (
    56  	// Plugin name of the external provisioner
    57  	externalPluginName = "example.com/nfs"
    58  )
    59  
    60  func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
    61  	cloud, err := gce.GetGCECloud()
    62  	if err != nil {
    63  		return err
    64  	}
    65  	diskName := volume.Spec.GCEPersistentDisk.PDName
    66  	disk, err := cloud.GetDiskByNameUnknownZone(diskName)
    67  	if err != nil {
    68  		return err
    69  	}
    70  
    71  	if !strings.HasSuffix(disk.Type, volumeType) {
    72  		return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
    73  	}
    74  	return nil
    75  }
    76  
    77  var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
    78  	f := framework.NewDefaultFramework("volume-provisioning")
    79  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    80  
    81  	// filled in BeforeEach
    82  	var c clientset.Interface
    83  	var timeouts *framework.TimeoutContext
    84  	var ns string
    85  
    86  	ginkgo.BeforeEach(func() {
    87  		c = f.ClientSet
    88  		ns = f.Namespace.Name
    89  		timeouts = f.Timeouts
    90  	})
    91  
    92  	f.Describe("DynamicProvisioner", framework.WithSlow(), feature.StorageProvider, func() {
    93  		ginkgo.It("should provision storage with different parameters", func(ctx context.Context) {
    94  
    95  			// This test checks that dynamic provisioning can provision a volume
    96  			// that can be used to persist data among pods.
    97  			tests := []testsuites.StorageClassTest{
    98  				// GCE/GKE
    99  				{
   100  					Name:           "SSD PD on GCE/GKE",
   101  					CloudProviders: []string{"gce", "gke"},
   102  					Timeouts:       f.Timeouts,
   103  					Provisioner:    "kubernetes.io/gce-pd",
   104  					Parameters: map[string]string{
   105  						"type": "pd-ssd",
   106  						"zone": getRandomClusterZone(ctx, c),
   107  					},
   108  					ClaimSize:    "1.5Gi",
   109  					ExpectedSize: "2Gi",
   110  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   111  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   112  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   113  
   114  						err := checkGCEPD(volume, "pd-ssd")
   115  						framework.ExpectNoError(err, "checkGCEPD pd-ssd")
   116  					},
   117  				},
   118  				{
   119  					Name:           "HDD PD on GCE/GKE",
   120  					CloudProviders: []string{"gce", "gke"},
   121  					Timeouts:       f.Timeouts,
   122  					Provisioner:    "kubernetes.io/gce-pd",
   123  					Parameters: map[string]string{
   124  						"type": "pd-standard",
   125  					},
   126  					ClaimSize:    "1.5Gi",
   127  					ExpectedSize: "2Gi",
   128  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   129  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   130  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   131  
   132  						err := checkGCEPD(volume, "pd-standard")
   133  						framework.ExpectNoError(err, "checkGCEPD pd-standard")
   134  					},
   135  				},
   136  				// AWS
   137  				{
   138  					Name:           "gp2 EBS on AWS",
   139  					CloudProviders: []string{"aws"},
   140  					Timeouts:       f.Timeouts,
   141  					Provisioner:    "kubernetes.io/aws-ebs",
   142  					Parameters: map[string]string{
   143  						"type": "gp2",
   144  						"zone": getRandomClusterZone(ctx, c),
   145  					},
   146  					ClaimSize:    "1.5Gi",
   147  					ExpectedSize: "2Gi",
   148  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   149  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   150  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   151  					},
   152  				},
   153  				{
   154  					Name:           "io1 EBS on AWS",
   155  					CloudProviders: []string{"aws"},
   156  					Timeouts:       f.Timeouts,
   157  					Provisioner:    "kubernetes.io/aws-ebs",
   158  					Parameters: map[string]string{
   159  						"type":      "io1",
   160  						"iopsPerGB": "50",
   161  					},
   162  					ClaimSize:    "3.5Gi",
   163  					ExpectedSize: "4Gi", // 4 GiB is minimum for io1
   164  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   165  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   166  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   167  					},
   168  				},
   169  				{
   170  					Name:           "sc1 EBS on AWS",
   171  					CloudProviders: []string{"aws"},
   172  					Timeouts:       f.Timeouts,
   173  					Provisioner:    "kubernetes.io/aws-ebs",
   174  					Parameters: map[string]string{
   175  						"type": "sc1",
   176  					},
   177  					ClaimSize:    "500Gi", // minimum for sc1
   178  					ExpectedSize: "500Gi",
   179  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   180  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   181  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   182  					},
   183  				},
   184  				{
   185  					Name:           "st1 EBS on AWS",
   186  					CloudProviders: []string{"aws"},
   187  					Timeouts:       f.Timeouts,
   188  					Provisioner:    "kubernetes.io/aws-ebs",
   189  					Parameters: map[string]string{
   190  						"type": "st1",
   191  					},
   192  					ClaimSize:    "500Gi", // minimum for st1
   193  					ExpectedSize: "500Gi",
   194  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   195  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   196  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   197  					},
   198  				},
   199  				{
   200  					Name:           "encrypted EBS on AWS",
   201  					CloudProviders: []string{"aws"},
   202  					Timeouts:       f.Timeouts,
   203  					Provisioner:    "kubernetes.io/aws-ebs",
   204  					Parameters: map[string]string{
   205  						"encrypted": "true",
   206  					},
   207  					ClaimSize:    "1Gi",
   208  					ExpectedSize: "1Gi",
   209  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   210  						volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   211  						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   212  					},
   213  				},
   214  				// OpenStack generic tests (works on all OpenStack deployments)
   215  				{
   216  					Name:           "generic Cinder volume on OpenStack",
   217  					CloudProviders: []string{"openstack"},
   218  					Timeouts:       f.Timeouts,
   219  					Provisioner:    "kubernetes.io/cinder",
   220  					Parameters:     map[string]string{},
   221  					ClaimSize:      "1.5Gi",
   222  					ExpectedSize:   "2Gi",
   223  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   224  						testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   225  					},
   226  				},
   227  				{
   228  					Name:           "Cinder volume with empty volume type and zone on OpenStack",
   229  					CloudProviders: []string{"openstack"},
   230  					Timeouts:       f.Timeouts,
   231  					Provisioner:    "kubernetes.io/cinder",
   232  					Parameters: map[string]string{
   233  						"type":         "",
   234  						"availability": "",
   235  					},
   236  					ClaimSize:    "1.5Gi",
   237  					ExpectedSize: "2Gi",
   238  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   239  						testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   240  					},
   241  				},
   242  				// vSphere generic test
   243  				{
   244  					Name:           "generic vSphere volume",
   245  					CloudProviders: []string{"vsphere"},
   246  					Timeouts:       f.Timeouts,
   247  					Provisioner:    "kubernetes.io/vsphere-volume",
   248  					Parameters:     map[string]string{},
   249  					ClaimSize:      "1.5Gi",
   250  					ExpectedSize:   "1.5Gi",
   251  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   252  						testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   253  					},
   254  				},
   255  				// Azure
   256  				{
   257  					Name:           "Azure disk volume with empty sku and location",
   258  					CloudProviders: []string{"azure"},
   259  					Timeouts:       f.Timeouts,
   260  					Provisioner:    "kubernetes.io/azure-disk",
   261  					Parameters:     map[string]string{},
   262  					ClaimSize:      "1Gi",
   263  					ExpectedSize:   "1Gi",
   264  					PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   265  						testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   266  					},
   267  				},
   268  			}
   269  
   270  			for i, t := range tests {
   271  				// Beware of closure, use local variables instead of those from
   272  				// outer scope
   273  				test := t
   274  
   275  				if !framework.ProviderIs(test.CloudProviders...) {
   276  					framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
   277  					continue
   278  				}
   279  
   280  				if zone, ok := test.Parameters["zone"]; ok {
   281  					gomega.Expect(zone).ToNot(gomega.BeEmpty(), "expect at least one zone")
   282  				}
   283  
   284  				ginkgo.By("Testing " + test.Name)
   285  				suffix := fmt.Sprintf("%d", i)
   286  				test.Client = c
   287  
   288  				// overwrite StorageClass spec with provisioned StorageClass
   289  				storageClass := testsuites.SetupStorageClass(ctx, test.Client, newStorageClass(test, ns, suffix))
   290  
   291  				test.Class = storageClass
   292  				test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   293  					ClaimSize:        test.ClaimSize,
   294  					StorageClassName: &test.Class.Name,
   295  					VolumeMode:       &test.VolumeMode,
   296  				}, ns)
   297  
   298  				test.TestDynamicProvisioning(ctx)
   299  			}
   300  		})
   301  
   302  		ginkgo.It("should provision storage with non-default reclaim policy Retain", func(ctx context.Context) {
   303  			e2eskipper.SkipUnlessProviderIs("gce", "gke")
   304  
   305  			test := testsuites.StorageClassTest{
   306  				Client:         c,
   307  				Name:           "HDD PD on GCE/GKE",
   308  				CloudProviders: []string{"gce", "gke"},
   309  				Provisioner:    "kubernetes.io/gce-pd",
   310  				Timeouts:       f.Timeouts,
   311  				Parameters: map[string]string{
   312  					"type": "pd-standard",
   313  				},
   314  				ClaimSize:    "1Gi",
   315  				ExpectedSize: "1Gi",
   316  				PvCheck: func(ctx context.Context, claim *v1.PersistentVolumeClaim) {
   317  					volume := testsuites.PVWriteReadSingleNodeCheck(ctx, c, f.Timeouts, claim, e2epod.NodeSelection{})
   318  					gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
   319  
   320  					err := checkGCEPD(volume, "pd-standard")
   321  					framework.ExpectNoError(err, "checkGCEPD")
   322  				},
   323  			}
   324  			test.Class = newStorageClass(test, ns, "reclaimpolicy")
   325  			retain := v1.PersistentVolumeReclaimRetain
   326  			test.Class.ReclaimPolicy = &retain
   327  			storageClass := testsuites.SetupStorageClass(ctx, test.Client, test.Class)
   328  			test.Class = storageClass
   329  
   330  			test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   331  				ClaimSize:        test.ClaimSize,
   332  				StorageClassName: &test.Class.Name,
   333  				VolumeMode:       &test.VolumeMode,
   334  			}, ns)
   335  
   336  			pv := test.TestDynamicProvisioning(ctx)
   337  
   338  			ginkgo.By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
   339  			framework.ExpectNoError(e2epv.WaitForPersistentVolumePhase(ctx, v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
   340  
   341  			ginkgo.By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
   342  			framework.ExpectNoError(e2epv.DeletePDWithRetry(ctx, pv.Spec.GCEPersistentDisk.PDName))
   343  
   344  			ginkgo.By(fmt.Sprintf("deleting the PV %q", pv.Name))
   345  			framework.ExpectNoError(e2epv.DeletePersistentVolume(ctx, c, pv.Name), "Failed to delete PV ", pv.Name)
   346  			framework.ExpectNoError(e2epv.WaitForPersistentVolumeDeleted(ctx, c, pv.Name, 1*time.Second, 30*time.Second))
   347  		})
   348  
   349  		ginkgo.It("should test that deleting a claim before the volume is provisioned deletes the volume.", func(ctx context.Context) {
   350  			// This case tests for the regressions of a bug fixed by PR #21268
   351  			// REGRESSION: Deleting the PVC before the PV is provisioned can result in the PV
   352  			// not being deleted.
   353  			// NOTE:  Polls until no PVs are detected, times out at 5 minutes.
   354  
   355  			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
   356  
   357  			const raceAttempts int = 100
   358  			var residualPVs []*v1.PersistentVolume
   359  			ginkgo.By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
   360  			test := testsuites.StorageClassTest{
   361  				Name:        "deletion race",
   362  				Provisioner: "", // Use a native one based on current cloud provider
   363  				Timeouts:    f.Timeouts,
   364  				ClaimSize:   "1Gi",
   365  			}
   366  
   367  			class := newStorageClass(test, ns, "race")
   368  			class, err := c.StorageV1().StorageClasses().Create(ctx, class, metav1.CreateOptions{})
   369  			framework.ExpectNoError(err)
   370  			ginkgo.DeferCleanup(deleteStorageClass, c, class.Name)
   371  
   372  			// To increase chance of detection, attempt multiple iterations
   373  			for i := 0; i < raceAttempts; i++ {
   374  				prefix := fmt.Sprintf("race-%d", i)
   375  				claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   376  					NamePrefix:       prefix,
   377  					ClaimSize:        test.ClaimSize,
   378  					StorageClassName: &class.Name,
   379  					VolumeMode:       &test.VolumeMode,
   380  				}, ns)
   381  				tmpClaim, err := e2epv.CreatePVC(ctx, c, ns, claim)
   382  				framework.ExpectNoError(err)
   383  				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(ctx, c, tmpClaim.Name, ns))
   384  			}
   385  
   386  			ginkgo.By(fmt.Sprintf("Checking for residual PersistentVolumes associated with StorageClass %s", class.Name))
   387  			residualPVs, err = waitForProvisionedVolumesDeleted(ctx, c, class.Name)
   388  			// Cleanup the test resources before breaking
   389  			ginkgo.DeferCleanup(deleteProvisionedVolumesAndDisks, c, residualPVs)
   390  			framework.ExpectNoError(err, "PersistentVolumes were not deleted as expected. %d remain", len(residualPVs))
   391  
   392  			framework.Logf("0 PersistentVolumes remain.")
   393  		})
   394  
   395  		ginkgo.It("deletion should be idempotent", func(ctx context.Context) {
   396  			// This test ensures that deletion of a volume is idempotent.
   397  			// It creates a PV with Retain policy, deletes underlying AWS / GCE
   398  			// volume and changes the reclaim policy to Delete.
   399  			// PV controller should delete the PV even though the underlying volume
   400  			// is already deleted.
   401  			e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
   402  			ginkgo.By("creating PD")
   403  			diskName, err := e2epv.CreatePDWithRetry(ctx)
   404  			framework.ExpectNoError(err)
   405  
   406  			ginkgo.By("creating PV")
   407  			pv := e2epv.MakePersistentVolume(e2epv.PersistentVolumeConfig{
   408  				NamePrefix: "volume-idempotent-delete-",
   409  				// Use Retain to keep the PV, the test will change it to Delete
   410  				// when the time comes.
   411  				ReclaimPolicy: v1.PersistentVolumeReclaimRetain,
   412  				AccessModes: []v1.PersistentVolumeAccessMode{
   413  					v1.ReadWriteOnce,
   414  				},
   415  				Capacity: "1Gi",
   416  				// PV is bound to non-existing PVC, so it's reclaim policy is
   417  				// executed immediately
   418  				Prebind: &v1.PersistentVolumeClaim{
   419  					ObjectMeta: metav1.ObjectMeta{
   420  						Name:      "dummy-claim-name",
   421  						Namespace: ns,
   422  						UID:       types.UID("01234567890"),
   423  					},
   424  				},
   425  			})
   426  			switch framework.TestContext.Provider {
   427  			case "aws":
   428  				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
   429  					AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
   430  						VolumeID: diskName,
   431  					},
   432  				}
   433  			case "gce", "gke":
   434  				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
   435  					GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
   436  						PDName: diskName,
   437  					},
   438  				}
   439  			}
   440  			pv, err = c.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{})
   441  			framework.ExpectNoError(err)
   442  
   443  			ginkgo.By("waiting for the PV to get Released")
   444  			err = e2epv.WaitForPersistentVolumePhase(ctx, v1.VolumeReleased, c, pv.Name, 2*time.Second, timeouts.PVReclaim)
   445  			framework.ExpectNoError(err)
   446  
   447  			ginkgo.By("deleting the PD")
   448  			err = e2epv.DeletePVSource(ctx, &pv.Spec.PersistentVolumeSource)
   449  			framework.ExpectNoError(err)
   450  
   451  			ginkgo.By("changing the PV reclaim policy")
   452  			pv, err = c.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{})
   453  			framework.ExpectNoError(err)
   454  			pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
   455  			pv, err = c.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{})
   456  			framework.ExpectNoError(err)
   457  
   458  			ginkgo.By("waiting for the PV to get deleted")
   459  			err = e2epv.WaitForPersistentVolumeDeleted(ctx, c, pv.Name, 5*time.Second, timeouts.PVDelete)
   460  			framework.ExpectNoError(err)
   461  		})
   462  	})
   463  
   464  	ginkgo.Describe("DynamicProvisioner External", func() {
   465  		f.It("should let an external dynamic provisioner create and delete persistent volumes", f.WithSlow(), func(ctx context.Context) {
   466  			// external dynamic provisioner pods need additional permissions provided by the
   467  			// persistent-volume-provisioner clusterrole and a leader-locking role
   468  			serviceAccountName := "default"
   469  			subject := rbacv1.Subject{
   470  				Kind:      rbacv1.ServiceAccountKind,
   471  				Namespace: ns,
   472  				Name:      serviceAccountName,
   473  			}
   474  
   475  			err := e2eauth.BindClusterRole(ctx, c.RbacV1(), "system:persistent-volume-provisioner", ns, subject)
   476  			framework.ExpectNoError(err)
   477  
   478  			roleName := "leader-locking-nfs-provisioner"
   479  			_, err = f.ClientSet.RbacV1().Roles(ns).Create(ctx, &rbacv1.Role{
   480  				ObjectMeta: metav1.ObjectMeta{
   481  					Name: roleName,
   482  				},
   483  				Rules: []rbacv1.PolicyRule{{
   484  					APIGroups: []string{""},
   485  					Resources: []string{"endpoints"},
   486  					Verbs:     []string{"get", "list", "watch", "create", "update", "patch"},
   487  				}},
   488  			}, metav1.CreateOptions{})
   489  			framework.ExpectNoError(err, "Failed to create leader-locking role")
   490  
   491  			err = e2eauth.BindRoleInNamespace(ctx, c.RbacV1(), roleName, ns, subject)
   492  			framework.ExpectNoError(err)
   493  
   494  			err = e2eauth.WaitForAuthorizationUpdate(ctx, c.AuthorizationV1(),
   495  				serviceaccount.MakeUsername(ns, serviceAccountName),
   496  				"", "get", schema.GroupResource{Group: "storage.k8s.io", Resource: "storageclasses"}, true)
   497  			framework.ExpectNoError(err, "Failed to update authorization")
   498  
   499  			ginkgo.By("creating an external dynamic provisioner pod")
   500  			pod := utils.StartExternalProvisioner(ctx, c, ns, externalPluginName)
   501  			ginkgo.DeferCleanup(e2epod.DeletePodOrFail, c, ns, pod.Name)
   502  
   503  			ginkgo.By("creating a StorageClass")
   504  			test := testsuites.StorageClassTest{
   505  				Client:       c,
   506  				Name:         "external provisioner test",
   507  				Provisioner:  externalPluginName,
   508  				Timeouts:     f.Timeouts,
   509  				ClaimSize:    "1500Mi",
   510  				ExpectedSize: "1500Mi",
   511  			}
   512  
   513  			storageClass := testsuites.SetupStorageClass(ctx, test.Client, newStorageClass(test, ns, "external"))
   514  			test.Class = storageClass
   515  
   516  			test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   517  				ClaimSize:        test.ClaimSize,
   518  				StorageClassName: &test.Class.Name,
   519  				VolumeMode:       &test.VolumeMode,
   520  			}, ns)
   521  
   522  			ginkgo.By("creating a claim with a external provisioning annotation")
   523  
   524  			test.TestDynamicProvisioning(ctx)
   525  		})
   526  	})
   527  
   528  	ginkgo.Describe("DynamicProvisioner Default", func() {
   529  		f.It("should create and delete default persistent volumes", f.WithSlow(), func(ctx context.Context) {
   530  			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
   531  			e2epv.SkipIfNoDefaultStorageClass(ctx, c)
   532  
   533  			ginkgo.By("creating a claim with no annotation")
   534  			test := testsuites.StorageClassTest{
   535  				Client:       c,
   536  				Name:         "default",
   537  				Timeouts:     f.Timeouts,
   538  				ClaimSize:    "2Gi",
   539  				ExpectedSize: "2Gi",
   540  			}
   541  
   542  			test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   543  				ClaimSize:  test.ClaimSize,
   544  				VolumeMode: &test.VolumeMode,
   545  			}, ns)
   546  			// NOTE: this test assumes that there's a default storageclass
   547  			test.Class = testsuites.SetupStorageClass(ctx, test.Client, nil)
   548  
   549  			test.TestDynamicProvisioning(ctx)
   550  		})
   551  
   552  		// Modifying the default storage class can be disruptive to other tests that depend on it
   553  		f.It("should be disabled by changing the default annotation", f.WithSerial(), f.WithDisruptive(), func(ctx context.Context) {
   554  			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
   555  			e2epv.SkipIfNoDefaultStorageClass(ctx, c)
   556  
   557  			scName, scErr := e2epv.GetDefaultStorageClassName(ctx, c)
   558  			framework.ExpectNoError(scErr)
   559  
   560  			test := testsuites.StorageClassTest{
   561  				Name:      "default",
   562  				Timeouts:  f.Timeouts,
   563  				ClaimSize: "2Gi",
   564  			}
   565  
   566  			ginkgo.By("setting the is-default StorageClass annotation to false")
   567  			verifyDefaultStorageClass(ctx, c, scName, true)
   568  			ginkgo.DeferCleanup(updateDefaultStorageClass, c, scName, "true")
   569  			updateDefaultStorageClass(ctx, c, scName, "false")
   570  
   571  			ginkgo.By("creating a claim with default storageclass and expecting it to timeout")
   572  			claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   573  				ClaimSize:  test.ClaimSize,
   574  				VolumeMode: &test.VolumeMode,
   575  			}, ns)
   576  			claim, err := c.CoreV1().PersistentVolumeClaims(ns).Create(ctx, claim, metav1.CreateOptions{})
   577  			framework.ExpectNoError(err)
   578  			ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, c, claim.Name, ns)
   579  
   580  			// The claim should timeout phase:Pending
   581  			err = e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, c, ns, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
   582  			framework.ExpectError(err)
   583  			framework.Logf(err.Error())
   584  			claim, err = c.CoreV1().PersistentVolumeClaims(ns).Get(ctx, claim.Name, metav1.GetOptions{})
   585  			framework.ExpectNoError(err)
   586  			gomega.Expect(claim.Status.Phase).To(gomega.Equal(v1.ClaimPending))
   587  		})
   588  
   589  		// Modifying the default storage class can be disruptive to other tests that depend on it
   590  		f.It("should be disabled by removing the default annotation", f.WithSerial(), f.WithDisruptive(), func(ctx context.Context) {
   591  			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
   592  			e2epv.SkipIfNoDefaultStorageClass(ctx, c)
   593  
   594  			scName, scErr := e2epv.GetDefaultStorageClassName(ctx, c)
   595  			framework.ExpectNoError(scErr)
   596  
   597  			test := testsuites.StorageClassTest{
   598  				Name:      "default",
   599  				Timeouts:  f.Timeouts,
   600  				ClaimSize: "2Gi",
   601  			}
   602  
   603  			ginkgo.By("removing the is-default StorageClass annotation")
   604  			verifyDefaultStorageClass(ctx, c, scName, true)
   605  			ginkgo.DeferCleanup(updateDefaultStorageClass, c, scName, "true")
   606  			updateDefaultStorageClass(ctx, c, scName, "")
   607  
   608  			ginkgo.By("creating a claim with default storageclass and expecting it to timeout")
   609  			claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   610  				ClaimSize:  test.ClaimSize,
   611  				VolumeMode: &test.VolumeMode,
   612  			}, ns)
   613  			claim, err := c.CoreV1().PersistentVolumeClaims(ns).Create(ctx, claim, metav1.CreateOptions{})
   614  			framework.ExpectNoError(err)
   615  			defer func() {
   616  				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(ctx, c, claim.Name, ns))
   617  			}()
   618  
   619  			// The claim should timeout phase:Pending
   620  			err = e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, c, ns, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
   621  			framework.ExpectError(err)
   622  			framework.Logf(err.Error())
   623  			claim, err = c.CoreV1().PersistentVolumeClaims(ns).Get(ctx, claim.Name, metav1.GetOptions{})
   624  			framework.ExpectNoError(err)
   625  			gomega.Expect(claim.Status.Phase).To(gomega.Equal(v1.ClaimPending))
   626  		})
   627  	})
   628  
   629  	ginkgo.Describe("Invalid AWS KMS key", func() {
   630  		ginkgo.It("should report an error and create no PV", func(ctx context.Context) {
   631  			e2eskipper.SkipUnlessProviderIs("aws")
   632  			test := testsuites.StorageClassTest{
   633  				Client:      c,
   634  				Name:        "AWS EBS with invalid KMS key",
   635  				Provisioner: "kubernetes.io/aws-ebs",
   636  				Timeouts:    f.Timeouts,
   637  				ClaimSize:   "2Gi",
   638  				Parameters:  map[string]string{"kmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/55555555-5555-5555-5555-555555555555"},
   639  			}
   640  
   641  			ginkgo.By("creating a StorageClass")
   642  			test.Class = testsuites.SetupStorageClass(ctx, test.Client, newStorageClass(test, ns, "invalid-aws"))
   643  
   644  			ginkgo.By("creating a claim object")
   645  			claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
   646  				ClaimSize:        test.ClaimSize,
   647  				StorageClassName: &test.Class.Name,
   648  				VolumeMode:       &test.VolumeMode,
   649  			}, ns)
   650  			claim, err := c.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(ctx, claim, metav1.CreateOptions{})
   651  			framework.ExpectNoError(err)
   652  			defer func() {
   653  				framework.Logf("deleting claim %q/%q", claim.Namespace, claim.Name)
   654  				err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(ctx, claim.Name, metav1.DeleteOptions{})
   655  				if err != nil && !apierrors.IsNotFound(err) {
   656  					framework.Failf("Error deleting claim %q. Error: %v", claim.Name, err)
   657  				}
   658  			}()
   659  
   660  			// Watch events until the message about invalid key appears.
   661  			// Event delivery is not reliable and it's used only as a quick way how to check if volume with wrong KMS
   662  			// key was not provisioned. If the event is not delivered, we check that the volume is not Bound for whole
   663  			// ClaimProvisionTimeout in the very same loop.
   664  			err = wait.Poll(time.Second, framework.ClaimProvisionTimeout, func() (bool, error) {
   665  				events, err := c.CoreV1().Events(claim.Namespace).List(ctx, metav1.ListOptions{})
   666  				if err != nil {
   667  					return false, fmt.Errorf("could not list PVC events in %s: %w", claim.Namespace, err)
   668  				}
   669  				for _, event := range events.Items {
   670  					if strings.Contains(event.Message, "failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key") {
   671  						return true, nil
   672  					}
   673  				}
   674  
   675  				pvc, err := c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(ctx, claim.Name, metav1.GetOptions{})
   676  				if err != nil {
   677  					return true, err
   678  				}
   679  				if pvc.Status.Phase != v1.ClaimPending {
   680  					// The PVC was bound to something, i.e. PV was created for wrong KMS key. That's bad!
   681  					return true, fmt.Errorf("PVC got unexpectedly %s (to PV %q)", pvc.Status.Phase, pvc.Spec.VolumeName)
   682  				}
   683  
   684  				return false, nil
   685  			})
   686  			if wait.Interrupted(err) {
   687  				framework.Logf("The test missed event about failed provisioning, but checked that no volume was provisioned for %v", framework.ClaimProvisionTimeout)
   688  				err = nil
   689  			}
   690  			framework.ExpectNoError(err, "Error waiting for PVC to fail provisioning: %v", err)
   691  		})
   692  	})
   693  })
   694  
   695  func verifyDefaultStorageClass(ctx context.Context, c clientset.Interface, scName string, expectedDefault bool) {
   696  	sc, err := c.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{})
   697  	framework.ExpectNoError(err)
   698  	gomega.Expect(storageutil.IsDefaultAnnotation(sc.ObjectMeta)).To(gomega.Equal(expectedDefault))
   699  }
   700  
   701  func updateDefaultStorageClass(ctx context.Context, c clientset.Interface, scName string, defaultStr string) {
   702  	sc, err := c.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{})
   703  	framework.ExpectNoError(err)
   704  
   705  	if defaultStr == "" {
   706  		delete(sc.Annotations, storageutil.BetaIsDefaultStorageClassAnnotation)
   707  		delete(sc.Annotations, storageutil.IsDefaultStorageClassAnnotation)
   708  	} else {
   709  		if sc.Annotations == nil {
   710  			sc.Annotations = make(map[string]string)
   711  		}
   712  		sc.Annotations[storageutil.BetaIsDefaultStorageClassAnnotation] = defaultStr
   713  		sc.Annotations[storageutil.IsDefaultStorageClassAnnotation] = defaultStr
   714  	}
   715  
   716  	_, err = c.StorageV1().StorageClasses().Update(ctx, sc, metav1.UpdateOptions{})
   717  	framework.ExpectNoError(err)
   718  
   719  	expectedDefault := false
   720  	if defaultStr == "true" {
   721  		expectedDefault = true
   722  	}
   723  	verifyDefaultStorageClass(ctx, c, scName, expectedDefault)
   724  }
   725  
   726  // waitForProvisionedVolumesDelete is a polling wrapper to scan all PersistentVolumes for any associated to the test's
   727  // StorageClass.  Returns either an error and nil values or the remaining PVs and their count.
   728  func waitForProvisionedVolumesDeleted(ctx context.Context, c clientset.Interface, scName string) ([]*v1.PersistentVolume, error) {
   729  	var remainingPVs []*v1.PersistentVolume
   730  
   731  	err := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) {
   732  		remainingPVs = []*v1.PersistentVolume{}
   733  
   734  		allPVs, err := c.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
   735  		if err != nil {
   736  			return true, err
   737  		}
   738  		for _, pv := range allPVs.Items {
   739  			if pv.Spec.StorageClassName == scName {
   740  				pv := pv
   741  				remainingPVs = append(remainingPVs, &pv)
   742  			}
   743  		}
   744  		if len(remainingPVs) > 0 {
   745  			return false, nil // Poll until no PVs remain
   746  		}
   747  		return true, nil // No PVs remain
   748  	})
   749  	if err != nil {
   750  		return remainingPVs, fmt.Errorf("Error waiting for PVs to be deleted: %w", err)
   751  	}
   752  	return nil, nil
   753  }
   754  
   755  // deleteStorageClass deletes the passed in StorageClass and catches errors other than "Not Found"
   756  func deleteStorageClass(ctx context.Context, c clientset.Interface, className string) {
   757  	err := c.StorageV1().StorageClasses().Delete(ctx, className, metav1.DeleteOptions{})
   758  	if err != nil && !apierrors.IsNotFound(err) {
   759  		framework.ExpectNoError(err)
   760  	}
   761  }
   762  
   763  // deleteProvisionedVolumes [gce||gke only]  iteratively deletes persistent volumes and attached GCE PDs.
   764  func deleteProvisionedVolumesAndDisks(ctx context.Context, c clientset.Interface, pvs []*v1.PersistentVolume) {
   765  	framework.Logf("Remaining PersistentVolumes:")
   766  	for i, pv := range pvs {
   767  		framework.Logf("\t%d) %s", i+1, pv.Name)
   768  	}
   769  	for _, pv := range pvs {
   770  		framework.ExpectNoError(e2epv.DeletePDWithRetry(ctx, pv.Spec.PersistentVolumeSource.GCEPersistentDisk.PDName))
   771  		framework.ExpectNoError(e2epv.DeletePersistentVolume(ctx, c, pv.Name))
   772  	}
   773  }
   774  
   775  func getRandomClusterZone(ctx context.Context, c clientset.Interface) string {
   776  	zones, err := e2enode.GetClusterZones(ctx, c)
   777  	zone := ""
   778  	framework.ExpectNoError(err)
   779  	if len(zones) != 0 {
   780  		zonesList := zones.UnsortedList()
   781  		zone = zonesList[rand.Intn(zones.Len())]
   782  	}
   783  	return zone
   784  }
   785
View as plain text