volume_resource.go

Documentation: k8s.io/kubernetes/test/e2e/storage/framework

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package framework
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"github.com/onsi/ginkgo/v2"
    25  	v1 "k8s.io/api/core/v1"
    26  	storagev1 "k8s.io/api/storage/v1"
    27  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    28  	"k8s.io/apimachinery/pkg/api/resource"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
    33  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    34  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    35  	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
    36  )
    37  
    38  // VolumeResource is a generic implementation of TestResource that wil be able to
    39  // be used in most of TestSuites.
    40  // See volume_io.go or volumes.go in test/e2e/storage/testsuites/ for how to use this resource.
    41  // Also, see subpath.go in the same directory for how to extend and use it.
    42  type VolumeResource struct {
    43  	Config    *PerTestConfig
    44  	Pattern   TestPattern
    45  	VolSource *v1.VolumeSource
    46  	Pvc       *v1.PersistentVolumeClaim
    47  	Pv        *v1.PersistentVolume
    48  	Sc        *storagev1.StorageClass
    49  
    50  	Volume TestVolume
    51  }
    52  
    53  // CreateVolumeResource constructs a VolumeResource for the current test. It knows how to deal with
    54  // different test pattern volume types.
    55  func CreateVolumeResource(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange) *VolumeResource {
    56  	return CreateVolumeResourceWithAccessModes(ctx, driver, config, pattern, testVolumeSizeRange, driver.GetDriverInfo().RequiredAccessModes)
    57  }
    58  
    59  // CreateVolumeResourceWithAccessModes constructs a VolumeResource for the current test with the provided access modes.
    60  func CreateVolumeResourceWithAccessModes(ctx context.Context, driver TestDriver, config *PerTestConfig, pattern TestPattern, testVolumeSizeRange e2evolume.SizeRange, accessModes []v1.PersistentVolumeAccessMode) *VolumeResource {
    61  	r := VolumeResource{
    62  		Config:  config,
    63  		Pattern: pattern,
    64  	}
    65  	dInfo := driver.GetDriverInfo()
    66  	f := config.Framework
    67  	cs := f.ClientSet
    68  
    69  	// Create volume for pre-provisioned volume tests
    70  	r.Volume = CreateVolume(ctx, driver, config, pattern.VolType)
    71  
    72  	switch pattern.VolType {
    73  	case InlineVolume:
    74  		framework.Logf("Creating resource for inline volume")
    75  		if iDriver, ok := driver.(InlineVolumeTestDriver); ok {
    76  			r.VolSource = iDriver.GetVolumeSource(false, pattern.FsType, r.Volume)
    77  		}
    78  	case PreprovisionedPV:
    79  		framework.Logf("Creating resource for pre-provisioned PV")
    80  		if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
    81  			pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, pattern.FsType, r.Volume)
    82  			if pvSource != nil {
    83  				r.Pv, r.Pvc = createPVCPV(ctx, f, dInfo.Name, pvSource, volumeNodeAffinity, pattern.VolMode, accessModes)
    84  				r.VolSource = storageutils.CreateVolumeSource(r.Pvc.Name, false /* readOnly */)
    85  			}
    86  		}
    87  	case DynamicPV, GenericEphemeralVolume:
    88  		framework.Logf("Creating resource for dynamic PV")
    89  		if dDriver, ok := driver.(DynamicPVTestDriver); ok {
    90  			var err error
    91  			driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange
    92  			claimSize, err := storageutils.GetSizeRangesIntersection(testVolumeSizeRange, driverVolumeSizeRange)
    93  			framework.ExpectNoError(err, "determine intersection of test size range %+v and driver size range %+v", testVolumeSizeRange, driverVolumeSizeRange)
    94  			framework.Logf("Using claimSize:%s, test suite supported size:%v, driver(%s) supported size:%v ", claimSize, testVolumeSizeRange, dDriver.GetDriverInfo().Name, testVolumeSizeRange)
    95  			r.Sc = dDriver.GetDynamicProvisionStorageClass(ctx, r.Config, pattern.FsType)
    96  
    97  			if pattern.BindingMode != "" {
    98  				r.Sc.VolumeBindingMode = &pattern.BindingMode
    99  			}
   100  			r.Sc.AllowVolumeExpansion = &pattern.AllowExpansion
   101  
   102  			ginkgo.By("creating a StorageClass " + r.Sc.Name)
   103  
   104  			r.Sc, err = cs.StorageV1().StorageClasses().Create(ctx, r.Sc, metav1.CreateOptions{})
   105  			framework.ExpectNoError(err)
   106  
   107  			switch pattern.VolType {
   108  			case DynamicPV:
   109  				r.Pv, r.Pvc = createPVCPVFromDynamicProvisionSC(
   110  					ctx, f, dInfo.Name, claimSize, r.Sc, pattern.VolMode, accessModes)
   111  				r.VolSource = storageutils.CreateVolumeSource(r.Pvc.Name, false /* readOnly */)
   112  			case GenericEphemeralVolume:
   113  				driverVolumeSizeRange := dDriver.GetDriverInfo().SupportedSizeRange
   114  				claimSize, err := storageutils.GetSizeRangesIntersection(testVolumeSizeRange, driverVolumeSizeRange)
   115  				framework.ExpectNoError(err, "determine intersection of test size range %+v and driver size range %+v", testVolumeSizeRange, driverVolumeSizeRange)
   116  				r.VolSource = createEphemeralVolumeSource(r.Sc.Name, pattern.VolMode, accessModes, claimSize)
   117  			}
   118  		}
   119  	case CSIInlineVolume:
   120  		framework.Logf("Creating resource for CSI ephemeral inline volume")
   121  		if eDriver, ok := driver.(EphemeralTestDriver); ok {
   122  			attributes, _, _ := eDriver.GetVolume(config, 0)
   123  			r.VolSource = &v1.VolumeSource{
   124  				CSI: &v1.CSIVolumeSource{
   125  					Driver:           eDriver.GetCSIDriverName(config),
   126  					VolumeAttributes: attributes,
   127  				},
   128  			}
   129  			if pattern.FsType != "" {
   130  				r.VolSource.CSI.FSType = &pattern.FsType
   131  			}
   132  		}
   133  	default:
   134  		framework.Failf("VolumeResource doesn't support: %s", pattern.VolType)
   135  	}
   136  
   137  	if r.VolSource == nil {
   138  		e2eskipper.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
   139  	}
   140  
   141  	return &r
   142  }
   143  
   144  func createEphemeralVolumeSource(scName string, volMode v1.PersistentVolumeMode, accessModes []v1.PersistentVolumeAccessMode, claimSize string) *v1.VolumeSource {
   145  	if len(accessModes) == 0 {
   146  		accessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}
   147  	}
   148  	if volMode == "" {
   149  		volMode = v1.PersistentVolumeFilesystem
   150  	}
   151  	return &v1.VolumeSource{
   152  		Ephemeral: &v1.EphemeralVolumeSource{
   153  			VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{
   154  				Spec: v1.PersistentVolumeClaimSpec{
   155  					StorageClassName: &scName,
   156  					AccessModes:      accessModes,
   157  					VolumeMode:       &volMode,
   158  					Resources: v1.VolumeResourceRequirements{
   159  						Requests: v1.ResourceList{
   160  							v1.ResourceStorage: resource.MustParse(claimSize),
   161  						},
   162  					},
   163  				},
   164  			},
   165  		},
   166  	}
   167  }
   168  
   169  // CleanupResource cleans up VolumeResource
   170  func (r *VolumeResource) CleanupResource(ctx context.Context) error {
   171  	f := r.Config.Framework
   172  	var cleanUpErrs []error
   173  	if r.Pvc != nil || r.Pv != nil {
   174  		switch r.Pattern.VolType {
   175  		case PreprovisionedPV:
   176  			ginkgo.By("Deleting pv and pvc")
   177  			if errs := e2epv.PVPVCCleanup(ctx, f.ClientSet, f.Namespace.Name, r.Pv, r.Pvc); len(errs) != 0 {
   178  				framework.Failf("Failed to delete PVC or PV: %v", utilerrors.NewAggregate(errs))
   179  			}
   180  		case DynamicPV:
   181  			ginkgo.By("Deleting pvc")
   182  			// We only delete the PVC so that PV (and disk) can be cleaned up by dynamic provisioner
   183  			if r.Pv != nil && r.Pv.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimDelete {
   184  				framework.Failf("Test framework does not currently support Dynamically Provisioned Persistent Volume %v specified with reclaim policy that isn't %v",
   185  					r.Pv.Name, v1.PersistentVolumeReclaimDelete)
   186  			}
   187  			if r.Pvc != nil {
   188  				cs := f.ClientSet
   189  				pv := r.Pv
   190  				if pv == nil && r.Pvc.Name != "" {
   191  					// This happens for late binding. Check whether we have a volume now that we need to wait for.
   192  					pvc, err := cs.CoreV1().PersistentVolumeClaims(r.Pvc.Namespace).Get(ctx, r.Pvc.Name, metav1.GetOptions{})
   193  					switch {
   194  					case err == nil:
   195  						if pvc.Spec.VolumeName != "" {
   196  							pv, err = cs.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
   197  							if err != nil {
   198  								cleanUpErrs = append(cleanUpErrs, fmt.Errorf("failed to find PV %v: %w", pvc.Spec.VolumeName, err))
   199  							}
   200  						}
   201  					case apierrors.IsNotFound(err):
   202  						// Without the PVC, we cannot locate the corresponding PV. Let's
   203  						// hope that it is gone.
   204  					default:
   205  						cleanUpErrs = append(cleanUpErrs, fmt.Errorf("failed to find PVC %v: %w", r.Pvc.Name, err))
   206  					}
   207  				}
   208  
   209  				err := e2epv.DeletePersistentVolumeClaim(ctx, f.ClientSet, r.Pvc.Name, f.Namespace.Name)
   210  				if err != nil {
   211  					cleanUpErrs = append(cleanUpErrs, fmt.Errorf("failed to delete PVC %v: %w", r.Pvc.Name, err))
   212  				}
   213  
   214  				if pv != nil {
   215  					err = e2epv.WaitForPersistentVolumeDeleted(ctx, f.ClientSet, pv.Name, 5*time.Second, f.Timeouts.PVDelete)
   216  					if err != nil {
   217  						cleanUpErrs = append(cleanUpErrs, fmt.Errorf(
   218  							"persistent Volume %v not deleted by dynamic provisioner: %w", pv.Name, err))
   219  					}
   220  				}
   221  			}
   222  		default:
   223  			framework.Failf("Found PVC (%v) or PV (%v) but not running Preprovisioned or Dynamic test pattern", r.Pvc, r.Pv)
   224  		}
   225  	}
   226  
   227  	if r.Sc != nil {
   228  		ginkgo.By("Deleting sc")
   229  		if err := storageutils.DeleteStorageClass(ctx, f.ClientSet, r.Sc.Name); err != nil {
   230  			cleanUpErrs = append(cleanUpErrs, fmt.Errorf("failed to delete StorageClass %v: %w", r.Sc.Name, err))
   231  		}
   232  	}
   233  
   234  	// Cleanup volume for pre-provisioned volume tests
   235  	if r.Volume != nil {
   236  		if err := storageutils.TryFunc(func() {
   237  			r.Volume.DeleteVolume(ctx)
   238  		}); err != nil {
   239  			cleanUpErrs = append(cleanUpErrs, fmt.Errorf("failed to delete Volume: %w", err))
   240  		}
   241  	}
   242  	return utilerrors.NewAggregate(cleanUpErrs)
   243  }
   244  
   245  func createPVCPV(
   246  	ctx context.Context,
   247  	f *framework.Framework,
   248  	name string,
   249  	pvSource *v1.PersistentVolumeSource,
   250  	volumeNodeAffinity *v1.VolumeNodeAffinity,
   251  	volMode v1.PersistentVolumeMode,
   252  	accessModes []v1.PersistentVolumeAccessMode,
   253  ) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
   254  	pvConfig := e2epv.PersistentVolumeConfig{
   255  		NamePrefix:       fmt.Sprintf("%s-", name),
   256  		StorageClassName: f.Namespace.Name,
   257  		PVSource:         *pvSource,
   258  		NodeAffinity:     volumeNodeAffinity,
   259  		AccessModes:      accessModes,
   260  	}
   261  
   262  	pvcConfig := e2epv.PersistentVolumeClaimConfig{
   263  		StorageClassName: &f.Namespace.Name,
   264  		AccessModes:      accessModes,
   265  	}
   266  
   267  	if volMode != "" {
   268  		pvConfig.VolumeMode = &volMode
   269  		pvcConfig.VolumeMode = &volMode
   270  	}
   271  
   272  	framework.Logf("Creating PVC and PV")
   273  	pv, pvc, err := e2epv.CreatePVCPV(ctx, f.ClientSet, f.Timeouts, pvConfig, pvcConfig, f.Namespace.Name, false)
   274  	framework.ExpectNoError(err, "PVC, PV creation failed")
   275  
   276  	err = e2epv.WaitOnPVandPVC(ctx, f.ClientSet, f.Timeouts, f.Namespace.Name, pv, pvc)
   277  	framework.ExpectNoError(err, "PVC, PV failed to bind")
   278  
   279  	return pv, pvc
   280  }
   281  
   282  func createPVCPVFromDynamicProvisionSC(
   283  	ctx context.Context,
   284  	f *framework.Framework,
   285  	name string,
   286  	claimSize string,
   287  	sc *storagev1.StorageClass,
   288  	volMode v1.PersistentVolumeMode,
   289  	accessModes []v1.PersistentVolumeAccessMode,
   290  ) (*v1.PersistentVolume, *v1.PersistentVolumeClaim) {
   291  	cs := f.ClientSet
   292  	ns := f.Namespace.Name
   293  
   294  	ginkgo.By("creating a claim")
   295  	pvcCfg := e2epv.PersistentVolumeClaimConfig{
   296  		NamePrefix:       name,
   297  		ClaimSize:        claimSize,
   298  		StorageClassName: &(sc.Name),
   299  		AccessModes:      accessModes,
   300  		VolumeMode:       &volMode,
   301  	}
   302  
   303  	pvc := e2epv.MakePersistentVolumeClaim(pvcCfg, ns)
   304  
   305  	var err error
   306  	pvc, err = e2epv.CreatePVC(ctx, cs, ns, pvc)
   307  	framework.ExpectNoError(err)
   308  
   309  	if !isDelayedBinding(sc) {
   310  		err = e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, cs, pvc.Namespace, pvc.Name, framework.Poll, f.Timeouts.ClaimProvision)
   311  		framework.ExpectNoError(err)
   312  	}
   313  
   314  	pvc, err = cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(ctx, pvc.Name, metav1.GetOptions{})
   315  	framework.ExpectNoError(err)
   316  
   317  	var pv *v1.PersistentVolume
   318  	if !isDelayedBinding(sc) {
   319  		pv, err = cs.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{})
   320  		framework.ExpectNoError(err)
   321  	}
   322  
   323  	return pv, pvc
   324  }
   325  
   326  func isDelayedBinding(sc *storagev1.StorageClass) bool {
   327  	if sc.VolumeBindingMode != nil {
   328  		return *sc.VolumeBindingMode == storagev1.VolumeBindingWaitForFirstConsumer
   329  	}
   330  	return false
   331  }
   332
View as plain text