external.go

Documentation: k8s.io/kubernetes/test/e2e/storage/external

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package external
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"flag"
    23  	"fmt"
    24  	"os"
    25  	"time"
    26  
    27  	storagev1 "k8s.io/api/storage/v1"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    30  	"k8s.io/apimachinery/pkg/runtime"
    31  	"k8s.io/apimachinery/pkg/runtime/schema"
    32  	"k8s.io/apimachinery/pkg/util/sets"
    33  	"k8s.io/client-go/kubernetes/scheme"
    34  	klog "k8s.io/klog/v2"
    35  	"k8s.io/kubernetes/test/e2e/framework"
    36  	e2econfig "k8s.io/kubernetes/test/e2e/framework/config"
    37  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    38  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    39  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    40  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    41  	"k8s.io/kubernetes/test/e2e/storage/testsuites"
    42  	"k8s.io/kubernetes/test/e2e/storage/utils"
    43  
    44  	"github.com/onsi/gomega"
    45  )
    46  
    47  // DriverDefinition needs to be filled in via a .yaml or .json
    48  // file. Its methods then implement the TestDriver interface, using
    49  // nothing but the information in this struct.
    50  type driverDefinition struct {
    51  	// DriverInfo is the static information that the storage testsuite
    52  	// expects from a test driver. See test/e2e/storage/testsuites/testdriver.go
    53  	// for details. The only field with a non-zero default is the list of
    54  	// supported file systems (SupportedFsType): it is set so that tests using
    55  	// the default file system are enabled.
    56  	DriverInfo storageframework.DriverInfo
    57  
    58  	// StorageClass must be set to enable dynamic provisioning tests.
    59  	// The default is to not run those tests.
    60  	StorageClass struct {
    61  		// FromName set to true enables the usage of a storage
    62  		// class with DriverInfo.Name as provisioner and no
    63  		// parameters.
    64  		FromName bool
    65  
    66  		// FromFile is used only when FromName is false.  It
    67  		// loads a storage class from the given .yaml or .json
    68  		// file. File names are resolved by the
    69  		// framework.testfiles package, which typically means
    70  		// that they can be absolute or relative to the test
    71  		// suite's --repo-root parameter.
    72  		//
    73  		// This can be used when the storage class is meant to have
    74  		// additional parameters.
    75  		FromFile string
    76  
    77  		// FromExistingClassName specifies the name of a pre-installed
    78  		// StorageClass that will be copied and used for the tests.
    79  		FromExistingClassName string
    80  	}
    81  
    82  	// SnapshotClass must be set to enable snapshotting tests.
    83  	// The default is to not run those tests.
    84  	SnapshotClass struct {
    85  		// FromName set to true enables the usage of a
    86  		// snapshotter class with DriverInfo.Name as provisioner.
    87  		FromName bool
    88  
    89  		// FromFile is used only when FromName is false.  It
    90  		// loads a snapshot class from the given .yaml or .json
    91  		// file. File names are resolved by the
    92  		// framework.testfiles package, which typically means
    93  		// that they can be absolute or relative to the test
    94  		// suite's --repo-root parameter.
    95  		//
    96  		// This can be used when the snapshot class is meant to have
    97  		// additional parameters.
    98  		FromFile string
    99  
   100  		// FromExistingClassName specifies the name of a pre-installed
   101  		// SnapshotClass that will be copied and used for the tests.
   102  		FromExistingClassName string
   103  	}
   104  
   105  	// InlineVolumes defines one or more volumes for use as inline
   106  	// ephemeral volumes. At least one such volume has to be
   107  	// defined to enable testing of inline ephemeral volumes.  If
   108  	// a test needs more volumes than defined, some of the defined
   109  	// volumes will be used multiple times.
   110  	//
   111  	// DriverInfo.Name is used as name of the driver in the inline volume.
   112  	InlineVolumes []struct {
   113  		// Attributes are passed as NodePublishVolumeReq.volume_context.
   114  		// Can be empty.
   115  		Attributes map[string]string
   116  		// Shared defines whether the resulting volume is
   117  		// shared between different pods (i.e.  changes made
   118  		// in one pod are visible in another)
   119  		Shared bool
   120  		// ReadOnly must be set to true if the driver does not
   121  		// support mounting as read/write.
   122  		ReadOnly bool
   123  	}
   124  
   125  	// ClientNodeName selects a specific node for scheduling test pods.
   126  	// Can be left empty. Most drivers should not need this and instead
   127  	// use topology to ensure that pods land on the right node(s).
   128  	ClientNodeName string
   129  
   130  	// Timeouts contains the custom timeouts used during the test execution.
   131  	// The values specified here will override the default values specified in
   132  	// the framework.TimeoutContext struct.
   133  	Timeouts map[string]string
   134  }
   135  
   136  func init() {
   137  	e2econfig.Flags.Var(testDriverParameter{}, "storage.testdriver", "name of a .yaml or .json file that defines a driver for storage testing, can be used more than once")
   138  }
   139  
   140  // testDriverParameter is used to hook loading of the driver
   141  // definition file and test instantiation into argument parsing: for
   142  // each of potentially many parameters, Set is called and then does
   143  // both immediately. There is no other code location between argument
   144  // parsing and starting of the test suite where those test could be
   145  // defined.
   146  type testDriverParameter struct {
   147  }
   148  
   149  var _ flag.Value = testDriverParameter{}
   150  
   151  func (t testDriverParameter) String() string {
   152  	return "<.yaml or .json file>"
   153  }
   154  
   155  func (t testDriverParameter) Set(filename string) error {
   156  	return AddDriverDefinition(filename)
   157  }
   158  
   159  // AddDriverDefinition defines ginkgo tests for CSI driver definition file.
   160  // Either --storage.testdriver cmdline argument or AddDriverDefinition can be used
   161  // to define the tests.
   162  func AddDriverDefinition(filename string) error {
   163  	driver, err := loadDriverDefinition(filename)
   164  	framework.Logf("Driver loaded from path [%s]: %+v", filename, driver)
   165  	if err != nil {
   166  		return err
   167  	}
   168  	if driver.DriverInfo.Name == "" {
   169  		return fmt.Errorf("%q: DriverInfo.Name not set", filename)
   170  	}
   171  
   172  	args := []interface{}{"External Storage"}
   173  	args = append(args, storageframework.GetDriverNameWithFeatureTags(driver)...)
   174  	args = append(args, func() {
   175  		storageframework.DefineTestSuites(driver, testsuites.CSISuites)
   176  	})
   177  	framework.Describe(args...)
   178  
   179  	return nil
   180  }
   181  
   182  func loadDriverDefinition(filename string) (*driverDefinition, error) {
   183  	if filename == "" {
   184  		return nil, fmt.Errorf("missing file name")
   185  	}
   186  	data, err := os.ReadFile(filename)
   187  	if err != nil {
   188  		return nil, err
   189  	}
   190  	// Some reasonable defaults follow.
   191  	driver := &driverDefinition{
   192  		DriverInfo: storageframework.DriverInfo{
   193  			SupportedFsType: sets.NewString(
   194  				"", // Default fsType
   195  			),
   196  			SupportedSizeRange: e2evolume.SizeRange{
   197  				Min: "5Gi",
   198  			},
   199  		},
   200  	}
   201  	// TODO: strict checking of the file content once https://github.com/kubernetes/kubernetes/pull/71589
   202  	// or something similar is merged.
   203  	if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), data, driver); err != nil {
   204  		return nil, fmt.Errorf("%s: %w", filename, err)
   205  	}
   206  
   207  	// To ensure backward compatibility: if controller expansion is enabled,
   208  	// then set both online and offline expansion to true
   209  	if _, ok := driver.GetDriverInfo().Capabilities[storageframework.CapOnlineExpansion]; !ok &&
   210  		driver.GetDriverInfo().Capabilities[storageframework.CapControllerExpansion] {
   211  		caps := driver.DriverInfo.Capabilities
   212  		caps[storageframework.CapOnlineExpansion] = true
   213  		driver.DriverInfo.Capabilities = caps
   214  	}
   215  	if _, ok := driver.GetDriverInfo().Capabilities[storageframework.CapOfflineExpansion]; !ok &&
   216  		driver.GetDriverInfo().Capabilities[storageframework.CapControllerExpansion] {
   217  		caps := driver.DriverInfo.Capabilities
   218  		caps[storageframework.CapOfflineExpansion] = true
   219  		driver.DriverInfo.Capabilities = caps
   220  	}
   221  	return driver, nil
   222  }
   223  
   224  var _ storageframework.TestDriver = &driverDefinition{}
   225  
   226  // We have to implement the interface because dynamic PV may or may
   227  // not be supported. driverDefinition.SkipUnsupportedTest checks that
   228  // based on the actual driver definition.
   229  var _ storageframework.DynamicPVTestDriver = &driverDefinition{}
   230  
   231  // Same for snapshotting.
   232  var _ storageframework.SnapshottableTestDriver = &driverDefinition{}
   233  
   234  // And for ephemeral volumes.
   235  var _ storageframework.EphemeralTestDriver = &driverDefinition{}
   236  
   237  var _ storageframework.CustomTimeoutsTestDriver = &driverDefinition{}
   238  
   239  // runtime.DecodeInto needs a runtime.Object but doesn't do any
   240  // deserialization of it and therefore none of the methods below need
   241  // an implementation.
   242  var _ runtime.Object = &driverDefinition{}
   243  
   244  func (d *driverDefinition) DeepCopyObject() runtime.Object {
   245  	return nil
   246  }
   247  
   248  func (d *driverDefinition) GetObjectKind() schema.ObjectKind {
   249  	return nil
   250  }
   251  
   252  func (d *driverDefinition) GetDriverInfo() *storageframework.DriverInfo {
   253  	return &d.DriverInfo
   254  }
   255  
   256  func (d *driverDefinition) SkipUnsupportedTest(pattern storageframework.TestPattern) {
   257  	supported := false
   258  	// TODO (?): add support for more volume types
   259  	switch pattern.VolType {
   260  	case "":
   261  		supported = true
   262  	case storageframework.DynamicPV, storageframework.GenericEphemeralVolume:
   263  		if d.StorageClass.FromName || d.StorageClass.FromFile != "" || d.StorageClass.FromExistingClassName != "" {
   264  			supported = true
   265  		}
   266  	case storageframework.CSIInlineVolume:
   267  		supported = len(d.InlineVolumes) != 0
   268  	}
   269  	if !supported {
   270  		e2eskipper.Skipf("Driver %q does not support volume type %q - skipping", d.DriverInfo.Name, pattern.VolType)
   271  	}
   272  
   273  }
   274  
   275  func (d *driverDefinition) GetDynamicProvisionStorageClass(ctx context.Context, e2econfig *storageframework.PerTestConfig, fsType string) *storagev1.StorageClass {
   276  	var (
   277  		sc  *storagev1.StorageClass
   278  		err error
   279  	)
   280  
   281  	f := e2econfig.Framework
   282  	switch {
   283  	case d.StorageClass.FromName:
   284  		sc = &storagev1.StorageClass{Provisioner: d.DriverInfo.Name}
   285  	case d.StorageClass.FromExistingClassName != "":
   286  		sc, err = f.ClientSet.StorageV1().StorageClasses().Get(ctx, d.StorageClass.FromExistingClassName, metav1.GetOptions{})
   287  		framework.ExpectNoError(err, "getting storage class %s", d.StorageClass.FromExistingClassName)
   288  	case d.StorageClass.FromFile != "":
   289  		var ok bool
   290  		items, err := utils.LoadFromManifests(d.StorageClass.FromFile)
   291  		framework.ExpectNoError(err, "load storage class from %s", d.StorageClass.FromFile)
   292  		gomega.Expect(items).To(gomega.HaveLen(1), "exactly one item from %s", d.StorageClass.FromFile)
   293  		err = utils.PatchItems(f, f.Namespace, items...)
   294  		framework.ExpectNoError(err, "patch items")
   295  
   296  		sc, ok = items[0].(*storagev1.StorageClass)
   297  		if !ok {
   298  			framework.Failf("storage class from %s", d.StorageClass.FromFile)
   299  		}
   300  	}
   301  
   302  	gomega.Expect(sc).ToNot(gomega.BeNil(), "storage class is unexpectantly nil")
   303  
   304  	if fsType != "" {
   305  		if sc.Parameters == nil {
   306  			sc.Parameters = map[string]string{}
   307  		}
   308  		// This limits the external storage test suite to only CSI drivers, which may need to be
   309  		// reconsidered if we eventually need to move in-tree storage tests out.
   310  		sc.Parameters["csi.storage.k8s.io/fstype"] = fsType
   311  	}
   312  	return storageframework.CopyStorageClass(sc, f.Namespace.Name, "e2e-sc")
   313  }
   314  
   315  func (d *driverDefinition) GetTimeouts() *framework.TimeoutContext {
   316  	timeouts := framework.NewTimeoutContext()
   317  	if d.Timeouts == nil {
   318  		return timeouts
   319  	}
   320  
   321  	// Use a temporary map to hold the timeouts specified in the manifest file
   322  	c := make(map[string]time.Duration)
   323  	for k, v := range d.Timeouts {
   324  		duration, err := time.ParseDuration(v)
   325  		if err != nil {
   326  			// We can't use ExpectNoError() because his method can be called out of an It(),
   327  			// so we simply log the error and return the default timeouts.
   328  			klog.Errorf("Could not parse duration for key %s, will use default values: %v", k, err)
   329  			return timeouts
   330  		}
   331  		c[k] = duration
   332  	}
   333  
   334  	// Convert the temporary map holding the custom timeouts to JSON
   335  	t, err := json.Marshal(c)
   336  	if err != nil {
   337  		klog.Errorf("Could not marshal custom timeouts, will use default values: %v", err)
   338  		return timeouts
   339  	}
   340  
   341  	// Override the default timeouts with the custom ones
   342  	err = json.Unmarshal(t, &timeouts)
   343  	if err != nil {
   344  		klog.Errorf("Could not unmarshal custom timeouts, will use default values: %v", err)
   345  		return timeouts
   346  	}
   347  
   348  	return timeouts
   349  }
   350  
   351  func loadSnapshotClass(filename string) (*unstructured.Unstructured, error) {
   352  	data, err := os.ReadFile(filename)
   353  	if err != nil {
   354  		return nil, err
   355  	}
   356  	snapshotClass := &unstructured.Unstructured{}
   357  
   358  	if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), data, snapshotClass); err != nil {
   359  		return nil, fmt.Errorf("%s: %w", filename, err)
   360  	}
   361  
   362  	return snapshotClass, nil
   363  }
   364  
   365  func (d *driverDefinition) GetSnapshotClass(ctx context.Context, e2econfig *storageframework.PerTestConfig, parameters map[string]string) *unstructured.Unstructured {
   366  	if !d.SnapshotClass.FromName && d.SnapshotClass.FromFile == "" && d.SnapshotClass.FromExistingClassName == "" {
   367  		e2eskipper.Skipf("Driver %q does not support snapshotting - skipping", d.DriverInfo.Name)
   368  	}
   369  
   370  	f := e2econfig.Framework
   371  	snapshotter := d.DriverInfo.Name
   372  	ns := e2econfig.Framework.Namespace.Name
   373  
   374  	switch {
   375  	case d.SnapshotClass.FromName:
   376  		// Do nothing (just use empty parameters)
   377  	case d.SnapshotClass.FromExistingClassName != "":
   378  		snapshotClass, err := f.DynamicClient.Resource(utils.SnapshotClassGVR).Get(ctx, d.SnapshotClass.FromExistingClassName, metav1.GetOptions{})
   379  		framework.ExpectNoError(err, "getting snapshot class %s", d.SnapshotClass.FromExistingClassName)
   380  
   381  		if params, ok := snapshotClass.Object["parameters"].(map[string]interface{}); ok {
   382  			for k, v := range params {
   383  				parameters[k] = v.(string)
   384  			}
   385  		}
   386  
   387  		if snapshotProvider, ok := snapshotClass.Object["driver"]; ok {
   388  			snapshotter = snapshotProvider.(string)
   389  		}
   390  	case d.SnapshotClass.FromFile != "":
   391  		snapshotClass, err := loadSnapshotClass(d.SnapshotClass.FromFile)
   392  		framework.ExpectNoError(err, "load snapshot class from %s", d.SnapshotClass.FromFile)
   393  
   394  		if params, ok := snapshotClass.Object["parameters"].(map[string]interface{}); ok {
   395  			for k, v := range params {
   396  				parameters[k] = v.(string)
   397  			}
   398  		}
   399  
   400  		if snapshotProvider, ok := snapshotClass.Object["driver"]; ok {
   401  			snapshotter = snapshotProvider.(string)
   402  		}
   403  	}
   404  
   405  	return utils.GenerateSnapshotClassSpec(snapshotter, parameters, ns)
   406  }
   407  
   408  func (d *driverDefinition) GetVolume(e2econfig *storageframework.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
   409  	if len(d.InlineVolumes) == 0 {
   410  		e2eskipper.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name)
   411  	}
   412  	e2evolume := d.InlineVolumes[volumeNumber%len(d.InlineVolumes)]
   413  	return e2evolume.Attributes, e2evolume.Shared, e2evolume.ReadOnly
   414  }
   415  
   416  func (d *driverDefinition) GetCSIDriverName(e2econfig *storageframework.PerTestConfig) string {
   417  	return d.DriverInfo.Name
   418  }
   419  
   420  func (d *driverDefinition) PrepareTest(ctx context.Context, f *framework.Framework) *storageframework.PerTestConfig {
   421  	e2econfig := &storageframework.PerTestConfig{
   422  		Driver:              d,
   423  		Prefix:              "external",
   424  		Framework:           f,
   425  		ClientNodeSelection: e2epod.NodeSelection{Name: d.ClientNodeName},
   426  	}
   427  
   428  	if framework.NodeOSDistroIs("windows") {
   429  		e2econfig.ClientNodeSelection.Selector = map[string]string{"kubernetes.io/os": "windows"}
   430  	} else {
   431  		e2econfig.ClientNodeSelection.Selector = map[string]string{"kubernetes.io/os": "linux"}
   432  	}
   433  
   434  	return e2econfig
   435  }
   436
View as plain text