volume_io.go

Documentation: k8s.io/kubernetes/test/e2e/storage/testsuites

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18   * This test checks that the plugin VolumeSources are working when pseudo-streaming
    19   * various write sizes to mounted files.
    20   */
    21  
    22  package testsuites
    23  
    24  import (
    25  	"context"
    26  	"fmt"
    27  	"math"
    28  	"path/filepath"
    29  	"strconv"
    30  	"strings"
    31  	"time"
    32  
    33  	"github.com/onsi/ginkgo/v2"
    34  
    35  	v1 "k8s.io/api/core/v1"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	"k8s.io/apimachinery/pkg/util/errors"
    38  	clientset "k8s.io/client-go/kubernetes"
    39  	"k8s.io/kubernetes/test/e2e/framework"
    40  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    41  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    42  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    43  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    44  	admissionapi "k8s.io/pod-security-admission/api"
    45  )
    46  
    47  // MD5 hashes of the test file corresponding to each file size.
    48  // Test files are generated in testVolumeIO()
    49  // If test file generation algorithm changes, these must be recomputed.
    50  var md5hashes = map[int64]string{
    51  	storageframework.FileSizeSmall:  "5c34c2813223a7ca05a3c2f38c0d1710",
    52  	storageframework.FileSizeMedium: "f2fa202b1ffeedda5f3a58bd1ae81104",
    53  	storageframework.FileSizeLarge:  "8d763edc71bd16217664793b5a15e403",
    54  }
    55  
    56  const mountPath = "/opt"
    57  
    58  type volumeIOTestSuite struct {
    59  	tsInfo storageframework.TestSuiteInfo
    60  }
    61  
    62  // InitCustomVolumeIOTestSuite returns volumeIOTestSuite that implements TestSuite interface
    63  // using custom test patterns
    64  func InitCustomVolumeIOTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
    65  	return &volumeIOTestSuite{
    66  		tsInfo: storageframework.TestSuiteInfo{
    67  			Name:         "volumeIO",
    68  			TestPatterns: patterns,
    69  			SupportedSizeRange: e2evolume.SizeRange{
    70  				Min: "1Mi",
    71  			},
    72  		},
    73  	}
    74  }
    75  
    76  // InitVolumeIOTestSuite returns volumeIOTestSuite that implements TestSuite interface
    77  // using testsuite default patterns
    78  func InitVolumeIOTestSuite() storageframework.TestSuite {
    79  	patterns := []storageframework.TestPattern{
    80  		storageframework.DefaultFsInlineVolume,
    81  		storageframework.DefaultFsPreprovisionedPV,
    82  		storageframework.DefaultFsDynamicPV,
    83  		storageframework.NtfsDynamicPV,
    84  	}
    85  	return InitCustomVolumeIOTestSuite(patterns)
    86  }
    87  
    88  func (t *volumeIOTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
    89  	return t.tsInfo
    90  }
    91  
    92  func (t *volumeIOTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
    93  	skipVolTypePatterns(pattern, driver, storageframework.NewVolTypeMap(
    94  		storageframework.PreprovisionedPV,
    95  		storageframework.InlineVolume))
    96  }
    97  
    98  func (t *volumeIOTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
    99  	type local struct {
   100  		config *storageframework.PerTestConfig
   101  
   102  		resource *storageframework.VolumeResource
   103  
   104  		migrationCheck *migrationOpCheck
   105  	}
   106  	var (
   107  		dInfo = driver.GetDriverInfo()
   108  		l     local
   109  	)
   110  
   111  	// Beware that it also registers an AfterEach which renders f unusable. Any code using
   112  	// f must run inside an It or Context callback.
   113  	f := framework.NewFrameworkWithCustomTimeouts("volumeio", storageframework.GetDriverTimeouts(driver))
   114  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
   115  
   116  	init := func(ctx context.Context) {
   117  		l = local{}
   118  
   119  		// Now do the more expensive test initialization.
   120  		l.config = driver.PrepareTest(ctx, f)
   121  		l.migrationCheck = newMigrationOpCheck(ctx, f.ClientSet, f.ClientConfig(), dInfo.InTreePluginName)
   122  
   123  		testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange
   124  		l.resource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, testVolumeSizeRange)
   125  		if l.resource.VolSource == nil {
   126  			e2eskipper.Skipf("Driver %q does not define volumeSource - skipping", dInfo.Name)
   127  		}
   128  
   129  	}
   130  
   131  	cleanup := func(ctx context.Context) {
   132  		var errs []error
   133  		if l.resource != nil {
   134  			errs = append(errs, l.resource.CleanupResource(ctx))
   135  			l.resource = nil
   136  		}
   137  
   138  		framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
   139  		l.migrationCheck.validateMigrationVolumeOpCounts(ctx)
   140  	}
   141  
   142  	f.It("should write files of various sizes, verify size, validate content", f.WithSlow(), func(ctx context.Context) {
   143  		init(ctx)
   144  		ginkgo.DeferCleanup(cleanup)
   145  
   146  		cs := f.ClientSet
   147  		fileSizes := createFileSizes(dInfo.MaxFileSize)
   148  		testFile := fmt.Sprintf("%s_io_test_%s", dInfo.Name, f.Namespace.Name)
   149  		var fsGroup *int64
   150  		if !framework.NodeOSDistroIs("windows") && dInfo.Capabilities[storageframework.CapFsGroup] {
   151  			fsGroupVal := int64(1234)
   152  			fsGroup = &fsGroupVal
   153  		}
   154  		podSec := v1.PodSecurityContext{
   155  			FSGroup: fsGroup,
   156  		}
   157  		err := testVolumeIO(ctx, f, cs, storageframework.ConvertTestConfig(l.config), *l.resource.VolSource, &podSec, testFile, fileSizes)
   158  		framework.ExpectNoError(err)
   159  	})
   160  }
   161  
   162  func createFileSizes(maxFileSize int64) []int64 {
   163  	allFileSizes := []int64{
   164  		storageframework.FileSizeSmall,
   165  		storageframework.FileSizeMedium,
   166  		storageframework.FileSizeLarge,
   167  	}
   168  	fileSizes := []int64{}
   169  
   170  	for _, size := range allFileSizes {
   171  		if size <= maxFileSize {
   172  			fileSizes = append(fileSizes, size)
   173  		}
   174  	}
   175  
   176  	return fileSizes
   177  }
   178  
   179  // Return the plugin's client pod spec. Use an InitContainer to setup the file i/o test env.
   180  func makePodSpec(config e2evolume.TestConfig, initCmd string, volsrc v1.VolumeSource, podSecContext *v1.PodSecurityContext) *v1.Pod {
   181  	var gracePeriod int64 = 1
   182  	volName := fmt.Sprintf("io-volume-%s", config.Namespace)
   183  	pod := &v1.Pod{
   184  		TypeMeta: metav1.TypeMeta{
   185  			Kind:       "Pod",
   186  			APIVersion: "v1",
   187  		},
   188  		ObjectMeta: metav1.ObjectMeta{
   189  			Name: config.Prefix + "-io-client",
   190  			Labels: map[string]string{
   191  				"role": config.Prefix + "-io-client",
   192  			},
   193  		},
   194  		Spec: v1.PodSpec{
   195  			InitContainers: []v1.Container{
   196  				{
   197  					Name:  config.Prefix + "-io-init",
   198  					Image: e2epod.GetDefaultTestImage(),
   199  					Command: []string{
   200  						"/bin/sh",
   201  						"-c",
   202  						initCmd,
   203  					},
   204  					VolumeMounts: []v1.VolumeMount{
   205  						{
   206  							Name:      volName,
   207  							MountPath: mountPath,
   208  						},
   209  					},
   210  				},
   211  			},
   212  			Containers: []v1.Container{
   213  				{
   214  					Name:  config.Prefix + "-io-client",
   215  					Image: e2epod.GetDefaultTestImage(),
   216  					Command: []string{
   217  						"/bin/sh",
   218  						"-c",
   219  						"sleep 3600", // keep pod alive until explicitly deleted
   220  					},
   221  					VolumeMounts: []v1.VolumeMount{
   222  						{
   223  							Name:      volName,
   224  							MountPath: mountPath,
   225  						},
   226  					},
   227  				},
   228  			},
   229  			TerminationGracePeriodSeconds: &gracePeriod,
   230  			SecurityContext:               podSecContext,
   231  			Volumes: []v1.Volume{
   232  				{
   233  					Name:         volName,
   234  					VolumeSource: volsrc,
   235  				},
   236  			},
   237  			RestartPolicy: v1.RestartPolicyNever, // want pod to fail if init container fails
   238  		},
   239  	}
   240  
   241  	e2epod.SetNodeSelection(&pod.Spec, config.ClientNodeSelection)
   242  	return pod
   243  }
   244  
   245  // Write `fsize` bytes to `fpath` in the pod, using dd and the `ddInput` file.
   246  func writeToFile(f *framework.Framework, pod *v1.Pod, fpath, ddInput string, fsize int64) error {
   247  	ginkgo.By(fmt.Sprintf("writing %d bytes to test file %s", fsize, fpath))
   248  	loopCnt := fsize / storageframework.MinFileSize
   249  	writeCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do dd if=%s bs=%d >>%s 2>/dev/null; let i+=1; done", loopCnt, ddInput, storageframework.MinFileSize, fpath)
   250  	stdout, stderr, err := e2evolume.PodExec(f, pod, writeCmd)
   251  	if err != nil {
   252  		return fmt.Errorf("error writing to volume using %q: %s\nstdout: %s\nstderr: %s", writeCmd, err, stdout, stderr)
   253  	}
   254  	return err
   255  }
   256  
   257  // Verify that the test file is the expected size and contains the expected content.
   258  func verifyFile(f *framework.Framework, pod *v1.Pod, fpath string, expectSize int64, ddInput string) error {
   259  	ginkgo.By("verifying file size")
   260  	rtnstr, stderr, err := e2evolume.PodExec(f, pod, fmt.Sprintf("stat -c %%s %s", fpath))
   261  	if err != nil || rtnstr == "" {
   262  		return fmt.Errorf("unable to get file size via `stat %s`: %v\nstdout: %s\nstderr: %s", fpath, err, rtnstr, stderr)
   263  	}
   264  	size, err := strconv.Atoi(strings.TrimSuffix(rtnstr, "\n"))
   265  	if err != nil {
   266  		return fmt.Errorf("unable to convert string %q to int: %w", rtnstr, err)
   267  	}
   268  	if int64(size) != expectSize {
   269  		return fmt.Errorf("size of file %s is %d, expected %d", fpath, size, expectSize)
   270  	}
   271  
   272  	ginkgo.By("verifying file hash")
   273  	rtnstr, stderr, err = e2evolume.PodExec(f, pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
   274  	if err != nil {
   275  		return fmt.Errorf("unable to test file hash via `md5sum %s`: %v\nstdout: %s\nstderr: %s", fpath, err, rtnstr, stderr)
   276  	}
   277  	actualHash := strings.TrimSuffix(rtnstr, "\n")
   278  	expectedHash, ok := md5hashes[expectSize]
   279  	if !ok {
   280  		return fmt.Errorf("File hash is unknown for file size %d. Was a new file size added to the test suite?",
   281  			expectSize)
   282  	}
   283  	if actualHash != expectedHash {
   284  		return fmt.Errorf("MD5 hash is incorrect for file %s with size %d. Expected: `%s`; Actual: `%s`",
   285  			fpath, expectSize, expectedHash, actualHash)
   286  	}
   287  
   288  	return nil
   289  }
   290  
   291  // Delete `fpath` to save some disk space on host. Delete errors are logged but ignored.
   292  func deleteFile(f *framework.Framework, pod *v1.Pod, fpath string) {
   293  	ginkgo.By(fmt.Sprintf("deleting test file %s...", fpath))
   294  	stdout, stderr, err := e2evolume.PodExec(f, pod, fmt.Sprintf("rm -f %s", fpath))
   295  	if err != nil {
   296  		// keep going, the test dir will be deleted when the volume is unmounted
   297  		framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test\nstdout: %s\nstderr: %s", fpath, err, stdout, stderr)
   298  	}
   299  }
   300  
   301  // Create the client pod and create files of the sizes passed in by the `fsizes` parameter. Delete the
   302  // client pod and the new files when done.
   303  // Note: the file name is appended to "/opt/<Prefix>/<namespace>", eg. "/opt/nfs/e2e-.../<file>".
   304  // Note: nil can be passed for the podSecContext parm, in which case it is ignored.
   305  // Note: `fsizes` values are enforced to each be at least `MinFileSize` and a multiple of `MinFileSize`
   306  //
   307  //	bytes.
   308  func testVolumeIO(ctx context.Context, f *framework.Framework, cs clientset.Interface, config e2evolume.TestConfig, volsrc v1.VolumeSource, podSecContext *v1.PodSecurityContext, file string, fsizes []int64) (err error) {
   309  	ddInput := filepath.Join(mountPath, fmt.Sprintf("%s-%s-dd_if", config.Prefix, config.Namespace))
   310  	writeBlk := strings.Repeat("abcdefghijklmnopqrstuvwxyz123456", 32) // 1KiB value
   311  	loopCnt := storageframework.MinFileSize / int64(len(writeBlk))
   312  	// initContainer cmd to create and fill dd's input file. The initContainer is used to create
   313  	// the `dd` input file which is currently 1MiB. Rather than store a 1MiB go value, a loop is
   314  	// used to create a 1MiB file in the target directory.
   315  	initCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do echo -n %s >>%s; let i+=1; done", loopCnt, writeBlk, ddInput)
   316  
   317  	clientPod := makePodSpec(config, initCmd, volsrc, podSecContext)
   318  
   319  	ginkgo.By(fmt.Sprintf("starting %s", clientPod.Name))
   320  	podsNamespacer := cs.CoreV1().Pods(config.Namespace)
   321  	clientPod, err = podsNamespacer.Create(ctx, clientPod, metav1.CreateOptions{})
   322  	if err != nil {
   323  		return fmt.Errorf("failed to create client pod %q: %w", clientPod.Name, err)
   324  	}
   325  	ginkgo.DeferCleanup(func(ctx context.Context) {
   326  		deleteFile(f, clientPod, ddInput)
   327  		ginkgo.By(fmt.Sprintf("deleting client pod %q...", clientPod.Name))
   328  		e := e2epod.DeletePodWithWait(ctx, cs, clientPod)
   329  		if e != nil {
   330  			framework.Logf("client pod failed to delete: %v", e)
   331  			if err == nil { // delete err is returned if err is not set
   332  				err = e
   333  			}
   334  		} else {
   335  			framework.Logf("sleeping a bit so kubelet can unmount and detach the volume")
   336  			time.Sleep(e2evolume.PodCleanupTimeout)
   337  		}
   338  	})
   339  
   340  	err = e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, clientPod.Name, clientPod.Namespace, f.Timeouts.PodStart)
   341  	if err != nil {
   342  		return fmt.Errorf("client pod %q not running: %w", clientPod.Name, err)
   343  	}
   344  
   345  	// create files of the passed-in file sizes and verify test file size and content
   346  	for _, fsize := range fsizes {
   347  		// file sizes must be a multiple of `MinFileSize`
   348  		if math.Mod(float64(fsize), float64(storageframework.MinFileSize)) != 0 {
   349  			fsize = fsize/storageframework.MinFileSize + storageframework.MinFileSize
   350  		}
   351  		fpath := filepath.Join(mountPath, fmt.Sprintf("%s-%d", file, fsize))
   352  		defer func() {
   353  			deleteFile(f, clientPod, fpath)
   354  		}()
   355  		if err = writeToFile(f, clientPod, fpath, ddInput, fsize); err != nil {
   356  			return err
   357  		}
   358  		if err = verifyFile(f, clientPod, fpath, fsize, ddInput); err != nil {
   359  			return err
   360  		}
   361  	}
   362  
   363  	return
   364  }
   365
View as plain text