flexvolume.go

Documentation: k8s.io/kubernetes/test/e2e/storage

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package storage
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"path"
    24  
    25  	"time"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	v1 "k8s.io/api/core/v1"
    29  	clientset "k8s.io/client-go/kubernetes"
    30  	"k8s.io/kubernetes/test/e2e/feature"
    31  	"k8s.io/kubernetes/test/e2e/framework"
    32  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    33  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    34  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    35  	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
    36  	e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles"
    37  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    38  	"k8s.io/kubernetes/test/e2e/storage/utils"
    39  	admissionapi "k8s.io/pod-security-admission/api"
    40  )
    41  
    42  const (
    43  	driverDir              = "test/e2e/testing-manifests/flexvolume/"
    44  	defaultVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
    45  	// TODO: change this and config-test.sh when default flex volume install path is changed for GCI
    46  	// On gci, root is read-only and controller-manager containerized. Assume
    47  	// controller-manager has started with --flex-volume-plugin-dir equal to this
    48  	// (see cluster/gce/config-test.sh)
    49  	gciVolumePluginDir = "/home/kubernetes/flexvolume"
    50  	detachTimeout      = 10 * time.Second
    51  )
    52  
    53  // testFlexVolume tests that a client pod using a given flexvolume driver
    54  // successfully mounts it and runs
    55  func testFlexVolume(ctx context.Context, driver string, config e2evolume.TestConfig, f *framework.Framework) {
    56  	tests := []e2evolume.Test{
    57  		{
    58  			Volume: v1.VolumeSource{
    59  				FlexVolume: &v1.FlexVolumeSource{
    60  					Driver: "k8s/" + driver,
    61  				},
    62  			},
    63  			File: "index.html",
    64  			// Must match content of examples/volumes/flexvolume/dummy(-attachable) domount
    65  			ExpectedContent: "Hello from flexvolume!",
    66  		},
    67  	}
    68  	e2evolume.TestVolumeClient(ctx, f, config, nil, "" /* fsType */, tests)
    69  }
    70  
    71  // installFlex installs the driver found at filePath on the node, and restarts
    72  // kubelet if 'restart' is true. If node is nil, installs on the master, and restarts
    73  // controller-manager if 'restart' is true.
    74  func installFlex(ctx context.Context, c clientset.Interface, node *v1.Node, vendor, driver, filePath string) {
    75  	flexDir := getFlexDir(c, node, vendor, driver)
    76  	flexFile := path.Join(flexDir, driver)
    77  
    78  	host := ""
    79  	var err error
    80  	if node != nil {
    81  		host, err = e2enode.GetSSHExternalIP(node)
    82  		if err != nil {
    83  			host, err = e2enode.GetSSHInternalIP(node)
    84  		}
    85  	} else {
    86  		instanceWithPort := framework.APIAddress()
    87  		hostName := getHostFromHostPort(instanceWithPort)
    88  		host = net.JoinHostPort(hostName, e2essh.SSHPort)
    89  	}
    90  
    91  	framework.ExpectNoError(err)
    92  
    93  	cmd := fmt.Sprintf("sudo mkdir -p %s", flexDir)
    94  	sshAndLog(ctx, cmd, host, true /*failOnError*/)
    95  
    96  	data, err := e2etestfiles.Read(filePath)
    97  	if err != nil {
    98  		framework.Fail(err.Error())
    99  	}
   100  	cmd = fmt.Sprintf("sudo tee <<'EOF' %s\n%s\nEOF", flexFile, string(data))
   101  	sshAndLog(ctx, cmd, host, true /*failOnError*/)
   102  
   103  	cmd = fmt.Sprintf("sudo chmod +x %s", flexFile)
   104  	sshAndLog(ctx, cmd, host, true /*failOnError*/)
   105  }
   106  
   107  func uninstallFlex(ctx context.Context, c clientset.Interface, node *v1.Node, vendor, driver string) {
   108  	flexDir := getFlexDir(c, node, vendor, driver)
   109  
   110  	host := ""
   111  	var err error
   112  	if node != nil {
   113  		host, err = e2enode.GetSSHExternalIP(node)
   114  		if err != nil {
   115  			host, err = e2enode.GetSSHInternalIP(node)
   116  		}
   117  	} else {
   118  		instanceWithPort := framework.APIAddress()
   119  		hostName := getHostFromHostPort(instanceWithPort)
   120  		host = net.JoinHostPort(hostName, e2essh.SSHPort)
   121  	}
   122  
   123  	if host == "" {
   124  		framework.Failf("Error getting node ip : %v", err)
   125  	}
   126  
   127  	cmd := fmt.Sprintf("sudo rm -r %s", flexDir)
   128  	sshAndLog(ctx, cmd, host, false /*failOnError*/)
   129  }
   130  
   131  func getFlexDir(c clientset.Interface, node *v1.Node, vendor, driver string) string {
   132  	volumePluginDir := defaultVolumePluginDir
   133  	if framework.ProviderIs("gce") {
   134  		volumePluginDir = gciVolumePluginDir
   135  	}
   136  	flexDir := path.Join(volumePluginDir, fmt.Sprintf("/%s~%s/", vendor, driver))
   137  	return flexDir
   138  }
   139  
   140  func sshAndLog(ctx context.Context, cmd, host string, failOnError bool) {
   141  	result, err := e2essh.SSH(ctx, cmd, host, framework.TestContext.Provider)
   142  	e2essh.LogResult(result)
   143  	framework.ExpectNoError(err)
   144  	if result.Code != 0 && failOnError {
   145  		framework.Failf("%s returned non-zero, stderr: %s", cmd, result.Stderr)
   146  	}
   147  }
   148  
   149  func getHostFromHostPort(hostPort string) string {
   150  	// try to split host and port
   151  	var host string
   152  	var err error
   153  	if host, _, err = net.SplitHostPort(hostPort); err != nil {
   154  		// if SplitHostPort returns an error, the entire hostport is considered as host
   155  		host = hostPort
   156  	}
   157  	return host
   158  }
   159  
   160  var _ = utils.SIGDescribe("Flexvolumes", func() {
   161  	f := framework.NewDefaultFramework("flexvolume")
   162  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
   163  
   164  	// note that namespace deletion is handled by delete-namespace flag
   165  
   166  	var cs clientset.Interface
   167  	var ns *v1.Namespace
   168  	var node *v1.Node
   169  	var config e2evolume.TestConfig
   170  	var suffix string
   171  
   172  	ginkgo.BeforeEach(func(ctx context.Context) {
   173  		e2eskipper.SkipUnlessProviderIs("gce", "local")
   174  		e2eskipper.SkipUnlessMasterOSDistroIs("debian", "ubuntu", "gci", "custom")
   175  		e2eskipper.SkipUnlessNodeOSDistroIs("debian", "ubuntu", "gci", "custom")
   176  		e2eskipper.SkipUnlessSSHKeyPresent()
   177  
   178  		cs = f.ClientSet
   179  		ns = f.Namespace
   180  		var err error
   181  		node, err = e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
   182  		framework.ExpectNoError(err)
   183  		config = e2evolume.TestConfig{
   184  			Namespace:           ns.Name,
   185  			Prefix:              "flex",
   186  			ClientNodeSelection: e2epod.NodeSelection{Name: node.Name},
   187  		}
   188  		suffix = ns.Name
   189  	})
   190  
   191  	ginkgo.It("should be mountable when non-attachable", func(ctx context.Context) {
   192  		driver := "dummy"
   193  		driverInstallAs := driver + "-" + suffix
   194  
   195  		ginkgo.By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
   196  		installFlex(ctx, cs, node, "k8s", driverInstallAs, path.Join(driverDir, driver))
   197  
   198  		testFlexVolume(ctx, driverInstallAs, config, f)
   199  
   200  		ginkgo.By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
   201  		uninstallFlex(ctx, cs, node, "k8s", driverInstallAs)
   202  	})
   203  
   204  	f.It("should be mountable when attachable", feature.Flexvolumes, func(ctx context.Context) {
   205  		driver := "dummy-attachable"
   206  		driverInstallAs := driver + "-" + suffix
   207  
   208  		ginkgo.By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
   209  		installFlex(ctx, cs, node, "k8s", driverInstallAs, path.Join(driverDir, driver))
   210  		ginkgo.By(fmt.Sprintf("installing flexvolume %s on master as %s", path.Join(driverDir, driver), driverInstallAs))
   211  		installFlex(ctx, cs, nil, "k8s", driverInstallAs, path.Join(driverDir, driver))
   212  
   213  		testFlexVolume(ctx, driverInstallAs, config, f)
   214  
   215  		// Detach might occur after pod deletion. Wait before deleting driver.
   216  		time.Sleep(detachTimeout)
   217  
   218  		ginkgo.By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
   219  		uninstallFlex(ctx, cs, node, "k8s", driverInstallAs)
   220  		ginkgo.By(fmt.Sprintf("uninstalling flexvolume %s from master", driverInstallAs))
   221  		uninstallFlex(ctx, cs, nil, "k8s", driverInstallAs)
   222  	})
   223  })
   224
View as plain text