local.go

Documentation: k8s.io/kubernetes/test/e2e/storage/utils

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package utils
    18  
    19  /*
    20   * Various local test resource implementations.
    21   */
    22  
    23  import (
    24  	"context"
    25  	"fmt"
    26  	"path/filepath"
    27  	"strings"
    28  
    29  	"github.com/onsi/ginkgo/v2"
    30  	v1 "k8s.io/api/core/v1"
    31  	"k8s.io/apimachinery/pkg/util/uuid"
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  )
    34  
    35  // LocalVolumeType represents type of local volume, e.g. tmpfs, directory,
    36  // block, etc.
    37  type LocalVolumeType string
    38  
    39  const (
    40  	// LocalVolumeDirectory reprensents a simple directory as local volume
    41  	LocalVolumeDirectory LocalVolumeType = "dir"
    42  	// LocalVolumeDirectoryLink is like LocalVolumeDirectory but it's a symbolic link to directory
    43  	LocalVolumeDirectoryLink LocalVolumeType = "dir-link"
    44  	// LocalVolumeDirectoryBindMounted is like LocalVolumeDirectory but bind mounted
    45  	LocalVolumeDirectoryBindMounted LocalVolumeType = "dir-bindmounted"
    46  	// LocalVolumeDirectoryLinkBindMounted is like LocalVolumeDirectory but it's a symbolic link to self bind mounted directory
    47  	// Note that bind mounting at symbolic link actually mounts at directory it
    48  	// links to
    49  	LocalVolumeDirectoryLinkBindMounted LocalVolumeType = "dir-link-bindmounted"
    50  	// LocalVolumeTmpfs represents a temporary filesystem to be used as local volume
    51  	LocalVolumeTmpfs LocalVolumeType = "tmpfs"
    52  	// LocalVolumeBlock represents a Block device, creates a local file, and maps it as a block device
    53  	LocalVolumeBlock LocalVolumeType = "block"
    54  	// LocalVolumeBlockFS represents a filesystem backed by a block device
    55  	LocalVolumeBlockFS LocalVolumeType = "blockfs"
    56  	// LocalVolumeGCELocalSSD represents a Filesystem backed by GCE Local SSD as local volume
    57  	LocalVolumeGCELocalSSD LocalVolumeType = "gce-localssd-scsi-fs"
    58  )
    59  
    60  // LocalTestResource represents test resource of a local volume.
    61  type LocalTestResource struct {
    62  	VolumeType LocalVolumeType
    63  	Node       *v1.Node
    64  	// Volume path, path to filesystem or block device on the node
    65  	Path string
    66  	// If volume is backed by a loop device, we create loop device storage file
    67  	// under this directory.
    68  	loopDir string
    69  }
    70  
    71  // LocalTestResourceManager represents interface to create/destroy local test resources on node
    72  type LocalTestResourceManager interface {
    73  	Create(ctx context.Context, node *v1.Node, volumeType LocalVolumeType, parameters map[string]string) *LocalTestResource
    74  	ExpandBlockDevice(ctx context.Context, ltr *LocalTestResource, mbToAdd int) error
    75  	Remove(ctx context.Context, ltr *LocalTestResource)
    76  }
    77  
    78  // ltrMgr implements LocalTestResourceManager
    79  type ltrMgr struct {
    80  	prefix   string
    81  	hostExec HostExec
    82  	// hostBase represents a writable directory on the host under which we
    83  	// create test directories
    84  	hostBase string
    85  }
    86  
    87  // NewLocalResourceManager returns a instance of LocalTestResourceManager
    88  func NewLocalResourceManager(prefix string, hostExec HostExec, hostBase string) LocalTestResourceManager {
    89  	return &ltrMgr{
    90  		prefix:   prefix,
    91  		hostExec: hostExec,
    92  		hostBase: hostBase,
    93  	}
    94  }
    95  
    96  // getTestDir returns a test dir under `hostBase` directory with randome name.
    97  func (l *ltrMgr) getTestDir() string {
    98  	testDirName := fmt.Sprintf("%s-%s", l.prefix, string(uuid.NewUUID()))
    99  	return filepath.Join(l.hostBase, testDirName)
   100  }
   101  
   102  func (l *ltrMgr) setupLocalVolumeTmpfs(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   103  	hostDir := l.getTestDir()
   104  	ginkgo.By(fmt.Sprintf("Creating tmpfs mount point on node %q at path %q", node.Name, hostDir))
   105  	err := l.hostExec.IssueCommand(ctx, fmt.Sprintf("mkdir -p %q && mount -t tmpfs -o size=10m tmpfs-%q %q", hostDir, hostDir, hostDir), node)
   106  	framework.ExpectNoError(err)
   107  	return &LocalTestResource{
   108  		Node: node,
   109  		Path: hostDir,
   110  	}
   111  }
   112  
   113  func (l *ltrMgr) cleanupLocalVolumeTmpfs(ctx context.Context, ltr *LocalTestResource) {
   114  	ginkgo.By(fmt.Sprintf("Unmount tmpfs mount point on node %q at path %q", ltr.Node.Name, ltr.Path))
   115  	err := l.hostExec.IssueCommand(ctx, fmt.Sprintf("umount %q", ltr.Path), ltr.Node)
   116  	framework.ExpectNoError(err)
   117  
   118  	ginkgo.By("Removing the test directory")
   119  	err = l.hostExec.IssueCommand(ctx, fmt.Sprintf("rm -r %s", ltr.Path), ltr.Node)
   120  	framework.ExpectNoError(err)
   121  }
   122  
   123  // createAndSetupLoopDevice creates an empty file and associates a loop devie with it.
   124  func (l *ltrMgr) createAndSetupLoopDevice(ctx context.Context, dir string, node *v1.Node, size int) {
   125  	ginkgo.By(fmt.Sprintf("Creating block device on node %q using path %q", node.Name, dir))
   126  	mkdirCmd := fmt.Sprintf("mkdir -p %s", dir)
   127  	count := size / 4096
   128  	// xfs requires at least 4096 blocks
   129  	if count < 4096 {
   130  		count = 4096
   131  	}
   132  	ddCmd := fmt.Sprintf("dd if=/dev/zero of=%s/file bs=4096 count=%d", dir, count)
   133  	losetupCmd := fmt.Sprintf("losetup -f %s/file", dir)
   134  	err := l.hostExec.IssueCommand(ctx, fmt.Sprintf("%s && %s && %s", mkdirCmd, ddCmd, losetupCmd), node)
   135  	framework.ExpectNoError(err)
   136  }
   137  
   138  // findLoopDevice finds loop device path by its associated storage directory.
   139  func (l *ltrMgr) findLoopDevice(ctx context.Context, dir string, node *v1.Node) string {
   140  	cmd := fmt.Sprintf("E2E_LOOP_DEV=$(losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", dir)
   141  	loopDevResult, err := l.hostExec.IssueCommandWithResult(ctx, cmd, node)
   142  	framework.ExpectNoError(err)
   143  	return strings.TrimSpace(loopDevResult)
   144  }
   145  
   146  func (l *ltrMgr) setupLocalVolumeBlock(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   147  	loopDir := l.getTestDir()
   148  	l.createAndSetupLoopDevice(ctx, loopDir, node, 20*1024*1024)
   149  	loopDev := l.findLoopDevice(ctx, loopDir, node)
   150  	return &LocalTestResource{
   151  		Node:    node,
   152  		Path:    loopDev,
   153  		loopDir: loopDir,
   154  	}
   155  }
   156  
   157  // teardownLoopDevice tears down loop device by its associated storage directory.
   158  func (l *ltrMgr) teardownLoopDevice(ctx context.Context, dir string, node *v1.Node) {
   159  	loopDev := l.findLoopDevice(ctx, dir, node)
   160  	ginkgo.By(fmt.Sprintf("Tear down block device %q on node %q at path %s/file", loopDev, node.Name, dir))
   161  	losetupDeleteCmd := fmt.Sprintf("losetup -d %s", loopDev)
   162  	err := l.hostExec.IssueCommand(ctx, losetupDeleteCmd, node)
   163  	framework.ExpectNoError(err)
   164  	return
   165  }
   166  
   167  func (l *ltrMgr) cleanupLocalVolumeBlock(ctx context.Context, ltr *LocalTestResource) {
   168  	l.teardownLoopDevice(ctx, ltr.loopDir, ltr.Node)
   169  	ginkgo.By(fmt.Sprintf("Removing the test directory %s", ltr.loopDir))
   170  	removeCmd := fmt.Sprintf("rm -r %s", ltr.loopDir)
   171  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   172  	framework.ExpectNoError(err)
   173  }
   174  
   175  func (l *ltrMgr) setupLocalVolumeBlockFS(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   176  	ltr := l.setupLocalVolumeBlock(ctx, node, parameters)
   177  	loopDev := ltr.Path
   178  	loopDir := ltr.loopDir
   179  	// Format and mount at loopDir and give others rwx for read/write testing
   180  	cmd := fmt.Sprintf("mkfs -t ext4 %s && mount -t ext4 %s %s && chmod o+rwx %s", loopDev, loopDev, loopDir, loopDir)
   181  	err := l.hostExec.IssueCommand(ctx, cmd, node)
   182  	framework.ExpectNoError(err)
   183  	return &LocalTestResource{
   184  		Node:    node,
   185  		Path:    loopDir,
   186  		loopDir: loopDir,
   187  	}
   188  }
   189  
   190  func (l *ltrMgr) cleanupLocalVolumeBlockFS(ctx context.Context, ltr *LocalTestResource) {
   191  	umountCmd := fmt.Sprintf("umount %s", ltr.Path)
   192  	err := l.hostExec.IssueCommand(ctx, umountCmd, ltr.Node)
   193  	framework.ExpectNoError(err)
   194  	l.cleanupLocalVolumeBlock(ctx, ltr)
   195  }
   196  
   197  func (l *ltrMgr) setupLocalVolumeDirectory(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   198  	hostDir := l.getTestDir()
   199  	mkdirCmd := fmt.Sprintf("mkdir -p %s", hostDir)
   200  	err := l.hostExec.IssueCommand(ctx, mkdirCmd, node)
   201  	framework.ExpectNoError(err)
   202  	return &LocalTestResource{
   203  		Node: node,
   204  		Path: hostDir,
   205  	}
   206  }
   207  
   208  func (l *ltrMgr) cleanupLocalVolumeDirectory(ctx context.Context, ltr *LocalTestResource) {
   209  	ginkgo.By("Removing the test directory")
   210  	removeCmd := fmt.Sprintf("rm -r %s", ltr.Path)
   211  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   212  	framework.ExpectNoError(err)
   213  }
   214  
   215  func (l *ltrMgr) setupLocalVolumeDirectoryLink(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   216  	hostDir := l.getTestDir()
   217  	hostDirBackend := hostDir + "-backend"
   218  	cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
   219  	err := l.hostExec.IssueCommand(ctx, cmd, node)
   220  	framework.ExpectNoError(err)
   221  	return &LocalTestResource{
   222  		Node: node,
   223  		Path: hostDir,
   224  	}
   225  }
   226  
   227  func (l *ltrMgr) cleanupLocalVolumeDirectoryLink(ctx context.Context, ltr *LocalTestResource) {
   228  	ginkgo.By("Removing the test directory")
   229  	hostDir := ltr.Path
   230  	hostDirBackend := hostDir + "-backend"
   231  	removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend)
   232  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   233  	framework.ExpectNoError(err)
   234  }
   235  
   236  func (l *ltrMgr) setupLocalVolumeDirectoryBindMounted(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   237  	hostDir := l.getTestDir()
   238  	cmd := fmt.Sprintf("mkdir %s && mount --bind %s %s", hostDir, hostDir, hostDir)
   239  	err := l.hostExec.IssueCommand(ctx, cmd, node)
   240  	framework.ExpectNoError(err)
   241  	return &LocalTestResource{
   242  		Node: node,
   243  		Path: hostDir,
   244  	}
   245  }
   246  
   247  func (l *ltrMgr) cleanupLocalVolumeDirectoryBindMounted(ctx context.Context, ltr *LocalTestResource) {
   248  	ginkgo.By("Removing the test directory")
   249  	hostDir := ltr.Path
   250  	removeCmd := fmt.Sprintf("umount %s && rm -r %s", hostDir, hostDir)
   251  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   252  	framework.ExpectNoError(err)
   253  }
   254  
   255  func (l *ltrMgr) setupLocalVolumeDirectoryLinkBindMounted(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   256  	hostDir := l.getTestDir()
   257  	hostDirBackend := hostDir + "-backend"
   258  	cmd := fmt.Sprintf("mkdir %s && mount --bind %s %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir)
   259  	err := l.hostExec.IssueCommand(ctx, cmd, node)
   260  	framework.ExpectNoError(err)
   261  	return &LocalTestResource{
   262  		Node: node,
   263  		Path: hostDir,
   264  	}
   265  }
   266  
   267  func (l *ltrMgr) cleanupLocalVolumeDirectoryLinkBindMounted(ctx context.Context, ltr *LocalTestResource) {
   268  	ginkgo.By("Removing the test directory")
   269  	hostDir := ltr.Path
   270  	hostDirBackend := hostDir + "-backend"
   271  	removeCmd := fmt.Sprintf("rm %s && umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
   272  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   273  	framework.ExpectNoError(err)
   274  }
   275  
   276  func (l *ltrMgr) setupLocalVolumeGCELocalSSD(ctx context.Context, node *v1.Node, parameters map[string]string) *LocalTestResource {
   277  	res, err := l.hostExec.IssueCommandWithResult(ctx, "ls /mnt/disks/by-uuid/google-local-ssds-scsi-fs/", node)
   278  	framework.ExpectNoError(err)
   279  	dirName := strings.Fields(res)[0]
   280  	hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName
   281  	return &LocalTestResource{
   282  		Node: node,
   283  		Path: hostDir,
   284  	}
   285  }
   286  
   287  func (l *ltrMgr) cleanupLocalVolumeGCELocalSSD(ctx context.Context, ltr *LocalTestResource) {
   288  	// This filesystem is attached in cluster initialization, we clean all files to make it reusable.
   289  	removeCmd := fmt.Sprintf("find '%s' -mindepth 1 -maxdepth 1 -print0 | xargs -r -0 rm -rf", ltr.Path)
   290  	err := l.hostExec.IssueCommand(ctx, removeCmd, ltr.Node)
   291  	framework.ExpectNoError(err)
   292  }
   293  
   294  func (l *ltrMgr) expandLocalVolumeBlockFS(ctx context.Context, ltr *LocalTestResource, mbToAdd int) error {
   295  	ddCmd := fmt.Sprintf("dd if=/dev/zero of=%s/file conv=notrunc oflag=append bs=1M count=%d", ltr.loopDir, mbToAdd)
   296  	loopDev := l.findLoopDevice(ctx, ltr.loopDir, ltr.Node)
   297  	losetupCmd := fmt.Sprintf("losetup -c %s", loopDev)
   298  	return l.hostExec.IssueCommand(ctx, fmt.Sprintf("%s && %s", ddCmd, losetupCmd), ltr.Node)
   299  }
   300  
   301  func (l *ltrMgr) ExpandBlockDevice(ctx context.Context, ltr *LocalTestResource, mbtoAdd int) error {
   302  	switch ltr.VolumeType {
   303  	case LocalVolumeBlockFS:
   304  		return l.expandLocalVolumeBlockFS(ctx, ltr, mbtoAdd)
   305  	}
   306  	return fmt.Errorf("Failed to expand local test resource, unsupported volume type: %s", ltr.VolumeType)
   307  }
   308  
   309  func (l *ltrMgr) Create(ctx context.Context, node *v1.Node, volumeType LocalVolumeType, parameters map[string]string) *LocalTestResource {
   310  	var ltr *LocalTestResource
   311  	switch volumeType {
   312  	case LocalVolumeDirectory:
   313  		ltr = l.setupLocalVolumeDirectory(ctx, node, parameters)
   314  	case LocalVolumeDirectoryLink:
   315  		ltr = l.setupLocalVolumeDirectoryLink(ctx, node, parameters)
   316  	case LocalVolumeDirectoryBindMounted:
   317  		ltr = l.setupLocalVolumeDirectoryBindMounted(ctx, node, parameters)
   318  	case LocalVolumeDirectoryLinkBindMounted:
   319  		ltr = l.setupLocalVolumeDirectoryLinkBindMounted(ctx, node, parameters)
   320  	case LocalVolumeTmpfs:
   321  		ltr = l.setupLocalVolumeTmpfs(ctx, node, parameters)
   322  	case LocalVolumeBlock:
   323  		ltr = l.setupLocalVolumeBlock(ctx, node, parameters)
   324  	case LocalVolumeBlockFS:
   325  		ltr = l.setupLocalVolumeBlockFS(ctx, node, parameters)
   326  	case LocalVolumeGCELocalSSD:
   327  		ltr = l.setupLocalVolumeGCELocalSSD(ctx, node, parameters)
   328  	default:
   329  		framework.Failf("Failed to create local test resource on node %q, unsupported volume type: %v is specified", node.Name, volumeType)
   330  		return nil
   331  	}
   332  	if ltr == nil {
   333  		framework.Failf("Failed to create local test resource on node %q, volume type: %v, parameters: %v", node.Name, volumeType, parameters)
   334  	}
   335  	ltr.VolumeType = volumeType
   336  	return ltr
   337  }
   338  
   339  func (l *ltrMgr) Remove(ctx context.Context, ltr *LocalTestResource) {
   340  	switch ltr.VolumeType {
   341  	case LocalVolumeDirectory:
   342  		l.cleanupLocalVolumeDirectory(ctx, ltr)
   343  	case LocalVolumeDirectoryLink:
   344  		l.cleanupLocalVolumeDirectoryLink(ctx, ltr)
   345  	case LocalVolumeDirectoryBindMounted:
   346  		l.cleanupLocalVolumeDirectoryBindMounted(ctx, ltr)
   347  	case LocalVolumeDirectoryLinkBindMounted:
   348  		l.cleanupLocalVolumeDirectoryLinkBindMounted(ctx, ltr)
   349  	case LocalVolumeTmpfs:
   350  		l.cleanupLocalVolumeTmpfs(ctx, ltr)
   351  	case LocalVolumeBlock:
   352  		l.cleanupLocalVolumeBlock(ctx, ltr)
   353  	case LocalVolumeBlockFS:
   354  		l.cleanupLocalVolumeBlockFS(ctx, ltr)
   355  	case LocalVolumeGCELocalSSD:
   356  		l.cleanupLocalVolumeGCELocalSSD(ctx, ltr)
   357  	default:
   358  		framework.Failf("Failed to remove local test resource, unsupported volume type: %v is specified", ltr.VolumeType)
   359  	}
   360  	return
   361  }
   362
View as plain text