capacity.go

Documentation: k8s.io/kubernetes/test/e2e/storage/testsuites

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package testsuites
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"time"
    24  
    25  	"github.com/onsi/ginkgo/v2"
    26  	"github.com/onsi/gomega"
    27  	"github.com/onsi/gomega/types"
    28  
    29  	storagev1 "k8s.io/api/storage/v1"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/client-go/kubernetes"
    32  	"k8s.io/kubernetes/test/e2e/framework"
    33  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    34  	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
    35  	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
    36  	admissionapi "k8s.io/pod-security-admission/api"
    37  )
    38  
    39  type capacityTestSuite struct {
    40  	tsInfo storageframework.TestSuiteInfo
    41  }
    42  
    43  // InitCustomCapacityTestSuite returns capacityTestSuite that implements TestSuite interface
    44  // using custom test patterns
    45  func InitCustomCapacityTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
    46  	return &capacityTestSuite{
    47  		tsInfo: storageframework.TestSuiteInfo{
    48  			Name:         "capacity",
    49  			TestPatterns: patterns,
    50  			SupportedSizeRange: e2evolume.SizeRange{
    51  				Min: "1Mi",
    52  			},
    53  		},
    54  	}
    55  }
    56  
    57  // InitCapacityTestSuite returns capacityTestSuite that implements TestSuite interface\
    58  // using test suite default patterns
    59  func InitCapacityTestSuite() storageframework.TestSuite {
    60  	patterns := []storageframework.TestPattern{
    61  		storageframework.DefaultFsDynamicPV,
    62  	}
    63  	return InitCustomCapacityTestSuite(patterns)
    64  }
    65  
    66  func (p *capacityTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
    67  	return p.tsInfo
    68  }
    69  
    70  func (p *capacityTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
    71  	// Check preconditions.
    72  	if pattern.VolType != storageframework.DynamicPV {
    73  		e2eskipper.Skipf("Suite %q does not support %v", p.tsInfo.Name, pattern.VolType)
    74  	}
    75  	dInfo := driver.GetDriverInfo()
    76  	if !dInfo.Capabilities[storageframework.CapCapacity] {
    77  		e2eskipper.Skipf("Driver %s doesn't publish storage capacity -- skipping", dInfo.Name)
    78  	}
    79  }
    80  
    81  func (p *capacityTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
    82  	var (
    83  		dInfo   = driver.GetDriverInfo()
    84  		dDriver storageframework.DynamicPVTestDriver
    85  		sc      *storagev1.StorageClass
    86  	)
    87  
    88  	// Beware that it also registers an AfterEach which renders f unusable. Any code using
    89  	// f must run inside an It or Context callback.
    90  	f := framework.NewFrameworkWithCustomTimeouts("capacity", storageframework.GetDriverTimeouts(driver))
    91  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    92  
    93  	init := func(ctx context.Context) {
    94  		dDriver, _ = driver.(storageframework.DynamicPVTestDriver)
    95  		// Now do the more expensive test initialization.
    96  		config := driver.PrepareTest(ctx, f)
    97  		sc = dDriver.GetDynamicProvisionStorageClass(ctx, config, pattern.FsType)
    98  		if sc == nil {
    99  			e2eskipper.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
   100  		}
   101  	}
   102  
   103  	ginkgo.It("provides storage capacity information", func(ctx context.Context) {
   104  		init(ctx)
   105  
   106  		timeout := time.Minute
   107  		pollInterval := time.Second
   108  		matchSC := HaveCapacitiesForClass(sc.Name)
   109  		listAll := gomega.Eventually(ctx, func() (*storagev1.CSIStorageCapacityList, error) {
   110  			return f.ClientSet.StorageV1().CSIStorageCapacities("").List(ctx, metav1.ListOptions{})
   111  		}, timeout, pollInterval)
   112  
   113  		// If we have further information about what storage
   114  		// capacity information to expect from the driver,
   115  		// then we can make the check more specific. The baseline
   116  		// is that it provides some arbitrary capacity for the
   117  		// storage class.
   118  		matcher := matchSC
   119  		if len(dInfo.TopologyKeys) == 1 {
   120  			// We can construct topology segments by
   121  			// collecting all values for this one key and
   122  			// then expect one CSIStorageCapacity object
   123  			// per value for the storage class.
   124  			//
   125  			// Local storage on a node will be covered by
   126  			// this checking. A more complex approach for
   127  			// drivers with multiple keys might be
   128  			// possible, too, but is not currently
   129  			// implemented.
   130  			matcher = HaveCapacitiesForClassAndNodes(ctx, f.ClientSet, sc.Provisioner, sc.Name, dInfo.TopologyKeys[0])
   131  		}
   132  
   133  		// Create storage class and wait for capacity information.
   134  		sc := SetupStorageClass(ctx, f.ClientSet, sc)
   135  		listAll.Should(MatchCapacities(matcher), "after creating storage class")
   136  
   137  		// Delete storage class again and wait for removal of storage capacity information.
   138  		err := f.ClientSet.StorageV1().StorageClasses().Delete(ctx, sc.Name, metav1.DeleteOptions{})
   139  		framework.ExpectNoError(err, "delete storage class")
   140  		listAll.ShouldNot(MatchCapacities(matchSC), "after deleting storage class")
   141  	})
   142  }
   143  
   144  func formatCapacities(capacities []storagev1.CSIStorageCapacity) []string {
   145  	lines := []string{}
   146  	for _, capacity := range capacities {
   147  		lines = append(lines, fmt.Sprintf("   %+v", capacity))
   148  	}
   149  	return lines
   150  }
   151  
   152  // MatchCapacities runs some kind of check against *storagev1.CSIStorageCapacityList.
   153  // In case of failure, all actual objects are appended to the failure message.
   154  func MatchCapacities(match types.GomegaMatcher) types.GomegaMatcher {
   155  	return matchCSIStorageCapacities{match: match}
   156  }
   157  
   158  type matchCSIStorageCapacities struct {
   159  	match types.GomegaMatcher
   160  }
   161  
   162  var _ types.GomegaMatcher = matchCSIStorageCapacities{}
   163  
   164  func (m matchCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
   165  	return m.match.Match(actual)
   166  }
   167  
   168  func (m matchCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
   169  	return m.match.FailureMessage(actual) + m.dump(actual)
   170  }
   171  
   172  func (m matchCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
   173  	return m.match.NegatedFailureMessage(actual) + m.dump(actual)
   174  }
   175  
   176  func (m matchCSIStorageCapacities) dump(actual interface{}) string {
   177  	capacities, ok := actual.(*storagev1.CSIStorageCapacityList)
   178  	if !ok || capacities == nil {
   179  		return ""
   180  	}
   181  	lines := []string{"\n\nall CSIStorageCapacity objects:"}
   182  	for _, capacity := range capacities.Items {
   183  		lines = append(lines, fmt.Sprintf("%+v", capacity))
   184  	}
   185  	return strings.Join(lines, "\n")
   186  }
   187  
   188  // CapacityMatcher can be used to compose different matchers where one
   189  // adds additional checks for CSIStorageCapacity objects already checked
   190  // by another.
   191  type CapacityMatcher interface {
   192  	types.GomegaMatcher
   193  	// MatchedCapacities returns all CSICapacityObjects which were
   194  	// found during the preceding Match call.
   195  	MatchedCapacities() []storagev1.CSIStorageCapacity
   196  }
   197  
   198  // HaveCapacitiesForClass filters all storage capacity objects in a *storagev1.CSIStorageCapacityList
   199  // by storage class. Success is when when there is at least one.
   200  func HaveCapacitiesForClass(scName string) CapacityMatcher {
   201  	return &haveCSIStorageCapacities{scName: scName}
   202  }
   203  
   204  type haveCSIStorageCapacities struct {
   205  	scName             string
   206  	matchingCapacities []storagev1.CSIStorageCapacity
   207  }
   208  
   209  var _ CapacityMatcher = &haveCSIStorageCapacities{}
   210  
   211  func (h *haveCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
   212  	capacities, ok := actual.(*storagev1.CSIStorageCapacityList)
   213  	if !ok {
   214  		return false, fmt.Errorf("expected *storagev1.CSIStorageCapacityList, got: %T", actual)
   215  	}
   216  	h.matchingCapacities = nil
   217  	for _, capacity := range capacities.Items {
   218  		if capacity.StorageClassName == h.scName {
   219  			h.matchingCapacities = append(h.matchingCapacities, capacity)
   220  		}
   221  	}
   222  	return len(h.matchingCapacities) > 0, nil
   223  }
   224  
   225  func (h *haveCSIStorageCapacities) MatchedCapacities() []storagev1.CSIStorageCapacity {
   226  	return h.matchingCapacities
   227  }
   228  
   229  func (h *haveCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
   230  	return fmt.Sprintf("no CSIStorageCapacity objects for storage class %q", h.scName)
   231  }
   232  
   233  func (h *haveCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
   234  	return fmt.Sprintf("CSIStorageCapacity objects for storage class %q:\n%s",
   235  		h.scName,
   236  		strings.Join(formatCapacities(h.matchingCapacities), "\n"),
   237  	)
   238  }
   239  
   240  // HaveCapacitiesForClassAndNodes matches objects by storage class name. It finds
   241  // all nodes on which the driver runs and expects one object per node.
   242  func HaveCapacitiesForClassAndNodes(ctx context.Context, client kubernetes.Interface, driverName, scName, topologyKey string) CapacityMatcher {
   243  	return &haveLocalStorageCapacities{
   244  		ctx:         ctx,
   245  		client:      client,
   246  		driverName:  driverName,
   247  		match:       HaveCapacitiesForClass(scName),
   248  		topologyKey: topologyKey,
   249  	}
   250  }
   251  
   252  type haveLocalStorageCapacities struct {
   253  	ctx         context.Context
   254  	client      kubernetes.Interface
   255  	driverName  string
   256  	match       CapacityMatcher
   257  	topologyKey string
   258  
   259  	matchSuccess          bool
   260  	expectedCapacities    []storagev1.CSIStorageCapacity
   261  	unexpectedCapacities  []storagev1.CSIStorageCapacity
   262  	missingTopologyValues []string
   263  }
   264  
   265  var _ CapacityMatcher = &haveLocalStorageCapacities{}
   266  
   267  func (h *haveLocalStorageCapacities) Match(actual interface{}) (success bool, err error) {
   268  	ctx := h.ctx
   269  	h.expectedCapacities = nil
   270  	h.unexpectedCapacities = nil
   271  	h.missingTopologyValues = nil
   272  
   273  	// First check with underlying matcher.
   274  	success, err = h.match.Match(actual)
   275  	h.matchSuccess = success
   276  	if !success || err != nil {
   277  		return
   278  	}
   279  
   280  	// Find all nodes on which the driver runs.
   281  	csiNodes, err := h.client.StorageV1().CSINodes().List(ctx, metav1.ListOptions{})
   282  	if err != nil {
   283  		return false, err
   284  	}
   285  	topologyValues := map[string]bool{}
   286  	for _, csiNode := range csiNodes.Items {
   287  		for _, driver := range csiNode.Spec.Drivers {
   288  			if driver.Name != h.driverName {
   289  				continue
   290  			}
   291  			node, err := h.client.CoreV1().Nodes().Get(ctx, csiNode.Name, metav1.GetOptions{})
   292  			if err != nil {
   293  				return false, err
   294  			}
   295  			value, ok := node.Labels[h.topologyKey]
   296  			if !ok || value == "" {
   297  				return false, fmt.Errorf("driver %q should run on node %q, but its topology label %q was not set",
   298  					h.driverName,
   299  					node.Name,
   300  					h.topologyKey)
   301  			}
   302  			topologyValues[value] = true
   303  			break
   304  		}
   305  	}
   306  	if len(topologyValues) == 0 {
   307  		return false, fmt.Errorf("driver %q not running on any node", h.driverName)
   308  	}
   309  
   310  	// Now check that for each topology value there is exactly one CSIStorageCapacity object.
   311  	remainingTopologyValues := map[string]bool{}
   312  	for value := range topologyValues {
   313  		remainingTopologyValues[value] = true
   314  	}
   315  	capacities := h.match.MatchedCapacities()
   316  	for _, capacity := range capacities {
   317  		if capacity.NodeTopology == nil ||
   318  			len(capacity.NodeTopology.MatchExpressions) > 0 ||
   319  			len(capacity.NodeTopology.MatchLabels) != 1 ||
   320  			!remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] {
   321  			h.unexpectedCapacities = append(h.unexpectedCapacities, capacity)
   322  			continue
   323  		}
   324  		remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] = false
   325  		h.expectedCapacities = append(h.expectedCapacities, capacity)
   326  	}
   327  
   328  	// Success is when there were no unexpected capacities and enough expected ones.
   329  	for value, remaining := range remainingTopologyValues {
   330  		if remaining {
   331  			h.missingTopologyValues = append(h.missingTopologyValues, value)
   332  		}
   333  	}
   334  	return len(h.unexpectedCapacities) == 0 && len(h.missingTopologyValues) == 0, nil
   335  }
   336  
   337  func (h *haveLocalStorageCapacities) MatchedCapacities() []storagev1.CSIStorageCapacity {
   338  	return h.match.MatchedCapacities()
   339  }
   340  
   341  func (h *haveLocalStorageCapacities) FailureMessage(actual interface{}) (message string) {
   342  	if !h.matchSuccess {
   343  		return h.match.FailureMessage(actual)
   344  	}
   345  	var lines []string
   346  	if len(h.unexpectedCapacities) != 0 {
   347  		lines = append(lines, "unexpected CSIStorageCapacity objects:")
   348  		lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
   349  	}
   350  	if len(h.missingTopologyValues) != 0 {
   351  		lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
   352  			h.topologyKey, h.missingTopologyValues,
   353  		))
   354  	}
   355  	return strings.Join(lines, "\n")
   356  }
   357  
   358  func (h *haveLocalStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
   359  	if h.matchSuccess {
   360  		return h.match.NegatedFailureMessage(actual)
   361  	}
   362  	// It's not entirely clear whether negating this check is useful. Just dump all info that we have.
   363  	var lines []string
   364  	if len(h.expectedCapacities) != 0 {
   365  		lines = append(lines, "expected CSIStorageCapacity objects:")
   366  		lines = append(lines, formatCapacities(h.expectedCapacities)...)
   367  	}
   368  	if len(h.unexpectedCapacities) != 0 {
   369  		lines = append(lines, "unexpected CSIStorageCapacity objects:")
   370  		lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
   371  	}
   372  	if len(h.missingTopologyValues) != 0 {
   373  		lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
   374  			h.topologyKey, h.missingTopologyValues,
   375  		))
   376  	}
   377  	return strings.Join(lines, "\n")
   378  }
   379
View as plain text