ginkgowrapper.go

Documentation: k8s.io/kubernetes/test/e2e/framework

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package framework
    18  
    19  import (
    20  	"fmt"
    21  	"path"
    22  	"reflect"
    23  	"regexp"
    24  	"slices"
    25  	"strings"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	"github.com/onsi/ginkgo/v2/types"
    29  
    30  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    31  	"k8s.io/apimachinery/pkg/util/sets"
    32  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    33  	"k8s.io/component-base/featuregate"
    34  )
    35  
    36  // Feature is the name of a certain feature that the cluster under test must have.
    37  // Such features are different from feature gates.
    38  type Feature string
    39  
    40  // Environment is the name for the environment in which a test can run, like
    41  // "Linux" or "Windows".
    42  type Environment string
    43  
    44  // NodeFeature is the name of a feature that a node must support. To be
    45  // removed, see
    46  // https://github.com/kubernetes/enhancements/tree/master/keps/sig-testing/3041-node-conformance-and-features#nodefeature.
    47  type NodeFeature string
    48  
    49  type Valid[T comparable] struct {
    50  	items  sets.Set[T]
    51  	frozen bool
    52  }
    53  
    54  // Add registers a new valid item name. The expected usage is
    55  //
    56  //	var SomeFeature = framework.ValidFeatures.Add("Some")
    57  //
    58  // during the init phase of an E2E suite. Individual tests should not register
    59  // their own, to avoid uncontrolled proliferation of new items. E2E suites can,
    60  // but don't have to, enforce that by freezing the set of valid names.
    61  func (v *Valid[T]) Add(item T) T {
    62  	if v.frozen {
    63  		RecordBug(NewBug(fmt.Sprintf(`registry %T is already frozen, "%v" must not be added anymore`, *v, item), 1))
    64  	}
    65  	if v.items == nil {
    66  		v.items = sets.New[T]()
    67  	}
    68  	if v.items.Has(item) {
    69  		RecordBug(NewBug(fmt.Sprintf(`registry %T already contains "%v", it must not be added again`, *v, item), 1))
    70  	}
    71  	v.items.Insert(item)
    72  	return item
    73  }
    74  
    75  func (v *Valid[T]) Freeze() {
    76  	v.frozen = true
    77  }
    78  
    79  // These variables contain the parameters that [WithFeature], [WithEnvironment]
    80  // and [WithNodeFeatures] accept. The framework itself has no pre-defined
    81  // constants. Test suites and tests may define their own and then add them here
    82  // before calling these With functions.
    83  var (
    84  	ValidFeatures     Valid[Feature]
    85  	ValidEnvironments Valid[Environment]
    86  	ValidNodeFeatures Valid[NodeFeature]
    87  )
    88  
    89  var errInterface = reflect.TypeOf((*error)(nil)).Elem()
    90  
    91  // IgnoreNotFound can be used to wrap an arbitrary function in a call to
    92  // [ginkgo.DeferCleanup]. When the wrapped function returns an error that
    93  // `apierrors.IsNotFound` considers as "not found", the error is ignored
    94  // instead of failing the test during cleanup. This is useful for cleanup code
    95  // that just needs to ensure that some object does not exist anymore.
    96  func IgnoreNotFound(in any) any {
    97  	inType := reflect.TypeOf(in)
    98  	inValue := reflect.ValueOf(in)
    99  	return reflect.MakeFunc(inType, func(args []reflect.Value) []reflect.Value {
   100  		out := inValue.Call(args)
   101  		if len(out) > 0 {
   102  			lastValue := out[len(out)-1]
   103  			last := lastValue.Interface()
   104  			if last != nil && lastValue.Type().Implements(errInterface) && apierrors.IsNotFound(last.(error)) {
   105  				out[len(out)-1] = reflect.Zero(errInterface)
   106  			}
   107  		}
   108  		return out
   109  	}).Interface()
   110  }
   111  
   112  // AnnotatedLocation can be used to provide more informative source code
   113  // locations by passing the result as additional parameter to a
   114  // BeforeEach/AfterEach/DeferCleanup/It/etc.
   115  func AnnotatedLocation(annotation string) types.CodeLocation {
   116  	return AnnotatedLocationWithOffset(annotation, 1)
   117  }
   118  
   119  // AnnotatedLocationWithOffset skips additional call stack levels. With 0 as offset
   120  // it is identical to [AnnotatedLocation].
   121  func AnnotatedLocationWithOffset(annotation string, offset int) types.CodeLocation {
   122  	codeLocation := types.NewCodeLocation(offset + 1)
   123  	codeLocation.FileName = path.Base(codeLocation.FileName)
   124  	codeLocation = types.NewCustomCodeLocation(annotation + " | " + codeLocation.String())
   125  	return codeLocation
   126  }
   127  
   128  // SIGDescribe returns a wrapper function for ginkgo.Describe which injects
   129  // the SIG name as annotation. The parameter should be lowercase with
   130  // no spaces and no sig- or SIG- prefix.
   131  func SIGDescribe(sig string) func(...interface{}) bool {
   132  	if !sigRE.MatchString(sig) || strings.HasPrefix(sig, "sig-") {
   133  		RecordBug(NewBug(fmt.Sprintf("SIG label must be lowercase, no spaces and no sig- prefix, got instead: %q", sig), 1))
   134  	}
   135  	return func(args ...interface{}) bool {
   136  		args = append([]interface{}{WithLabel("sig-" + sig)}, args...)
   137  		return registerInSuite(ginkgo.Describe, args)
   138  	}
   139  }
   140  
   141  var sigRE = regexp.MustCompile(`^[a-z]+(-[a-z]+)*$`)
   142  
   143  // ConformanceIt is wrapper function for ginkgo It.  Adds "[Conformance]" tag and makes static analysis easier.
   144  func ConformanceIt(args ...interface{}) bool {
   145  	args = append(args, ginkgo.Offset(1), WithConformance())
   146  	return It(args...)
   147  }
   148  
   149  // It is a wrapper around [ginkgo.It] which supports framework With* labels as
   150  // optional arguments in addition to those already supported by ginkgo itself,
   151  // like [ginkgo.Label] and [ginkgo.Offset].
   152  //
   153  // Text and arguments may be mixed. The final text is a concatenation
   154  // of the text arguments and special tags from the With functions.
   155  func It(args ...interface{}) bool {
   156  	return registerInSuite(ginkgo.It, args)
   157  }
   158  
   159  // It is a shorthand for the corresponding package function.
   160  func (f *Framework) It(args ...interface{}) bool {
   161  	return registerInSuite(ginkgo.It, args)
   162  }
   163  
   164  // Describe is a wrapper around [ginkgo.Describe] which supports framework
   165  // With* labels as optional arguments in addition to those already supported by
   166  // ginkgo itself, like [ginkgo.Label] and [ginkgo.Offset].
   167  //
   168  // Text and arguments may be mixed. The final text is a concatenation
   169  // of the text arguments and special tags from the With functions.
   170  func Describe(args ...interface{}) bool {
   171  	return registerInSuite(ginkgo.Describe, args)
   172  }
   173  
   174  // Describe is a shorthand for the corresponding package function.
   175  func (f *Framework) Describe(args ...interface{}) bool {
   176  	return registerInSuite(ginkgo.Describe, args)
   177  }
   178  
   179  // Context is a wrapper around [ginkgo.Context] which supports framework With*
   180  // labels as optional arguments in addition to those already supported by
   181  // ginkgo itself, like [ginkgo.Label] and [ginkgo.Offset].
   182  //
   183  // Text and arguments may be mixed. The final text is a concatenation
   184  // of the text arguments and special tags from the With functions.
   185  func Context(args ...interface{}) bool {
   186  	return registerInSuite(ginkgo.Context, args)
   187  }
   188  
   189  // Context is a shorthand for the corresponding package function.
   190  func (f *Framework) Context(args ...interface{}) bool {
   191  	return registerInSuite(ginkgo.Context, args)
   192  }
   193  
   194  // registerInSuite is the common implementation of all wrapper functions. It
   195  // expects to be called through one intermediate wrapper.
   196  func registerInSuite(ginkgoCall func(string, ...interface{}) bool, args []interface{}) bool {
   197  	var ginkgoArgs []interface{}
   198  	var offset ginkgo.Offset
   199  	var texts []string
   200  
   201  	addLabel := func(label string) {
   202  		texts = append(texts, fmt.Sprintf("[%s]", label))
   203  		ginkgoArgs = append(ginkgoArgs, ginkgo.Label(label))
   204  	}
   205  
   206  	haveEmptyStrings := false
   207  	for _, arg := range args {
   208  		switch arg := arg.(type) {
   209  		case label:
   210  			fullLabel := strings.Join(arg.parts, ":")
   211  			addLabel(fullLabel)
   212  			if arg.extra != "" {
   213  				addLabel(arg.extra)
   214  			}
   215  			if fullLabel == "Serial" {
   216  				ginkgoArgs = append(ginkgoArgs, ginkgo.Serial)
   217  			}
   218  		case ginkgo.Offset:
   219  			offset = arg
   220  		case string:
   221  			if arg == "" {
   222  				haveEmptyStrings = true
   223  			}
   224  			texts = append(texts, arg)
   225  		default:
   226  			ginkgoArgs = append(ginkgoArgs, arg)
   227  		}
   228  	}
   229  	offset += 2 // This function and its direct caller.
   230  
   231  	// Now that we have the final offset, we can record bugs.
   232  	if haveEmptyStrings {
   233  		RecordBug(NewBug("empty strings as separators are unnecessary and need to be removed", int(offset)))
   234  	}
   235  
   236  	// Enforce that text snippets to not start or end with spaces because
   237  	// those lead to double spaces when concatenating below.
   238  	for _, text := range texts {
   239  		if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") {
   240  			RecordBug(NewBug(fmt.Sprintf("trailing or leading spaces are unnecessary and need to be removed: %q", text), int(offset)))
   241  		}
   242  	}
   243  
   244  	ginkgoArgs = append(ginkgoArgs, offset)
   245  	text := strings.Join(texts, " ")
   246  	return ginkgoCall(text, ginkgoArgs...)
   247  }
   248  
   249  var (
   250  	tagRe                 = regexp.MustCompile(`\[.*?\]`)
   251  	deprecatedTags        = sets.New("Conformance", "Flaky", "NodeConformance", "Disruptive", "Serial", "Slow")
   252  	deprecatedTagPrefixes = sets.New("Environment", "Feature", "NodeFeature", "FeatureGate")
   253  	deprecatedStability   = sets.New("Alpha", "Beta")
   254  )
   255  
   256  // validateSpecs checks that the test specs were registered as intended.
   257  func validateSpecs(specs types.SpecReports) {
   258  	checked := sets.New[call]()
   259  
   260  	for _, spec := range specs {
   261  		for i, text := range spec.ContainerHierarchyTexts {
   262  			c := call{
   263  				text:     text,
   264  				location: spec.ContainerHierarchyLocations[i],
   265  			}
   266  			if checked.Has(c) {
   267  				// No need to check the same container more than once.
   268  				continue
   269  			}
   270  			checked.Insert(c)
   271  			validateText(c.location, text, spec.ContainerHierarchyLabels[i])
   272  		}
   273  		c := call{
   274  			text:     spec.LeafNodeText,
   275  			location: spec.LeafNodeLocation,
   276  		}
   277  		if !checked.Has(c) {
   278  			validateText(spec.LeafNodeLocation, spec.LeafNodeText, spec.LeafNodeLabels)
   279  			checked.Insert(c)
   280  		}
   281  	}
   282  }
   283  
   284  // call acts as (mostly) unique identifier for a container node call like
   285  // Describe or Context. It's not perfect because theoretically a line might
   286  // have multiple calls with the same text, but that isn't a problem in
   287  // practice.
   288  type call struct {
   289  	text     string
   290  	location types.CodeLocation
   291  }
   292  
   293  // validateText checks for some known tags that should not be added through the
   294  // plain text strings anymore. Eventually, all such tags should get replaced
   295  // with the new APIs.
   296  func validateText(location types.CodeLocation, text string, labels []string) {
   297  	for _, tag := range tagRe.FindAllString(text, -1) {
   298  		if tag == "[]" {
   299  			recordTextBug(location, "[] in plain text is invalid")
   300  			continue
   301  		}
   302  		// Strip square brackets.
   303  		tag = tag[1 : len(tag)-1]
   304  		if slices.Contains(labels, tag) {
   305  			// Okay, was also set as label.
   306  			continue
   307  		}
   308  		if deprecatedTags.Has(tag) {
   309  			recordTextBug(location, fmt.Sprintf("[%s] in plain text is deprecated and must be added through With%s instead", tag, tag))
   310  		}
   311  		if deprecatedStability.Has(tag) {
   312  			recordTextBug(location, fmt.Sprintf("[%s] in plain text is deprecated and must be added by defining the feature gate through WithFeatureGate instead", tag))
   313  		}
   314  		if index := strings.Index(tag, ":"); index > 0 {
   315  			prefix := tag[:index]
   316  			if deprecatedTagPrefixes.Has(prefix) {
   317  				recordTextBug(location, fmt.Sprintf("[%s] in plain text is deprecated and must be added through With%s(%s) instead", tag, prefix, tag[index+1:]))
   318  			}
   319  		}
   320  	}
   321  }
   322  
   323  func recordTextBug(location types.CodeLocation, message string) {
   324  	RecordBug(Bug{FileName: location.FileName, LineNumber: location.LineNumber, Message: message})
   325  }
   326  
   327  // WithEnvironment specifies that a certain test or group of tests only works
   328  // with a feature available. The return value must be passed as additional
   329  // argument to [framework.It], [framework.Describe], [framework.Context].
   330  //
   331  // The feature must be listed in ValidFeatures.
   332  func WithFeature(name Feature) interface{} {
   333  	return withFeature(name)
   334  }
   335  
   336  // WithFeature is a shorthand for the corresponding package function.
   337  func (f *Framework) WithFeature(name Feature) interface{} {
   338  	return withFeature(name)
   339  }
   340  
   341  func withFeature(name Feature) interface{} {
   342  	if !ValidFeatures.items.Has(name) {
   343  		RecordBug(NewBug(fmt.Sprintf("WithFeature: unknown feature %q", name), 2))
   344  	}
   345  	return newLabel("Feature", string(name))
   346  }
   347  
   348  // WithFeatureGate specifies that a certain test or group of tests depends on a
   349  // feature gate being enabled. The return value must be passed as additional
   350  // argument to [framework.It], [framework.Describe], [framework.Context].
   351  //
   352  // The feature gate must be listed in
   353  // [k8s.io/apiserver/pkg/util/feature.DefaultMutableFeatureGate]. Once a
   354  // feature gate gets removed from there, the WithFeatureGate calls using it
   355  // also need to be removed.
   356  func WithFeatureGate(featureGate featuregate.Feature) interface{} {
   357  	return withFeatureGate(featureGate)
   358  }
   359  
   360  // WithFeatureGate is a shorthand for the corresponding package function.
   361  func (f *Framework) WithFeatureGate(featureGate featuregate.Feature) interface{} {
   362  	return withFeatureGate(featureGate)
   363  }
   364  
   365  func withFeatureGate(featureGate featuregate.Feature) interface{} {
   366  	spec, ok := utilfeature.DefaultMutableFeatureGate.GetAll()[featureGate]
   367  	if !ok {
   368  		RecordBug(NewBug(fmt.Sprintf("WithFeatureGate: the feature gate %q is unknown", featureGate), 2))
   369  	}
   370  
   371  	// We use mixed case (i.e. Beta instead of BETA). GA feature gates have no level string.
   372  	var level string
   373  	if spec.PreRelease != "" {
   374  		level = string(spec.PreRelease)
   375  		level = strings.ToUpper(level[0:1]) + strings.ToLower(level[1:])
   376  	}
   377  
   378  	l := newLabel("FeatureGate", string(featureGate))
   379  	l.extra = level
   380  	return l
   381  }
   382  
   383  // WithEnvironment specifies that a certain test or group of tests only works
   384  // in a certain environment. The return value must be passed as additional
   385  // argument to [framework.It], [framework.Describe], [framework.Context].
   386  //
   387  // The environment must be listed in ValidEnvironments.
   388  func WithEnvironment(name Environment) interface{} {
   389  	return withEnvironment(name)
   390  }
   391  
   392  // WithEnvironment is a shorthand for the corresponding package function.
   393  func (f *Framework) WithEnvironment(name Environment) interface{} {
   394  	return withEnvironment(name)
   395  }
   396  
   397  func withEnvironment(name Environment) interface{} {
   398  	if !ValidEnvironments.items.Has(name) {
   399  		RecordBug(NewBug(fmt.Sprintf("WithEnvironment: unknown environment %q", name), 2))
   400  	}
   401  	return newLabel("Environment", string(name))
   402  }
   403  
   404  // WithNodeFeature specifies that a certain test or group of tests only works
   405  // if the node supports a certain feature. The return value must be passed as
   406  // additional argument to [framework.It], [framework.Describe],
   407  // [framework.Context].
   408  //
   409  // The environment must be listed in ValidNodeFeatures.
   410  func WithNodeFeature(name NodeFeature) interface{} {
   411  	return withNodeFeature(name)
   412  }
   413  
   414  // WithNodeFeature is a shorthand for the corresponding package function.
   415  func (f *Framework) WithNodeFeature(name NodeFeature) interface{} {
   416  	return withNodeFeature(name)
   417  }
   418  
   419  func withNodeFeature(name NodeFeature) interface{} {
   420  	if !ValidNodeFeatures.items.Has(name) {
   421  		RecordBug(NewBug(fmt.Sprintf("WithNodeFeature: unknown environment %q", name), 2))
   422  	}
   423  	return newLabel("NodeFeature", string(name))
   424  }
   425  
   426  // WithConformace specifies that a certain test or group of tests must pass in
   427  // all conformant Kubernetes clusters. The return value must be passed as
   428  // additional argument to [framework.It], [framework.Describe],
   429  // [framework.Context].
   430  func WithConformance() interface{} {
   431  	return withConformance()
   432  }
   433  
   434  // WithConformance is a shorthand for the corresponding package function.
   435  func (f *Framework) WithConformance() interface{} {
   436  	return withConformance()
   437  }
   438  
   439  func withConformance() interface{} {
   440  	return newLabel("Conformance")
   441  }
   442  
   443  // WithNodeConformance specifies that a certain test or group of tests for node
   444  // functionality that does not depend on runtime or Kubernetes distro specific
   445  // behavior. The return value must be passed as additional argument to
   446  // [framework.It], [framework.Describe], [framework.Context].
   447  func WithNodeConformance() interface{} {
   448  	return withNodeConformance()
   449  }
   450  
   451  // WithNodeConformance is a shorthand for the corresponding package function.
   452  func (f *Framework) WithNodeConformance() interface{} {
   453  	return withNodeConformance()
   454  }
   455  
   456  func withNodeConformance() interface{} {
   457  	return newLabel("NodeConformance")
   458  }
   459  
   460  // WithDisruptive specifies that a certain test or group of tests temporarily
   461  // affects the functionality of the Kubernetes cluster. The return value must
   462  // be passed as additional argument to [framework.It], [framework.Describe],
   463  // [framework.Context].
   464  func WithDisruptive() interface{} {
   465  	return withDisruptive()
   466  }
   467  
   468  // WithDisruptive is a shorthand for the corresponding package function.
   469  func (f *Framework) WithDisruptive() interface{} {
   470  	return withDisruptive()
   471  }
   472  
   473  func withDisruptive() interface{} {
   474  	return newLabel("Disruptive")
   475  }
   476  
   477  // WithSerial specifies that a certain test or group of tests must not run in
   478  // parallel with other tests. The return value must be passed as additional
   479  // argument to [framework.It], [framework.Describe], [framework.Context].
   480  //
   481  // Starting with ginkgo v2, serial and parallel tests can be executed in the
   482  // same invocation. Ginkgo itself will ensure that the serial tests run
   483  // sequentially.
   484  func WithSerial() interface{} {
   485  	return withSerial()
   486  }
   487  
   488  // WithSerial is a shorthand for the corresponding package function.
   489  func (f *Framework) WithSerial() interface{} {
   490  	return withSerial()
   491  }
   492  
   493  func withSerial() interface{} {
   494  	return newLabel("Serial")
   495  }
   496  
   497  // WithSlow specifies that a certain test or group of tests must not run in
   498  // parallel with other tests. The return value must be passed as additional
   499  // argument to [framework.It], [framework.Describe], [framework.Context].
   500  func WithSlow() interface{} {
   501  	return withSlow()
   502  }
   503  
   504  // WithSlow is a shorthand for the corresponding package function.
   505  func (f *Framework) WithSlow() interface{} {
   506  	return WithSlow()
   507  }
   508  
   509  func withSlow() interface{} {
   510  	return newLabel("Slow")
   511  }
   512  
   513  // WithLabel is a wrapper around [ginkgo.Label]. Besides adding an arbitrary
   514  // label to a test, it also injects the label in square brackets into the test
   515  // name.
   516  func WithLabel(label string) interface{} {
   517  	return withLabel(label)
   518  }
   519  
   520  // WithLabel is a shorthand for the corresponding package function.
   521  func (f *Framework) WithLabel(label string) interface{} {
   522  	return withLabel(label)
   523  }
   524  
   525  func withLabel(label string) interface{} {
   526  	return newLabel(label)
   527  }
   528  
   529  // WithFlaky specifies that a certain test or group of tests are failing randomly.
   530  // These tests are usually filtered out and ran separately from other tests.
   531  func WithFlaky() interface{} {
   532  	return withFlaky()
   533  }
   534  
   535  // WithFlaky is a shorthand for the corresponding package function.
   536  func (f *Framework) WithFlaky() interface{} {
   537  	return withFlaky()
   538  }
   539  
   540  func withFlaky() interface{} {
   541  	return newLabel("Flaky")
   542  }
   543  
   544  type label struct {
   545  	// parts get concatenated with ":" to build the full label.
   546  	parts []string
   547  	// extra is an optional fully-formed extra label.
   548  	extra string
   549  	// explanation gets set for each label to help developers
   550  	// who pass a label to a ginkgo function. They need to use
   551  	// the corresponding framework function instead.
   552  	explanation string
   553  }
   554  
   555  func newLabel(parts ...string) label {
   556  	return label{
   557  		parts:       parts,
   558  		explanation: "If you see this as part of an 'Unknown Decorator' error from Ginkgo, then you need to replace the ginkgo.It/Context/Describe call with the corresponding framework.It/Context/Describe or (if available) f.It/Context/Describe.",
   559  	}
   560  }
   561  
   562  // TagsEqual can be used to check whether two tags are the same.
   563  // It's safe to compare e.g. the result of WithSlow() against the result
   564  // of WithSerial(), the result will be false. False is also returned
   565  // when a parameter is some completely different value.
   566  func TagsEqual(a, b interface{}) bool {
   567  	al, ok := a.(label)
   568  	if !ok {
   569  		return false
   570  	}
   571  	bl, ok := b.(label)
   572  	if !ok {
   573  		return false
   574  	}
   575  	if al.extra != bl.extra {
   576  		return false
   577  	}
   578  	return slices.Equal(al.parts, bl.parts)
   579  }
   580
View as plain text