framework.go

Documentation: k8s.io/kubernetes/test/e2e/framework

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package framework contains provider-independent helper code for
    18  // building and running E2E tests with Ginkgo. The actual Ginkgo test
    19  // suites gets assembled by combining this framework, the optional
    20  // provider support code and specific tests via a separate .go file
    21  // like Kubernetes' test/e2e.go.
    22  package framework
    23  
    24  import (
    25  	"context"
    26  	"fmt"
    27  	"math/rand"
    28  	"os"
    29  	"path"
    30  	"reflect"
    31  	"strings"
    32  	"time"
    33  
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  
    36  	v1 "k8s.io/api/core/v1"
    37  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    38  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    39  	"k8s.io/apimachinery/pkg/labels"
    40  	"k8s.io/apimachinery/pkg/runtime/schema"
    41  	"k8s.io/apimachinery/pkg/util/wait"
    42  	v1svc "k8s.io/client-go/applyconfigurations/core/v1"
    43  	"k8s.io/client-go/discovery"
    44  	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
    45  	"k8s.io/client-go/dynamic"
    46  	clientset "k8s.io/client-go/kubernetes"
    47  	"k8s.io/client-go/kubernetes/scheme"
    48  	"k8s.io/client-go/rest"
    49  	"k8s.io/client-go/restmapper"
    50  	scaleclient "k8s.io/client-go/scale"
    51  	admissionapi "k8s.io/pod-security-admission/api"
    52  
    53  	"github.com/onsi/ginkgo/v2"
    54  )
    55  
    56  const (
    57  	// DefaultNamespaceDeletionTimeout is timeout duration for waiting for a namespace deletion.
    58  	DefaultNamespaceDeletionTimeout = 5 * time.Minute
    59  	defaultServiceAccountName       = "default"
    60  )
    61  
    62  var (
    63  	// NewFrameworkExtensions lists functions that get called by
    64  	// NewFramework after constructing a new framework and after
    65  	// calling ginkgo.BeforeEach for the framework.
    66  	//
    67  	// This can be used by extensions of the core framework to modify
    68  	// settings in the framework instance or to add additional callbacks
    69  	// with ginkgo.BeforeEach/AfterEach/DeferCleanup.
    70  	//
    71  	// When a test runs, functions will be invoked in this order:
    72  	// - BeforeEaches defined by tests before f.NewDefaultFramework
    73  	//   in the order in which they were defined (first-in-first-out)
    74  	// - f.BeforeEach
    75  	// - BeforeEaches defined by tests after f.NewDefaultFramework
    76  	// - It callback
    77  	// - all AfterEaches in the order in which they were defined
    78  	// - all DeferCleanups with the order reversed (first-in-last-out)
    79  	// - f.AfterEach
    80  	//
    81  	// Because a test might skip test execution in a BeforeEach that runs
    82  	// before f.BeforeEach, AfterEach callbacks that depend on the
    83  	// framework instance must check whether it was initialized. They can
    84  	// do that by checking f.ClientSet for nil. DeferCleanup callbacks
    85  	// don't need to do this because they get defined when the test
    86  	// runs.
    87  	NewFrameworkExtensions []func(f *Framework)
    88  )
    89  
    90  // Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
    91  // Eventual goal is to merge this with integration test framework.
    92  //
    93  // You can configure the pod security level for your test by setting the `NamespacePodSecurityLevel`
    94  // which will set all three of pod security admission enforce, warn and audit labels on the namespace.
    95  // The default pod security profile is "restricted".
    96  // Each of the labels can be overridden by using more specific NamespacePodSecurity* attributes of this
    97  // struct.
    98  type Framework struct {
    99  	BaseName string
   100  
   101  	// Set together with creating the ClientSet and the namespace.
   102  	// Guaranteed to be unique in the cluster even when running the same
   103  	// test multiple times in parallel.
   104  	UniqueName string
   105  
   106  	clientConfig                     *rest.Config
   107  	ClientSet                        clientset.Interface
   108  	KubemarkExternalClusterClientSet clientset.Interface
   109  
   110  	DynamicClient dynamic.Interface
   111  
   112  	ScalesGetter scaleclient.ScalesGetter
   113  
   114  	SkipNamespaceCreation            bool            // Whether to skip creating a namespace
   115  	SkipSecretCreation               bool            // Whether to skip creating secret for a test
   116  	Namespace                        *v1.Namespace   // Every test has at least one namespace unless creation is skipped
   117  	namespacesToDelete               []*v1.Namespace // Some tests have more than one.
   118  	NamespaceDeletionTimeout         time.Duration
   119  	NamespacePodSecurityEnforceLevel admissionapi.Level // The pod security enforcement level for namespaces to be applied.
   120  	NamespacePodSecurityWarnLevel    admissionapi.Level // The pod security warn (client logging) level for namespaces to be applied.
   121  	NamespacePodSecurityAuditLevel   admissionapi.Level // The pod security audit (server logging) level for namespaces to be applied.
   122  	NamespacePodSecurityLevel        admissionapi.Level // The pod security level to be used for all of enforcement, warn and audit. Can be rewritten by more specific configuration attributes.
   123  
   124  	// Flaky operation failures in an e2e test can be captured through this.
   125  	flakeReport *FlakeReport
   126  
   127  	// configuration for framework's client
   128  	Options Options
   129  
   130  	// Place where various additional data is stored during test run to be printed to ReportDir,
   131  	// or stdout if ReportDir is not set once test ends.
   132  	TestSummaries []TestDataSummary
   133  
   134  	// Timeouts contains the custom timeouts used during the test execution.
   135  	Timeouts *TimeoutContext
   136  
   137  	// DumpAllNamespaceInfo is invoked by the framework to record
   138  	// information about a namespace after a test failure.
   139  	DumpAllNamespaceInfo DumpAllNamespaceInfoAction
   140  }
   141  
   142  // DumpAllNamespaceInfoAction is called after each failed test for namespaces
   143  // created for the test.
   144  type DumpAllNamespaceInfoAction func(ctx context.Context, f *Framework, namespace string)
   145  
   146  // TestDataSummary is an interface for managing test data.
   147  type TestDataSummary interface {
   148  	SummaryKind() string
   149  	PrintHumanReadable() string
   150  	PrintJSON() string
   151  }
   152  
   153  // Options is a struct for managing test framework options.
   154  type Options struct {
   155  	ClientQPS    float32
   156  	ClientBurst  int
   157  	GroupVersion *schema.GroupVersion
   158  }
   159  
   160  // NewFrameworkWithCustomTimeouts makes a framework with custom timeouts.
   161  // For timeout values that are zero the normal default value continues to
   162  // be used.
   163  func NewFrameworkWithCustomTimeouts(baseName string, timeouts *TimeoutContext) *Framework {
   164  	f := NewDefaultFramework(baseName)
   165  	in := reflect.ValueOf(timeouts).Elem()
   166  	out := reflect.ValueOf(f.Timeouts).Elem()
   167  	for i := 0; i < in.NumField(); i++ {
   168  		value := in.Field(i)
   169  		if !value.IsZero() {
   170  			out.Field(i).Set(value)
   171  		}
   172  	}
   173  	return f
   174  }
   175  
   176  // NewDefaultFramework makes a new framework and sets up a BeforeEach which
   177  // initializes the framework instance. It cleans up with a DeferCleanup,
   178  // which runs last, so a AfterEach in the test still has a valid framework
   179  // instance.
   180  func NewDefaultFramework(baseName string) *Framework {
   181  	options := Options{
   182  		ClientQPS:   20,
   183  		ClientBurst: 50,
   184  	}
   185  	return NewFramework(baseName, options, nil)
   186  }
   187  
   188  // NewFramework creates a test framework.
   189  func NewFramework(baseName string, options Options, client clientset.Interface) *Framework {
   190  	f := &Framework{
   191  		BaseName:  baseName,
   192  		Options:   options,
   193  		ClientSet: client,
   194  		Timeouts:  NewTimeoutContext(),
   195  	}
   196  
   197  	// The order is important here: if the extension calls ginkgo.BeforeEach
   198  	// itself, then it can be sure that f.BeforeEach already ran when its
   199  	// own callback gets invoked.
   200  	ginkgo.BeforeEach(f.BeforeEach, AnnotatedLocation("set up framework"))
   201  	for _, extension := range NewFrameworkExtensions {
   202  		extension(f)
   203  	}
   204  
   205  	return f
   206  }
   207  
   208  // BeforeEach gets a client and makes a namespace.
   209  func (f *Framework) BeforeEach(ctx context.Context) {
   210  	// DeferCleanup, in contrast to AfterEach, triggers execution in
   211  	// first-in-last-out order. This ensures that the framework instance
   212  	// remains valid as long as possible.
   213  	//
   214  	// In addition, AfterEach will not be called if a test never gets here.
   215  	ginkgo.DeferCleanup(f.AfterEach, AnnotatedLocation("tear down framework"))
   216  
   217  	// Registered later and thus runs before deleting namespaces.
   218  	ginkgo.DeferCleanup(f.dumpNamespaceInfo, AnnotatedLocation("dump namespaces"))
   219  
   220  	ginkgo.By("Creating a kubernetes client")
   221  	config, err := LoadConfig()
   222  	ExpectNoError(err)
   223  
   224  	config.QPS = f.Options.ClientQPS
   225  	config.Burst = f.Options.ClientBurst
   226  	if f.Options.GroupVersion != nil {
   227  		config.GroupVersion = f.Options.GroupVersion
   228  	}
   229  	if TestContext.KubeAPIContentType != "" {
   230  		config.ContentType = TestContext.KubeAPIContentType
   231  	}
   232  	f.clientConfig = rest.CopyConfig(config)
   233  	f.ClientSet, err = clientset.NewForConfig(config)
   234  	ExpectNoError(err)
   235  	f.DynamicClient, err = dynamic.NewForConfig(config)
   236  	ExpectNoError(err)
   237  
   238  	// create scales getter, set GroupVersion and NegotiatedSerializer to default values
   239  	// as they are required when creating a REST client.
   240  	if config.GroupVersion == nil {
   241  		config.GroupVersion = &schema.GroupVersion{}
   242  	}
   243  	if config.NegotiatedSerializer == nil {
   244  		config.NegotiatedSerializer = scheme.Codecs
   245  	}
   246  	restClient, err := rest.RESTClientFor(config)
   247  	ExpectNoError(err)
   248  	discoClient, err := discovery.NewDiscoveryClientForConfig(config)
   249  	ExpectNoError(err)
   250  	cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoClient)
   251  	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient)
   252  	restMapper.Reset()
   253  	resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient)
   254  	f.ScalesGetter = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver)
   255  
   256  	TestContext.CloudConfig.Provider.FrameworkBeforeEach(f)
   257  
   258  	if !f.SkipNamespaceCreation {
   259  		ginkgo.By(fmt.Sprintf("Building a namespace api object, basename %s", f.BaseName))
   260  		namespace, err := f.CreateNamespace(ctx, f.BaseName, map[string]string{
   261  			"e2e-framework": f.BaseName,
   262  		})
   263  		ExpectNoError(err)
   264  
   265  		f.Namespace = namespace
   266  
   267  		if TestContext.VerifyServiceAccount {
   268  			ginkgo.By("Waiting for a default service account to be provisioned in namespace")
   269  			err = WaitForDefaultServiceAccountInNamespace(ctx, f.ClientSet, namespace.Name)
   270  			ExpectNoError(err)
   271  			ginkgo.By("Waiting for kube-root-ca.crt to be provisioned in namespace")
   272  			err = WaitForKubeRootCAInNamespace(ctx, f.ClientSet, namespace.Name)
   273  			ExpectNoError(err)
   274  		} else {
   275  			Logf("Skipping waiting for service account")
   276  		}
   277  
   278  		f.UniqueName = f.Namespace.GetName()
   279  	} else {
   280  		// not guaranteed to be unique, but very likely
   281  		f.UniqueName = fmt.Sprintf("%s-%08x", f.BaseName, rand.Int31())
   282  	}
   283  
   284  	f.flakeReport = NewFlakeReport()
   285  }
   286  
   287  func (f *Framework) dumpNamespaceInfo(ctx context.Context) {
   288  	if !ginkgo.CurrentSpecReport().Failed() {
   289  		return
   290  	}
   291  	if !TestContext.DumpLogsOnFailure {
   292  		return
   293  	}
   294  	if f.DumpAllNamespaceInfo == nil {
   295  		return
   296  	}
   297  	ginkgo.By("dump namespace information after failure", func() {
   298  		if !f.SkipNamespaceCreation {
   299  			for _, ns := range f.namespacesToDelete {
   300  				f.DumpAllNamespaceInfo(ctx, f, ns.Name)
   301  			}
   302  		}
   303  	})
   304  }
   305  
   306  // printSummaries prints summaries of tests.
   307  func printSummaries(summaries []TestDataSummary, testBaseName string) {
   308  	now := time.Now()
   309  	for i := range summaries {
   310  		Logf("Printing summary: %v", summaries[i].SummaryKind())
   311  		switch TestContext.OutputPrintType {
   312  		case "hr":
   313  			if TestContext.ReportDir == "" {
   314  				Logf(summaries[i].PrintHumanReadable())
   315  			} else {
   316  				// TODO: learn to extract test name and append it to the kind instead of timestamp.
   317  				filePath := path.Join(TestContext.ReportDir, summaries[i].SummaryKind()+"_"+testBaseName+"_"+now.Format(time.RFC3339)+".txt")
   318  				if err := os.WriteFile(filePath, []byte(summaries[i].PrintHumanReadable()), 0644); err != nil {
   319  					Logf("Failed to write file %v with test performance data: %v", filePath, err)
   320  				}
   321  			}
   322  		case "json":
   323  			fallthrough
   324  		default:
   325  			if TestContext.OutputPrintType != "json" {
   326  				Logf("Unknown output type: %v. Printing JSON", TestContext.OutputPrintType)
   327  			}
   328  			if TestContext.ReportDir == "" {
   329  				Logf("%v JSON\n%v", summaries[i].SummaryKind(), summaries[i].PrintJSON())
   330  				Logf("Finished")
   331  			} else {
   332  				// TODO: learn to extract test name and append it to the kind instead of timestamp.
   333  				filePath := path.Join(TestContext.ReportDir, summaries[i].SummaryKind()+"_"+testBaseName+"_"+now.Format(time.RFC3339)+".json")
   334  				Logf("Writing to %s", filePath)
   335  				if err := os.WriteFile(filePath, []byte(summaries[i].PrintJSON()), 0644); err != nil {
   336  					Logf("Failed to write file %v with test performance data: %v", filePath, err)
   337  				}
   338  			}
   339  		}
   340  	}
   341  }
   342  
   343  // AfterEach deletes the namespace, after reading its events.
   344  func (f *Framework) AfterEach(ctx context.Context) {
   345  	// This should not happen. Given ClientSet is a public field a test must have updated it!
   346  	// Error out early before any API calls during cleanup.
   347  	if f.ClientSet == nil {
   348  		Failf("The framework ClientSet must not be nil at this point")
   349  	}
   350  
   351  	// DeleteNamespace at the very end in defer, to avoid any
   352  	// expectation failures preventing deleting the namespace.
   353  	defer func() {
   354  		nsDeletionErrors := map[string]error{}
   355  		// Whether to delete namespace is determined by 3 factors: delete-namespace flag, delete-namespace-on-failure flag and the test result
   356  		// if delete-namespace set to false, namespace will always be preserved.
   357  		// if delete-namespace is true and delete-namespace-on-failure is false, namespace will be preserved if test failed.
   358  		if TestContext.DeleteNamespace && (TestContext.DeleteNamespaceOnFailure || !ginkgo.CurrentSpecReport().Failed()) {
   359  			for _, ns := range f.namespacesToDelete {
   360  				ginkgo.By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name))
   361  				if err := f.ClientSet.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}); err != nil {
   362  					if !apierrors.IsNotFound(err) {
   363  						nsDeletionErrors[ns.Name] = err
   364  
   365  						// Dump namespace if we are unable to delete the namespace and the dump was not already performed.
   366  						if !ginkgo.CurrentSpecReport().Failed() && TestContext.DumpLogsOnFailure && f.DumpAllNamespaceInfo != nil {
   367  							f.DumpAllNamespaceInfo(ctx, f, ns.Name)
   368  						}
   369  					} else {
   370  						Logf("Namespace %v was already deleted", ns.Name)
   371  					}
   372  				}
   373  			}
   374  		} else {
   375  			if !TestContext.DeleteNamespace {
   376  				Logf("Found DeleteNamespace=false, skipping namespace deletion!")
   377  			} else {
   378  				Logf("Found DeleteNamespaceOnFailure=false and current test failed, skipping namespace deletion!")
   379  			}
   380  		}
   381  
   382  		// Unsetting this is relevant for a following test that uses
   383  		// the same instance because it might not reach f.BeforeEach
   384  		// when some other BeforeEach skips the test first.
   385  		f.Namespace = nil
   386  		f.clientConfig = nil
   387  		f.ClientSet = nil
   388  		f.namespacesToDelete = nil
   389  
   390  		// if we had errors deleting, report them now.
   391  		if len(nsDeletionErrors) != 0 {
   392  			messages := []string{}
   393  			for namespaceKey, namespaceErr := range nsDeletionErrors {
   394  				messages = append(messages, fmt.Sprintf("Couldn't delete ns: %q: %s (%#v)", namespaceKey, namespaceErr, namespaceErr))
   395  			}
   396  			Failf(strings.Join(messages, ","))
   397  		}
   398  	}()
   399  
   400  	TestContext.CloudConfig.Provider.FrameworkAfterEach(f)
   401  
   402  	// Report any flakes that were observed in the e2e test and reset.
   403  	if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 {
   404  		f.TestSummaries = append(f.TestSummaries, f.flakeReport)
   405  		f.flakeReport = nil
   406  	}
   407  
   408  	printSummaries(f.TestSummaries, f.BaseName)
   409  }
   410  
   411  // DeleteNamespace can be used to delete a namespace. Additionally it can be used to
   412  // dump namespace information so as it can be used as an alternative of framework
   413  // deleting the namespace towards the end.
   414  func (f *Framework) DeleteNamespace(ctx context.Context, name string) {
   415  	defer func() {
   416  		err := f.ClientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
   417  		if err != nil && !apierrors.IsNotFound(err) {
   418  			Logf("error deleting namespace %s: %v", name, err)
   419  			return
   420  		}
   421  		err = WaitForNamespacesDeleted(ctx, f.ClientSet, []string{name}, DefaultNamespaceDeletionTimeout)
   422  		if err != nil {
   423  			Logf("error deleting namespace %s: %v", name, err)
   424  			return
   425  		}
   426  		// remove deleted namespace from namespacesToDelete map
   427  		for i, ns := range f.namespacesToDelete {
   428  			if ns == nil {
   429  				continue
   430  			}
   431  			if ns.Name == name {
   432  				f.namespacesToDelete = append(f.namespacesToDelete[:i], f.namespacesToDelete[i+1:]...)
   433  			}
   434  		}
   435  	}()
   436  	// if current test failed then we should dump namespace information
   437  	if !f.SkipNamespaceCreation && ginkgo.CurrentSpecReport().Failed() && TestContext.DumpLogsOnFailure && f.DumpAllNamespaceInfo != nil {
   438  		f.DumpAllNamespaceInfo(ctx, f, name)
   439  	}
   440  
   441  }
   442  
   443  // CreateNamespace creates a namespace for e2e testing.
   444  func (f *Framework) CreateNamespace(ctx context.Context, baseName string, labels map[string]string) (*v1.Namespace, error) {
   445  	createTestingNS := TestContext.CreateTestingNS
   446  	if createTestingNS == nil {
   447  		createTestingNS = CreateTestingNS
   448  	}
   449  
   450  	if labels == nil {
   451  		labels = make(map[string]string)
   452  	} else {
   453  		labelsCopy := make(map[string]string)
   454  		for k, v := range labels {
   455  			labelsCopy[k] = v
   456  		}
   457  		labels = labelsCopy
   458  	}
   459  
   460  	labels[admissionapi.EnforceLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityEnforceLevel, f.NamespacePodSecurityLevel)
   461  	labels[admissionapi.WarnLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityWarnLevel, f.NamespacePodSecurityLevel)
   462  	labels[admissionapi.AuditLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityAuditLevel, f.NamespacePodSecurityLevel)
   463  
   464  	ns, err := createTestingNS(ctx, baseName, f.ClientSet, labels)
   465  	// check ns instead of err to see if it's nil as we may
   466  	// fail to create serviceAccount in it.
   467  	f.AddNamespacesToDelete(ns)
   468  
   469  	if TestContext.E2EDockerConfigFile != "" && !f.SkipSecretCreation {
   470  		// With the Secret created, the default service account (in the new namespace)
   471  		// is patched with the secret and can then be referenced by all the pods spawned by E2E process, and repository authentication should be successful.
   472  		secret, err := f.createSecretFromDockerConfig(ctx, ns.Name)
   473  		if err != nil {
   474  			return ns, fmt.Errorf("failed to create secret from docker config file: %v", err)
   475  		}
   476  
   477  		serviceAccountClient := f.ClientSet.CoreV1().ServiceAccounts(ns.Name)
   478  		serviceAccountConfig := v1svc.ServiceAccount(defaultServiceAccountName, ns.Name)
   479  		serviceAccountConfig.ImagePullSecrets = append(serviceAccountConfig.ImagePullSecrets, v1svc.LocalObjectReferenceApplyConfiguration{Name: &secret.Name})
   480  
   481  		svc, err := serviceAccountClient.Apply(ctx, serviceAccountConfig, metav1.ApplyOptions{FieldManager: "e2e-framework"})
   482  		if err != nil {
   483  			return ns, fmt.Errorf("failed to patch imagePullSecret [%s] to service account [%s]: %v", secret.Name, svc.Name, err)
   484  		}
   485  
   486  	}
   487  
   488  	return ns, err
   489  }
   490  
   491  func firstNonEmptyPSaLevelOrRestricted(levelConfig ...admissionapi.Level) string {
   492  	for _, l := range levelConfig {
   493  		if len(l) > 0 {
   494  			return string(l)
   495  		}
   496  	}
   497  	return string(admissionapi.LevelRestricted)
   498  }
   499  
   500  // createSecretFromDockerConfig creates a secret using the private image registry credentials.
   501  // The credentials are provided by --e2e-docker-config-file flag.
   502  func (f *Framework) createSecretFromDockerConfig(ctx context.Context, namespace string) (*v1.Secret, error) {
   503  	contents, err := os.ReadFile(TestContext.E2EDockerConfigFile)
   504  	if err != nil {
   505  		return nil, fmt.Errorf("error reading docker config file: %v", err)
   506  	}
   507  
   508  	secretObject := &v1.Secret{
   509  		Data: map[string][]byte{v1.DockerConfigJsonKey: contents},
   510  		Type: v1.SecretTypeDockerConfigJson,
   511  	}
   512  	secretObject.GenerateName = "registry-cred"
   513  	Logf("create image pull secret %s", secretObject.Name)
   514  
   515  	secret, err := f.ClientSet.CoreV1().Secrets(namespace).Create(ctx, secretObject, metav1.CreateOptions{})
   516  
   517  	return secret, err
   518  }
   519  
   520  // RecordFlakeIfError records flakeness info if error happens.
   521  // NOTE: This function is not used at any places yet, but we are in progress for https://github.com/kubernetes/kubernetes/issues/66239 which requires this. Please don't remove this.
   522  func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
   523  	f.flakeReport.RecordFlakeIfError(err, optionalDescription...)
   524  }
   525  
   526  // AddNamespacesToDelete adds one or more namespaces to be deleted when the test
   527  // completes.
   528  func (f *Framework) AddNamespacesToDelete(namespaces ...*v1.Namespace) {
   529  	for _, ns := range namespaces {
   530  		if ns == nil {
   531  			continue
   532  		}
   533  		f.namespacesToDelete = append(f.namespacesToDelete, ns)
   534  
   535  	}
   536  }
   537  
   538  // ClientConfig an externally accessible method for reading the kube client config.
   539  func (f *Framework) ClientConfig() *rest.Config {
   540  	ret := rest.CopyConfig(f.clientConfig)
   541  	// json is least common denominator
   542  	ret.ContentType = runtime.ContentTypeJSON
   543  	ret.AcceptContentTypes = runtime.ContentTypeJSON
   544  	return ret
   545  }
   546  
   547  // KubeUser is a struct for managing kubernetes user info.
   548  type KubeUser struct {
   549  	Name string `yaml:"name"`
   550  	User struct {
   551  		Username string `yaml:"username"`
   552  		Password string `yaml:"password" datapolicy:"password"`
   553  		Token    string `yaml:"token" datapolicy:"token"`
   554  	} `yaml:"user"`
   555  }
   556  
   557  // KubeCluster is a struct for managing kubernetes cluster info.
   558  type KubeCluster struct {
   559  	Name    string `yaml:"name"`
   560  	Cluster struct {
   561  		CertificateAuthorityData string `yaml:"certificate-authority-data"`
   562  		Server                   string `yaml:"server"`
   563  	} `yaml:"cluster"`
   564  }
   565  
   566  // KubeConfig is a struct for managing kubernetes config.
   567  type KubeConfig struct {
   568  	Contexts []struct {
   569  		Name    string `yaml:"name"`
   570  		Context struct {
   571  			Cluster string `yaml:"cluster"`
   572  			User    string
   573  		} `yaml:"context"`
   574  	} `yaml:"contexts"`
   575  
   576  	Clusters []KubeCluster `yaml:"clusters"`
   577  
   578  	Users []KubeUser `yaml:"users"`
   579  }
   580  
   581  // FindUser returns user info which is the specified user name.
   582  func (kc *KubeConfig) FindUser(name string) *KubeUser {
   583  	for _, user := range kc.Users {
   584  		if user.Name == name {
   585  			return &user
   586  		}
   587  	}
   588  	return nil
   589  }
   590  
   591  // FindCluster returns cluster info which is the specified cluster name.
   592  func (kc *KubeConfig) FindCluster(name string) *KubeCluster {
   593  	for _, cluster := range kc.Clusters {
   594  		if cluster.Name == name {
   595  			return &cluster
   596  		}
   597  	}
   598  	return nil
   599  }
   600  
   601  // PodStateVerification represents a verification of pod state.
   602  // Any time you have a set of pods that you want to operate against or query,
   603  // this struct can be used to declaratively identify those pods.
   604  type PodStateVerification struct {
   605  	// Optional: only pods that have k=v labels will pass this filter.
   606  	Selectors map[string]string
   607  
   608  	// Required: The phases which are valid for your pod.
   609  	ValidPhases []v1.PodPhase
   610  
   611  	// Optional: only pods passing this function will pass the filter
   612  	// Verify a pod.
   613  	// As an optimization, in addition to specifying filter (boolean),
   614  	// this function allows specifying an error as well.
   615  	// The error indicates that the polling of the pod spectrum should stop.
   616  	Verify func(v1.Pod) (bool, error)
   617  
   618  	// Optional: only pods with this name will pass the filter.
   619  	PodName string
   620  }
   621  
   622  // ClusterVerification is a struct for a verification of cluster state.
   623  type ClusterVerification struct {
   624  	client    clientset.Interface
   625  	namespace *v1.Namespace // pointer rather than string, since ns isn't created until before each.
   626  	podState  PodStateVerification
   627  }
   628  
   629  // NewClusterVerification creates a new cluster verification.
   630  func (f *Framework) NewClusterVerification(namespace *v1.Namespace, filter PodStateVerification) *ClusterVerification {
   631  	return &ClusterVerification{
   632  		f.ClientSet,
   633  		namespace,
   634  		filter,
   635  	}
   636  }
   637  
   638  func passesPodNameFilter(pod v1.Pod, name string) bool {
   639  	return name == "" || strings.Contains(pod.Name, name)
   640  }
   641  
   642  func passesVerifyFilter(pod v1.Pod, verify func(p v1.Pod) (bool, error)) (bool, error) {
   643  	if verify == nil {
   644  		return true, nil
   645  	}
   646  
   647  	verified, err := verify(pod)
   648  	// If an error is returned, by definition, pod verification fails
   649  	if err != nil {
   650  		return false, err
   651  	}
   652  	return verified, nil
   653  }
   654  
   655  func passesPhasesFilter(pod v1.Pod, validPhases []v1.PodPhase) bool {
   656  	passesPhaseFilter := false
   657  	for _, phase := range validPhases {
   658  		if pod.Status.Phase == phase {
   659  			passesPhaseFilter = true
   660  		}
   661  	}
   662  	return passesPhaseFilter
   663  }
   664  
   665  // filterLabels returns a list of pods which have labels.
   666  func filterLabels(ctx context.Context, selectors map[string]string, cli clientset.Interface, ns string) (*v1.PodList, error) {
   667  	var err error
   668  	var selector labels.Selector
   669  	var pl *v1.PodList
   670  	// List pods based on selectors.  This might be a tiny optimization rather then filtering
   671  	// everything manually.
   672  	if len(selectors) > 0 {
   673  		selector = labels.SelectorFromSet(labels.Set(selectors))
   674  		options := metav1.ListOptions{LabelSelector: selector.String()}
   675  		pl, err = cli.CoreV1().Pods(ns).List(ctx, options)
   676  	} else {
   677  		pl, err = cli.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
   678  	}
   679  	return pl, err
   680  }
   681  
   682  // filter filters pods which pass a filter.  It can be used to compose
   683  // the more useful abstractions like ForEach, WaitFor, and so on, which
   684  // can be used directly by tests.
   685  func (p *PodStateVerification) filter(ctx context.Context, c clientset.Interface, namespace *v1.Namespace) ([]v1.Pod, error) {
   686  	if len(p.ValidPhases) == 0 || namespace == nil {
   687  		panic(fmt.Errorf("Need to specify a valid pod phases (%v) and namespace (%v). ", p.ValidPhases, namespace))
   688  	}
   689  
   690  	ns := namespace.Name
   691  	pl, err := filterLabels(ctx, p.Selectors, c, ns) // Build an v1.PodList to operate against.
   692  	Logf("Selector matched %v pods for %v", len(pl.Items), p.Selectors)
   693  	if len(pl.Items) == 0 || err != nil {
   694  		return pl.Items, err
   695  	}
   696  
   697  	unfilteredPods := pl.Items
   698  	filteredPods := []v1.Pod{}
   699  ReturnPodsSoFar:
   700  	// Next: Pod must match at least one of the states that the user specified
   701  	for _, pod := range unfilteredPods {
   702  		if !(passesPhasesFilter(pod, p.ValidPhases) && passesPodNameFilter(pod, p.PodName)) {
   703  			continue
   704  		}
   705  		passesVerify, err := passesVerifyFilter(pod, p.Verify)
   706  		if err != nil {
   707  			Logf("Error detected on %v : %v !", pod.Name, err)
   708  			break ReturnPodsSoFar
   709  		}
   710  		if passesVerify {
   711  			filteredPods = append(filteredPods, pod)
   712  		}
   713  	}
   714  	return filteredPods, err
   715  }
   716  
   717  // WaitFor waits for some minimum number of pods to be verified, according to the PodStateVerification
   718  // definition.
   719  func (cl *ClusterVerification) WaitFor(ctx context.Context, atLeast int, timeout time.Duration) ([]v1.Pod, error) {
   720  	pods := []v1.Pod{}
   721  	var returnedErr error
   722  
   723  	err := wait.PollWithContext(ctx, 1*time.Second, timeout, func(ctx context.Context) (bool, error) {
   724  		pods, returnedErr = cl.podState.filter(ctx, cl.client, cl.namespace)
   725  
   726  		// Failure
   727  		if returnedErr != nil {
   728  			Logf("Cutting polling short: We got an error from the pod filtering layer.")
   729  			// stop polling if the pod filtering returns an error.  that should never happen.
   730  			// it indicates, for example, that the client is broken or something non-pod related.
   731  			return false, returnedErr
   732  		}
   733  		Logf("Found %v / %v", len(pods), atLeast)
   734  
   735  		// Success
   736  		if len(pods) >= atLeast {
   737  			return true, nil
   738  		}
   739  		// Keep trying...
   740  		return false, nil
   741  	})
   742  	Logf("WaitFor completed with timeout %v.  Pods found = %v out of %v", timeout, len(pods), atLeast)
   743  	return pods, err
   744  }
   745  
   746  // WaitForOrFail provides a shorthand WaitFor with failure as an option if anything goes wrong.
   747  func (cl *ClusterVerification) WaitForOrFail(ctx context.Context, atLeast int, timeout time.Duration) {
   748  	pods, err := cl.WaitFor(ctx, atLeast, timeout)
   749  	if err != nil || len(pods) < atLeast {
   750  		Failf("Verified %v of %v pods , error : %v", len(pods), atLeast, err)
   751  	}
   752  }
   753  
   754  // ForEach runs a function against every verifiable pod.  Be warned that this doesn't wait for "n" pods to verify,
   755  // so it may return very quickly if you have strict pod state requirements.
   756  //
   757  // For example, if you require at least 5 pods to be running before your test will pass,
   758  // its smart to first call "clusterVerification.WaitFor(5)" before you call clusterVerification.ForEach.
   759  func (cl *ClusterVerification) ForEach(ctx context.Context, podFunc func(v1.Pod)) error {
   760  	pods, err := cl.podState.filter(ctx, cl.client, cl.namespace)
   761  	if err == nil {
   762  		if len(pods) == 0 {
   763  			Failf("No pods matched the filter.")
   764  		}
   765  		Logf("ForEach: Found %v pods from the filter.  Now looping through them.", len(pods))
   766  		for _, p := range pods {
   767  			podFunc(p)
   768  		}
   769  	} else {
   770  		Logf("ForEach: Something went wrong when filtering pods to execute against: %v", err)
   771  	}
   772  
   773  	return err
   774  }
   775
View as plain text