envtest.go

Documentation: edge-infra.dev/test/f2/x/ktest/envtest

     1  // Package envtest helps to set up various pieces of controller-runtime's
     2  // envtest library to simplify writing K8s controller tests
     3  package envtest
     4  
     5  import (
     6  	"errors"
     7  	"flag"
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  	"time"
    12  
    13  	"github.com/bazelbuild/rules_go/go/runfiles"
    14  	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    15  	"sigs.k8s.io/controller-runtime/pkg/envtest"
    16  
    17  	"edge-infra.dev/pkg/lib/build/bazel"
    18  	"edge-infra.dev/test/f2/integration"
    19  	"edge-infra.dev/test/fixtures"
    20  )
    21  
    22  // Environment is an alias for envtest.Environment
    23  type Environment = envtest.Environment
    24  
    25  var (
    26  	ControlPlaneStartTimeout time.Duration
    27  	ControlPlaneStopTimeout  time.Duration
    28  	AttachControlPlaneOutput bool
    29  )
    30  
    31  func BindFlags(fs *flag.FlagSet) {
    32  	fs.DurationVar(&ControlPlaneStartTimeout,
    33  		"envtest-start-timeout",
    34  		time.Second*60,
    35  		"timeout for envtest control plane to start up",
    36  	)
    37  	fs.DurationVar(&ControlPlaneStopTimeout,
    38  		"envtest-stop-timeout",
    39  		time.Second*60,
    40  		"timeout for envtest control plane to shut down",
    41  	)
    42  	fs.BoolVar(&AttachControlPlaneOutput,
    43  		"envtest-attach-control-plane-output",
    44  		false,
    45  		"whether or not to attach kube-apiserver + etcd output to test output",
    46  	)
    47  }
    48  
    49  // SetupEnvtestTools attempts to configure environment variables based on
    50  // where envtest binary dependencies (etcd, kube-apiserver, kubectl) will be
    51  // when they are provided by Bazel by default. It respects existing values set
    52  // for those binaries, see: https://book.kubebuilder.io/reference/envtest.html#environment-variables
    53  func SetupEnvtestTools() error {
    54  	return errors.Join(
    55  		maybeSetEnv("TEST_ASSET_ETCD", "etcd", "hack/tools/etcd"),
    56  		maybeSetEnv("TEST_ASSET_KUBE_APISERVER", "kube-apiserver", "hack/tools/kube-apiserver"),
    57  		maybeSetEnv("TEST_ASSET_KUBECTL", "kubectl", "hack/tools/kubectl"),
    58  	)
    59  }
    60  
    61  func maybeSetEnv(key, bin, runfilePath string) error {
    62  	// If the specific key has been set or the generic path for kubebuilder test
    63  	// assets has been set, don't try to update the path based on the Bazel test
    64  	// sandbox
    65  	if os.Getenv(key) != "" || os.Getenv("KUBEBUILDER_ASSETS") != "" {
    66  		return nil
    67  	}
    68  	if !bazel.IsBazelTest() {
    69  		return fmt.Errorf(`failed to find integration test dependency %q.
    70  Either re-run this test using "bazel test" or set the %s environment variable`, bin, key)
    71  	}
    72  	p, err := runfiles.Rlocation(filepath.Join(os.Getenv(bazel.TestWorkspace), runfilePath))
    73  	if err != nil {
    74  		return fmt.Errorf("failed to look up test dependency %q: %w. ensure that "+
    75  			"it is present in this test targets 'data' attribute", bin, err)
    76  	}
    77  	os.Setenv(key, p)
    78  	return nil
    79  }
    80  
    81  // Option defines publicly visible options for instantiating envtest
    82  type Option func(*envtestOpts)
    83  
    84  type envtestOpts struct {
    85  	loadCRDs bool
    86  }
    87  
    88  // WithoutCRDs will skip CRD loading during envtest setup. Useful if you need
    89  // to apply CRDs during your test
    90  func WithoutCRDs() Option {
    91  	return func(o *envtestOpts) {
    92  		o.loadCRDs = false
    93  	}
    94  }
    95  
    96  // New creates a envtest.Environment that is:
    97  //   - safe to be used when multiple controller test suites are ran in parallel
    98  //   - has all of our CRD fixtures loaded
    99  //   - honors our test flags for attaching control plane output and the control
   100  //     plane timeouts
   101  func New(opts ...Option) (*envtest.Environment, error) {
   102  	o := envtestOpts{loadCRDs: true}
   103  	for _, opt := range opts {
   104  		opt(&o)
   105  	}
   106  
   107  	crds := []*v1.CustomResourceDefinition{}
   108  
   109  	if o.loadCRDs {
   110  		var err error
   111  		crds, err = loadCRDs()
   112  		if err != nil {
   113  			return nil, err
   114  		}
   115  	}
   116  
   117  	n := false
   118  	env := &envtest.Environment{
   119  		CRDs:                     crds,
   120  		ControlPlaneStartTimeout: ControlPlaneStartTimeout,
   121  		ControlPlaneStopTimeout:  ControlPlaneStopTimeout,
   122  		AttachControlPlaneOutput: AttachControlPlaneOutput,
   123  		UseExistingCluster:       &n,
   124  	}
   125  
   126  	if integration.IsL2() {
   127  		y := true
   128  		env.UseExistingCluster = &y
   129  	}
   130  
   131  	return env, nil
   132  }
   133  
   134  // wrapper around fixtures.LoadCRDs to only load Edge CRDs if the current test
   135  // is an integration test.  this is done because we do version skew testing of
   136  // all third party components, and the specific version we are installing may
   137  // contain CRDs which conflict with the vendored test fixtures generated from
   138  // the version in our go.mod
   139  func loadCRDs() ([]*v1.CustomResourceDefinition, error) {
   140  	if integration.IsL2() {
   141  		return fixtures.LoadCRDs(fixtures.Only("edge"))
   142  	}
   143  	return fixtures.LoadCRDs()
   144  }
   145  
   146  // Setup combines `New()`, `SetupEnvtestTools()` and starts the created
   147  // envtest.Environment, can be used to reduce test setup boilerplate.
   148  func Setup(opts ...Option) (*envtest.Environment, error) {
   149  	// TODO: this could potentially be avoided in integration scenarios, but not sure
   150  	// how much (if any) envtest relies on kubectl.  kube-apiserver and etcd (the
   151  	// other envtest tools) are not needed
   152  	if err := SetupEnvtestTools(); err != nil {
   153  		return nil, err
   154  	}
   155  
   156  	// create and start environment
   157  	testEnv, err := New(opts...)
   158  	if err != nil {
   159  		return nil, fmt.Errorf("failed to create envtest Environment: %w", err)
   160  	}
   161  
   162  	_, err = testEnv.Start()
   163  	if err != nil {
   164  		return nil, fmt.Errorf("failed to start envtest Environment: %w", err)
   165  	}
   166  
   167  	// specify a request timeout in order to avoid hanging up the K8s API server
   168  	// on shutdown:https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598
   169  	testEnv.Config.Timeout = 20 * time.Second
   170  
   171  	return testEnv, nil
   172  }
   173
View as plain text