envtest.go

Documentation: edge-infra.dev/test/framework/k8s/envtest

     1  // Package envtest helps to set up various pieces of
     2  // sigs.k8s.io/controller-runtime/pkg/envtest framework to simplify
     3  // writing K8s controller tests
     4  package envtest
     5  
     6  import (
     7  	"fmt"
     8  	"os"
     9  	"os/exec"
    10  	"path"
    11  	"time"
    12  
    13  	"github.com/bazelbuild/rules_go/go/runfiles"
    14  	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    15  	"sigs.k8s.io/controller-runtime/pkg/envtest"
    16  
    17  	"edge-infra.dev/test/fixtures"
    18  	"edge-infra.dev/test/framework/config"
    19  	"edge-infra.dev/test/framework/integration"
    20  )
    21  
    22  // Environment is an alias for envtest.Environment
    23  type Environment = envtest.Environment
    24  
    25  var cfg struct {
    26  	ControlPlaneStartTimeout time.Duration `default:"60s" description:"timeout for envtest control plane to start up"`
    27  	ControlPlaneStopTimeout  time.Duration `default:"60s" description:"timeout for envtest control plane to stop"`
    28  	AttachControlPlaneOutput bool          `default:"false" description:"whether or not to attach kube-apiserver + etcd output to test output"`
    29  }
    30  
    31  func init() {
    32  	_ = config.AddOptions(&cfg, "envtest")
    33  }
    34  
    35  // SetupEnvtestTools attempts to configure environment variables based on
    36  // where envtest binary dependencies (etcd, kube-apiserver, kubectl) will be
    37  // when they are provided by Bazel, by default.  It respects existing values set
    38  // for those binaries, see: https://book.kubebuilder.io/reference/envtest.html#environment-variables
    39  func SetupEnvtestTools() {
    40  	maybeSetEnv("TEST_ASSET_ETCD", "etcd", "edge_infra", "hack", "tools", "etcd")
    41  	maybeSetEnv("TEST_ASSET_KUBE_APISERVER", "kube-apiserver", "edge_infra", "hack", "tools", "kube-apiserver")
    42  	maybeSetEnv("TEST_ASSET_KUBECTL", "kubectl", "edge_infra", "hack", "tools", "kubectl")
    43  }
    44  
    45  func maybeSetEnv(key, bin string, partials ...string) {
    46  	if os.Getenv(key) != "" {
    47  		return
    48  	}
    49  	p, err := getPath(bin, partials...)
    50  	if err != nil {
    51  		panic(fmt.Sprintf(`Failed to find integration test dependency %q.
    52  Either re-run this test using "bazel test" or set the %s environment variable.`, bin, key))
    53  	}
    54  	os.Setenv(key, p)
    55  }
    56  
    57  func getPath(name string, partials ...string) (string, error) {
    58  	bazelPath, err := runfiles.Rlocation(path.Join(partials...))
    59  	if err != nil {
    60  		return "", err
    61  	}
    62  	p, err := exec.LookPath(bazelPath)
    63  	if err == nil {
    64  		return p, nil
    65  	}
    66  
    67  	return exec.LookPath(name)
    68  }
    69  
    70  // Option defines publicly visible options for instantiating envtest
    71  type Option func(*envtestOpts)
    72  
    73  type envtestOpts struct {
    74  	loadCRDs bool
    75  }
    76  
    77  // WithoutCRDs will skip CRD loading during envtest setup.  Useful if you need
    78  // to apply CRDs during your test
    79  func WithoutCRDs() Option {
    80  	return func(o *envtestOpts) {
    81  		o.loadCRDs = false
    82  	}
    83  }
    84  
    85  // New creates a envtest.Environment that is:
    86  //   - safe to be used when multiple controller test suites are ran in parallel
    87  //   - has all of our CRD fixtures loaded
    88  //   - honors our test flags for attaching control plane output and the control
    89  //     plane timeouts
    90  func New(opts ...Option) (*envtest.Environment, error) {
    91  	o := envtestOpts{loadCRDs: true}
    92  	for _, opt := range opts {
    93  		opt(&o)
    94  	}
    95  
    96  	crds := []*v1.CustomResourceDefinition{}
    97  
    98  	if o.loadCRDs {
    99  		var err error
   100  		crds, err = loadCRDs()
   101  		if err != nil {
   102  			return nil, err
   103  		}
   104  	}
   105  
   106  	env := &envtest.Environment{
   107  		CRDs:                     crds,
   108  		ControlPlaneStartTimeout: cfg.ControlPlaneStartTimeout,
   109  		ControlPlaneStopTimeout:  cfg.ControlPlaneStopTimeout,
   110  		AttachControlPlaneOutput: cfg.AttachControlPlaneOutput,
   111  	}
   112  
   113  	if integration.IsIntegrationTest() {
   114  		y := true
   115  		env.UseExistingCluster = &y
   116  	}
   117  
   118  	return env, nil
   119  }
   120  
   121  // wrapper around fixtures.LoadCRDs to only load Edge CRDs if the current test
   122  // is an integration test.  this is done because we do version skew testing of
   123  // all third party components, and the specific version we are installing may
   124  // contain CRDs which conflict with the vendored test fixtures generated from
   125  // the version in our go.mod
   126  func loadCRDs() ([]*v1.CustomResourceDefinition, error) {
   127  	if integration.IsIntegrationTest() {
   128  		return fixtures.LoadCRDs(fixtures.Only("edge"))
   129  	}
   130  	return fixtures.LoadCRDs()
   131  }
   132  
   133  // Setup combines `New()`, `SetupEnvtestTools()` and starts the created
   134  // envtest.Environment, can be used to reduce test setup boilerplate.
   135  func Setup(opts ...Option) *envtest.Environment {
   136  	// TODO: this could potentially be avoided in integration scenarios, but not sure
   137  	// how much (if any) envtest relies on kubectl.  kube-apiserver and etcd (the
   138  	// other envtest tools) are not needed
   139  	SetupEnvtestTools()
   140  
   141  	// create and start environment
   142  	testEnv, err := New(opts...)
   143  	if err != nil {
   144  		panic("failed to create envtest Environment: " + err.Error())
   145  	}
   146  
   147  	_, err = testEnv.Start()
   148  	if err != nil {
   149  		panic("failed to start envtest Environment: " + err.Error())
   150  	}
   151  
   152  	// specify a request timeout in order to avoid hanging up the K8s API server
   153  	// on shutdown:https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598
   154  	testEnv.Config.Timeout = 20 * time.Second
   155  
   156  	return testEnv
   157  }
   158
View as plain text