// Package envtest helps to set up various pieces of // sigs.k8s.io/controller-runtime/pkg/envtest framework to simplify // writing K8s controller tests package envtest import ( "fmt" "os" "os/exec" "path" "time" "github.com/bazelbuild/rules_go/go/runfiles" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "sigs.k8s.io/controller-runtime/pkg/envtest" "edge-infra.dev/test/fixtures" "edge-infra.dev/test/framework/config" "edge-infra.dev/test/framework/integration" ) // Environment is an alias for envtest.Environment type Environment = envtest.Environment var cfg struct { ControlPlaneStartTimeout time.Duration `default:"60s" description:"timeout for envtest control plane to start up"` ControlPlaneStopTimeout time.Duration `default:"60s" description:"timeout for envtest control plane to stop"` AttachControlPlaneOutput bool `default:"false" description:"whether or not to attach kube-apiserver + etcd output to test output"` } func init() { _ = config.AddOptions(&cfg, "envtest") } // SetupEnvtestTools attempts to configure environment variables based on // where envtest binary dependencies (etcd, kube-apiserver, kubectl) will be // when they are provided by Bazel, by default. It respects existing values set // for those binaries, see: https://book.kubebuilder.io/reference/envtest.html#environment-variables func SetupEnvtestTools() { maybeSetEnv("TEST_ASSET_ETCD", "etcd", "edge_infra", "hack", "tools", "etcd") maybeSetEnv("TEST_ASSET_KUBE_APISERVER", "kube-apiserver", "edge_infra", "hack", "tools", "kube-apiserver") maybeSetEnv("TEST_ASSET_KUBECTL", "kubectl", "edge_infra", "hack", "tools", "kubectl") } func maybeSetEnv(key, bin string, partials ...string) { if os.Getenv(key) != "" { return } p, err := getPath(bin, partials...) if err != nil { panic(fmt.Sprintf(`Failed to find integration test dependency %q. Either re-run this test using "bazel test" or set the %s environment variable.`, bin, key)) } os.Setenv(key, p) } func getPath(name string, partials ...string) (string, error) { bazelPath, err := runfiles.Rlocation(path.Join(partials...)) if err != nil { return "", err } p, err := exec.LookPath(bazelPath) if err == nil { return p, nil } return exec.LookPath(name) } // Option defines publicly visible options for instantiating envtest type Option func(*envtestOpts) type envtestOpts struct { loadCRDs bool } // WithoutCRDs will skip CRD loading during envtest setup. Useful if you need // to apply CRDs during your test func WithoutCRDs() Option { return func(o *envtestOpts) { o.loadCRDs = false } } // New creates a envtest.Environment that is: // - safe to be used when multiple controller test suites are ran in parallel // - has all of our CRD fixtures loaded // - honors our test flags for attaching control plane output and the control // plane timeouts func New(opts ...Option) (*envtest.Environment, error) { o := envtestOpts{loadCRDs: true} for _, opt := range opts { opt(&o) } crds := []*v1.CustomResourceDefinition{} if o.loadCRDs { var err error crds, err = loadCRDs() if err != nil { return nil, err } } env := &envtest.Environment{ CRDs: crds, ControlPlaneStartTimeout: cfg.ControlPlaneStartTimeout, ControlPlaneStopTimeout: cfg.ControlPlaneStopTimeout, AttachControlPlaneOutput: cfg.AttachControlPlaneOutput, } if integration.IsIntegrationTest() { y := true env.UseExistingCluster = &y } return env, nil } // wrapper around fixtures.LoadCRDs to only load Edge CRDs if the current test // is an integration test. this is done because we do version skew testing of // all third party components, and the specific version we are installing may // contain CRDs which conflict with the vendored test fixtures generated from // the version in our go.mod func loadCRDs() ([]*v1.CustomResourceDefinition, error) { if integration.IsIntegrationTest() { return fixtures.LoadCRDs(fixtures.Only("edge")) } return fixtures.LoadCRDs() } // Setup combines `New()`, `SetupEnvtestTools()` and starts the created // envtest.Environment, can be used to reduce test setup boilerplate. func Setup(opts ...Option) *envtest.Environment { // TODO: this could potentially be avoided in integration scenarios, but not sure // how much (if any) envtest relies on kubectl. kube-apiserver and etcd (the // other envtest tools) are not needed SetupEnvtestTools() // create and start environment testEnv, err := New(opts...) if err != nil { panic("failed to create envtest Environment: " + err.Error()) } _, err = testEnv.Start() if err != nil { panic("failed to start envtest Environment: " + err.Error()) } // specify a request timeout in order to avoid hanging up the K8s API server // on shutdown:https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598 testEnv.Config.Timeout = 20 * time.Second return testEnv }