// Package envtest helps to set up various pieces of controller-runtime's // envtest library to simplify writing K8s controller tests package envtest import ( "errors" "flag" "fmt" "os" "path/filepath" "time" "github.com/bazelbuild/rules_go/go/runfiles" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "sigs.k8s.io/controller-runtime/pkg/envtest" "edge-infra.dev/pkg/lib/build/bazel" "edge-infra.dev/test/f2/integration" "edge-infra.dev/test/fixtures" ) // Environment is an alias for envtest.Environment type Environment = envtest.Environment var ( ControlPlaneStartTimeout time.Duration ControlPlaneStopTimeout time.Duration AttachControlPlaneOutput bool ) func BindFlags(fs *flag.FlagSet) { fs.DurationVar(&ControlPlaneStartTimeout, "envtest-start-timeout", time.Second*60, "timeout for envtest control plane to start up", ) fs.DurationVar(&ControlPlaneStopTimeout, "envtest-stop-timeout", time.Second*60, "timeout for envtest control plane to shut down", ) fs.BoolVar(&AttachControlPlaneOutput, "envtest-attach-control-plane-output", false, "whether or not to attach kube-apiserver + etcd output to test output", ) } // SetupEnvtestTools attempts to configure environment variables based on // where envtest binary dependencies (etcd, kube-apiserver, kubectl) will be // when they are provided by Bazel by default. It respects existing values set // for those binaries, see: https://book.kubebuilder.io/reference/envtest.html#environment-variables func SetupEnvtestTools() error { return errors.Join( maybeSetEnv("TEST_ASSET_ETCD", "etcd", "hack/tools/etcd"), maybeSetEnv("TEST_ASSET_KUBE_APISERVER", "kube-apiserver", "hack/tools/kube-apiserver"), maybeSetEnv("TEST_ASSET_KUBECTL", "kubectl", "hack/tools/kubectl"), ) } func maybeSetEnv(key, bin, runfilePath string) error { // If the specific key has been set or the generic path for kubebuilder test // assets has been set, don't try to update the path based on the Bazel test // sandbox if os.Getenv(key) != "" || os.Getenv("KUBEBUILDER_ASSETS") != "" { return nil } if !bazel.IsBazelTest() { return fmt.Errorf(`failed to find integration test dependency %q. Either re-run this test using "bazel test" or set the %s environment variable`, bin, key) } p, err := runfiles.Rlocation(filepath.Join(os.Getenv(bazel.TestWorkspace), runfilePath)) if err != nil { return fmt.Errorf("failed to look up test dependency %q: %w. ensure that "+ "it is present in this test targets 'data' attribute", bin, err) } os.Setenv(key, p) return nil } // Option defines publicly visible options for instantiating envtest type Option func(*envtestOpts) type envtestOpts struct { loadCRDs bool } // WithoutCRDs will skip CRD loading during envtest setup. Useful if you need // to apply CRDs during your test func WithoutCRDs() Option { return func(o *envtestOpts) { o.loadCRDs = false } } // New creates a envtest.Environment that is: // - safe to be used when multiple controller test suites are ran in parallel // - has all of our CRD fixtures loaded // - honors our test flags for attaching control plane output and the control // plane timeouts func New(opts ...Option) (*envtest.Environment, error) { o := envtestOpts{loadCRDs: true} for _, opt := range opts { opt(&o) } crds := []*v1.CustomResourceDefinition{} if o.loadCRDs { var err error crds, err = loadCRDs() if err != nil { return nil, err } } n := false env := &envtest.Environment{ CRDs: crds, ControlPlaneStartTimeout: ControlPlaneStartTimeout, ControlPlaneStopTimeout: ControlPlaneStopTimeout, AttachControlPlaneOutput: AttachControlPlaneOutput, UseExistingCluster: &n, } if integration.IsL2() { y := true env.UseExistingCluster = &y } return env, nil } // wrapper around fixtures.LoadCRDs to only load Edge CRDs if the current test // is an integration test. this is done because we do version skew testing of // all third party components, and the specific version we are installing may // contain CRDs which conflict with the vendored test fixtures generated from // the version in our go.mod func loadCRDs() ([]*v1.CustomResourceDefinition, error) { if integration.IsL2() { return fixtures.LoadCRDs(fixtures.Only("edge")) } return fixtures.LoadCRDs() } // Setup combines `New()`, `SetupEnvtestTools()` and starts the created // envtest.Environment, can be used to reduce test setup boilerplate. func Setup(opts ...Option) (*envtest.Environment, error) { // TODO: this could potentially be avoided in integration scenarios, but not sure // how much (if any) envtest relies on kubectl. kube-apiserver and etcd (the // other envtest tools) are not needed if err := SetupEnvtestTools(); err != nil { return nil, err } // create and start environment testEnv, err := New(opts...) if err != nil { return nil, fmt.Errorf("failed to create envtest Environment: %w", err) } _, err = testEnv.Start() if err != nil { return nil, fmt.Errorf("failed to start envtest Environment: %w", err) } // specify a request timeout in order to avoid hanging up the K8s API server // on shutdown:https://github.com/kubernetes-sigs/controller-runtime/issues/1571#issuecomment-945535598 testEnv.Config.Timeout = 20 * time.Second return testEnv, nil }