server.go

Documentation: sigs.k8s.io/controller-runtime/pkg/envtest

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package envtest
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"strings"
    24  	"time"
    25  
    26  	corev1 "k8s.io/api/core/v1"
    27  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    28  	"k8s.io/apimachinery/pkg/runtime"
    29  	"k8s.io/apimachinery/pkg/types"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	"k8s.io/client-go/kubernetes/scheme"
    32  	"k8s.io/client-go/rest"
    33  	"sigs.k8s.io/controller-runtime/pkg/client"
    34  
    35  	"sigs.k8s.io/controller-runtime/pkg/client/config"
    36  	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
    37  	"sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane"
    38  	"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
    39  )
    40  
    41  var log = logf.RuntimeLog.WithName("test-env")
    42  
    43  /*
    44  It's possible to override some defaults, by setting the following environment variables:
    45  * USE_EXISTING_CLUSTER (boolean): if set to true, envtest will use an existing cluster
    46  * TEST_ASSET_KUBE_APISERVER (string): path to the api-server binary to use
    47  * TEST_ASSET_ETCD (string): path to the etcd binary to use
    48  * TEST_ASSET_KUBECTL (string): path to the kubectl binary to use
    49  * KUBEBUILDER_ASSETS (string): directory containing the binaries to use (api-server, etcd and kubectl). Defaults to /usr/local/kubebuilder/bin.
    50  * KUBEBUILDER_CONTROLPLANE_START_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s.
    51  * KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s.
    52  * KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT (boolean): if set to true, the control plane's stdout and stderr are attached to os.Stdout and os.Stderr
    53  */
    54  const (
    55  	envUseExistingCluster = "USE_EXISTING_CLUSTER"
    56  	envStartTimeout       = "KUBEBUILDER_CONTROLPLANE_START_TIMEOUT"
    57  	envStopTimeout        = "KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT"
    58  	envAttachOutput       = "KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT"
    59  	StartTimeout          = 60
    60  	StopTimeout           = 60
    61  
    62  	defaultKubebuilderControlPlaneStartTimeout = 20 * time.Second
    63  	defaultKubebuilderControlPlaneStopTimeout  = 20 * time.Second
    64  )
    65  
    66  // internal types we expose as part of our public API.
    67  type (
    68  	// ControlPlane is the re-exported ControlPlane type from the internal testing package.
    69  	ControlPlane = controlplane.ControlPlane
    70  
    71  	// APIServer is the re-exported APIServer from the internal testing package.
    72  	APIServer = controlplane.APIServer
    73  
    74  	// Etcd is the re-exported Etcd from the internal testing package.
    75  	Etcd = controlplane.Etcd
    76  
    77  	// User represents a Kubernetes user to provision for auth purposes.
    78  	User = controlplane.User
    79  
    80  	// AuthenticatedUser represets a Kubernetes user that's been provisioned.
    81  	AuthenticatedUser = controlplane.AuthenticatedUser
    82  
    83  	// ListenAddr indicates the address and port that the API server should listen on.
    84  	ListenAddr = process.ListenAddr
    85  
    86  	// SecureServing contains details describing how the API server should serve
    87  	// its secure endpoint.
    88  	SecureServing = controlplane.SecureServing
    89  
    90  	// Authn is an authentication method that can be used with the control plane to
    91  	// provision users.
    92  	Authn = controlplane.Authn
    93  
    94  	// Arguments allows configuring a process's flags.
    95  	Arguments = process.Arguments
    96  
    97  	// Arg is a single flag with one or more values.
    98  	Arg = process.Arg
    99  )
   100  
   101  var (
   102  	// EmptyArguments constructs a new set of flags with nothing set.
   103  	//
   104  	// This is mostly useful for testing helper methods -- you'll want to call
   105  	// Configure on the APIServer (or etcd) to configure their arguments.
   106  	EmptyArguments = process.EmptyArguments
   107  )
   108  
   109  // Environment creates a Kubernetes test environment that will start / stop the Kubernetes control plane and
   110  // install extension APIs.
   111  type Environment struct {
   112  	// ControlPlane is the ControlPlane including the apiserver and etcd
   113  	ControlPlane controlplane.ControlPlane
   114  
   115  	// Scheme is used to determine if conversion webhooks should be enabled
   116  	// for a particular CRD / object.
   117  	//
   118  	// Conversion webhooks are going to be enabled if an object in the scheme
   119  	// implements Hub and Spoke conversions.
   120  	//
   121  	// If nil, scheme.Scheme is used.
   122  	Scheme *runtime.Scheme
   123  
   124  	// Config can be used to talk to the apiserver.  It's automatically
   125  	// populated if not set using the standard controller-runtime config
   126  	// loading.
   127  	Config *rest.Config
   128  
   129  	// CRDInstallOptions are the options for installing CRDs.
   130  	CRDInstallOptions CRDInstallOptions
   131  
   132  	// WebhookInstallOptions are the options for installing webhooks.
   133  	WebhookInstallOptions WebhookInstallOptions
   134  
   135  	// ErrorIfCRDPathMissing provides an interface for the underlying
   136  	// CRDInstallOptions.ErrorIfPathMissing. It prevents silent failures
   137  	// for missing CRD paths.
   138  	ErrorIfCRDPathMissing bool
   139  
   140  	// CRDs is a list of CRDs to install.
   141  	// If both this field and CRDs field in CRDInstallOptions are specified, the
   142  	// values are merged.
   143  	CRDs []*apiextensionsv1.CustomResourceDefinition
   144  
   145  	// CRDDirectoryPaths is a list of paths containing CRD yaml or json configs.
   146  	// If both this field and Paths field in CRDInstallOptions are specified, the
   147  	// values are merged.
   148  	CRDDirectoryPaths []string
   149  
   150  	// BinaryAssetsDirectory is the path where the binaries required for the envtest are
   151  	// located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS.
   152  	BinaryAssetsDirectory string
   153  
   154  	// UseExistingCluster indicates that this environments should use an
   155  	// existing kubeconfig, instead of trying to stand up a new control plane.
   156  	// This is useful in cases that need aggregated API servers and the like.
   157  	UseExistingCluster *bool
   158  
   159  	// ControlPlaneStartTimeout is the maximum duration each controlplane component
   160  	// may take to start. It defaults to the KUBEBUILDER_CONTROLPLANE_START_TIMEOUT
   161  	// environment variable or 20 seconds if unspecified
   162  	ControlPlaneStartTimeout time.Duration
   163  
   164  	// ControlPlaneStopTimeout is the maximum duration each controlplane component
   165  	// may take to stop. It defaults to the KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT
   166  	// environment variable or 20 seconds if unspecified
   167  	ControlPlaneStopTimeout time.Duration
   168  
   169  	// AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr.
   170  	// Enable this to get more visibility of the testing control plane.
   171  	// It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable.
   172  	AttachControlPlaneOutput bool
   173  }
   174  
   175  // Stop stops a running server.
   176  // Previously installed CRDs, as listed in CRDInstallOptions.CRDs, will be uninstalled
   177  // if CRDInstallOptions.CleanUpAfterUse are set to true.
   178  func (te *Environment) Stop() error {
   179  	if te.CRDInstallOptions.CleanUpAfterUse {
   180  		if err := UninstallCRDs(te.Config, te.CRDInstallOptions); err != nil {
   181  			return err
   182  		}
   183  	}
   184  
   185  	if err := te.WebhookInstallOptions.Cleanup(); err != nil {
   186  		return err
   187  	}
   188  
   189  	if te.useExistingCluster() {
   190  		return nil
   191  	}
   192  
   193  	return te.ControlPlane.Stop()
   194  }
   195  
   196  // Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on.
   197  func (te *Environment) Start() (*rest.Config, error) {
   198  	if te.useExistingCluster() {
   199  		log.V(1).Info("using existing cluster")
   200  		if te.Config == nil {
   201  			// we want to allow people to pass in their own config, so
   202  			// only load a config if it hasn't already been set.
   203  			log.V(1).Info("automatically acquiring client configuration")
   204  
   205  			var err error
   206  			te.Config, err = config.GetConfig()
   207  			if err != nil {
   208  				return nil, fmt.Errorf("unable to get configuration for existing cluster: %w", err)
   209  			}
   210  		}
   211  	} else {
   212  		apiServer := te.ControlPlane.GetAPIServer()
   213  
   214  		if te.ControlPlane.Etcd == nil {
   215  			te.ControlPlane.Etcd = &controlplane.Etcd{}
   216  		}
   217  
   218  		if os.Getenv(envAttachOutput) == "true" {
   219  			te.AttachControlPlaneOutput = true
   220  		}
   221  		if te.AttachControlPlaneOutput {
   222  			if apiServer.Out == nil {
   223  				apiServer.Out = os.Stdout
   224  			}
   225  			if apiServer.Err == nil {
   226  				apiServer.Err = os.Stderr
   227  			}
   228  			if te.ControlPlane.Etcd.Out == nil {
   229  				te.ControlPlane.Etcd.Out = os.Stdout
   230  			}
   231  			if te.ControlPlane.Etcd.Err == nil {
   232  				te.ControlPlane.Etcd.Err = os.Stderr
   233  			}
   234  		}
   235  
   236  		apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
   237  		te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory)
   238  		te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory)
   239  
   240  		if err := te.defaultTimeouts(); err != nil {
   241  			return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err)
   242  		}
   243  		te.ControlPlane.Etcd.StartTimeout = te.ControlPlaneStartTimeout
   244  		te.ControlPlane.Etcd.StopTimeout = te.ControlPlaneStopTimeout
   245  		apiServer.StartTimeout = te.ControlPlaneStartTimeout
   246  		apiServer.StopTimeout = te.ControlPlaneStopTimeout
   247  
   248  		log.V(1).Info("starting control plane")
   249  		if err := te.startControlPlane(); err != nil {
   250  			return nil, fmt.Errorf("unable to start control plane itself: %w", err)
   251  		}
   252  
   253  		// Create the *rest.Config for creating new clients
   254  		baseConfig := &rest.Config{
   255  			// gotta go fast during tests -- we don't really care about overwhelming our test API server
   256  			QPS:   1000.0,
   257  			Burst: 2000.0,
   258  		}
   259  
   260  		adminInfo := User{Name: "admin", Groups: []string{"system:masters"}}
   261  		adminUser, err := te.ControlPlane.AddUser(adminInfo, baseConfig)
   262  		if err != nil {
   263  			return te.Config, fmt.Errorf("unable to provision admin user: %w", err)
   264  		}
   265  		te.Config = adminUser.Config()
   266  	}
   267  
   268  	// Set the default scheme if nil.
   269  	if te.Scheme == nil {
   270  		te.Scheme = scheme.Scheme
   271  	}
   272  
   273  	// If we are bringing etcd up for the first time, it can take some time for the
   274  	// default namespace to actually be created and seen as available to the apiserver
   275  	if err := te.waitForDefaultNamespace(te.Config); err != nil {
   276  		return nil, fmt.Errorf("default namespace didn't register within deadline: %w", err)
   277  	}
   278  
   279  	// Call PrepWithoutInstalling to setup certificates first
   280  	// and have them available to patch CRD conversion webhook as well.
   281  	if err := te.WebhookInstallOptions.PrepWithoutInstalling(); err != nil {
   282  		return nil, err
   283  	}
   284  
   285  	log.V(1).Info("installing CRDs")
   286  	if te.CRDInstallOptions.Scheme == nil {
   287  		te.CRDInstallOptions.Scheme = te.Scheme
   288  	}
   289  	te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs)
   290  	te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths)
   291  	te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing
   292  	te.CRDInstallOptions.WebhookOptions = te.WebhookInstallOptions
   293  	crds, err := InstallCRDs(te.Config, te.CRDInstallOptions)
   294  	if err != nil {
   295  		return te.Config, fmt.Errorf("unable to install CRDs onto control plane: %w", err)
   296  	}
   297  	te.CRDs = crds
   298  
   299  	log.V(1).Info("installing webhooks")
   300  	if err := te.WebhookInstallOptions.Install(te.Config); err != nil {
   301  		return nil, fmt.Errorf("unable to install webhooks onto control plane: %w", err)
   302  	}
   303  	return te.Config, nil
   304  }
   305  
   306  // AddUser provisions a new user for connecting to this Environment.  The user will
   307  // have the specified name & belong to the specified groups.
   308  //
   309  // If you specify a "base" config, the returned REST Config will contain those
   310  // settings as well as any required by the authentication method.  You can use
   311  // this to easily specify options like QPS.
   312  //
   313  // This is effectively a convinience alias for ControlPlane.AddUser -- see that
   314  // for more low-level details.
   315  func (te *Environment) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) {
   316  	return te.ControlPlane.AddUser(user, baseConfig)
   317  }
   318  
   319  func (te *Environment) startControlPlane() error {
   320  	numTries, maxRetries := 0, 5
   321  	var err error
   322  	for ; numTries < maxRetries; numTries++ {
   323  		// Start the control plane - retry if it fails
   324  		err = te.ControlPlane.Start()
   325  		if err == nil {
   326  			break
   327  		}
   328  		log.Error(err, "unable to start the controlplane", "tries", numTries)
   329  	}
   330  	if numTries == maxRetries {
   331  		return fmt.Errorf("failed to start the controlplane. retried %d times: %w", numTries, err)
   332  	}
   333  	return nil
   334  }
   335  
   336  func (te *Environment) waitForDefaultNamespace(config *rest.Config) error {
   337  	cs, err := client.New(config, client.Options{})
   338  	if err != nil {
   339  		return fmt.Errorf("unable to create client: %w", err)
   340  	}
   341  	// It shouldn't take longer than 5s for the default namespace to be brought up in etcd
   342  	return wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*50, time.Second*5, true, func(ctx context.Context) (bool, error) {
   343  		if err = cs.Get(ctx, types.NamespacedName{Name: "default"}, &corev1.Namespace{}); err != nil {
   344  			return false, nil //nolint:nilerr
   345  		}
   346  		return true, nil
   347  	})
   348  }
   349  
   350  func (te *Environment) defaultTimeouts() error {
   351  	var err error
   352  	if te.ControlPlaneStartTimeout == 0 {
   353  		if envVal := os.Getenv(envStartTimeout); envVal != "" {
   354  			te.ControlPlaneStartTimeout, err = time.ParseDuration(envVal)
   355  			if err != nil {
   356  				return err
   357  			}
   358  		} else {
   359  			te.ControlPlaneStartTimeout = defaultKubebuilderControlPlaneStartTimeout
   360  		}
   361  	}
   362  
   363  	if te.ControlPlaneStopTimeout == 0 {
   364  		if envVal := os.Getenv(envStopTimeout); envVal != "" {
   365  			te.ControlPlaneStopTimeout, err = time.ParseDuration(envVal)
   366  			if err != nil {
   367  				return err
   368  			}
   369  		} else {
   370  			te.ControlPlaneStopTimeout = defaultKubebuilderControlPlaneStopTimeout
   371  		}
   372  	}
   373  	return nil
   374  }
   375  
   376  func (te *Environment) useExistingCluster() bool {
   377  	if te.UseExistingCluster == nil {
   378  		return strings.ToLower(os.Getenv(envUseExistingCluster)) == "true"
   379  	}
   380  	return *te.UseExistingCluster
   381  }
   382  
   383  // DefaultKubeAPIServerFlags exposes the default args for the APIServer so that
   384  // you can use those to append your own additional arguments.
   385  //
   386  // Deprecated: use APIServer.Configure() instead.
   387  var DefaultKubeAPIServerFlags = controlplane.APIServerDefaultArgs //nolint:staticcheck
   388
View as plain text