testserver.go

Documentation: k8s.io/apiextensions-apiserver/pkg/cmd/server/testing

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package testing
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"os"
    24  	"path/filepath"
    25  	"runtime"
    26  	"time"
    27  
    28  	"github.com/spf13/pflag"
    29  
    30  	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
    31  	"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
    32  	generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
    35  	genericapiserver "k8s.io/apiserver/pkg/server"
    36  	"k8s.io/apiserver/pkg/storage/storagebackend"
    37  	"k8s.io/apiserver/pkg/util/openapi"
    38  	"k8s.io/client-go/kubernetes"
    39  	restclient "k8s.io/client-go/rest"
    40  	logsapi "k8s.io/component-base/logs/api/v1"
    41  	"k8s.io/klog/v2"
    42  )
    43  
    44  func init() {
    45  	// If instantiated more than once or together with other servers, the
    46  	// servers would try to modify the global logging state. This must get
    47  	// ignored during testing.
    48  	logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged
    49  }
    50  
    51  // TearDownFunc is to be called to tear down a test server.
    52  type TearDownFunc func()
    53  
    54  // TestServerInstanceOptions Instance options the TestServer
    55  type TestServerInstanceOptions struct {
    56  }
    57  
    58  // TestServer return values supplied by kube-test-ApiServer
    59  type TestServer struct {
    60  	ClientConfig    *restclient.Config                              // Rest client config
    61  	ServerOpts      *options.CustomResourceDefinitionsServerOptions // ServerOpts
    62  	TearDownFn      TearDownFunc                                    // TearDown function
    63  	TmpDir          string                                          // Temp Dir used, by the apiserver
    64  	CompletedConfig extensionsapiserver.CompletedConfig
    65  }
    66  
    67  // Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
    68  type Logger interface {
    69  	Errorf(format string, args ...interface{})
    70  	Fatalf(format string, args ...interface{})
    71  	Logf(format string, args ...interface{})
    72  }
    73  
    74  // NewDefaultTestServerOptions Default options for TestServer instances
    75  func NewDefaultTestServerOptions() *TestServerInstanceOptions {
    76  	return &TestServerInstanceOptions{}
    77  }
    78  
    79  // StartTestServer starts a apiextensions-apiserver. A rest client config and a tear-down func,
    80  // and location of the tmpdir are returned.
    81  //
    82  // Note: we return a tear-down func instead of a stop channel because the later will leak temporary
    83  // files that because Golang testing's call to os.Exit will not give a stop channel go routine
    84  // enough time to remove temporary files.
    85  func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
    86  	stopCh := make(chan struct{})
    87  	var errCh chan error
    88  	tearDown := func() {
    89  		// Closing stopCh is stopping apiextensions apiserver and its
    90  		// delegates, which itself is cleaning up after itself,
    91  		// including shutting down its storage layer.
    92  		close(stopCh)
    93  
    94  		// If the apiextensions apiserver was started, let's wait for
    95  		// it to shutdown clearly.
    96  		if errCh != nil {
    97  			err, ok := <-errCh
    98  			if ok && err != nil {
    99  				klog.Errorf("Failed to shutdown test server clearly: %v", err)
   100  			}
   101  		}
   102  
   103  		if len(result.TmpDir) != 0 {
   104  			os.RemoveAll(result.TmpDir)
   105  		}
   106  	}
   107  	defer func() {
   108  		if result.TearDownFn == nil {
   109  			tearDown()
   110  		}
   111  	}()
   112  
   113  	result.TmpDir, err = os.MkdirTemp("", "apiextensions-apiserver")
   114  	if err != nil {
   115  		return result, fmt.Errorf("failed to create temp dir: %v", err)
   116  	}
   117  
   118  	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
   119  
   120  	s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
   121  	s.AddFlags(fs)
   122  
   123  	s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
   124  	if err != nil {
   125  		return result, fmt.Errorf("failed to create listener: %v", err)
   126  	}
   127  	s.RecommendedOptions.SecureServing.ServerCert.CertDirectory = result.TmpDir
   128  	s.RecommendedOptions.SecureServing.ExternalAddress = s.RecommendedOptions.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device
   129  
   130  	pkgPath, err := pkgPath(t)
   131  	if err != nil {
   132  		return result, err
   133  	}
   134  	s.RecommendedOptions.SecureServing.ServerCert.FixtureDirectory = filepath.Join(pkgPath, "testdata")
   135  
   136  	if storageConfig != nil {
   137  		s.RecommendedOptions.Etcd.StorageConfig = *storageConfig
   138  	}
   139  	s.APIEnablement.RuntimeConfig.Set("api/all=true")
   140  
   141  	fs.Parse(customFlags)
   142  
   143  	if err := s.Complete(); err != nil {
   144  		return result, fmt.Errorf("failed to set default options: %v", err)
   145  	}
   146  	if err := s.Validate(); err != nil {
   147  		return result, fmt.Errorf("failed to validate options: %v", err)
   148  	}
   149  
   150  	t.Logf("runtime-config=%v", s.APIEnablement.RuntimeConfig)
   151  	t.Logf("Starting apiextensions-apiserver on port %d...", s.RecommendedOptions.SecureServing.BindPort)
   152  
   153  	config, err := s.Config()
   154  	if err != nil {
   155  		return result, fmt.Errorf("failed to create config from options: %v", err)
   156  	}
   157  
   158  	getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions)
   159  	namer := openapinamer.NewDefinitionNamer(extensionsapiserver.Scheme)
   160  	config.GenericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitions, namer)
   161  
   162  	completedConfig := config.Complete()
   163  	server, err := completedConfig.New(genericapiserver.NewEmptyDelegate())
   164  	if err != nil {
   165  		return result, fmt.Errorf("failed to create server: %v", err)
   166  	}
   167  
   168  	errCh = make(chan error)
   169  	go func(stopCh <-chan struct{}) {
   170  		defer close(errCh)
   171  
   172  		if err := server.GenericAPIServer.PrepareRun().Run(stopCh); err != nil {
   173  			errCh <- err
   174  		}
   175  	}(stopCh)
   176  
   177  	t.Logf("Waiting for /healthz to be ok...")
   178  
   179  	client, err := kubernetes.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
   180  	if err != nil {
   181  		return result, fmt.Errorf("failed to create a client: %v", err)
   182  	}
   183  	err = wait.Poll(100*time.Millisecond, time.Minute, func() (bool, error) {
   184  		select {
   185  		case err := <-errCh:
   186  			return false, err
   187  		default:
   188  		}
   189  
   190  		result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
   191  		status := 0
   192  		result.StatusCode(&status)
   193  		if status == 200 {
   194  			return true, nil
   195  		}
   196  		return false, nil
   197  	})
   198  	if err != nil {
   199  		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
   200  	}
   201  
   202  	// from here the caller must call tearDown
   203  	result.ClientConfig = server.GenericAPIServer.LoopbackClientConfig
   204  	result.ServerOpts = s
   205  	result.TearDownFn = tearDown
   206  	result.CompletedConfig = completedConfig
   207  
   208  	return result, nil
   209  }
   210  
   211  // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
   212  func StartTestServerOrDie(t Logger, instanceOptions *TestServerInstanceOptions, flags []string, storageConfig *storagebackend.Config) *TestServer {
   213  	result, err := StartTestServer(t, instanceOptions, flags, storageConfig)
   214  	if err == nil {
   215  		return &result
   216  	}
   217  
   218  	t.Fatalf("failed to launch server: %v", err)
   219  	return nil
   220  }
   221  
   222  func createLocalhostListenerOnFreePort() (net.Listener, int, error) {
   223  	ln, err := net.Listen("tcp", "127.0.0.1:0")
   224  	if err != nil {
   225  		return nil, 0, err
   226  	}
   227  
   228  	// get port
   229  	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
   230  	if !ok {
   231  		ln.Close()
   232  		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
   233  	}
   234  
   235  	return ln, tcpAddr.Port, nil
   236  }
   237  
   238  // pkgPath returns the absolute file path to this package's directory. With go
   239  // test, we can just look at the runtime call stack. However, bazel compiles go
   240  // binaries with the -trimpath option so the simple approach fails however we
   241  // can consult environment variables to derive the path.
   242  //
   243  // The approach taken here works for both go test and bazel on the assumption
   244  // that if and only if trimpath is passed, we are running under bazel.
   245  func pkgPath(t Logger) (string, error) {
   246  	_, thisFile, _, ok := runtime.Caller(0)
   247  	if !ok {
   248  		return "", fmt.Errorf("failed to get current file")
   249  	}
   250  
   251  	pkgPath := filepath.Dir(thisFile)
   252  
   253  	// If we find bazel env variables, then -trimpath was passed so we need to
   254  	// construct the path from the environment.
   255  	if testSrcdir, testWorkspace := os.Getenv("TEST_SRCDIR"), os.Getenv("TEST_WORKSPACE"); testSrcdir != "" && testWorkspace != "" {
   256  		t.Logf("Detected bazel env varaiables: TEST_SRCDIR=%q TEST_WORKSPACE=%q", testSrcdir, testWorkspace)
   257  		pkgPath = filepath.Join(testSrcdir, testWorkspace, pkgPath)
   258  	}
   259  
   260  	// If the path is still not absolute, something other than bazel compiled
   261  	// with -trimpath.
   262  	if !filepath.IsAbs(pkgPath) {
   263  		return "", fmt.Errorf("can't construct an absolute path from %q", pkgPath)
   264  	}
   265  
   266  	t.Logf("Resolved testserver package path to: %q", pkgPath)
   267  
   268  	return pkgPath, nil
   269  }
   270
View as plain text