test_server.go

Documentation: k8s.io/kubernetes/test/integration/framework

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package framework
    18  
    19  import (
    20  	"context"
    21  	"net"
    22  	"net/http"
    23  	"os"
    24  	"path"
    25  	"strings"
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/google/uuid"
    30  
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	genericapiserver "k8s.io/apiserver/pkg/server"
    35  	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
    36  	client "k8s.io/client-go/kubernetes"
    37  	"k8s.io/client-go/rest"
    38  	"k8s.io/client-go/util/cert"
    39  	"k8s.io/kubernetes/cmd/kube-apiserver/app"
    40  	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
    41  	"k8s.io/kubernetes/pkg/controlplane"
    42  	"k8s.io/kubernetes/test/utils"
    43  	netutils "k8s.io/utils/net"
    44  )
    45  
    46  // This key is for testing purposes only and is not considered secure.
    47  const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY-----
    48  MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49
    49  AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
    50  /IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg==
    51  -----END EC PRIVATE KEY-----`
    52  
    53  // TestServerSetup holds configuration information for a kube-apiserver test server.
    54  type TestServerSetup struct {
    55  	ModifyServerRunOptions func(*options.ServerRunOptions)
    56  	ModifyServerConfig     func(*controlplane.Config)
    57  }
    58  
    59  type TearDownFunc func()
    60  
    61  // StartTestServer runs a kube-apiserver, optionally calling out to the setup.ModifyServerRunOptions and setup.ModifyServerConfig functions
    62  // TODO (pohly): convert to ktesting contexts
    63  func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (client.Interface, *rest.Config, TearDownFunc) {
    64  	ctx, cancel := context.WithCancel(ctx)
    65  
    66  	certDir, err := os.MkdirTemp("", "test-integration-"+strings.ReplaceAll(t.Name(), "/", "_"))
    67  	if err != nil {
    68  		t.Fatalf("Couldn't create temp dir: %v", err)
    69  	}
    70  
    71  	var errCh chan error
    72  	tearDownFn := func() {
    73  		// Calling cancel function is stopping apiserver and cleaning up
    74  		// after itself, including shutting down its storage layer.
    75  		cancel()
    76  
    77  		// If the apiserver was started, let's wait for it to
    78  		// shutdown clearly.
    79  		if errCh != nil {
    80  			err, ok := <-errCh
    81  			if ok && err != nil {
    82  				t.Error(err)
    83  			}
    84  		}
    85  		if err := os.RemoveAll(certDir); err != nil {
    86  			t.Log(err)
    87  		}
    88  	}
    89  
    90  	_, defaultServiceClusterIPRange, _ := netutils.ParseCIDRSloppy("10.0.0.0/24")
    91  	proxySigningKey, err := utils.NewPrivateKey()
    92  	if err != nil {
    93  		t.Fatal(err)
    94  	}
    95  	proxySigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "front-proxy-ca"}, proxySigningKey)
    96  	if err != nil {
    97  		t.Fatal(err)
    98  	}
    99  	proxyCACertFile, _ := os.CreateTemp(certDir, "proxy-ca.crt")
   100  	if err := os.WriteFile(proxyCACertFile.Name(), utils.EncodeCertPEM(proxySigningCert), 0644); err != nil {
   101  		t.Fatal(err)
   102  	}
   103  	defer proxyCACertFile.Close()
   104  	clientSigningKey, err := utils.NewPrivateKey()
   105  	if err != nil {
   106  		t.Fatal(err)
   107  	}
   108  	clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey)
   109  	if err != nil {
   110  		t.Fatal(err)
   111  	}
   112  	clientCACertFile, _ := os.CreateTemp(certDir, "client-ca.crt")
   113  	if err := os.WriteFile(clientCACertFile.Name(), utils.EncodeCertPEM(clientSigningCert), 0644); err != nil {
   114  		t.Fatal(err)
   115  	}
   116  	defer clientCACertFile.Close()
   117  	listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{})
   118  	if err != nil {
   119  		t.Fatal(err)
   120  	}
   121  
   122  	saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")
   123  	if err != nil {
   124  		t.Fatalf("create temp file failed: %v", err)
   125  	}
   126  	defer saSigningKeyFile.Close()
   127  	if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
   128  		t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err)
   129  	}
   130  
   131  	opts := options.NewServerRunOptions()
   132  	opts.SecureServing.Listener = listener
   133  	opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
   134  	opts.SecureServing.ServerCert.CertDirectory = certDir
   135  	opts.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
   136  	opts.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry")
   137  	opts.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()}
   138  	opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String()
   139  	opts.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"}
   140  	opts.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"}
   141  	opts.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"}
   142  	opts.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"}
   143  	opts.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name()
   144  	opts.Authentication.APIAudiences = []string{"https://foo.bar.example.com"}
   145  	opts.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
   146  	opts.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
   147  	opts.Authentication.ClientCert.ClientCA = clientCACertFile.Name()
   148  	opts.Authorization.Modes = []string{"Node", "RBAC"}
   149  
   150  	if setup.ModifyServerRunOptions != nil {
   151  		setup.ModifyServerRunOptions(opts)
   152  	}
   153  
   154  	completedOptions, err := opts.Complete()
   155  	if err != nil {
   156  		t.Fatal(err)
   157  	}
   158  
   159  	if errs := completedOptions.Validate(); len(errs) != 0 {
   160  		t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs))
   161  	}
   162  
   163  	kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions)
   164  	if err != nil {
   165  		t.Fatal(err)
   166  	}
   167  
   168  	if setup.ModifyServerConfig != nil {
   169  		setup.ModifyServerConfig(kubeAPIServerConfig)
   170  	}
   171  	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(genericapiserver.NewEmptyDelegate())
   172  	if err != nil {
   173  		t.Fatal(err)
   174  	}
   175  
   176  	errCh = make(chan error)
   177  	go func() {
   178  		defer close(errCh)
   179  		if err := kubeAPIServer.GenericAPIServer.PrepareRun().Run(ctx.Done()); err != nil {
   180  			errCh <- err
   181  		}
   182  	}()
   183  
   184  	// Adjust the loopback config for external use (external server name and CA)
   185  	kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)
   186  	kubeAPIServerClientConfig.CAFile = path.Join(certDir, "apiserver.crt")
   187  	kubeAPIServerClientConfig.CAData = nil
   188  	kubeAPIServerClientConfig.ServerName = ""
   189  
   190  	// wait for health
   191  	err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
   192  		select {
   193  		case err := <-errCh:
   194  			return false, err
   195  		default:
   196  		}
   197  
   198  		healthzConfig := rest.CopyConfig(kubeAPIServerClientConfig)
   199  		healthzConfig.ContentType = ""
   200  		healthzConfig.AcceptContentTypes = ""
   201  		kubeClient, err := client.NewForConfig(healthzConfig)
   202  		if err != nil {
   203  			// this happens because we race the API server start
   204  			t.Log(err)
   205  			return false, nil
   206  		}
   207  
   208  		healthStatus := 0
   209  		kubeClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do(ctx).StatusCode(&healthStatus)
   210  		if healthStatus != http.StatusOK {
   211  			return false, nil
   212  		}
   213  
   214  		if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}); err != nil {
   215  			return false, nil
   216  		}
   217  		if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}); err != nil {
   218  			return false, nil
   219  		}
   220  
   221  		return true, nil
   222  	})
   223  	if err != nil {
   224  		t.Fatal(err)
   225  	}
   226  
   227  	kubeAPIServerClient, err := client.NewForConfig(kubeAPIServerClientConfig)
   228  	if err != nil {
   229  		t.Fatal(err)
   230  	}
   231  
   232  	return kubeAPIServerClient, kubeAPIServerClientConfig, tearDownFn
   233  }
   234
View as plain text