serving_test.go

Documentation: k8s.io/kubernetes/test/integration/serving

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package serving
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"fmt"
    24  	"io"
    25  	"net/http"
    26  	"os"
    27  	"path"
    28  	"strings"
    29  	"testing"
    30  
    31  	"k8s.io/apiserver/pkg/server"
    32  	"k8s.io/apiserver/pkg/server/options"
    33  	cloudprovider "k8s.io/cloud-provider"
    34  	cloudctrlmgrtesting "k8s.io/cloud-provider/app/testing"
    35  	"k8s.io/cloud-provider/fake"
    36  	"k8s.io/klog/v2/ktesting"
    37  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    38  	kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
    39  	kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing"
    40  	"k8s.io/kubernetes/test/integration/framework"
    41  )
    42  
    43  type componentTester interface {
    44  	StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error)
    45  }
    46  
    47  type kubeControllerManagerTester struct{}
    48  
    49  func (kubeControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) {
    50  	// avoid starting any controller loops, we're just testing serving
    51  	customFlags = append([]string{"--controllers="}, customFlags...)
    52  	gotResult, err := kubectrlmgrtesting.StartTestServer(ctx, customFlags)
    53  	if err != nil {
    54  		return nil, nil, nil, err
    55  	}
    56  	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err
    57  }
    58  
    59  type cloudControllerManagerTester struct{}
    60  
    61  func (cloudControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) {
    62  	gotResult, err := cloudctrlmgrtesting.StartTestServer(ctx, customFlags)
    63  	if err != nil {
    64  		return nil, nil, nil, err
    65  	}
    66  	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err
    67  }
    68  
    69  type kubeSchedulerTester struct{}
    70  
    71  func (kubeSchedulerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) {
    72  	gotResult, err := kubeschedulertesting.StartTestServer(ctx, customFlags)
    73  	if err != nil {
    74  		return nil, nil, nil, err
    75  	}
    76  	return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err
    77  }
    78  
    79  func TestComponentSecureServingAndAuth(t *testing.T) {
    80  	if !cloudprovider.IsCloudProvider("fake") {
    81  		cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory)
    82  	}
    83  
    84  	// Insulate this test from picking up in-cluster config when run inside a pod
    85  	// We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing
    86  	if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 {
    87  		t.Setenv("KUBERNETES_SERVICE_HOST", "")
    88  	}
    89  
    90  	// authenticate to apiserver via bearer token
    91  	token := "flwqkenfjasasdfmwerasd" // Fake token for testing.
    92  	tokenFile, err := os.CreateTemp("", "kubeconfig")
    93  	if err != nil {
    94  		t.Fatal(err)
    95  	}
    96  	tokenFile.WriteString(fmt.Sprintf(`
    97  %s,system:kube-controller-manager,system:kube-controller-manager,""
    98  `, token))
    99  	tokenFile.Close()
   100  
   101  	// start apiserver
   102  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{
   103  		"--token-auth-file", tokenFile.Name(),
   104  		"--authorization-mode", "RBAC",
   105  	}, framework.SharedEtcd())
   106  	defer server.TearDownFn()
   107  
   108  	// create kubeconfig for the apiserver
   109  	apiserverConfig, err := os.CreateTemp("", "kubeconfig")
   110  	if err != nil {
   111  		t.Fatal(err)
   112  	}
   113  	apiserverConfig.WriteString(fmt.Sprintf(`
   114  apiVersion: v1
   115  kind: Config
   116  clusters:
   117  - cluster:
   118      server: %s
   119      certificate-authority: %s
   120    name: integration
   121  contexts:
   122  - context:
   123      cluster: integration
   124      user: controller-manager
   125    name: default-context
   126  current-context: default-context
   127  users:
   128  - name: controller-manager
   129    user:
   130      token: %s
   131  `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token))
   132  	apiserverConfig.Close()
   133  
   134  	// create BROKEN kubeconfig for the apiserver
   135  	brokenApiserverConfig, err := os.CreateTemp("", "kubeconfig")
   136  	if err != nil {
   137  		t.Fatal(err)
   138  	}
   139  	brokenApiserverConfig.WriteString(fmt.Sprintf(`
   140  apiVersion: v1
   141  kind: Config
   142  clusters:
   143  - cluster:
   144      server: %s
   145      certificate-authority: %s
   146    name: integration
   147  contexts:
   148  - context:
   149      cluster: integration
   150      user: controller-manager
   151    name: default-context
   152  current-context: default-context
   153  users:
   154  - name: controller-manager
   155    user:
   156      token: WRONGTOKEN
   157  `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile))
   158  	brokenApiserverConfig.Close()
   159  
   160  	tests := []struct {
   161  		name       string
   162  		tester     componentTester
   163  		extraFlags []string
   164  	}{
   165  		{"kube-controller-manager", kubeControllerManagerTester{}, nil},
   166  		{"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake", "--webhook-secure-port=0"}},
   167  		{"kube-scheduler", kubeSchedulerTester{}, nil},
   168  	}
   169  
   170  	for _, tt := range tests {
   171  		t.Run(tt.name, func(t *testing.T) {
   172  			testComponentWithSecureServing(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags)
   173  		})
   174  	}
   175  }
   176  
   177  func testComponentWithSecureServing(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) {
   178  	tests := []struct {
   179  		name           string
   180  		flags          []string
   181  		path           string
   182  		anonymous      bool // to use the token or not
   183  		wantErr        bool
   184  		wantSecureCode *int
   185  	}{
   186  		{"no-flags", nil, "/healthz", false, true, nil},
   187  		{"/healthz without authn/authz", []string{
   188  			"--kubeconfig", kubeconfig,
   189  			"--leader-elect=false",
   190  		}, "/healthz", true, false, intPtr(http.StatusOK)},
   191  		{"/metrics without authn/authz", []string{
   192  			"--kubeconfig", kubeconfig,
   193  			"--leader-elect=false",
   194  		}, "/metrics", true, false, intPtr(http.StatusForbidden)},
   195  		{"authorization skipped for /healthz with authn/authz", []string{
   196  			"--authentication-kubeconfig", kubeconfig,
   197  			"--authorization-kubeconfig", kubeconfig,
   198  			"--kubeconfig", kubeconfig,
   199  			"--leader-elect=false",
   200  		}, "/healthz", false, false, intPtr(http.StatusOK)},
   201  		{"authorization skipped for /healthz with BROKEN authn/authz", []string{
   202  			"--authentication-skip-lookup", // to survive inaccessible extensions-apiserver-authentication configmap
   203  			"--authentication-kubeconfig", brokenKubeconfig,
   204  			"--authorization-kubeconfig", brokenKubeconfig,
   205  			"--kubeconfig", kubeconfig,
   206  			"--leader-elect=false",
   207  		}, "/healthz", false, false, intPtr(http.StatusOK)},
   208  		{"not authorized /metrics with BROKEN authn/authz", []string{
   209  			"--authentication-kubeconfig", kubeconfig,
   210  			"--authorization-kubeconfig", brokenKubeconfig,
   211  			"--kubeconfig", kubeconfig,
   212  			"--leader-elect=false",
   213  		}, "/metrics", false, false, intPtr(http.StatusInternalServerError)},
   214  		{"always-allowed /metrics with BROKEN authn/authz", []string{
   215  			"--authentication-skip-lookup", // to survive inaccessible extensions-apiserver-authentication configmap
   216  			"--authentication-kubeconfig", brokenKubeconfig,
   217  			"--authorization-kubeconfig", brokenKubeconfig,
   218  			"--authorization-always-allow-paths", "/healthz,/metrics",
   219  			"--kubeconfig", kubeconfig,
   220  			"--leader-elect=false",
   221  		}, "/metrics", false, false, intPtr(http.StatusOK)},
   222  	}
   223  	for _, tt := range tests {
   224  		t.Run(tt.name, func(t *testing.T) {
   225  			_, ctx := ktesting.NewTestContext(t)
   226  			secureOptions, secureInfo, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...))
   227  			if tearDownFn != nil {
   228  				defer tearDownFn()
   229  			}
   230  			if (err != nil) != tt.wantErr {
   231  				t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr)
   232  			}
   233  			if err != nil {
   234  				return
   235  			}
   236  
   237  			if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got {
   238  				t.Errorf("SecureServing enabled: expected=%v got=%v", want, got)
   239  			} else if want {
   240  				url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path)
   241  				url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::]
   242  
   243  				// read self-signed server cert disk
   244  				pool := x509.NewCertPool()
   245  				serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt")
   246  				serverCert, err := os.ReadFile(serverCertPath)
   247  				if err != nil {
   248  					t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err)
   249  				}
   250  				pool.AppendCertsFromPEM(serverCert)
   251  				tr := &http.Transport{
   252  					TLSClientConfig: &tls.Config{
   253  						RootCAs: pool,
   254  					},
   255  				}
   256  
   257  				client := &http.Client{Transport: tr}
   258  				req, err := http.NewRequest("GET", url, nil)
   259  				if err != nil {
   260  					t.Fatal(err)
   261  				}
   262  				if !tt.anonymous {
   263  					req.Header.Add("Authorization", fmt.Sprintf("Token %s", token))
   264  				}
   265  				r, err := client.Do(req)
   266  				if err != nil {
   267  					t.Fatalf("failed to GET %s from component: %v", tt.path, err)
   268  				}
   269  
   270  				body, err := io.ReadAll(r.Body)
   271  				if err != nil {
   272  					t.Fatalf("failed to read response body: %v", err)
   273  				}
   274  				defer r.Body.Close()
   275  				if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected {
   276  					t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body))
   277  				}
   278  			}
   279  		})
   280  	}
   281  }
   282  
   283  func intPtr(x int) *int {
   284  	return &x
   285  }
   286  
   287  func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) {
   288  	return &fake.Cloud{
   289  		DisableRoutes: true, // disable routes for server tests, otherwise --cluster-cidr is required
   290  	}, nil
   291  }
   292
View as plain text