podlogs_test.go

Documentation: k8s.io/kubernetes/test/integration/apiserver/podlogs

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package podlogs
    18  
    19  import (
    20  	"context"
    21  	"crypto/ecdsa"
    22  	"crypto/elliptic"
    23  	"crypto/rand"
    24  	"crypto/tls"
    25  	"crypto/x509"
    26  	"crypto/x509/pkix"
    27  	"encoding/pem"
    28  	"fmt"
    29  	"math"
    30  	"math/big"
    31  	"net"
    32  	"net/http"
    33  	"net/http/httptest"
    34  	"net/url"
    35  	"os"
    36  	"strconv"
    37  	"strings"
    38  	"testing"
    39  	"time"
    40  
    41  	corev1 "k8s.io/api/core/v1"
    42  	"k8s.io/apimachinery/pkg/api/errors"
    43  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    44  	"k8s.io/apimachinery/pkg/util/wait"
    45  	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
    46  	"k8s.io/apiserver/pkg/endpoints/filters"
    47  	"k8s.io/apiserver/pkg/server/dynamiccertificates"
    48  	"k8s.io/client-go/kubernetes"
    49  	"k8s.io/client-go/transport"
    50  	certutil "k8s.io/client-go/util/cert"
    51  	"k8s.io/client-go/util/keyutil"
    52  	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
    53  	"k8s.io/kubernetes/test/integration/framework"
    54  	"k8s.io/kubernetes/test/utils/ktesting"
    55  )
    56  
    57  func TestInsecurePodLogs(t *testing.T) {
    58  	badCA := writeDataToTempFile(t, []byte(`
    59  -----BEGIN CERTIFICATE-----
    60  MIIDMDCCAhigAwIBAgIIHNPD7sig7YIwDQYJKoZIhvcNAQELBQAwNjESMBAGA1UE
    61  CxMJb3BlbnNoaWZ0MSAwHgYDVQQDExdhZG1pbi1rdWJlY29uZmlnLXNpZ25lcjAe
    62  Fw0xOTA1MzAxNTA3MzlaFw0yOTA1MjcxNTA3MzlaMDYxEjAQBgNVBAsTCW9wZW5z
    63  aGlmdDEgMB4GA1UEAxMXYWRtaW4ta3ViZWNvbmZpZy1zaWduZXIwggEiMA0GCSqG
    64  SIb3DQEBAQUAA4IBDwAwggEKAoIBAQD0dHk23lHRcuq06FzYDOl9J9+s8pnGxqA3
    65  IPcARI6ag/98aYe3ENwAB5e1i7AU2F2WiDZgj444w374XLdVgIK8zgQEm9yoqrlc
    66  +/ayO7ceKklrKHOMwh63LvGLEOqzhol2nFmBhXAZt+HyIoZHXN0IqlA92196+Dml
    67  0WOn1F4ce6JbAtEceFHPgLeI7KFmVaPz2796pBXh23ii6r7WvV1Rn9MKlMSBJQR4
    68  0LZzu9/j+GdnFXewdLAAMfgPzwEqv6h3PzvtUCjgdraHEm8Rs7s15S3PUmLK4RQS
    69  PsThx5BhJEGd/W6EzQ3BKoQfochhu3mnAQtW1J07CullySQ5Gg9fAgMBAAGjQjBA
    70  MA4GA1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQkTaaw
    71  YJSZ5k2Wd+OsM4GFMTGdqzANBgkqhkiG9w0BAQsFAAOCAQEAHK7+zBZPLqK+f9DT
    72  UEnpwRmZ0aeGS4YgbGIkqpjxJymVOwkRd5A1wslvVfGZ6yOQthF6KlCmqnPyJJMR
    73  I7FHw8j0h2ci90fEQ6IS90Y/ZJXkcgiK9Ncwa35GFGs8QrBxN4leGhtm84BnnBHN
    74  cTWpa4zcBwru0CRG7iHc66VX16X8jHB1iFeZ5W/FgY4MsE+G1Vze4mCXSPVI4BZ2
    75  /qlAgogjBivvSwQ9SFuCszg7IPjvT2ksm+Cf+8eT4YBqW41F85vBGR+FYK14yIla
    76  Bgqc+dJN9xS9Ah5gLiGQJ6C4niUA11piCpvMsy+j/LQ1Erx47KMar5fuMXYk7iPq
    77  1vqIwg==
    78  -----END CERTIFICATE-----
    79  `))
    80  
    81  	tCtx := ktesting.Init(t)
    82  	clientSet, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
    83  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
    84  			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
    85  			// I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation
    86  			opts.KubeletConfig.TLSClientConfig.CAFile = badCA
    87  		},
    88  	})
    89  	defer tearDownFn()
    90  
    91  	fakeKubeletServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
    92  		w.Write([]byte("fake-log"))
    93  		w.WriteHeader(http.StatusOK)
    94  	}))
    95  	defer fakeKubeletServer.Close()
    96  
    97  	pod := prepareFakeNodeAndPod(tCtx, t, clientSet, fakeKubeletServer)
    98  
    99  	insecureResult := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{InsecureSkipTLSVerifyBackend: true}).Do(context.TODO())
   100  	if err := insecureResult.Error(); err != nil {
   101  		t.Fatal(err)
   102  	}
   103  	insecureStatusCode := 0
   104  	insecureResult.StatusCode(&insecureStatusCode)
   105  	if insecureStatusCode != http.StatusOK {
   106  		t.Fatal(insecureStatusCode)
   107  	}
   108  
   109  	secureResult := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).Do(tCtx)
   110  	if err := secureResult.Error(); err == nil || !strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
   111  		t.Fatal(err)
   112  	}
   113  	secureStatusCode := 0
   114  	secureResult.StatusCode(&secureStatusCode)
   115  	if secureStatusCode == http.StatusOK {
   116  		raw, rawErr := secureResult.Raw()
   117  		if rawErr != nil {
   118  			t.Log(rawErr)
   119  		}
   120  		t.Log(string(raw))
   121  		t.Fatal(secureStatusCode)
   122  	}
   123  }
   124  
   125  func prepareFakeNodeAndPod(ctx context.Context, t *testing.T, clientSet kubernetes.Interface, fakeKubeletServer *httptest.Server) *corev1.Pod {
   126  	t.Helper()
   127  
   128  	fakeKubeletURL, err := url.Parse(fakeKubeletServer.URL)
   129  	if err != nil {
   130  		t.Fatal(err)
   131  	}
   132  	fakeKubeletHost, fakeKubeletPortStr, err := net.SplitHostPort(fakeKubeletURL.Host)
   133  	if err != nil {
   134  		t.Fatal(err)
   135  	}
   136  	fakeKubeletPort, err := strconv.ParseUint(fakeKubeletPortStr, 10, 32)
   137  	if err != nil {
   138  		t.Fatal(err)
   139  	}
   140  
   141  	node, err := clientSet.CoreV1().Nodes().Create(ctx, &corev1.Node{
   142  		ObjectMeta: metav1.ObjectMeta{Name: "fake"},
   143  	}, metav1.CreateOptions{})
   144  	if err != nil {
   145  		t.Fatal(err)
   146  	}
   147  	node.Status = corev1.NodeStatus{
   148  		Addresses: []corev1.NodeAddress{
   149  			{
   150  				Type:    corev1.NodeExternalIP,
   151  				Address: fakeKubeletHost,
   152  			},
   153  		},
   154  		DaemonEndpoints: corev1.NodeDaemonEndpoints{
   155  			KubeletEndpoint: corev1.DaemonEndpoint{
   156  				Port: int32(fakeKubeletPort),
   157  			},
   158  		},
   159  	}
   160  	node, err = clientSet.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{})
   161  	if err != nil {
   162  		t.Fatal(err)
   163  	}
   164  
   165  	_, err = clientSet.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
   166  		ObjectMeta: metav1.ObjectMeta{Name: "ns"},
   167  	}, metav1.CreateOptions{})
   168  	if err != nil {
   169  		t.Fatal(err)
   170  	}
   171  
   172  	_, err = clientSet.CoreV1().ServiceAccounts("ns").Create(ctx, &corev1.ServiceAccount{
   173  		ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "ns"},
   174  	}, metav1.CreateOptions{})
   175  	if err != nil {
   176  		t.Fatal(err)
   177  	}
   178  
   179  	falseRef := false
   180  	pod, err := clientSet.CoreV1().Pods("ns").Create(ctx, &corev1.Pod{
   181  		ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "ns"},
   182  		Spec: corev1.PodSpec{
   183  			Containers: []corev1.Container{
   184  				{
   185  					Name:  "foo",
   186  					Image: "some/image:latest",
   187  				},
   188  			},
   189  			NodeName:                     node.Name,
   190  			AutomountServiceAccountToken: &falseRef,
   191  		},
   192  	}, metav1.CreateOptions{})
   193  	if err != nil {
   194  		t.Fatal(err)
   195  	}
   196  
   197  	return pod
   198  }
   199  
   200  func TestPodLogsKubeletClientCertReload(t *testing.T) {
   201  	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
   202  	t.Cleanup(cancel)
   203  
   204  	origCertCallbackRefreshDuration := transport.CertCallbackRefreshDuration
   205  	origDialerStopCh := transport.DialerStopCh
   206  	transport.CertCallbackRefreshDuration = time.Second // make client cert reloading fast
   207  	transport.DialerStopCh = ctx.Done()
   208  	t.Cleanup(func() {
   209  		transport.CertCallbackRefreshDuration = origCertCallbackRefreshDuration
   210  		transport.DialerStopCh = origDialerStopCh
   211  	})
   212  
   213  	// create a CA to sign the API server's kubelet client cert
   214  	startingCerts := generateClientCert(t)
   215  
   216  	dynamicCAContentFromFile, err := dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", startingCerts.caFile)
   217  	if err != nil {
   218  		t.Fatal(err)
   219  	}
   220  	if err := dynamicCAContentFromFile.RunOnce(ctx); err != nil {
   221  		t.Fatal(err)
   222  	}
   223  	go dynamicCAContentFromFile.Run(ctx, 1)
   224  	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
   225  		ClientCertificateCAContentProvider: dynamicCAContentFromFile,
   226  	}
   227  	authenticator, _, err := authenticatorConfig.New()
   228  	if err != nil {
   229  		t.Fatal(err)
   230  	}
   231  
   232  	// this fake kubelet will perform per request authentication using the configured CA (which is dynamically reloaded)
   233  	fakeKubeletServer := httptest.NewUnstartedServer(
   234  		filters.WithAuthentication(
   235  			http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
   236  				_, _ = w.Write([]byte("pod-logs-here"))
   237  			}),
   238  			authenticator,
   239  			http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
   240  				w.WriteHeader(http.StatusUnauthorized)
   241  			}),
   242  			nil,
   243  			nil,
   244  		),
   245  	)
   246  	fakeKubeletServer.TLS = &tls.Config{ClientAuth: tls.RequestClientCert}
   247  	fakeKubeletServer.StartTLS()
   248  	t.Cleanup(fakeKubeletServer.Close)
   249  
   250  	kubeletCA := writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{
   251  		Type:  "CERTIFICATE",
   252  		Bytes: fakeKubeletServer.Certificate().Raw,
   253  	}))
   254  
   255  	clientSet, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{
   256  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   257  			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
   258  			opts.KubeletConfig.TLSClientConfig.CAFile = kubeletCA
   259  			opts.KubeletConfig.TLSClientConfig.CertFile = startingCerts.clientCertFile
   260  			opts.KubeletConfig.TLSClientConfig.KeyFile = startingCerts.clientCertKeyFile
   261  		},
   262  	})
   263  	t.Cleanup(tearDownFn)
   264  
   265  	pod := prepareFakeNodeAndPod(ctx, t, clientSet, fakeKubeletServer)
   266  
   267  	// verify that the starting state works as expected
   268  	podLogs, err := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx)
   269  	if err != nil {
   270  		t.Fatal(err)
   271  	}
   272  	if l := string(podLogs); l != "pod-logs-here" {
   273  		t.Fatalf("unexpected pod logs: %s", l)
   274  	}
   275  
   276  	// generate a new CA and overwrite the existing CA that the kubelet is using for request authentication
   277  	newCerts := generateClientCert(t)
   278  	if err := os.Rename(newCerts.caFile, startingCerts.caFile); err != nil {
   279  		t.Fatal(err)
   280  	}
   281  
   282  	// wait until the kubelet observes the new CA
   283  	if err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
   284  		_, errLog := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx)
   285  		if errors.IsUnauthorized(errLog) {
   286  			return true, nil
   287  		}
   288  		return false, errLog
   289  	}); err != nil {
   290  		t.Fatal(err)
   291  	}
   292  
   293  	// now update the API server's kubelet client cert to use the new cert
   294  	if err := os.Rename(newCerts.clientCertFile, startingCerts.clientCertFile); err != nil {
   295  		t.Fatal(err)
   296  	}
   297  	if err := os.Rename(newCerts.clientCertKeyFile, startingCerts.clientCertKeyFile); err != nil {
   298  		t.Fatal(err)
   299  	}
   300  
   301  	// confirm that the API server observes the new client cert and closes existing connections to use it
   302  	if err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
   303  		fixedPodLogs, errLog := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx)
   304  		if errors.IsUnauthorized(errLog) {
   305  			t.Log("api server has not observed new client cert")
   306  			return false, nil
   307  		}
   308  		if errLog != nil {
   309  			return false, errLog
   310  		}
   311  		if l := string(fixedPodLogs); l != "pod-logs-here" {
   312  			return false, fmt.Errorf("unexpected pod logs: %s", l)
   313  		}
   314  		return true, nil
   315  	}); err != nil {
   316  		t.Fatal(err)
   317  	}
   318  }
   319  
   320  type testCerts struct {
   321  	caFile, clientCertFile, clientCertKeyFile string
   322  }
   323  
   324  func generateClientCert(t *testing.T) testCerts {
   325  	t.Helper()
   326  
   327  	caPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
   328  	if err != nil {
   329  		t.Fatal(err)
   330  	}
   331  	caCert, err := certutil.NewSelfSignedCACert(certutil.Config{CommonName: "test-ca"}, caPrivateKey)
   332  	if err != nil {
   333  		t.Fatal(err)
   334  	}
   335  
   336  	clientCertKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
   337  	if err != nil {
   338  		t.Fatal(err)
   339  	}
   340  
   341  	clientCertKeyBytes, err := keyutil.MarshalPrivateKeyToPEM(clientCertKey)
   342  	if err != nil {
   343  		t.Fatal(err)
   344  	}
   345  
   346  	// returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max).
   347  	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64-1))
   348  	if err != nil {
   349  		t.Fatal(err)
   350  	}
   351  	serial = new(big.Int).Add(serial, big.NewInt(1))
   352  	certTmpl := x509.Certificate{
   353  		Subject: pkix.Name{
   354  			CommonName: "the-api-server-user",
   355  		},
   356  		NotBefore:    caCert.NotBefore,
   357  		SerialNumber: serial,
   358  		NotAfter:     time.Now().Add(time.Hour).UTC(),
   359  		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
   360  		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
   361  	}
   362  	clientCertDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, clientCertKey.Public(), caPrivateKey)
   363  	if err != nil {
   364  		t.Fatal(err)
   365  	}
   366  
   367  	clientCert, err := x509.ParseCertificate(clientCertDERBytes)
   368  	if err != nil {
   369  		t.Fatal(err)
   370  	}
   371  
   372  	return testCerts{
   373  		caFile: writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{
   374  			Type:  "CERTIFICATE",
   375  			Bytes: caCert.Raw,
   376  		})),
   377  		clientCertFile: writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{
   378  			Type:  "CERTIFICATE",
   379  			Bytes: clientCert.Raw,
   380  		})),
   381  		clientCertKeyFile: writeDataToTempFile(t, clientCertKeyBytes),
   382  	}
   383  }
   384  
   385  func writeDataToTempFile(t *testing.T, data []byte) string {
   386  	t.Helper()
   387  
   388  	file, err := os.CreateTemp("", "pod-logs-test-")
   389  	if err != nil {
   390  		t.Fatal(err)
   391  	}
   392  	if _, err := file.Write(data); err != nil {
   393  		t.Fatal(err)
   394  	}
   395  	t.Cleanup(func() {
   396  		_ = os.Remove(file.Name())
   397  	})
   398  	return file.Name()
   399  }
   400
View as plain text