certreload_test.go

Documentation: k8s.io/kubernetes/test/integration/apiserver/certreload

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package podlogs
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/rand"
    23  	"crypto/rsa"
    24  	"crypto/x509"
    25  	"crypto/x509/pkix"
    26  	"encoding/pem"
    27  	"fmt"
    28  	"math/big"
    29  	"os"
    30  	"path"
    31  	"strings"
    32  	"testing"
    33  	"time"
    34  
    35  	authorizationv1 "k8s.io/api/authorization/v1"
    36  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    37  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/client-go/kubernetes"
    40  	"k8s.io/client-go/kubernetes/scheme"
    41  	"k8s.io/client-go/rest"
    42  	"k8s.io/client-go/util/cert"
    43  	"k8s.io/component-base/cli/flag"
    44  	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
    45  	"k8s.io/kubernetes/test/integration/framework"
    46  	"k8s.io/kubernetes/test/utils/ktesting"
    47  )
    48  
    49  type caWithClient struct {
    50  	CACert     []byte
    51  	ClientCert []byte
    52  	ClientKey  []byte
    53  }
    54  
    55  func newTestCAWithClient(caSubject pkix.Name, caSerial *big.Int, clientSubject pkix.Name, subjectSerial *big.Int) (*caWithClient, error) {
    56  	ca := &x509.Certificate{
    57  		SerialNumber:          caSerial,
    58  		Subject:               caSubject,
    59  		NotBefore:             time.Now(),
    60  		NotAfter:              time.Now().Add(24 * time.Hour),
    61  		IsCA:                  true,
    62  		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
    63  		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
    64  		BasicConstraintsValid: true,
    65  	}
    66  
    67  	caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
    68  	if err != nil {
    69  		return nil, err
    70  	}
    71  
    72  	caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
    73  	if err != nil {
    74  		return nil, err
    75  	}
    76  
    77  	caPEM := new(bytes.Buffer)
    78  	err = pem.Encode(caPEM, &pem.Block{
    79  		Type:  "CERTIFICATE",
    80  		Bytes: caBytes,
    81  	})
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	clientCert := &x509.Certificate{
    87  		SerialNumber: subjectSerial,
    88  		Subject:      clientSubject,
    89  		NotBefore:    time.Now(),
    90  		NotAfter:     time.Now().Add(24 * time.Hour),
    91  		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
    92  		KeyUsage:     x509.KeyUsageDigitalSignature,
    93  	}
    94  
    95  	clientCertPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
    96  	if err != nil {
    97  		return nil, err
    98  	}
    99  
   100  	clientCertPrivateKeyPEM := new(bytes.Buffer)
   101  	err = pem.Encode(clientCertPrivateKeyPEM, &pem.Block{
   102  		Type:  "RSA PRIVATE KEY",
   103  		Bytes: x509.MarshalPKCS1PrivateKey(clientCertPrivateKey),
   104  	})
   105  	if err != nil {
   106  		return nil, err
   107  	}
   108  
   109  	clientCertBytes, err := x509.CreateCertificate(rand.Reader, clientCert, ca, &clientCertPrivateKey.PublicKey, caPrivateKey)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  
   114  	clientCertPEM := new(bytes.Buffer)
   115  	err = pem.Encode(clientCertPEM, &pem.Block{
   116  		Type:  "CERTIFICATE",
   117  		Bytes: clientCertBytes,
   118  	})
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  
   123  	return &caWithClient{
   124  		CACert:     caPEM.Bytes(),
   125  		ClientCert: clientCertPEM.Bytes(),
   126  		ClientKey:  clientCertPrivateKeyPEM.Bytes(),
   127  	}, nil
   128  }
   129  
   130  func TestClientCAUpdate(t *testing.T) {
   131  	testClientCA(t, false)
   132  }
   133  
   134  func TestClientCARecreate(t *testing.T) {
   135  	testClientCA(t, true)
   136  }
   137  
   138  func testClientCA(t *testing.T, recreate bool) {
   139  	tCtx := ktesting.Init(t)
   140  
   141  	frontProxyCA, err := newTestCAWithClient(
   142  		pkix.Name{
   143  			CommonName: "test-front-proxy-ca",
   144  		},
   145  		big.NewInt(43),
   146  		pkix.Name{
   147  			CommonName:   "test-aggregated-apiserver",
   148  			Organization: []string{"system:masters"},
   149  		},
   150  		big.NewInt(86),
   151  	)
   152  	if err != nil {
   153  		t.Error(err)
   154  		return
   155  	}
   156  
   157  	clientCA, err := newTestCAWithClient(
   158  		pkix.Name{
   159  			CommonName: "test-client-ca",
   160  		},
   161  		big.NewInt(42),
   162  		pkix.Name{
   163  			CommonName:   "system:admin",
   164  			Organization: []string{"system:masters"},
   165  		},
   166  		big.NewInt(84),
   167  	)
   168  	if err != nil {
   169  		t.Error(err)
   170  		return
   171  	}
   172  
   173  	clientCAFilename := ""
   174  	frontProxyCAFilename := ""
   175  
   176  	kubeClient, kubeconfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   177  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   178  			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
   179  			clientCAFilename = opts.Authentication.ClientCert.ClientCA
   180  			frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile
   181  			opts.Authentication.RequestHeader.AllowedNames = append(opts.Authentication.RequestHeader.AllowedNames, "test-aggregated-apiserver")
   182  		},
   183  	})
   184  	defer tearDownFn()
   185  
   186  	// wait for request header info
   187  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
   188  	if err != nil {
   189  		t.Fatal(err)
   190  	}
   191  	// wait for client cert info
   192  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
   193  	if err != nil {
   194  		t.Fatal(err)
   195  	}
   196  
   197  	if recreate {
   198  		if err := os.Remove(path.Join(clientCAFilename)); err != nil {
   199  			t.Fatal(err)
   200  		}
   201  		if err := os.Remove(path.Join(frontProxyCAFilename)); err != nil {
   202  			t.Fatal(err)
   203  		}
   204  	}
   205  
   206  	// when we run this the second time, we know which one we are expecting
   207  	if err := os.WriteFile(clientCAFilename, clientCA.CACert, 0644); err != nil {
   208  		t.Fatal(err)
   209  	}
   210  	if err := os.WriteFile(frontProxyCAFilename, frontProxyCA.CACert, 0644); err != nil {
   211  		t.Fatal(err)
   212  	}
   213  
   214  	time.Sleep(4 * time.Second)
   215  
   216  	acceptableCAs, err := cert.GetClientCANamesForURL(kubeconfig.Host)
   217  	if err != nil {
   218  		t.Fatal(err)
   219  	}
   220  
   221  	expectedCAs := []string{"test-client-ca", "test-front-proxy-ca"}
   222  	if len(expectedCAs) != len(acceptableCAs) {
   223  		t.Fatal(strings.Join(acceptableCAs, ":"))
   224  	}
   225  	for i := range expectedCAs {
   226  		if !strings.Contains(acceptableCAs[i], expectedCAs[i]) {
   227  			t.Errorf("expected %q, got %q", expectedCAs[i], acceptableCAs[i])
   228  		}
   229  	}
   230  
   231  	// wait for updated request header info that contains both
   232  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 2))
   233  	if err != nil {
   234  		t.Error(err)
   235  	}
   236  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", string(frontProxyCA.CACert), 1))
   237  	if err != nil {
   238  		t.Error(err)
   239  	}
   240  	// wait for updated client cert info that contains both
   241  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "-----BEGIN CERTIFICATE-----", 2))
   242  	if err != nil {
   243  		t.Error(err)
   244  	}
   245  	err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", string(clientCA.CACert), 1))
   246  	if err != nil {
   247  		t.Error(err)
   248  	}
   249  
   250  	// Test an aggregated apiserver client (signed by the new front proxy CA) is authorized
   251  	extensionApiserverClient, err := kubernetes.NewForConfig(&rest.Config{
   252  		Host: kubeconfig.Host,
   253  		TLSClientConfig: rest.TLSClientConfig{
   254  			CAData:     kubeconfig.TLSClientConfig.CAData,
   255  			CAFile:     kubeconfig.TLSClientConfig.CAFile,
   256  			ServerName: kubeconfig.TLSClientConfig.ServerName,
   257  			KeyData:    frontProxyCA.ClientKey,
   258  			CertData:   frontProxyCA.ClientCert,
   259  		},
   260  	})
   261  	if err != nil {
   262  		t.Error(err)
   263  		return
   264  	}
   265  
   266  	// Call an endpoint to make sure we are authenticated
   267  	err = extensionApiserverClient.AuthorizationV1().RESTClient().
   268  		Post().
   269  		Resource("subjectaccessreviews").
   270  		VersionedParams(&metav1.CreateOptions{}, scheme.ParameterCodec).
   271  		Body(&authorizationv1.SubjectAccessReview{
   272  			Spec: authorizationv1.SubjectAccessReviewSpec{
   273  				ResourceAttributes: &authorizationv1.ResourceAttributes{
   274  					Verb:      "create",
   275  					Resource:  "pods",
   276  					Namespace: "default",
   277  				},
   278  				User: "deads2k",
   279  			},
   280  		}).
   281  		SetHeader("X-Remote-User", "test-aggregated-apiserver").
   282  		SetHeader("X-Remote-Group", "system:masters").
   283  		Do(context.Background()).
   284  		Into(&authorizationv1.SubjectAccessReview{})
   285  	if err != nil {
   286  		t.Error(err)
   287  	}
   288  
   289  	// Test a client signed by the new ClientCA is authorized
   290  	testClient, err := kubernetes.NewForConfig(&rest.Config{
   291  		Host: kubeconfig.Host,
   292  		TLSClientConfig: rest.TLSClientConfig{
   293  			CAData:     kubeconfig.TLSClientConfig.CAData,
   294  			CAFile:     kubeconfig.TLSClientConfig.CAFile,
   295  			ServerName: kubeconfig.TLSClientConfig.ServerName,
   296  			KeyData:    clientCA.ClientKey,
   297  			CertData:   clientCA.ClientCert,
   298  		},
   299  	})
   300  	if err != nil {
   301  		t.Error(err)
   302  		return
   303  	}
   304  
   305  	// Call an endpoint to make sure we are authenticated
   306  	_, err = testClient.CoreV1().Nodes().List(tCtx, metav1.ListOptions{})
   307  	if err != nil {
   308  		t.Error(err)
   309  	}
   310  }
   311  
   312  func waitForConfigMapCAContent(t *testing.T, kubeClient kubernetes.Interface, key, content string, count int) func() (bool, error) {
   313  	return func() (bool, error) {
   314  		clusterAuthInfo, err := kubeClient.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "extension-apiserver-authentication", metav1.GetOptions{})
   315  		if apierrors.IsNotFound(err) {
   316  			return false, nil
   317  		}
   318  		if err != nil {
   319  			return false, err
   320  		}
   321  
   322  		ca := clusterAuthInfo.Data[key]
   323  		if strings.Count(ca, content) == count {
   324  			return true, nil
   325  		}
   326  		t.Log(ca)
   327  		return false, nil
   328  	}
   329  }
   330  
   331  var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
   332  MIIEowIBAAKCAQEA13f50PPWuR/InxLIoJjHdNSG+jVUd25CY7ZL2J023X2BAY+1
   333  M6jkLR6C2nSFZnn58ubiB74/d1g/Fg1Twd419iR615A013f+qOoyFx3LFHxU1S6e
   334  v22fgJ6ntK/+4QD5MwNgOwD8k1jN2WxHqNWn16IF4Tidbv8M9A35YHAdtYDYaOJC
   335  kzjVztzRw1y6bKRakpMXxHylQyWmAKDJ2GSbRTbGtjr7Ji54WBfG43k94tO5X8K4
   336  VGbz/uxrKe1IFMHNOlrjR438dbOXusksx9EIqDA9a42J3qjr5NKSqzCIbgBFl6qu
   337  45V3A7cdRI/sJ2G1aqlWIXh2fAQiaFQAEBrPfwIDAQABAoIBAAZbxgWCjJ2d8H+x
   338  QDZtC8XI18redAWqPU9P++ECkrHqmDoBkalanJEwS1BDDATAKL4gTh9IX/sXoZT3
   339  A7e+5PzEitN9r/GD2wIFF0FTYcDTAnXgEFM52vEivXQ5lV3yd2gn+1kCaHG4typp
   340  ZZv34iIc5+uDjjHOWQWCvA86f8XxX5EfYH+GkjfixTtN2xhWWlfi9vzYeESS4Jbt
   341  tqfH0iEaZ1Bm/qvb8vFgKiuSTOoSpaf+ojAdtPtXDjf1bBtQQG+RSQkP59O/taLM
   342  FCVuRrU8EtdB0+9anwmAP+O2UqjL5izA578lQtdIh13jHtGEgOcnfGNUphK11y9r
   343  Mg5V28ECgYEA9fwI6Xy1Rb9b9irp4bU5Ec99QXa4x2bxld5cDdNOZWJQu9OnaIbg
   344  kw/1SyUkZZCGMmibM/BiWGKWoDf8E+rn/ujGOtd70sR9U0A94XMPqEv7iHxhpZmD
   345  rZuSz4/snYbOWCZQYXFoD/nqOwE7Atnz7yh+Jti0qxBQ9bmkb9o0QW8CgYEA4D3d
   346  okzodg5QQ1y9L0J6jIC6YysoDedveYZMd4Un9bKlZEJev4OwiT4xXmSGBYq/7dzo
   347  OJOvN6qgPfibr27mSB8NkAk6jL/VdJf3thWxNYmjF4E3paLJ24X31aSipN1Ta6K3
   348  KKQUQRvixVoI1q+8WHAubBDEqvFnNYRHD+AjKvECgYBkekjhpvEcxme4DBtw+OeQ
   349  4OJXJTmhKemwwB12AERboWc88d3GEqIVMEWQJmHRotFOMfCDrMNfOxYv5+5t7FxL
   350  gaXHT1Hi7CQNJ4afWrKgmjjqrXPtguGIvq2fXzjVt8T9uNjIlNxe+kS1SXFjXsgH
   351  ftDY6VgTMB0B4ozKq6UAvQKBgQDER8K5buJHe+3rmMCMHn+Qfpkndr4ftYXQ9Kn4
   352  MFiy6sV0hdfTgRzEdOjXu9vH/BRVy3iFFVhYvIR42iTEIal2VaAUhM94Je5cmSyd
   353  eE1eFHTqfRPNazmPaqttmSc4cfa0D4CNFVoZR6RupIl6Cect7jvkIaVUD+wMXxWo
   354  osOFsQKBgDLwVhZWoQ13RV/jfQxS3veBUnHJwQJ7gKlL1XZ16mpfEOOVnJF7Es8j
   355  TIIXXYhgSy/XshUbsgXQ+YGliye/rXSCTXHBXvWShOqxEMgeMYMRkcm8ZLp/DH7C
   356  kC2pemkLPUJqgSh1PASGcJbDJIvFGUfP69tUCYpHpk3nHzexuAg3
   357  -----END RSA PRIVATE KEY-----`)
   358  
   359  var serverCert = []byte(`-----BEGIN CERTIFICATE-----
   360  MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
   361  BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
   362  DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
   363  b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
   364  AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
   365  dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
   366  r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
   367  XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
   368  7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
   369  j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
   370  BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
   371  hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
   372  ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
   373  ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
   374  T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
   375  bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
   376  M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
   377  YkNtGc1RUDHwecCTFpJtPb7Yu/E=
   378  -----END CERTIFICATE-----`)
   379  
   380  var anotherServerKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
   381  MIIJKAIBAAKCAgEAlZJORzCjbzF1SaCXFHyitudlE+q3Z5bGhS2TpXG6d6Laoqtw
   382  No4UC3i+TnMndtrP2pNkV/ZYivsp1fHz92FqFAT+XpYcG8pLm4iL0k0UufOWdLPT
   383  X87HCJjKZ4r7fmzstyjqK4sv9I3ye1jKi0VE1BLrF1KVvEE/1PXCug68EBP/aF06
   384  +uvcr6o8hbMYzgdKSzhRYm9C3kGcawNofqAD/Kk/zn+pMk4Bloy4UgtXFXgj2bEn
   385  mVE+tRWyLv2+TONlmLnXaBW3/MvZtKC3mIs2KG+6aBNuY8PdWzWvMtp30r/ibgnH
   386  zuMKtvXJ5XRhTaST4QYXNbGwb1bIV1ylnX8zdXPEQkuYTQDctaYQCe0RXt1I9Fp3
   387  gVQRxyTM+0IetbsU0k9VvBwQ07mgU8Rik3DxVnfbuJY/wREnERTkgv6ojtRwiszr
   388  GIY5x36peRs30CqRMv3uJtqC/FU6nCQbHxwssQyB/umN6L7bcpsQFDydeK95hvRQ
   389  y6tb2v/vMcw7MMo5kSFUHjoL5Zc4DObwiqs+p7F7S0WIJMBzJOcjmgCMzgZ7Jmc7
   390  bMmrm43GLzOaVLIjuPVVpOp7YgJ/lqRf7K3hZXrMdaXkCm01aL8L59d+3Vfdjp3H
   391  HvmYpCh8bc+Kjs/nR9Rc+2JKK/H13LH3W5Cr8Fnc/FP6TgbvvNwsQV01gG8CAwEA
   392  AQKCAgBLBQn8DPo8YDsqxcBhRy45vQ/mkHiTHX3O+JAwkD1tmiI9Ku3qfxKwukwB
   393  fyKRK6jLQdg3gljgxJ80Ltol/xc8mVCYUoQgsDOB/FfdEEpQBkw1lqhzSnxr5G7I
   394  xl3kCHAmYgAp/PL9n2C620sj1YdzM1X06bgupy+D+gxEU/WhvtYBG5nklv6moSUg
   395  DjdnxyJNXh7710Bbx97Tke8Ma+f0B1P4l/FeSN/lCgm9JPD11L9uhbuN28EvBIXN
   396  qfmUCQ5BLx1KmHIi+n/kaCQN/+0XFQsS/oQEyA2znNaWFBu7egDxHji4nQoXwGoW
   397  i2vujJibafmkNc5/2bA8mTx8JXvCLhU2L9j2ZumpKOda0g+pfMauesL+9rvZdqwW
   398  gjdjndOHZlg3qm40hGCDBVmmV3mdnvXrk1BbuB4Y0N7qGo3PyYtJHGwJILaNQVGR
   399  Sj75uTatxJwFXsqSaJaErV3Q90IiyXX4AOFGnWHOs29GEwtnDbCvT/rzqutTYSXD
   400  Yv0XFDznzJelhZTH7FbaW3FW3YGEG1ER/0MtKpsAH4i7H9q3KKK8yrzUsgUkGwXt
   401  xtoLckh91xilPIGbzARdELTEdHrjlFL+qaz3PIqEQScWz3WBu2JcIzGbp6PQfMZ+
   402  FZXarEb/ADZuX0+WoKFYR5jzwMoQfF/fxe2Ib/37ETNw4BgfSQKCAQEAxOw64XgO
   403  nUVJslzGK/H5fqTVpD1rfRmvVAiSDLAuWpClbpDZXqEPuoPPYsiccuUWu9VkJE1F
   404  6MZEexGx1jFkN08QUHD1Bobzu6ThaBc2PrWHRjFGKM60d0AkhOiL4N04FGwVeCN6
   405  xzIJFk1E4VOOo1+lzeAWRvi1lwuWTgQi+m25nwBJtmYdBLGeS+DXy80Fi6deECei
   406  ipDzJ4rxJsZ61uqBeYC4CfuHW9m5rCzJWPMMMFrPdl3OxEyZzKng4Co5EYc5i/QH
   407  piXD6IJayKcTPRK3tBJZp2YCIIdtQLcjAwmDEDowQtelHkbTihXMGRarf3VcOEoN
   408  ozMRgcLEEynuKwKCAQEAwnF5ZkkJEL/1MCOZ6PZfSKl35ZMIz/4Umk8hOMAQGhCT
   409  cnxlDUfGSBu4OihdBbIuBSBsYDjgcev8uyiIPDVy0FIkBKRGfgrNCLDh19aHljvE
   410  bUc3akvbft0mro86AvSd/Rpc7sj841bru37RDUm6AJOtIvb6DWUpMOZgMm0WMmSI
   411  kNs/UT+7rqg+AZPP8lumnJIFnRK38xOehQAaS1FHWGP//38py8yo8eXpMsoCWMch
   412  c+kZD2jsAYV+SWjjkZjcrv/52+asd4AotRXIShV8E8xItQeq6vLHKOaIe0tC2Y44
   413  ONAKiu4dgABt1voy8I5J63MwgeNmgAUS+KsgUclYzQKCAQEAlt/3bPAzIkQH5uQ1
   414  4U2PvnxEQ4XbaQnYzyWR4K7LlQ/l8ASCxoHYLyr2JdVWKKFk/ZzNERMzUNk3dqNk
   415  AZvuEII/GaKx2MJk04vMN5gxM3KZpinyeymEEynN0RbqtOpJITx+ZoGofB3V4IRr
   416  FciTLJEH0+iwqMe9OXDjQ/rfYcfXw/7QezNZYFNF2RT3wWnfqdQduXrkig3sfotx
   417  oCfJzgf2E0WPu/Y/CxyRqVzXF5N/7zxkX2gYF0YpQCmX5afz+X4FlTju81lT9DyL
   418  mdiIYO6KWSkGD7+UOaAJEOA/rwAGrtQmTdAy7jONt+pjaYV4+DrO4UG7mSJzc1vq
   419  JlSl6QKCAQARqwPv8mT7e6XI2QNMMs7XqGZ3mtOrKpguqVAIexM7exQazAjWmxX+
   420  SV6FElPZh6Y82wRd/e0PDPVrADTY27ZyDXSuY0rwewTEbGYpGZo6YXXoxBbZ9sic
   421  D3ZLWEJaMGYGsJWPMP4hni1PXSebwH5BPSn3Sl/QRcfnZJeLHXRt4cqy9uka9eKU
   422  7T6tIAQ+LmvGQFJ4QlIqqTa3ORoqi9kiw/tn+OMQXKlhSZXWApsR/A4jHSQkzVDc
   423  loeyHfDHsw8ia6oFfEFhnmiUg8UuTiN3HRHiOS8jqCnGoqP2KBGL+StMpkK++wH9
   424  NozEgvmL+DHpTg8zTjlrGortw4btR5FlAoIBABVni+EsGA5K/PM1gIct2pDm+6Kq
   425  UCYScTwIjftuwKLk/KqermG9QJLiJouKO3ZSz7iCelu87Dx1cKeXrc2LQ1pnQzCB
   426  JnI6BCT+zRnQFXjLokJXD2hIS2hXhqV6/9FRXLKKMYePcDxWt/etLNGmpLnhDfb3
   427  sMOH/9pnaGmtk36Ce03Hh7E1C6io/MKfTq+KKUV1UGwO1BdNQCiclkYzAUqn1O+Y
   428  c8BaeGKc2c6as8DKrPTGGQGmzo/ZUxQVfVFl2g7+HXISWBBcui/G5gtnU1afZqbW
   429  mTmDoqs4510vhlkhN9XZ0DyhewDIqNNGEY2vS1x2fJz1XC2Eve4KpSyUsiE=
   430  -----END RSA PRIVATE KEY-----
   431  `)
   432  
   433  var anotherServerCert = []byte(`-----BEGIN CERTIFICATE-----
   434  MIIFJjCCAw6gAwIBAgIJAOcEAbv8NslfMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV
   435  BAYTAlVTMQswCQYDVQQIDAJDQTETMBEGA1UECgwKQWNtZSwgSW5jLjEPMA0GA1UE
   436  AwwGc29tZUNBMCAXDTE4MDYwODEzMzkyNFoYDzIyMTgwNDIxMTMzOTI0WjBDMQsw
   437  CQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExEzARBgNVBAoMCkFjbWUsIEluYy4xEjAQ
   438  BgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
   439  AJWSTkcwo28xdUmglxR8orbnZRPqt2eWxoUtk6Vxunei2qKrcDaOFAt4vk5zJ3ba
   440  z9qTZFf2WIr7KdXx8/dhahQE/l6WHBvKS5uIi9JNFLnzlnSz01/OxwiYymeK+35s
   441  7Lco6iuLL/SN8ntYyotFRNQS6xdSlbxBP9T1wroOvBAT/2hdOvrr3K+qPIWzGM4H
   442  Sks4UWJvQt5BnGsDaH6gA/ypP85/qTJOAZaMuFILVxV4I9mxJ5lRPrUVsi79vkzj
   443  ZZi512gVt/zL2bSgt5iLNihvumgTbmPD3Vs1rzLad9K/4m4Jx87jCrb1yeV0YU2k
   444  k+EGFzWxsG9WyFdcpZ1/M3VzxEJLmE0A3LWmEAntEV7dSPRad4FUEcckzPtCHrW7
   445  FNJPVbwcENO5oFPEYpNw8VZ327iWP8ERJxEU5IL+qI7UcIrM6xiGOcd+qXkbN9Aq
   446  kTL97ibagvxVOpwkGx8cLLEMgf7pjei+23KbEBQ8nXiveYb0UMurW9r/7zHMOzDK
   447  OZEhVB46C+WXOAzm8IqrPqexe0tFiCTAcyTnI5oAjM4GeyZnO2zJq5uNxi8zmlSy
   448  I7j1VaTqe2ICf5akX+yt4WV6zHWl5AptNWi/C+fXft1X3Y6dxx75mKQofG3Pio7P
   449  50fUXPtiSivx9dyx91uQq/BZ3PxT+k4G77zcLEFdNYBvAgMBAAGjHjAcMBoGA1Ud
   450  EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEABL8kffi7
   451  48qSD+/l/UwCYdmqta1vAbOkvLnPtfXe1XlDpJipNuPxUBc8nNTemtrbg0erNJnC
   452  jQHodqmdKBJJOdaEKTwAGp5pYvvjlU3WasmhfJy+QwOWgeqjJcTUo3+DEaHRls16
   453  AZXlsp3hB6z0gzR/qzUuZwpMbL477JpuZtAcwLYeVvLG8bQRyWyEy8JgGDoYSn8s
   454  Z16s+r6AX+cnL/2GHkZ+oc3iuXJbnac4xfWTKDiYnyzK6RWRnoyro7X0jiPz6XX3
   455  wyoWzB1uMSCXscrW6ZcKyKqz75lySLuwGxOMhX4nGOoYHY0ZtrYn5WK2ZAJxsQnn
   456  8QcjPB0nq37U7ifk1uebmuXe99iqyKnWaLvlcpe+HnO5pVxFkSQEf7Zh+hEnRDkN
   457  IBzLFnqwDS1ug/oQ1aSvc8oBh2ylKDJuGtPNqGKibNJyb2diXO/aEUOKRUKPAxKa
   458  dbKsc4Y1bhZNN3/MICMoyghwAOiuwUQMR5uhxTkQmZUwNrPFa+eW6GvyoYLFUsZs
   459  hZfWLNGD5mLADElxs0HF7F9Zk6pSocTDXba4d4lfxsq88SyZZ7PbjJYFRfLQPzd1
   460  CfvpRPqolEmZo1Y5Q644PELYiJRKpBxmX5GtC5j5eaUD9XdGKvXsGhb0m0gW75rq
   461  iUnnLkZt2ya1cDJDiCnJjo7r5KxMo0XXFDc=
   462  -----END CERTIFICATE-----
   463  `)
   464  
   465  func TestServingCertUpdate(t *testing.T) {
   466  	testServingCert(t, false)
   467  }
   468  
   469  func TestServingCertRecreate(t *testing.T) {
   470  	testServingCert(t, true)
   471  }
   472  
   473  func testServingCert(t *testing.T, recreate bool) {
   474  	tCtx := ktesting.Init(t)
   475  
   476  	var servingCertPath string
   477  
   478  	_, kubeconfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   479  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   480  			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
   481  			servingCertPath = opts.SecureServing.ServerCert.CertDirectory
   482  		},
   483  	})
   484  	defer tearDownFn()
   485  
   486  	if recreate {
   487  		if err := os.Remove(path.Join(servingCertPath, "apiserver.key")); err != nil {
   488  			t.Fatal(err)
   489  		}
   490  		if err := os.Remove(path.Join(servingCertPath, "apiserver.crt")); err != nil {
   491  			t.Fatal(err)
   492  		}
   493  	}
   494  
   495  	if err := os.WriteFile(path.Join(servingCertPath, "apiserver.key"), serverKey, 0644); err != nil {
   496  		t.Fatal(err)
   497  	}
   498  	if err := os.WriteFile(path.Join(servingCertPath, "apiserver.crt"), serverCert, 0644); err != nil {
   499  		t.Fatal(err)
   500  	}
   501  
   502  	time.Sleep(4 * time.Second)
   503  
   504  	// get the certs we're actually serving with
   505  	_, actualCerts, err := cert.GetServingCertificatesForURL(kubeconfig.Host, "")
   506  	if err != nil {
   507  		t.Fatal(err)
   508  	}
   509  	if err := checkServingCerts(serverCert, actualCerts); err != nil {
   510  		t.Fatal(err)
   511  	}
   512  }
   513  
   514  func TestSNICert(t *testing.T) {
   515  	var servingCertPath string
   516  
   517  	tCtx := ktesting.Init(t)
   518  
   519  	_, kubeconfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   520  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   521  			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
   522  			servingCertPath = opts.SecureServing.ServerCert.CertDirectory
   523  
   524  			if err := os.WriteFile(path.Join(servingCertPath, "foo.key"), anotherServerKey, 0644); err != nil {
   525  				t.Fatal(err)
   526  			}
   527  			if err := os.WriteFile(path.Join(servingCertPath, "foo.crt"), anotherServerCert, 0644); err != nil {
   528  				t.Fatal(err)
   529  			}
   530  
   531  			opts.SecureServing.SNICertKeys = []flag.NamedCertKey{{
   532  				Names:    []string{"foo"},
   533  				CertFile: path.Join(servingCertPath, "foo.crt"),
   534  				KeyFile:  path.Join(servingCertPath, "foo.key"),
   535  			}}
   536  		},
   537  	})
   538  	defer tearDownFn()
   539  
   540  	// When we run this the second time, we know which one we are expecting.
   541  	_, actualCerts, err := cert.GetServingCertificatesForURL(kubeconfig.Host, "foo")
   542  	if err != nil {
   543  		t.Fatal(err)
   544  	}
   545  	if err := checkServingCerts(anotherServerCert, actualCerts); err != nil {
   546  		t.Fatal(err)
   547  	}
   548  
   549  	if err := os.WriteFile(path.Join(servingCertPath, "foo.key"), serverKey, 0644); err != nil {
   550  		t.Fatal(err)
   551  	}
   552  	if err := os.WriteFile(path.Join(servingCertPath, "foo.crt"), serverCert, 0644); err != nil {
   553  		t.Fatal(err)
   554  	}
   555  
   556  	time.Sleep(4 * time.Second)
   557  
   558  	_, actualCerts, err = cert.GetServingCertificatesForURL(kubeconfig.Host, "foo")
   559  	if err != nil {
   560  		t.Fatal(err)
   561  	}
   562  	if err := checkServingCerts(serverCert, actualCerts); err != nil {
   563  		t.Fatal(err)
   564  	}
   565  }
   566  
   567  func checkServingCerts(expectedBytes []byte, actual [][]byte) error {
   568  	expectedCerts, err := cert.ParseCertsPEM(expectedBytes)
   569  	if err != nil {
   570  		return err
   571  	}
   572  	expected := [][]byte{}
   573  	for _, curr := range expectedCerts {
   574  		currBytes, err := cert.EncodeCertificates(curr)
   575  		if err != nil {
   576  			return err
   577  		}
   578  		expected = append(expected, []byte(strings.TrimSpace(string(currBytes))))
   579  	}
   580  
   581  	if len(expected) != len(actual) {
   582  		var certs []string
   583  		for _, a := range actual {
   584  			certs = append(certs, string(a))
   585  		}
   586  		return fmt.Errorf("unexpected number of certs %d vs %d: %v", len(expected), len(actual), strings.Join(certs, "\n"))
   587  	}
   588  	for i := range expected {
   589  		if !bytes.Equal(actual[i], expected[i]) {
   590  			return fmt.Errorf("expected %q, got %q", string(expected[i]), string(actual[i]))
   591  		}
   592  	}
   593  	return nil
   594  }
   595
View as plain text