cert_rotation_test.go

Documentation: k8s.io/kubernetes/test/integration/client

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package client
    18  
    19  import (
    20  	"context"
    21  	"crypto/rand"
    22  	"crypto/rsa"
    23  	"crypto/x509"
    24  	"crypto/x509/pkix"
    25  	"encoding/pem"
    26  	"errors"
    27  	"math"
    28  	"math/big"
    29  	"os"
    30  	"path"
    31  	"testing"
    32  	"time"
    33  
    34  	"github.com/stretchr/testify/assert"
    35  	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	clientset "k8s.io/client-go/kubernetes"
    37  	"k8s.io/client-go/transport"
    38  	"k8s.io/client-go/util/cert"
    39  	apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    40  	"k8s.io/kubernetes/test/integration/framework"
    41  	"k8s.io/kubernetes/test/utils"
    42  )
    43  
    44  func TestCertRotation(t *testing.T) {
    45  	stopCh := make(chan struct{})
    46  	defer close(stopCh)
    47  
    48  	transport.CertCallbackRefreshDuration = 1 * time.Second
    49  	transport.DialerStopCh = stopCh
    50  
    51  	certDir := os.TempDir()
    52  	clientCAFilename, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir)
    53  
    54  	server := apiservertesting.StartTestServerOrDie(t, apiservertesting.NewDefaultTestServerOptions(), []string{
    55  		"--client-ca-file=" + clientCAFilename,
    56  	}, framework.SharedEtcd())
    57  	defer server.TearDownFn()
    58  
    59  	clientCertFilename, clientKeyFilename := writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second)
    60  
    61  	kubeconfig := server.ClientConfig
    62  	kubeconfig.CertFile = clientCertFilename
    63  	kubeconfig.KeyFile = clientKeyFilename
    64  	kubeconfig.BearerToken = ""
    65  
    66  	client := clientset.NewForConfigOrDie(kubeconfig)
    67  	ctx := context.Background()
    68  
    69  	w, err := client.CoreV1().ServiceAccounts("default").Watch(ctx, v1.ListOptions{})
    70  	if err != nil {
    71  		t.Fatal(err)
    72  	}
    73  
    74  	select {
    75  	case <-w.ResultChan():
    76  		t.Fatal("Watch closed before rotation")
    77  	default:
    78  	}
    79  
    80  	writeCerts(t, clientSigningCert, clientSigningKey, certDir, 5*time.Minute)
    81  
    82  	time.Sleep(10 * time.Second)
    83  
    84  	// Should have had a rotation; connections will have been closed
    85  	select {
    86  	case _, ok := <-w.ResultChan():
    87  		assert.Equal(t, false, ok)
    88  	default:
    89  		t.Fatal("Watch wasn't closed despite rotation")
    90  	}
    91  
    92  	// Wait for old cert to expire (30s)
    93  	time.Sleep(30 * time.Second)
    94  
    95  	// Ensure we make requests with the new cert
    96  	_, err = client.CoreV1().ServiceAccounts("default").List(ctx, v1.ListOptions{})
    97  	if err != nil {
    98  		t.Fatal(err)
    99  	}
   100  }
   101  
   102  func TestCertRotationContinuousRequests(t *testing.T) {
   103  	stopCh := make(chan struct{})
   104  	defer close(stopCh)
   105  
   106  	transport.CertCallbackRefreshDuration = 1 * time.Second
   107  	transport.DialerStopCh = stopCh
   108  
   109  	certDir := os.TempDir()
   110  	clientCAFilename, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir)
   111  
   112  	server := apiservertesting.StartTestServerOrDie(t, apiservertesting.NewDefaultTestServerOptions(), []string{
   113  		"--client-ca-file=" + clientCAFilename,
   114  	}, framework.SharedEtcd())
   115  	defer server.TearDownFn()
   116  
   117  	clientCertFilename, clientKeyFilename := writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second)
   118  
   119  	kubeconfig := server.ClientConfig
   120  	kubeconfig.CertFile = clientCertFilename
   121  	kubeconfig.KeyFile = clientKeyFilename
   122  	kubeconfig.BearerToken = ""
   123  
   124  	client := clientset.NewForConfigOrDie(kubeconfig)
   125  
   126  	ctx, cancel := context.WithCancel(context.Background())
   127  
   128  	go func() {
   129  		time.Sleep(10 * time.Second)
   130  
   131  		writeCerts(t, clientSigningCert, clientSigningKey, certDir, 5*time.Minute)
   132  
   133  		// Wait for old cert to expire (30s)
   134  		time.Sleep(30 * time.Second)
   135  		cancel()
   136  	}()
   137  
   138  	for range time.Tick(time.Second) {
   139  		_, err := client.CoreV1().ServiceAccounts("default").List(ctx, v1.ListOptions{})
   140  		if err != nil {
   141  			// client may wrap the context.Canceled error, so we can't
   142  			// do 'err == ctx.Err()', instead use 'errors.Is'.
   143  			if errors.Is(err, context.Canceled) {
   144  				return
   145  			}
   146  
   147  			t.Fatal(err)
   148  		}
   149  	}
   150  }
   151  
   152  func writeCACertFiles(t *testing.T, certDir string) (string, *x509.Certificate, *rsa.PrivateKey) {
   153  	clientSigningKey, err := utils.NewPrivateKey()
   154  	if err != nil {
   155  		t.Fatal(err)
   156  	}
   157  	clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey)
   158  	if err != nil {
   159  		t.Fatal(err)
   160  	}
   161  
   162  	clientCAFilename := path.Join(certDir, "ca.crt")
   163  
   164  	if err := os.WriteFile(clientCAFilename, utils.EncodeCertPEM(clientSigningCert), 0644); err != nil {
   165  		t.Fatal(err)
   166  	}
   167  
   168  	return clientCAFilename, clientSigningCert, clientSigningKey
   169  }
   170  
   171  func writeCerts(t *testing.T, clientSigningCert *x509.Certificate, clientSigningKey *rsa.PrivateKey, certDir string, duration time.Duration) (string, string) {
   172  	clientKey, err := utils.NewPrivateKey()
   173  	if err != nil {
   174  		t.Fatal(err)
   175  	}
   176  
   177  	privBytes, err := x509.MarshalPKCS8PrivateKey(clientKey)
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  
   182  	if err := os.WriteFile(path.Join(certDir, "client.key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}), 0666); err != nil {
   183  		t.Fatal(err)
   184  	}
   185  
   186  	// returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max).
   187  	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64-1))
   188  	if err != nil {
   189  		t.Fatal(err)
   190  	}
   191  	serial = new(big.Int).Add(serial, big.NewInt(1))
   192  
   193  	certTmpl := x509.Certificate{
   194  		Subject: pkix.Name{
   195  			CommonName:   "foo",
   196  			Organization: []string{"system:masters"},
   197  		},
   198  		SerialNumber: serial,
   199  		NotBefore:    clientSigningCert.NotBefore,
   200  		NotAfter:     time.Now().Add(duration).UTC(),
   201  		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
   202  		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
   203  	}
   204  
   205  	certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, clientSigningCert, clientKey.Public(), clientSigningKey)
   206  	if err != nil {
   207  		t.Fatal(err)
   208  	}
   209  
   210  	if err := os.WriteFile(path.Join(certDir, "client.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDERBytes}), 0666); err != nil {
   211  		t.Fatal(err)
   212  	}
   213  
   214  	return path.Join(certDir, "client.crt"), path.Join(certDir, "client.key")
   215  }
   216
View as plain text