server_bootstrap_test.go

Documentation: k8s.io/kubernetes/cmd/kubelet/app

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"crypto/ecdsa"
    21  	"crypto/elliptic"
    22  	"crypto/rand"
    23  	"crypto/x509"
    24  	"crypto/x509/pkix"
    25  	"encoding/json"
    26  	"encoding/pem"
    27  	"io"
    28  	"math/big"
    29  	"net/http"
    30  	"net/http/httptest"
    31  	"os"
    32  	"path/filepath"
    33  	"sync"
    34  	"testing"
    35  	"time"
    36  
    37  	certapi "k8s.io/api/certificates/v1"
    38  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    39  	"k8s.io/apimachinery/pkg/runtime"
    40  	"k8s.io/apimachinery/pkg/types"
    41  	restclient "k8s.io/client-go/rest"
    42  	certutil "k8s.io/client-go/util/cert"
    43  	capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1"
    44  	"k8s.io/kubernetes/pkg/controller/certificates/authority"
    45  )
    46  
    47  // Test_buildClientCertificateManager validates that we can build a local client cert
    48  // manager that will use the bootstrap client until we get a valid cert, then use our
    49  // provided identity on subsequent requests.
    50  func Test_buildClientCertificateManager(t *testing.T) {
    51  	testDir, err := os.MkdirTemp("", "kubeletcert")
    52  	if err != nil {
    53  		t.Fatal(err)
    54  	}
    55  	defer func() { os.RemoveAll(testDir) }()
    56  
    57  	serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
    58  	if err != nil {
    59  		t.Fatal(err)
    60  	}
    61  	serverCA, err := certutil.NewSelfSignedCACert(certutil.Config{
    62  		CommonName: "the-test-framework",
    63  	}, serverPrivateKey)
    64  	if err != nil {
    65  		t.Fatal(err)
    66  	}
    67  	server := &csrSimulator{
    68  		t:                t,
    69  		serverPrivateKey: serverPrivateKey,
    70  		serverCA:         serverCA,
    71  	}
    72  	s := httptest.NewServer(server)
    73  	defer s.Close()
    74  
    75  	config1 := &restclient.Config{
    76  		UserAgent: "FirstClient",
    77  		Host:      s.URL,
    78  	}
    79  	config2 := &restclient.Config{
    80  		UserAgent: "SecondClient",
    81  		Host:      s.URL,
    82  	}
    83  
    84  	nodeName := types.NodeName("test")
    85  	m, err := buildClientCertificateManager(config1, config2, testDir, nodeName)
    86  	if err != nil {
    87  		t.Fatal(err)
    88  	}
    89  	defer m.Stop()
    90  	r := m.(rotater)
    91  
    92  	// get an expired CSR (simulating historical output)
    93  	server.backdate = 2 * time.Hour
    94  	server.SetExpectUserAgent("FirstClient")
    95  	ok, err := r.RotateCerts()
    96  	if !ok || err != nil {
    97  		t.Fatalf("unexpected rotation err: %t %v", ok, err)
    98  	}
    99  	if cert := m.Current(); cert != nil {
   100  		t.Fatalf("Unexpected cert, should be expired: %#v", cert)
   101  	}
   102  	fi := getFileInfo(testDir)
   103  	if len(fi) != 2 {
   104  		t.Fatalf("Unexpected directory contents: %#v", fi)
   105  	}
   106  
   107  	// if m.Current() == nil, then we try again and get a valid
   108  	// client
   109  	server.backdate = 0
   110  	server.SetExpectUserAgent("FirstClient")
   111  	if ok, err := r.RotateCerts(); !ok || err != nil {
   112  		t.Fatalf("unexpected rotation err: %t %v", ok, err)
   113  	}
   114  	if cert := m.Current(); cert == nil {
   115  		t.Fatalf("Unexpected cert, should be valid: %#v", cert)
   116  	}
   117  	fi = getFileInfo(testDir)
   118  	if len(fi) != 2 {
   119  		t.Fatalf("Unexpected directory contents: %#v", fi)
   120  	}
   121  
   122  	// if m.Current() != nil, then we should use the second client
   123  	server.SetExpectUserAgent("SecondClient")
   124  	if ok, err := r.RotateCerts(); !ok || err != nil {
   125  		t.Fatalf("unexpected rotation err: %t %v", ok, err)
   126  	}
   127  	if cert := m.Current(); cert == nil {
   128  		t.Fatalf("Unexpected cert, should be valid: %#v", cert)
   129  	}
   130  	fi = getFileInfo(testDir)
   131  	if len(fi) != 2 {
   132  		t.Fatalf("Unexpected directory contents: %#v", fi)
   133  	}
   134  }
   135  
   136  func Test_buildClientCertificateManager_populateCertDir(t *testing.T) {
   137  	testDir, err := os.MkdirTemp("", "kubeletcert")
   138  	if err != nil {
   139  		t.Fatal(err)
   140  	}
   141  	defer func() { os.RemoveAll(testDir) }()
   142  
   143  	// when no cert is provided, write nothing to disk
   144  	config1 := &restclient.Config{
   145  		UserAgent: "FirstClient",
   146  		Host:      "http://localhost",
   147  	}
   148  	config2 := &restclient.Config{
   149  		UserAgent: "SecondClient",
   150  		Host:      "http://localhost",
   151  	}
   152  	nodeName := types.NodeName("test")
   153  	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
   154  		t.Fatal(err)
   155  	}
   156  	fi := getFileInfo(testDir)
   157  	if len(fi) != 0 {
   158  		t.Fatalf("Unexpected directory contents: %#v", fi)
   159  	}
   160  
   161  	// an invalid cert should be ignored
   162  	config2.CertData = []byte("invalid contents")
   163  	config2.KeyData = []byte("invalid contents")
   164  	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err == nil {
   165  		t.Fatal("unexpected non error")
   166  	}
   167  	fi = getFileInfo(testDir)
   168  	if len(fi) != 0 {
   169  		t.Fatalf("Unexpected directory contents: %#v", fi)
   170  	}
   171  
   172  	// an expired client certificate should be written to disk, because the cert manager can
   173  	// use config1 to refresh it and the cert manager won't return it for clients.
   174  	config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour))
   175  	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
   176  		t.Fatal(err)
   177  	}
   178  	fi = getFileInfo(testDir)
   179  	if len(fi) != 2 {
   180  		t.Fatalf("Unexpected directory contents: %#v", fi)
   181  	}
   182  
   183  	// a valid, non-expired client certificate should be written to disk
   184  	config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-time.Hour), time.Now().Add(24*time.Hour))
   185  	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
   186  		t.Fatal(err)
   187  	}
   188  	fi = getFileInfo(testDir)
   189  	if len(fi) != 2 {
   190  		t.Fatalf("Unexpected directory contents: %#v", fi)
   191  	}
   192  
   193  }
   194  
   195  func getFileInfo(dir string) map[string]os.FileInfo {
   196  	fi := make(map[string]os.FileInfo)
   197  	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   198  		if path == dir {
   199  			return nil
   200  		}
   201  		fi[path] = info
   202  		if !info.IsDir() {
   203  			os.Remove(path)
   204  		}
   205  		return nil
   206  	})
   207  	return fi
   208  }
   209  
   210  type rotater interface {
   211  	RotateCerts() (bool, error)
   212  }
   213  
   214  func getCSR(req *http.Request) (*certapi.CertificateSigningRequest, error) {
   215  	if req.Body == nil {
   216  		return nil, nil
   217  	}
   218  	body, err := io.ReadAll(req.Body)
   219  	if err != nil {
   220  		return nil, err
   221  	}
   222  	csr := &certapi.CertificateSigningRequest{}
   223  	if err := json.Unmarshal(body, csr); err != nil {
   224  		return nil, err
   225  	}
   226  	return csr, nil
   227  }
   228  
   229  func mustMarshal(obj interface{}) []byte {
   230  	data, err := json.Marshal(obj)
   231  	if err != nil {
   232  		panic(err)
   233  	}
   234  	return data
   235  }
   236  
   237  type csrSimulator struct {
   238  	t *testing.T
   239  
   240  	serverPrivateKey *ecdsa.PrivateKey
   241  	serverCA         *x509.Certificate
   242  	backdate         time.Duration
   243  
   244  	userAgentLock   sync.Mutex
   245  	expectUserAgent string
   246  
   247  	lock sync.Mutex
   248  	csr  *certapi.CertificateSigningRequest
   249  }
   250  
   251  func (s *csrSimulator) SetExpectUserAgent(a string) {
   252  	s.userAgentLock.Lock()
   253  	defer s.userAgentLock.Unlock()
   254  	s.expectUserAgent = a
   255  }
   256  func (s *csrSimulator) ExpectUserAgent() string {
   257  	s.userAgentLock.Lock()
   258  	defer s.userAgentLock.Unlock()
   259  	return s.expectUserAgent
   260  }
   261  
   262  func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   263  	s.lock.Lock()
   264  	defer s.lock.Unlock()
   265  	t := s.t
   266  
   267  	// filter out timeouts as csrSimulator don't support them
   268  	q := req.URL.Query()
   269  	q.Del("timeout")
   270  	q.Del("timeoutSeconds")
   271  	q.Del("allowWatchBookmarks")
   272  	req.URL.RawQuery = q.Encode()
   273  
   274  	t.Logf("Request %q %q %q", req.Method, req.URL, req.UserAgent())
   275  
   276  	if a := s.ExpectUserAgent(); len(a) > 0 && req.UserAgent() != a {
   277  		t.Errorf("Unexpected user agent: %s", req.UserAgent())
   278  	}
   279  
   280  	switch {
   281  	case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests":
   282  		csr, err := getCSR(req)
   283  		if err != nil {
   284  			t.Fatal(err)
   285  		}
   286  		if csr.Name == "" {
   287  			csr.Name = "test-csr"
   288  		}
   289  
   290  		csr.UID = types.UID("1")
   291  		csr.ResourceVersion = "1"
   292  		data := mustMarshal(csr)
   293  		w.Header().Set("Content-Type", "application/json")
   294  		w.Write(data)
   295  
   296  		csr = csr.DeepCopy()
   297  		csr.ResourceVersion = "2"
   298  		ca := &authority.CertificateAuthority{
   299  			Certificate: s.serverCA,
   300  			PrivateKey:  s.serverPrivateKey,
   301  		}
   302  		cr, err := capihelper.ParseCSR(csr.Spec.Request)
   303  		if err != nil {
   304  			t.Fatal(err)
   305  		}
   306  		der, err := ca.Sign(cr.Raw, authority.PermissiveSigningPolicy{
   307  			TTL:      time.Hour,
   308  			Backdate: s.backdate,
   309  		})
   310  		if err != nil {
   311  			t.Fatal(err)
   312  		}
   313  		csr.Status.Certificate = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
   314  		csr.Status.Conditions = []certapi.CertificateSigningRequestCondition{
   315  			{Type: certapi.CertificateApproved},
   316  		}
   317  		s.csr = csr
   318  
   319  	case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && (req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0" || req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr"):
   320  		if s.csr == nil {
   321  			t.Fatalf("no csr")
   322  		}
   323  		csr := s.csr.DeepCopy()
   324  
   325  		data := mustMarshal(&certapi.CertificateSigningRequestList{
   326  			ListMeta: metav1.ListMeta{
   327  				ResourceVersion: "2",
   328  			},
   329  			Items: []certapi.CertificateSigningRequest{
   330  				*csr,
   331  			},
   332  		})
   333  		w.Header().Set("Content-Type", "application/json")
   334  		w.Write(data)
   335  
   336  	case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true":
   337  		if s.csr == nil {
   338  			t.Fatalf("no csr")
   339  		}
   340  		csr := s.csr.DeepCopy()
   341  
   342  		data := mustMarshal(&metav1.WatchEvent{
   343  			Type: "ADDED",
   344  			Object: runtime.RawExtension{
   345  				Raw: mustMarshal(csr),
   346  			},
   347  		})
   348  		w.Header().Set("Content-Type", "application/json")
   349  		w.Write(data)
   350  
   351  	default:
   352  		t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
   353  	}
   354  }
   355  
   356  // genClientCert generates an x509 certificate for testing. Certificate and key
   357  // are returned in PEM encoding.
   358  func genClientCert(t *testing.T, from, to time.Time) ([]byte, []byte) {
   359  	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
   360  	if err != nil {
   361  		t.Fatal(err)
   362  	}
   363  	keyRaw, err := x509.MarshalECPrivateKey(key)
   364  	if err != nil {
   365  		t.Fatal(err)
   366  	}
   367  	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
   368  	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
   369  	if err != nil {
   370  		t.Fatal(err)
   371  	}
   372  	cert := &x509.Certificate{
   373  		SerialNumber: serialNumber,
   374  		Subject:      pkix.Name{Organization: []string{"Acme Co"}},
   375  		NotBefore:    from,
   376  		NotAfter:     to,
   377  
   378  		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
   379  		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
   380  		BasicConstraintsValid: true,
   381  	}
   382  	certRaw, err := x509.CreateCertificate(rand.Reader, cert, cert, key.Public(), key)
   383  	if err != nil {
   384  		t.Fatal(err)
   385  	}
   386  	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certRaw}),
   387  		pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw})
   388  }
   389
View as plain text