certificate_manager.go

Documentation: k8s.io/client-go/util/certificate

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package certificate
    18  
    19  import (
    20  	"context"
    21  	"crypto/ecdsa"
    22  	"crypto/elliptic"
    23  	cryptorand "crypto/rand"
    24  	"crypto/rsa"
    25  	"crypto/tls"
    26  	"crypto/x509"
    27  	"encoding/pem"
    28  	"errors"
    29  	"fmt"
    30  	"reflect"
    31  	"sync"
    32  	"time"
    33  
    34  	"k8s.io/klog/v2"
    35  
    36  	certificates "k8s.io/api/certificates/v1"
    37  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    38  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    39  	"k8s.io/apimachinery/pkg/util/sets"
    40  	"k8s.io/apimachinery/pkg/util/wait"
    41  	clientset "k8s.io/client-go/kubernetes"
    42  	"k8s.io/client-go/util/cert"
    43  	"k8s.io/client-go/util/certificate/csr"
    44  	"k8s.io/client-go/util/keyutil"
    45  )
    46  
    47  var (
    48  	// certificateWaitTimeout controls the amount of time we wait for certificate
    49  	// approval in one iteration.
    50  	certificateWaitTimeout = 15 * time.Minute
    51  
    52  	kubeletServingUsagesWithEncipherment = []certificates.KeyUsage{
    53  		// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
    54  		//
    55  		// Digital signature allows the certificate to be used to verify
    56  		// digital signatures used during TLS negotiation.
    57  		certificates.UsageDigitalSignature,
    58  		// KeyEncipherment allows the cert/key pair to be used to encrypt
    59  		// keys, including the symmetric keys negotiated during TLS setup
    60  		// and used for data transfer.
    61  		certificates.UsageKeyEncipherment,
    62  		// ServerAuth allows the cert to be used by a TLS server to
    63  		// authenticate itself to a TLS client.
    64  		certificates.UsageServerAuth,
    65  	}
    66  	kubeletServingUsagesNoEncipherment = []certificates.KeyUsage{
    67  		// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
    68  		//
    69  		// Digital signature allows the certificate to be used to verify
    70  		// digital signatures used during TLS negotiation.
    71  		certificates.UsageDigitalSignature,
    72  		// ServerAuth allows the cert to be used by a TLS server to
    73  		// authenticate itself to a TLS client.
    74  		certificates.UsageServerAuth,
    75  	}
    76  	DefaultKubeletServingGetUsages = func(privateKey interface{}) []certificates.KeyUsage {
    77  		switch privateKey.(type) {
    78  		case *rsa.PrivateKey:
    79  			return kubeletServingUsagesWithEncipherment
    80  		default:
    81  			return kubeletServingUsagesNoEncipherment
    82  		}
    83  	}
    84  	kubeletClientUsagesWithEncipherment = []certificates.KeyUsage{
    85  		// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
    86  		//
    87  		// Digital signature allows the certificate to be used to verify
    88  		// digital signatures used during TLS negotiation.
    89  		certificates.UsageDigitalSignature,
    90  		// KeyEncipherment allows the cert/key pair to be used to encrypt
    91  		// keys, including the symmetric keys negotiated during TLS setup
    92  		// and used for data transfer.
    93  		certificates.UsageKeyEncipherment,
    94  		// ClientAuth allows the cert to be used by a TLS client to
    95  		// authenticate itself to the TLS server.
    96  		certificates.UsageClientAuth,
    97  	}
    98  	kubeletClientUsagesNoEncipherment = []certificates.KeyUsage{
    99  		// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
   100  		//
   101  		// Digital signature allows the certificate to be used to verify
   102  		// digital signatures used during TLS negotiation.
   103  		certificates.UsageDigitalSignature,
   104  		// ClientAuth allows the cert to be used by a TLS client to
   105  		// authenticate itself to the TLS server.
   106  		certificates.UsageClientAuth,
   107  	}
   108  	DefaultKubeletClientGetUsages = func(privateKey interface{}) []certificates.KeyUsage {
   109  		switch privateKey.(type) {
   110  		case *rsa.PrivateKey:
   111  			return kubeletClientUsagesWithEncipherment
   112  		default:
   113  			return kubeletClientUsagesNoEncipherment
   114  		}
   115  	}
   116  )
   117  
   118  // Manager maintains and updates the certificates in use by this certificate
   119  // manager. In the background it communicates with the API server to get new
   120  // certificates for certificates about to expire.
   121  type Manager interface {
   122  	// Start the API server status sync loop.
   123  	Start()
   124  	// Stop the cert manager loop.
   125  	Stop()
   126  	// Current returns the currently selected certificate from the
   127  	// certificate manager, as well as the associated certificate and key data
   128  	// in PEM format.
   129  	Current() *tls.Certificate
   130  	// ServerHealthy returns true if the manager is able to communicate with
   131  	// the server. This allows a caller to determine whether the cert manager
   132  	// thinks it can potentially talk to the API server. The cert manager may
   133  	// be very conservative and only return true if recent communication has
   134  	// occurred with the server.
   135  	ServerHealthy() bool
   136  }
   137  
   138  // Config is the set of configuration parameters available for a new Manager.
   139  type Config struct {
   140  	// ClientsetFn will be used to create a clientset for
   141  	// creating/fetching new certificate requests generated when a key rotation occurs.
   142  	// The function will never be invoked in parallel.
   143  	// It is passed the current client certificate if one exists.
   144  	ClientsetFn ClientsetFunc
   145  	// Template is the CertificateRequest that will be used as a template for
   146  	// generating certificate signing requests for all new keys generated as
   147  	// part of rotation. It follows the same rules as the template parameter of
   148  	// crypto.x509.CreateCertificateRequest in the Go standard libraries.
   149  	Template *x509.CertificateRequest
   150  	// GetTemplate returns the CertificateRequest that will be used as a template for
   151  	// generating certificate signing requests for all new keys generated as
   152  	// part of rotation. It follows the same rules as the template parameter of
   153  	// crypto.x509.CreateCertificateRequest in the Go standard libraries.
   154  	// If no template is available, nil may be returned, and no certificate will be requested.
   155  	// If specified, takes precedence over Template.
   156  	GetTemplate func() *x509.CertificateRequest
   157  	// SignerName is the name of the certificate signer that should sign certificates
   158  	// generated by the manager.
   159  	SignerName string
   160  	// RequestedCertificateLifetime is the requested lifetime length for certificates generated by the manager.
   161  	// Optional.
   162  	// This will set the spec.expirationSeconds field on the CSR.  Controlling the lifetime of
   163  	// the issued certificate is not guaranteed as the signer may choose to ignore the request.
   164  	RequestedCertificateLifetime *time.Duration
   165  	// Usages is the types of usages that certificates generated by the manager
   166  	// can be used for. It is mutually exclusive with GetUsages.
   167  	Usages []certificates.KeyUsage
   168  	// GetUsages is dynamic way to get the types of usages that certificates generated by the manager
   169  	// can be used for. If Usages is not nil, GetUsages has to be nil, vice versa.
   170  	// It is mutually exclusive with Usages.
   171  	GetUsages func(privateKey interface{}) []certificates.KeyUsage
   172  	// CertificateStore is a persistent store where the current cert/key is
   173  	// kept and future cert/key pairs will be persisted after they are
   174  	// generated.
   175  	CertificateStore Store
   176  	// BootstrapCertificatePEM is the certificate data that will be returned
   177  	// from the Manager if the CertificateStore doesn't have any cert/key pairs
   178  	// currently available and has not yet had a chance to get a new cert/key
   179  	// pair from the API. If the CertificateStore does have a cert/key pair,
   180  	// this will be ignored. If there is no cert/key pair available in the
   181  	// CertificateStore, as soon as Start is called, it will request a new
   182  	// cert/key pair from the CertificateSigningRequestClient. This is intended
   183  	// to allow the first boot of a component to be initialized using a
   184  	// generic, multi-use cert/key pair which will be quickly replaced with a
   185  	// unique cert/key pair.
   186  	BootstrapCertificatePEM []byte
   187  	// BootstrapKeyPEM is the key data that will be returned from the Manager
   188  	// if the CertificateStore doesn't have any cert/key pairs currently
   189  	// available. If the CertificateStore does have a cert/key pair, this will
   190  	// be ignored. If the bootstrap cert/key pair are used, they will be
   191  	// rotated at the first opportunity, possibly well in advance of expiring.
   192  	// This is intended to allow the first boot of a component to be
   193  	// initialized using a generic, multi-use cert/key pair which will be
   194  	// quickly replaced with a unique cert/key pair.
   195  	BootstrapKeyPEM []byte `datapolicy:"security-key"`
   196  	// CertificateRotation will record a metric showing the time in seconds
   197  	// that certificates lived before being rotated. This metric is a histogram
   198  	// because there is value in keeping a history of rotation cadences. It
   199  	// allows one to setup monitoring and alerting of unexpected rotation
   200  	// behavior and track trends in rotation frequency.
   201  	CertificateRotation Histogram
   202  	// CertifcateRenewFailure will record a metric that keeps track of
   203  	// certificate renewal failures.
   204  	CertificateRenewFailure Counter
   205  	// Name is an optional string that will be used when writing log output
   206  	// or returning errors from manager methods. If not set, SignerName will
   207  	// be used, if SignerName is not set, if Usages includes client auth the
   208  	// name will be "client auth", otherwise the value will be "server".
   209  	Name string
   210  	// Logf is an optional function that log output will be sent to from the
   211  	// certificate manager. If not set it will use klog.V(2)
   212  	Logf func(format string, args ...interface{})
   213  }
   214  
   215  // Store is responsible for getting and updating the current certificate.
   216  // Depending on the concrete implementation, the backing store for this
   217  // behavior may vary.
   218  type Store interface {
   219  	// Current returns the currently selected certificate, as well as the
   220  	// associated certificate and key data in PEM format. If the Store doesn't
   221  	// have a cert/key pair currently, it should return a NoCertKeyError so
   222  	// that the Manager can recover by using bootstrap certificates to request
   223  	// a new cert/key pair.
   224  	Current() (*tls.Certificate, error)
   225  	// Update accepts the PEM data for the cert/key pair and makes the new
   226  	// cert/key pair the 'current' pair, that will be returned by future calls
   227  	// to Current().
   228  	Update(cert, key []byte) (*tls.Certificate, error)
   229  }
   230  
   231  // Gauge will record the remaining lifetime of the certificate each time it is
   232  // updated.
   233  type Gauge interface {
   234  	Set(float64)
   235  }
   236  
   237  // Histogram will record the time a rotated certificate was used before being
   238  // rotated.
   239  type Histogram interface {
   240  	Observe(float64)
   241  }
   242  
   243  // Counter will wrap a counter with labels
   244  type Counter interface {
   245  	Inc()
   246  }
   247  
   248  // NoCertKeyError indicates there is no cert/key currently available.
   249  type NoCertKeyError string
   250  
   251  // ClientsetFunc returns a new clientset for discovering CSR API availability and requesting CSRs.
   252  // It is passed the current certificate if one is available and valid.
   253  type ClientsetFunc func(current *tls.Certificate) (clientset.Interface, error)
   254  
   255  func (e *NoCertKeyError) Error() string { return string(*e) }
   256  
   257  type manager struct {
   258  	getTemplate func() *x509.CertificateRequest
   259  
   260  	// lastRequestLock guards lastRequestCancel and lastRequest
   261  	lastRequestLock   sync.Mutex
   262  	lastRequestCancel context.CancelFunc
   263  	lastRequest       *x509.CertificateRequest
   264  
   265  	dynamicTemplate              bool
   266  	signerName                   string
   267  	requestedCertificateLifetime *time.Duration
   268  	getUsages                    func(privateKey interface{}) []certificates.KeyUsage
   269  	forceRotation                bool
   270  
   271  	certStore Store
   272  
   273  	certificateRotation     Histogram
   274  	certificateRenewFailure Counter
   275  
   276  	// the following variables must only be accessed under certAccessLock
   277  	certAccessLock sync.RWMutex
   278  	cert           *tls.Certificate
   279  	serverHealth   bool
   280  
   281  	// the clientFn must only be accessed under the clientAccessLock
   282  	clientAccessLock sync.Mutex
   283  	clientsetFn      ClientsetFunc
   284  	stopCh           chan struct{}
   285  	stopped          bool
   286  
   287  	// Set to time.Now but can be stubbed out for testing
   288  	now func() time.Time
   289  
   290  	name string
   291  	logf func(format string, args ...interface{})
   292  }
   293  
   294  // NewManager returns a new certificate manager. A certificate manager is
   295  // responsible for being the authoritative source of certificates in the
   296  // Kubelet and handling updates due to rotation.
   297  func NewManager(config *Config) (Manager, error) {
   298  	cert, forceRotation, err := getCurrentCertificateOrBootstrap(
   299  		config.CertificateStore,
   300  		config.BootstrapCertificatePEM,
   301  		config.BootstrapKeyPEM)
   302  	if err != nil {
   303  		return nil, err
   304  	}
   305  
   306  	getTemplate := config.GetTemplate
   307  	if getTemplate == nil {
   308  		getTemplate = func() *x509.CertificateRequest { return config.Template }
   309  	}
   310  
   311  	if config.GetUsages != nil && config.Usages != nil {
   312  		return nil, errors.New("cannot specify both GetUsages and Usages")
   313  	}
   314  	if config.GetUsages == nil && config.Usages == nil {
   315  		return nil, errors.New("either GetUsages or Usages should be specified")
   316  	}
   317  	var getUsages func(interface{}) []certificates.KeyUsage
   318  	if config.GetUsages != nil {
   319  		getUsages = config.GetUsages
   320  	} else {
   321  		getUsages = func(interface{}) []certificates.KeyUsage { return config.Usages }
   322  	}
   323  	m := manager{
   324  		stopCh:                       make(chan struct{}),
   325  		clientsetFn:                  config.ClientsetFn,
   326  		getTemplate:                  getTemplate,
   327  		dynamicTemplate:              config.GetTemplate != nil,
   328  		signerName:                   config.SignerName,
   329  		requestedCertificateLifetime: config.RequestedCertificateLifetime,
   330  		getUsages:                    getUsages,
   331  		certStore:                    config.CertificateStore,
   332  		cert:                         cert,
   333  		forceRotation:                forceRotation,
   334  		certificateRotation:          config.CertificateRotation,
   335  		certificateRenewFailure:      config.CertificateRenewFailure,
   336  		now:                          time.Now,
   337  	}
   338  
   339  	name := config.Name
   340  	if len(name) == 0 {
   341  		name = m.signerName
   342  	}
   343  	if len(name) == 0 {
   344  		usages := getUsages(nil)
   345  		switch {
   346  		case hasKeyUsage(usages, certificates.UsageClientAuth):
   347  			name = string(certificates.UsageClientAuth)
   348  		default:
   349  			name = "certificate"
   350  		}
   351  	}
   352  
   353  	m.name = name
   354  	m.logf = config.Logf
   355  	if m.logf == nil {
   356  		m.logf = func(format string, args ...interface{}) { klog.V(2).Infof(format, args...) }
   357  	}
   358  
   359  	return &m, nil
   360  }
   361  
   362  // Current returns the currently selected certificate from the certificate
   363  // manager. This can be nil if the manager was initialized without a
   364  // certificate and has not yet received one from the
   365  // CertificateSigningRequestClient, or if the current cert has expired.
   366  func (m *manager) Current() *tls.Certificate {
   367  	m.certAccessLock.RLock()
   368  	defer m.certAccessLock.RUnlock()
   369  	if m.cert != nil && m.cert.Leaf != nil && m.now().After(m.cert.Leaf.NotAfter) {
   370  		m.logf("%s: Current certificate is expired", m.name)
   371  		return nil
   372  	}
   373  	return m.cert
   374  }
   375  
   376  // ServerHealthy returns true if the cert manager believes the server
   377  // is currently alive.
   378  func (m *manager) ServerHealthy() bool {
   379  	m.certAccessLock.RLock()
   380  	defer m.certAccessLock.RUnlock()
   381  	return m.serverHealth
   382  }
   383  
   384  // Stop terminates the manager.
   385  func (m *manager) Stop() {
   386  	m.clientAccessLock.Lock()
   387  	defer m.clientAccessLock.Unlock()
   388  	if m.stopped {
   389  		return
   390  	}
   391  	close(m.stopCh)
   392  	m.stopped = true
   393  }
   394  
   395  // Start will start the background work of rotating the certificates.
   396  func (m *manager) Start() {
   397  	// Certificate rotation depends on access to the API server certificate
   398  	// signing API, so don't start the certificate manager if we don't have a
   399  	// client.
   400  	if m.clientsetFn == nil {
   401  		m.logf("%s: Certificate rotation is not enabled, no connection to the apiserver", m.name)
   402  		return
   403  	}
   404  	m.logf("%s: Certificate rotation is enabled", m.name)
   405  
   406  	templateChanged := make(chan struct{})
   407  	go wait.Until(func() {
   408  		deadline := m.nextRotationDeadline()
   409  		if sleepInterval := deadline.Sub(m.now()); sleepInterval > 0 {
   410  			m.logf("%s: Waiting %v for next certificate rotation", m.name, sleepInterval)
   411  
   412  			timer := time.NewTimer(sleepInterval)
   413  			defer timer.Stop()
   414  
   415  			select {
   416  			case <-timer.C:
   417  				// unblock when deadline expires
   418  			case <-templateChanged:
   419  				_, lastRequestTemplate := m.getLastRequest()
   420  				if reflect.DeepEqual(lastRequestTemplate, m.getTemplate()) {
   421  					// if the template now matches what we last requested, restart the rotation deadline loop
   422  					return
   423  				}
   424  				m.logf("%s: Certificate template changed, rotating", m.name)
   425  			}
   426  		}
   427  
   428  		// Don't enter rotateCerts and trigger backoff if we don't even have a template to request yet
   429  		if m.getTemplate() == nil {
   430  			return
   431  		}
   432  
   433  		backoff := wait.Backoff{
   434  			Duration: 2 * time.Second,
   435  			Factor:   2,
   436  			Jitter:   0.1,
   437  			Steps:    5,
   438  		}
   439  		if err := wait.ExponentialBackoff(backoff, m.rotateCerts); err != nil {
   440  			utilruntime.HandleError(fmt.Errorf("%s: Reached backoff limit, still unable to rotate certs: %v", m.name, err))
   441  			wait.PollInfinite(32*time.Second, m.rotateCerts)
   442  		}
   443  	}, time.Second, m.stopCh)
   444  
   445  	if m.dynamicTemplate {
   446  		go wait.Until(func() {
   447  			// check if the current template matches what we last requested
   448  			lastRequestCancel, lastRequestTemplate := m.getLastRequest()
   449  
   450  			if !m.certSatisfiesTemplate() && !reflect.DeepEqual(lastRequestTemplate, m.getTemplate()) {
   451  				// if the template is different, queue up an interrupt of the rotation deadline loop.
   452  				// if we've requested a CSR that matches the new template by the time the interrupt is handled, the interrupt is disregarded.
   453  				if lastRequestCancel != nil {
   454  					// if we're currently waiting on a submitted request that no longer matches what we want, stop waiting
   455  					lastRequestCancel()
   456  				}
   457  				select {
   458  				case templateChanged <- struct{}{}:
   459  				case <-m.stopCh:
   460  				}
   461  			}
   462  		}, time.Second, m.stopCh)
   463  	}
   464  }
   465  
   466  func getCurrentCertificateOrBootstrap(
   467  	store Store,
   468  	bootstrapCertificatePEM []byte,
   469  	bootstrapKeyPEM []byte) (cert *tls.Certificate, shouldRotate bool, errResult error) {
   470  
   471  	currentCert, err := store.Current()
   472  	if err == nil {
   473  		// if the current cert is expired, fall back to the bootstrap cert
   474  		if currentCert.Leaf != nil && time.Now().Before(currentCert.Leaf.NotAfter) {
   475  			return currentCert, false, nil
   476  		}
   477  	} else {
   478  		if _, ok := err.(*NoCertKeyError); !ok {
   479  			return nil, false, err
   480  		}
   481  	}
   482  
   483  	if bootstrapCertificatePEM == nil || bootstrapKeyPEM == nil {
   484  		return nil, true, nil
   485  	}
   486  
   487  	bootstrapCert, err := tls.X509KeyPair(bootstrapCertificatePEM, bootstrapKeyPEM)
   488  	if err != nil {
   489  		return nil, false, err
   490  	}
   491  	if len(bootstrapCert.Certificate) < 1 {
   492  		return nil, false, fmt.Errorf("no cert/key data found")
   493  	}
   494  
   495  	certs, err := x509.ParseCertificates(bootstrapCert.Certificate[0])
   496  	if err != nil {
   497  		return nil, false, fmt.Errorf("unable to parse certificate data: %v", err)
   498  	}
   499  	if len(certs) < 1 {
   500  		return nil, false, fmt.Errorf("no cert data found")
   501  	}
   502  	bootstrapCert.Leaf = certs[0]
   503  
   504  	if _, err := store.Update(bootstrapCertificatePEM, bootstrapKeyPEM); err != nil {
   505  		utilruntime.HandleError(fmt.Errorf("unable to set the cert/key pair to the bootstrap certificate: %v", err))
   506  	}
   507  
   508  	return &bootstrapCert, true, nil
   509  }
   510  
   511  func (m *manager) getClientset() (clientset.Interface, error) {
   512  	current := m.Current()
   513  	m.clientAccessLock.Lock()
   514  	defer m.clientAccessLock.Unlock()
   515  	return m.clientsetFn(current)
   516  }
   517  
   518  // RotateCerts is exposed for testing only and is not a part of the public interface.
   519  // Returns true if it changed the cert, false otherwise. Error is only returned in
   520  // exceptional cases.
   521  func (m *manager) RotateCerts() (bool, error) {
   522  	return m.rotateCerts()
   523  }
   524  
   525  // rotateCerts attempts to request a client cert from the server, wait a reasonable
   526  // period of time for it to be signed, and then update the cert on disk. If it cannot
   527  // retrieve a cert, it will return false. It will only return error in exceptional cases.
   528  // This method also keeps track of "server health" by interpreting the responses it gets
   529  // from the server on the various calls it makes.
   530  // TODO: return errors, have callers handle and log them correctly
   531  func (m *manager) rotateCerts() (bool, error) {
   532  	m.logf("%s: Rotating certificates", m.name)
   533  
   534  	template, csrPEM, keyPEM, privateKey, err := m.generateCSR()
   535  	if err != nil {
   536  		utilruntime.HandleError(fmt.Errorf("%s: Unable to generate a certificate signing request: %v", m.name, err))
   537  		if m.certificateRenewFailure != nil {
   538  			m.certificateRenewFailure.Inc()
   539  		}
   540  		return false, nil
   541  	}
   542  
   543  	// request the client each time
   544  	clientSet, err := m.getClientset()
   545  	if err != nil {
   546  		utilruntime.HandleError(fmt.Errorf("%s: Unable to load a client to request certificates: %v", m.name, err))
   547  		if m.certificateRenewFailure != nil {
   548  			m.certificateRenewFailure.Inc()
   549  		}
   550  		return false, nil
   551  	}
   552  
   553  	getUsages := m.getUsages
   554  	if m.getUsages == nil {
   555  		getUsages = DefaultKubeletClientGetUsages
   556  	}
   557  	usages := getUsages(privateKey)
   558  	// Call the Certificate Signing Request API to get a certificate for the
   559  	// new private key
   560  	reqName, reqUID, err := csr.RequestCertificate(clientSet, csrPEM, "", m.signerName, m.requestedCertificateLifetime, usages, privateKey)
   561  	if err != nil {
   562  		utilruntime.HandleError(fmt.Errorf("%s: Failed while requesting a signed certificate from the control plane: %v", m.name, err))
   563  		if m.certificateRenewFailure != nil {
   564  			m.certificateRenewFailure.Inc()
   565  		}
   566  		return false, m.updateServerError(err)
   567  	}
   568  
   569  	ctx, cancel := context.WithTimeout(context.Background(), certificateWaitTimeout)
   570  	defer cancel()
   571  
   572  	// Once we've successfully submitted a CSR for this template, record that we did so
   573  	m.setLastRequest(cancel, template)
   574  
   575  	// Wait for the certificate to be signed. This interface and internal timout
   576  	// is a remainder after the old design using raw watch wrapped with backoff.
   577  	crtPEM, err := csr.WaitForCertificate(ctx, clientSet, reqName, reqUID)
   578  	if err != nil {
   579  		utilruntime.HandleError(fmt.Errorf("%s: certificate request was not signed: %v", m.name, err))
   580  		if m.certificateRenewFailure != nil {
   581  			m.certificateRenewFailure.Inc()
   582  		}
   583  		return false, nil
   584  	}
   585  
   586  	cert, err := m.certStore.Update(crtPEM, keyPEM)
   587  	if err != nil {
   588  		utilruntime.HandleError(fmt.Errorf("%s: Unable to store the new cert/key pair: %v", m.name, err))
   589  		if m.certificateRenewFailure != nil {
   590  			m.certificateRenewFailure.Inc()
   591  		}
   592  		return false, nil
   593  	}
   594  
   595  	if old := m.updateCached(cert); old != nil && m.certificateRotation != nil {
   596  		m.certificateRotation.Observe(m.now().Sub(old.Leaf.NotBefore).Seconds())
   597  	}
   598  
   599  	return true, nil
   600  }
   601  
   602  // Check that the current certificate on disk satisfies the requests from the
   603  // current template.
   604  //
   605  // Note that extra items in the certificate's SAN or orgs that don't exist in
   606  // the template will not trigger a renewal.
   607  //
   608  // Requires certAccessLock to be locked.
   609  func (m *manager) certSatisfiesTemplateLocked() bool {
   610  	if m.cert == nil {
   611  		return false
   612  	}
   613  
   614  	if template := m.getTemplate(); template != nil {
   615  		if template.Subject.CommonName != m.cert.Leaf.Subject.CommonName {
   616  			m.logf("%s: Current certificate CN (%s) does not match requested CN (%s)", m.name, m.cert.Leaf.Subject.CommonName, template.Subject.CommonName)
   617  			return false
   618  		}
   619  
   620  		currentDNSNames := sets.NewString(m.cert.Leaf.DNSNames...)
   621  		desiredDNSNames := sets.NewString(template.DNSNames...)
   622  		missingDNSNames := desiredDNSNames.Difference(currentDNSNames)
   623  		if len(missingDNSNames) > 0 {
   624  			m.logf("%s: Current certificate is missing requested DNS names %v", m.name, missingDNSNames.List())
   625  			return false
   626  		}
   627  
   628  		currentIPs := sets.NewString()
   629  		for _, ip := range m.cert.Leaf.IPAddresses {
   630  			currentIPs.Insert(ip.String())
   631  		}
   632  		desiredIPs := sets.NewString()
   633  		for _, ip := range template.IPAddresses {
   634  			desiredIPs.Insert(ip.String())
   635  		}
   636  		missingIPs := desiredIPs.Difference(currentIPs)
   637  		if len(missingIPs) > 0 {
   638  			m.logf("%s: Current certificate is missing requested IP addresses %v", m.name, missingIPs.List())
   639  			return false
   640  		}
   641  
   642  		currentOrgs := sets.NewString(m.cert.Leaf.Subject.Organization...)
   643  		desiredOrgs := sets.NewString(template.Subject.Organization...)
   644  		missingOrgs := desiredOrgs.Difference(currentOrgs)
   645  		if len(missingOrgs) > 0 {
   646  			m.logf("%s: Current certificate is missing requested orgs %v", m.name, missingOrgs.List())
   647  			return false
   648  		}
   649  	}
   650  
   651  	return true
   652  }
   653  
   654  func (m *manager) certSatisfiesTemplate() bool {
   655  	m.certAccessLock.RLock()
   656  	defer m.certAccessLock.RUnlock()
   657  	return m.certSatisfiesTemplateLocked()
   658  }
   659  
   660  // nextRotationDeadline returns a value for the threshold at which the
   661  // current certificate should be rotated, 80%+/-10% of the expiration of the
   662  // certificate.
   663  func (m *manager) nextRotationDeadline() time.Time {
   664  	// forceRotation is not protected by locks
   665  	if m.forceRotation {
   666  		m.forceRotation = false
   667  		return m.now()
   668  	}
   669  
   670  	m.certAccessLock.RLock()
   671  	defer m.certAccessLock.RUnlock()
   672  
   673  	if !m.certSatisfiesTemplateLocked() {
   674  		return m.now()
   675  	}
   676  
   677  	notAfter := m.cert.Leaf.NotAfter
   678  	totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore))
   679  	deadline := m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration))
   680  
   681  	m.logf("%s: Certificate expiration is %v, rotation deadline is %v", m.name, notAfter, deadline)
   682  	return deadline
   683  }
   684  
   685  // jitteryDuration uses some jitter to set the rotation threshold so each node
   686  // will rotate at approximately 70-90% of the total lifetime of the
   687  // certificate.  With jitter, if a number of nodes are added to a cluster at
   688  // approximately the same time (such as cluster creation time), they won't all
   689  // try to rotate certificates at the same time for the rest of the life of the
   690  // cluster.
   691  //
   692  // This function is represented as a variable to allow replacement during testing.
   693  var jitteryDuration = func(totalDuration float64) time.Duration {
   694  	return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
   695  }
   696  
   697  // updateCached sets the most recent retrieved cert and returns the old cert.
   698  // It also sets the server as assumed healthy.
   699  func (m *manager) updateCached(cert *tls.Certificate) *tls.Certificate {
   700  	m.certAccessLock.Lock()
   701  	defer m.certAccessLock.Unlock()
   702  	m.serverHealth = true
   703  	old := m.cert
   704  	m.cert = cert
   705  	return old
   706  }
   707  
   708  // updateServerError takes an error returned by the server and infers
   709  // the health of the server based on the error. It will return nil if
   710  // the error does not require immediate termination of any wait loops,
   711  // and otherwise it will return the error.
   712  func (m *manager) updateServerError(err error) error {
   713  	m.certAccessLock.Lock()
   714  	defer m.certAccessLock.Unlock()
   715  	switch {
   716  	case apierrors.IsUnauthorized(err):
   717  		// SSL terminating proxies may report this error instead of the master
   718  		m.serverHealth = true
   719  	case apierrors.IsUnexpectedServerError(err):
   720  		// generally indicates a proxy or other load balancer problem, rather than a problem coming
   721  		// from the master
   722  		m.serverHealth = false
   723  	default:
   724  		// Identify known errors that could be expected for a cert request that
   725  		// indicate everything is working normally
   726  		m.serverHealth = apierrors.IsNotFound(err) || apierrors.IsForbidden(err)
   727  	}
   728  	return nil
   729  }
   730  
   731  func (m *manager) generateCSR() (template *x509.CertificateRequest, csrPEM []byte, keyPEM []byte, key interface{}, err error) {
   732  	// Generate a new private key.
   733  	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
   734  	if err != nil {
   735  		return nil, nil, nil, nil, fmt.Errorf("%s: unable to generate a new private key: %v", m.name, err)
   736  	}
   737  	der, err := x509.MarshalECPrivateKey(privateKey)
   738  	if err != nil {
   739  		return nil, nil, nil, nil, fmt.Errorf("%s: unable to marshal the new key to DER: %v", m.name, err)
   740  	}
   741  
   742  	keyPEM = pem.EncodeToMemory(&pem.Block{Type: keyutil.ECPrivateKeyBlockType, Bytes: der})
   743  
   744  	template = m.getTemplate()
   745  	if template == nil {
   746  		return nil, nil, nil, nil, fmt.Errorf("%s: unable to create a csr, no template available", m.name)
   747  	}
   748  	csrPEM, err = cert.MakeCSRFromTemplate(privateKey, template)
   749  	if err != nil {
   750  		return nil, nil, nil, nil, fmt.Errorf("%s: unable to create a csr from the private key: %v", m.name, err)
   751  	}
   752  	return template, csrPEM, keyPEM, privateKey, nil
   753  }
   754  
   755  func (m *manager) getLastRequest() (context.CancelFunc, *x509.CertificateRequest) {
   756  	m.lastRequestLock.Lock()
   757  	defer m.lastRequestLock.Unlock()
   758  	return m.lastRequestCancel, m.lastRequest
   759  }
   760  
   761  func (m *manager) setLastRequest(cancel context.CancelFunc, r *x509.CertificateRequest) {
   762  	m.lastRequestLock.Lock()
   763  	defer m.lastRequestLock.Unlock()
   764  	m.lastRequestCancel = cancel
   765  	m.lastRequest = r
   766  }
   767  
   768  func hasKeyUsage(usages []certificates.KeyUsage, usage certificates.KeyUsage) bool {
   769  	for _, u := range usages {
   770  		if u == usage {
   771  			return true
   772  		}
   773  	}
   774  	return false
   775  }
   776
View as plain text