token.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/discovery/token

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package token
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"fmt"
    23  	"time"
    24  
    25  	"github.com/pkg/errors"
    26  
    27  	v1 "k8s.io/api/core/v1"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/util/wait"
    30  	clientset "k8s.io/client-go/kubernetes"
    31  	"k8s.io/client-go/tools/clientcmd"
    32  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    33  	certutil "k8s.io/client-go/util/cert"
    34  	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
    35  	bootstrap "k8s.io/cluster-bootstrap/token/jws"
    36  	"k8s.io/klog/v2"
    37  
    38  	bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
    39  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    40  	kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
    41  	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
    42  	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
    43  	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
    44  )
    45  
    46  // BootstrapUser defines bootstrap user name
    47  const BootstrapUser = "token-bootstrap-client"
    48  
    49  // RetrieveValidatedConfigInfo connects to the API Server and tries to fetch the cluster-info ConfigMap
    50  // It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if CACertHashes is not empty)
    51  // validating the cluster CA against a set of pinned public keys
    52  func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery, timeout time.Duration) (*clientcmdapi.Config, error) {
    53  	return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval, timeout)
    54  }
    55  
    56  // retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo.
    57  // It accepts an optional clientset that can be used for testing purposes.
    58  func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval, timeout time.Duration) (*clientcmdapi.Config, error) {
    59  	token, err := bootstraptokenv1.NewBootstrapTokenString(cfg.BootstrapToken.Token)
    60  	if err != nil {
    61  		return nil, err
    62  	}
    63  
    64  	// Load the CACertHashes into a pubkeypin.Set
    65  	pubKeyPins := pubkeypin.NewSet()
    66  	if err = pubKeyPins.Allow(cfg.BootstrapToken.CACertHashes...); err != nil {
    67  		return nil, errors.Wrap(err, "invalid discovery token CA certificate hash")
    68  	}
    69  
    70  	// Make sure the interval is not bigger than the duration
    71  	if interval > timeout {
    72  		interval = timeout
    73  	}
    74  
    75  	endpoint := cfg.BootstrapToken.APIServerEndpoint
    76  	insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1.DefaultClusterName)
    77  	clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster
    78  
    79  	klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q", endpoint)
    80  	insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, timeout)
    81  	if err != nil {
    82  		return nil, err
    83  	}
    84  
    85  	// Validate the token in the cluster info
    86  	insecureKubeconfigBytes, err := validateClusterInfoToken(insecureClusterInfo, token)
    87  	if err != nil {
    88  		return nil, err
    89  	}
    90  
    91  	// Load the insecure config
    92  	insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes)
    93  	if err != nil {
    94  		return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
    95  	}
    96  
    97  	// The ConfigMap should contain a single cluster
    98  	if len(insecureConfig.Clusters) != 1 {
    99  		return nil, errors.Errorf("expected the kubeconfig file in the %s ConfigMap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters))
   100  	}
   101  
   102  	// If no TLS root CA pinning was specified, we're done
   103  	if pubKeyPins.Empty() {
   104  		klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q", endpoint)
   105  		return insecureConfig, nil
   106  	}
   107  
   108  	// Load and validate the cluster CA from the insecure kubeconfig
   109  	clusterCABytes, err := validateClusterCA(insecureConfig, pubKeyPins)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  
   114  	// Now that we know the cluster CA, connect back a second time validating with that CA
   115  	secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName)
   116  
   117  	klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key", endpoint)
   118  	secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, timeout)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  
   123  	// Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time
   124  	secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey])
   125  	if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) {
   126  		return nil, errors.Errorf("the second kubeconfig from the %s ConfigMap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo)
   127  	}
   128  
   129  	secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes)
   130  	if err != nil {
   131  		return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
   132  	}
   133  
   134  	klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q", endpoint)
   135  
   136  	return secureKubeconfig, nil
   137  }
   138  
   139  // buildInsecureBootstrapKubeConfig makes a kubeconfig object that connects insecurely to the API Server for bootstrapping purposes
   140  func buildInsecureBootstrapKubeConfig(endpoint, clustername string) *clientcmdapi.Config {
   141  	controlPlaneEndpoint := fmt.Sprintf("https://%s", endpoint)
   142  	bootstrapConfig := kubeconfigutil.CreateBasic(controlPlaneEndpoint, clustername, BootstrapUser, []byte{})
   143  	bootstrapConfig.Clusters[clustername].InsecureSkipTLSVerify = true
   144  	return bootstrapConfig
   145  }
   146  
   147  // buildSecureBootstrapKubeConfig makes a kubeconfig object that connects securely to the API Server for bootstrapping purposes (validating with the specified CA)
   148  func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername string) *clientcmdapi.Config {
   149  	controlPlaneEndpoint := fmt.Sprintf("https://%s", endpoint)
   150  	bootstrapConfig := kubeconfigutil.CreateBasic(controlPlaneEndpoint, clustername, BootstrapUser, caCert)
   151  	return bootstrapConfig
   152  }
   153  
   154  // validateClusterInfoToken validates that the JWS token present in the cluster info ConfigMap is valid
   155  func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *bootstraptokenv1.BootstrapTokenString) ([]byte, error) {
   156  	insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey]
   157  	if !ok || len(insecureKubeconfigString) == 0 {
   158  		return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect",
   159  			bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo)
   160  	}
   161  
   162  	detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]
   163  	if !ok || len(detachedJWSToken) == 0 {
   164  		return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID)
   165  	}
   166  
   167  	if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) {
   168  		return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server")
   169  	}
   170  
   171  	return []byte(insecureKubeconfigString), nil
   172  }
   173  
   174  // validateClusterCA validates the cluster CA found in the insecure kubeconfig
   175  func validateClusterCA(insecureConfig *clientcmdapi.Config, pubKeyPins *pubkeypin.Set) ([]byte, error) {
   176  	var clusterCABytes []byte
   177  	for _, cluster := range insecureConfig.Clusters {
   178  		clusterCABytes = cluster.CertificateAuthorityData
   179  	}
   180  
   181  	clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes)
   182  	if err != nil {
   183  		return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
   184  	}
   185  
   186  	// Validate the cluster CA public key against the pinned set
   187  	err = pubKeyPins.CheckAny(clusterCAs)
   188  	if err != nil {
   189  		return nil, errors.Wrapf(err, "cluster CA found in %s ConfigMap is invalid", bootstrapapi.ConfigMapClusterInfo)
   190  	}
   191  
   192  	return clusterCABytes, nil
   193  }
   194  
   195  // getClusterInfo creates a client from the given kubeconfig if the given client is nil,
   196  // and requests the cluster info ConfigMap using PollImmediate.
   197  // If a client is provided it will be used instead.
   198  func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *bootstraptokenv1.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) {
   199  	var cm *v1.ConfigMap
   200  	var err error
   201  
   202  	// Create client from kubeconfig
   203  	if client == nil {
   204  		client, err = kubeconfigutil.ToClientSet(kubeconfig)
   205  		if err != nil {
   206  			return nil, err
   207  		}
   208  	}
   209  
   210  	klog.V(1).Infof("[discovery] Waiting for the cluster-info ConfigMap to receive a JWS signature"+
   211  		"for token ID %q", token.ID)
   212  
   213  	var lastError error
   214  	err = wait.PollUntilContextTimeout(context.Background(),
   215  		interval, duration, true,
   216  		func(ctx context.Context) (bool, error) {
   217  			cm, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).
   218  				Get(ctx, bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
   219  			if err != nil {
   220  				lastError = errors.Wrapf(err, "failed to request the cluster-info ConfigMap")
   221  				klog.V(1).Infof("[discovery] Retrying due to error: %v", lastError)
   222  				return false, nil
   223  			}
   224  			// Even if the ConfigMap is available the JWS signature is patched-in a bit later.
   225  			if _, ok := cm.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]; !ok {
   226  				lastError = errors.Errorf("could not find a JWS signature in the cluster-info ConfigMap"+
   227  					" for token ID %q", token.ID)
   228  				klog.V(1).Infof("[discovery] Retrying due to error: %v", lastError)
   229  				return false, nil
   230  			}
   231  			return true, nil
   232  		})
   233  	if err != nil {
   234  		return nil, lastError
   235  	}
   236  
   237  	return cm, nil
   238  }
   239
View as plain text