kubeconfig.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubeconfig
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto"
    23  	"crypto/x509"
    24  	"fmt"
    25  	"io"
    26  	"os"
    27  	"path/filepath"
    28  	"time"
    29  
    30  	"github.com/pkg/errors"
    31  
    32  	rbac "k8s.io/api/rbac/v1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  	clientset "k8s.io/client-go/kubernetes"
    37  	"k8s.io/client-go/tools/clientcmd"
    38  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    39  	certutil "k8s.io/client-go/util/cert"
    40  	"k8s.io/client-go/util/keyutil"
    41  	"k8s.io/klog/v2"
    42  
    43  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    44  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    45  	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
    46  	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
    47  	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
    48  	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
    49  )
    50  
    51  const (
    52  	errInvalid = "invalid argument"
    53  	errExist   = "file already exists"
    54  )
    55  
    56  // clientCertAuth struct holds info required to build a client certificate to provide authentication info in a kubeconfig object
    57  type clientCertAuth struct {
    58  	CAKey         crypto.Signer
    59  	Organizations []string
    60  }
    61  
    62  // tokenAuth struct holds info required to use a token to provide authentication info in a kubeconfig object
    63  type tokenAuth struct {
    64  	Token string `datapolicy:"token"`
    65  }
    66  
    67  // kubeConfigSpec struct holds info required to build a KubeConfig object
    68  type kubeConfigSpec struct {
    69  	CACert         *x509.Certificate
    70  	APIServer      string
    71  	ClientName     string
    72  	TokenAuth      *tokenAuth      `datapolicy:"token"`
    73  	ClientCertAuth *clientCertAuth `datapolicy:"security-key"`
    74  }
    75  
    76  // CreateJoinControlPlaneKubeConfigFiles will create and write to disk the kubeconfig files required by kubeadm
    77  // join --control-plane workflow, plus the admin kubeconfig file used by the administrator and kubeadm itself; the
    78  // kubelet.conf file must not be created because it will be created and signed by the kubelet TLS bootstrap process.
    79  // When not using external CA mode, if a kubeconfig file already exists it is used only if evaluated equal,
    80  // otherwise an error is returned. For external CA mode, the creation of kubeconfig files is skipped.
    81  func CreateJoinControlPlaneKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error {
    82  	var externalCA bool
    83  	caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)
    84  	if _, err := os.Stat(caKeyPath); os.IsNotExist(err) {
    85  		externalCA = true
    86  	}
    87  
    88  	files := []string{
    89  		kubeadmconstants.AdminKubeConfigFileName,
    90  		kubeadmconstants.ControllerManagerKubeConfigFileName,
    91  		kubeadmconstants.SchedulerKubeConfigFileName,
    92  	}
    93  
    94  	for _, file := range files {
    95  		if externalCA {
    96  			fmt.Printf("[kubeconfig] External CA mode: Using user provided %s\n", file)
    97  			continue
    98  		}
    99  		if err := createKubeConfigFiles(outDir, cfg, file); err != nil {
   100  			return err
   101  		}
   102  	}
   103  	return nil
   104  }
   105  
   106  // CreateKubeConfigFileFunc defines a function type used for creating kubeconfig files.
   107  type CreateKubeConfigFileFunc func(string, string, *kubeadmapi.InitConfiguration) error
   108  
   109  // CreateKubeConfigFile creates a kubeconfig file.
   110  // If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
   111  func CreateKubeConfigFile(kubeConfigFileName string, outDir string, cfg *kubeadmapi.InitConfiguration) error {
   112  	klog.V(1).Infof("creating kubeconfig file for %s", kubeConfigFileName)
   113  	return createKubeConfigFiles(outDir, cfg, kubeConfigFileName)
   114  }
   115  
   116  // createKubeConfigFiles creates all the requested kubeconfig files.
   117  // If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned.
   118  func createKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration, kubeConfigFileNames ...string) error {
   119  
   120  	// gets the KubeConfigSpecs, actualized for the current InitConfiguration
   121  	specs, err := getKubeConfigSpecs(cfg)
   122  	if err != nil {
   123  		return err
   124  	}
   125  
   126  	for _, kubeConfigFileName := range kubeConfigFileNames {
   127  		// retrieves the KubeConfigSpec for given kubeConfigFileName
   128  		spec, exists := specs[kubeConfigFileName]
   129  		if !exists {
   130  			return errors.Errorf("couldn't retrieve KubeConfigSpec for %s", kubeConfigFileName)
   131  		}
   132  
   133  		// builds the KubeConfig object
   134  		config, err := buildKubeConfigFromSpec(spec, cfg.ClusterName, nil)
   135  		if err != nil {
   136  			return err
   137  		}
   138  
   139  		// writes the kubeconfig to disk if it does not exist
   140  		if err = createKubeConfigFileIfNotExists(outDir, kubeConfigFileName, config); err != nil {
   141  			return err
   142  		}
   143  	}
   144  
   145  	return nil
   146  }
   147  
   148  // getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current InitConfiguration
   149  // NB. this method holds the information about how kubeadm creates kubeconfig files.
   150  func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) {
   151  	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
   152  	if os.IsNotExist(errors.Cause(err)) {
   153  		return nil, errors.Wrap(err, "the CA files do not exist, please run `kubeadm init phase certs ca` to generate it")
   154  	}
   155  	if err != nil {
   156  		return nil, errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
   157  	}
   158  	// Validate period
   159  	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
   160  
   161  	configs, err := getKubeConfigSpecsBase(cfg)
   162  	if err != nil {
   163  		return nil, err
   164  	}
   165  	for _, spec := range configs {
   166  		spec.CACert = caCert
   167  		spec.ClientCertAuth.CAKey = caKey
   168  	}
   169  	return configs, nil
   170  }
   171  
   172  // buildKubeConfigFromSpec creates a kubeconfig object for the given kubeConfigSpec
   173  func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string, notAfter *time.Time) (*clientcmdapi.Config, error) {
   174  
   175  	// If this kubeconfig should use token
   176  	if spec.TokenAuth != nil {
   177  		// create a kubeconfig with a token
   178  		return kubeconfigutil.CreateWithToken(
   179  			spec.APIServer,
   180  			clustername,
   181  			spec.ClientName,
   182  			pkiutil.EncodeCertPEM(spec.CACert),
   183  			spec.TokenAuth.Token,
   184  		), nil
   185  	}
   186  
   187  	// otherwise, create a client certs
   188  	clientCertConfig := newClientCertConfigFromKubeConfigSpec(spec, notAfter)
   189  
   190  	clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CACert, spec.ClientCertAuth.CAKey, &clientCertConfig)
   191  	if err != nil {
   192  		return nil, errors.Wrapf(err, "failure while creating %s client certificate", spec.ClientName)
   193  	}
   194  
   195  	encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(clientKey)
   196  	if err != nil {
   197  		return nil, errors.Wrapf(err, "failed to marshal private key to PEM")
   198  	}
   199  	// create a kubeconfig with the client certs
   200  	return kubeconfigutil.CreateWithCerts(
   201  		spec.APIServer,
   202  		clustername,
   203  		spec.ClientName,
   204  		pkiutil.EncodeCertPEM(spec.CACert),
   205  		encodedClientKey,
   206  		pkiutil.EncodeCertPEM(clientCert),
   207  	), nil
   208  }
   209  
   210  func newClientCertConfigFromKubeConfigSpec(spec *kubeConfigSpec, notAfter *time.Time) pkiutil.CertConfig {
   211  	return pkiutil.CertConfig{
   212  		Config: certutil.Config{
   213  			CommonName:   spec.ClientName,
   214  			Organization: spec.ClientCertAuth.Organizations,
   215  			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
   216  		},
   217  		NotAfter: notAfter,
   218  	}
   219  }
   220  
   221  // validateKubeConfig check if the kubeconfig file exist and has the expected CA and server URL
   222  func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) error {
   223  	kubeConfigFilePath := filepath.Join(outDir, filename)
   224  
   225  	if _, err := os.Stat(kubeConfigFilePath); err != nil {
   226  		return err
   227  	}
   228  
   229  	// The kubeconfig already exists, let's check if it has got the same CA and server URL
   230  	currentConfig, err := clientcmd.LoadFromFile(kubeConfigFilePath)
   231  	if err != nil {
   232  		return errors.Wrapf(err, "failed to load kubeconfig file %s that already exists on disk", kubeConfigFilePath)
   233  	}
   234  
   235  	expectedCtx, exists := config.Contexts[config.CurrentContext]
   236  	if !exists {
   237  		return errors.Errorf("failed to find expected context %s", config.CurrentContext)
   238  	}
   239  	expectedCluster := expectedCtx.Cluster
   240  	currentCtx, exists := currentConfig.Contexts[currentConfig.CurrentContext]
   241  	if !exists {
   242  		return errors.Errorf("failed to find CurrentContext in Contexts of the kubeconfig file %s", kubeConfigFilePath)
   243  	}
   244  	currentCluster := currentCtx.Cluster
   245  	if currentConfig.Clusters[currentCluster] == nil {
   246  		return errors.Errorf("failed to find the given CurrentContext Cluster in Clusters of the kubeconfig file %s", kubeConfigFilePath)
   247  	}
   248  
   249  	// Make sure the compared CAs are whitespace-trimmed. The function clientcmd.LoadFromFile() just decodes
   250  	// the base64 CA and places it raw in the v1.Config object. In case the user has extra whitespace
   251  	// in the CA they used to create a kubeconfig this comparison to a generated v1.Config will otherwise fail.
   252  	caCurrent := bytes.TrimSpace(currentConfig.Clusters[currentCluster].CertificateAuthorityData)
   253  	if len(caCurrent) == 0 {
   254  		// fallback to load CA cert data from external CA file
   255  		clusterCAFilePath := currentConfig.Clusters[currentCluster].CertificateAuthority
   256  		if len(clusterCAFilePath) > 0 {
   257  			clusterCABytes, err := os.ReadFile(clusterCAFilePath)
   258  			if err != nil {
   259  				klog.Warningf("failed to load CA cert from %q for kubeconfig %q, %v", clusterCAFilePath, kubeConfigFilePath, err)
   260  			} else {
   261  				caCurrent = bytes.TrimSpace(clusterCABytes)
   262  			}
   263  		}
   264  	}
   265  	caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData)
   266  
   267  	// If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale
   268  	if !bytes.Equal(caCurrent, caExpected) {
   269  		return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath)
   270  	}
   271  	// If the current API Server location on disk doesn't match the expected API server, show a warning
   272  	if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server {
   273  		klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s",
   274  			kubeConfigFilePath, config.Clusters[expectedCluster].Server, currentConfig.Clusters[currentCluster].Server)
   275  	}
   276  
   277  	return nil
   278  }
   279  
   280  // createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path.
   281  // If there already is a kubeconfig file at the given path; kubeadm tries to load it and check if the values in the
   282  // existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
   283  // but if a file exists but has old content or isn't a kubeconfig file, this function returns an error.
   284  func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error {
   285  	kubeConfigFilePath := filepath.Join(outDir, filename)
   286  
   287  	err := validateKubeConfig(outDir, filename, config)
   288  	if err != nil {
   289  		// Check if the file exist, and if it doesn't, just write it to disk
   290  		if !os.IsNotExist(err) {
   291  			return err
   292  		}
   293  		fmt.Printf("[kubeconfig] Writing %q kubeconfig file\n", filename)
   294  		err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config)
   295  		return errors.Wrapf(err, "failed to save kubeconfig file %q on disk", kubeConfigFilePath)
   296  	}
   297  	// kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid)
   298  	// Basically, if we find a kubeconfig file with the same path; the same CA cert and the same server URL;
   299  	// kubeadm thinks those files are equal and doesn't bother writing a new file
   300  	fmt.Printf("[kubeconfig] Using existing kubeconfig file: %q\n", kubeConfigFilePath)
   301  
   302  	return nil
   303  }
   304  
   305  // WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info  - to the given writer.
   306  func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName string, organizations []string, notAfter *time.Time) error {
   307  
   308  	// creates the KubeConfigSpecs, actualized for the current InitConfiguration
   309  	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
   310  	if err != nil {
   311  		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
   312  	}
   313  	// Validate period
   314  	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
   315  
   316  	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
   317  	if err != nil {
   318  		return err
   319  	}
   320  
   321  	spec := &kubeConfigSpec{
   322  		ClientName: clientName,
   323  		APIServer:  controlPlaneEndpoint,
   324  		CACert:     caCert,
   325  		ClientCertAuth: &clientCertAuth{
   326  			CAKey:         caKey,
   327  			Organizations: organizations,
   328  		},
   329  	}
   330  
   331  	return writeKubeConfigFromSpec(out, spec, cfg.ClusterName, notAfter)
   332  }
   333  
   334  // WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer.
   335  func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName, token string, notAfter *time.Time) error {
   336  
   337  	// creates the KubeConfigSpecs, actualized for the current InitConfiguration
   338  	caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
   339  	if err != nil {
   340  		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
   341  	}
   342  	// Validate period
   343  	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
   344  
   345  	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
   346  	if err != nil {
   347  		return err
   348  	}
   349  
   350  	spec := &kubeConfigSpec{
   351  		ClientName: clientName,
   352  		APIServer:  controlPlaneEndpoint,
   353  		CACert:     caCert,
   354  		TokenAuth: &tokenAuth{
   355  			Token: token,
   356  		},
   357  	}
   358  
   359  	return writeKubeConfigFromSpec(out, spec, cfg.ClusterName, notAfter)
   360  }
   361  
   362  // writeKubeConfigFromSpec creates a kubeconfig object from a kubeConfigSpec and writes it to the given writer.
   363  func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername string, notAfter *time.Time) error {
   364  
   365  	// builds the KubeConfig object
   366  	config, err := buildKubeConfigFromSpec(spec, clustername, notAfter)
   367  	if err != nil {
   368  		return err
   369  	}
   370  
   371  	// writes the kubeconfig to disk if it not exists
   372  	configBytes, err := clientcmd.Write(*config)
   373  	if err != nil {
   374  		return errors.Wrap(err, "failure while serializing admin kubeconfig")
   375  	}
   376  
   377  	fmt.Fprintln(out, string(configBytes))
   378  	return nil
   379  }
   380  
   381  // ValidateKubeconfigsForExternalCA check if the kubeconfig file exist and has the expected CA and server URL using kubeadmapi.InitConfiguration.
   382  func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error {
   383  	// Creates a kubeconfig file with the target CA and server URL
   384  	// to be used as a input for validating user provided kubeconfig files
   385  	caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
   386  	if err != nil {
   387  		return errors.Wrapf(err, "the CA file couldn't be loaded")
   388  	}
   389  	// Validate period
   390  	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
   391  
   392  	// validate user provided kubeconfig files for the scheduler and controller-manager
   393  	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
   394  	if err != nil {
   395  		return err
   396  	}
   397  
   398  	validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
   399  	kubeConfigFileNamesLocal := []string{
   400  		kubeadmconstants.ControllerManagerKubeConfigFileName,
   401  		kubeadmconstants.SchedulerKubeConfigFileName,
   402  	}
   403  
   404  	for _, kubeConfigFileName := range kubeConfigFileNamesLocal {
   405  		if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigLocal); err != nil {
   406  			return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName)
   407  		}
   408  	}
   409  
   410  	// validate user provided kubeconfig files for the kubelet and admin
   411  	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
   412  	if err != nil {
   413  		return err
   414  	}
   415  
   416  	validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
   417  	kubeConfigFileNamesCPE := []string{
   418  		kubeadmconstants.AdminKubeConfigFileName,
   419  		kubeadmconstants.SuperAdminKubeConfigFileName,
   420  		kubeadmconstants.KubeletKubeConfigFileName,
   421  	}
   422  
   423  	for _, kubeConfigFileName := range kubeConfigFileNamesCPE {
   424  		if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigCPE); err != nil {
   425  			return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName)
   426  		}
   427  	}
   428  
   429  	return nil
   430  }
   431  
   432  func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) {
   433  	controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
   434  	if err != nil {
   435  		return nil, err
   436  	}
   437  	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
   438  	if err != nil {
   439  		return nil, err
   440  	}
   441  
   442  	return map[string]*kubeConfigSpec{
   443  		kubeadmconstants.AdminKubeConfigFileName: {
   444  			APIServer:  controlPlaneEndpoint,
   445  			ClientName: "kubernetes-admin",
   446  			ClientCertAuth: &clientCertAuth{
   447  				Organizations: []string{kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding},
   448  			},
   449  		},
   450  		kubeadmconstants.SuperAdminKubeConfigFileName: {
   451  			APIServer:  controlPlaneEndpoint,
   452  			ClientName: "kubernetes-super-admin",
   453  			ClientCertAuth: &clientCertAuth{
   454  				Organizations: []string{kubeadmconstants.SystemPrivilegedGroup},
   455  			},
   456  		},
   457  		kubeadmconstants.KubeletKubeConfigFileName: {
   458  			APIServer:  controlPlaneEndpoint,
   459  			ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
   460  			ClientCertAuth: &clientCertAuth{
   461  				Organizations: []string{kubeadmconstants.NodesGroup},
   462  			},
   463  		},
   464  		kubeadmconstants.ControllerManagerKubeConfigFileName: {
   465  			APIServer:      localAPIEndpoint,
   466  			ClientName:     kubeadmconstants.ControllerManagerUser,
   467  			ClientCertAuth: &clientCertAuth{},
   468  		},
   469  		kubeadmconstants.SchedulerKubeConfigFileName: {
   470  			APIServer:      localAPIEndpoint,
   471  			ClientName:     kubeadmconstants.SchedulerUser,
   472  			ClientCertAuth: &clientCertAuth{},
   473  		},
   474  	}, nil
   475  }
   476  
   477  func createKubeConfigAndCSR(kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration, name string, spec *kubeConfigSpec) error {
   478  	if kubeConfigDir == "" {
   479  		return errors.Errorf("%s: kubeConfigDir was empty", errInvalid)
   480  	}
   481  	if kubeadmConfig == nil {
   482  		return errors.Errorf("%s: kubeadmConfig was nil", errInvalid)
   483  	}
   484  	if name == "" {
   485  		return errors.Errorf("%s: name was empty", errInvalid)
   486  	}
   487  	if spec == nil {
   488  		return errors.Errorf("%s: spec was nil", errInvalid)
   489  	}
   490  	kubeConfigPath := filepath.Join(kubeConfigDir, name)
   491  	if _, err := os.Stat(kubeConfigPath); err == nil {
   492  		return errors.Errorf("%s: kube config: %s", errExist, kubeConfigPath)
   493  	} else if !os.IsNotExist(err) {
   494  		return errors.Wrapf(err, "unexpected error while checking if file exists: %s", kubeConfigPath)
   495  	}
   496  	if pkiutil.CSROrKeyExist(kubeConfigDir, name) {
   497  		return errors.Errorf("%s: csr: %s", errExist, kubeConfigPath)
   498  	}
   499  
   500  	clientCertConfig := newClientCertConfigFromKubeConfigSpec(spec, nil)
   501  
   502  	clientKey, err := pkiutil.NewPrivateKey(clientCertConfig.EncryptionAlgorithm)
   503  	if err != nil {
   504  		return err
   505  	}
   506  	clientCSR, err := pkiutil.NewCSR(clientCertConfig, clientKey)
   507  	if err != nil {
   508  		return err
   509  	}
   510  
   511  	encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(clientKey)
   512  	if err != nil {
   513  		return err
   514  	}
   515  
   516  	var (
   517  		emptyCACert     []byte
   518  		emptyClientCert []byte
   519  	)
   520  
   521  	// create a kubeconfig with the client certs
   522  	config := kubeconfigutil.CreateWithCerts(
   523  		spec.APIServer,
   524  		kubeadmConfig.ClusterName,
   525  		spec.ClientName,
   526  		emptyCACert,
   527  		encodedClientKey,
   528  		emptyClientCert,
   529  	)
   530  
   531  	if err := kubeconfigutil.WriteToDisk(kubeConfigPath, config); err != nil {
   532  		return err
   533  	}
   534  	// Write CSR to disk
   535  	if err := pkiutil.WriteCSR(kubeConfigDir, name, clientCSR); err != nil {
   536  		return err
   537  	}
   538  	return nil
   539  }
   540  
   541  // CreateDefaultKubeConfigsAndCSRFiles is used in ExternalCA mode to create
   542  // kubeconfig files and adjacent CSR files.
   543  func CreateDefaultKubeConfigsAndCSRFiles(out io.Writer, kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration) error {
   544  	kubeConfigs, err := getKubeConfigSpecsBase(kubeadmConfig)
   545  	if err != nil {
   546  		return err
   547  	}
   548  	if out != nil {
   549  		fmt.Fprintf(out, "generating keys and CSRs in %s\n", kubeConfigDir)
   550  	}
   551  	for name, spec := range kubeConfigs {
   552  		if err := createKubeConfigAndCSR(kubeConfigDir, kubeadmConfig, name, spec); err != nil {
   553  			return err
   554  		}
   555  		if out != nil {
   556  			fmt.Fprintf(out, "  %s\n", name)
   557  		}
   558  	}
   559  	return nil
   560  }
   561  
   562  // EnsureRBACFunc defines a function type that can be passed to EnsureAdminClusterRoleBinding().
   563  type EnsureRBACFunc func(context.Context, clientset.Interface, clientset.Interface, time.Duration, time.Duration) (clientset.Interface, error)
   564  
   565  // EnsureAdminClusterRoleBinding constructs a client from admin.conf and optionally
   566  // constructs a client from super-admin.conf if the file exists. It then proceeds
   567  // to pass the clients to EnsureAdminClusterRoleBindingImpl. The function returns a
   568  // usable client from admin.conf with RBAC properly constructed or an error.
   569  func EnsureAdminClusterRoleBinding(outDir string, ensureRBACFunc EnsureRBACFunc) (clientset.Interface, error) {
   570  	var (
   571  		err                           error
   572  		adminClient, superAdminClient clientset.Interface
   573  	)
   574  
   575  	// Create a client from admin.conf.
   576  	adminClient, err = kubeconfigutil.ClientSetFromFile(filepath.Join(outDir, kubeadmconstants.AdminKubeConfigFileName))
   577  	if err != nil {
   578  		return nil, err
   579  	}
   580  
   581  	// Create a client from super-admin.conf.
   582  	superAdminPath := filepath.Join(outDir, kubeadmconstants.SuperAdminKubeConfigFileName)
   583  	if _, err := os.Stat(superAdminPath); err == nil {
   584  		superAdminClient, err = kubeconfigutil.ClientSetFromFile(superAdminPath)
   585  		if err != nil {
   586  			return nil, err
   587  		}
   588  	}
   589  
   590  	if ensureRBACFunc == nil {
   591  		ensureRBACFunc = EnsureAdminClusterRoleBindingImpl
   592  	}
   593  
   594  	ctx := context.Background()
   595  	return ensureRBACFunc(
   596  		ctx, adminClient, superAdminClient,
   597  		kubeadmconstants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration,
   598  	)
   599  }
   600  
   601  // EnsureAdminClusterRoleBindingImpl first attempts to see if the ClusterRoleBinding
   602  // kubeadm:cluster-admins exists by using adminClient. If it already exists,
   603  // it would mean the adminClient is usable. If it does not, attempt to create
   604  // the ClusterRoleBinding by using superAdminClient.
   605  func EnsureAdminClusterRoleBindingImpl(ctx context.Context, adminClient, superAdminClient clientset.Interface,
   606  	retryInterval, retryTimeout time.Duration) (clientset.Interface, error) {
   607  
   608  	klog.V(1).Infof("ensuring that the ClusterRoleBinding for the %s Group exists",
   609  		kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding)
   610  
   611  	var (
   612  		err, lastError     error
   613  		crbExists          bool
   614  		clusterRoleBinding = &rbac.ClusterRoleBinding{
   615  			ObjectMeta: metav1.ObjectMeta{
   616  				Name: kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
   617  			},
   618  			RoleRef: rbac.RoleRef{
   619  				APIGroup: rbac.GroupName,
   620  				Kind:     "ClusterRole",
   621  				Name:     "cluster-admin",
   622  			},
   623  			Subjects: []rbac.Subject{
   624  				{
   625  					Kind: rbac.GroupKind,
   626  					Name: kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
   627  				},
   628  			},
   629  		}
   630  	)
   631  
   632  	// First try to create the CRB with the admin.conf client. If the admin.conf contains a User bound
   633  	// to the built-in super-user group, this will pass. In all other cases an error will be returned.
   634  	// The poll here is required to ensure the API server is reachable during "kubeadm init" workflows.
   635  	err = wait.PollUntilContextTimeout(
   636  		ctx,
   637  		retryInterval,
   638  		retryTimeout,
   639  		true, func(ctx context.Context) (bool, error) {
   640  			if _, err := adminClient.RbacV1().ClusterRoleBindings().Create(
   641  				ctx,
   642  				clusterRoleBinding,
   643  				metav1.CreateOptions{},
   644  			); err != nil {
   645  				if apierrors.IsForbidden(err) {
   646  					// If it encounters a forbidden error this means that the API server was reached
   647  					// but the CRB is missing - i.e. the admin.conf user does not have permissions
   648  					// to create its own permission RBAC yet.
   649  					return true, nil
   650  				} else if apierrors.IsAlreadyExists(err) {
   651  					// If the CRB exists it means the admin.conf already has the right
   652  					// permissions; return.
   653  					crbExists = true
   654  					return true, nil
   655  				} else {
   656  					// Retry on any other error type.
   657  					lastError = errors.Wrap(err, "unable to create ClusterRoleBinding")
   658  					return false, nil
   659  				}
   660  			}
   661  			crbExists = true
   662  			return true, nil
   663  		})
   664  	if err != nil {
   665  		return nil, lastError
   666  	}
   667  
   668  	// The CRB was created or already existed; return the admin.conf client.
   669  	if crbExists {
   670  		return adminClient, nil
   671  	}
   672  
   673  	// If the superAdminClient is nil at this point we cannot proceed creating the CRB; return an error.
   674  	if superAdminClient == nil {
   675  		return nil, errors.Errorf("the ClusterRoleBinding for the %s Group is missing but there is no %s to create it",
   676  			kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
   677  			kubeadmconstants.SuperAdminKubeConfigFileName)
   678  	}
   679  
   680  	// Create the ClusterRoleBinding with the super-admin.conf client.
   681  	klog.V(1).Infof("creating the ClusterRoleBinding for the %s Group by using %s",
   682  		kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
   683  		kubeadmconstants.SuperAdminKubeConfigFileName)
   684  
   685  	err = wait.PollUntilContextTimeout(
   686  		ctx,
   687  		retryInterval,
   688  		retryTimeout,
   689  		true, func(ctx context.Context) (bool, error) {
   690  			if _, err := superAdminClient.RbacV1().ClusterRoleBindings().Create(
   691  				ctx,
   692  				clusterRoleBinding,
   693  				metav1.CreateOptions{},
   694  			); err != nil {
   695  				lastError = err
   696  				if apierrors.IsAlreadyExists(err) {
   697  					klog.V(5).Infof("ClusterRoleBinding %s already exists.", kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding)
   698  					return true, nil
   699  				}
   700  				// Retry on any other type of error.
   701  				return false, nil
   702  			}
   703  			return true, nil
   704  		})
   705  	if err != nil {
   706  		return nil, errors.Wrapf(lastError, "unable to create the %s ClusterRoleBinding by using %s",
   707  			kubeadmconstants.ClusterAdminsGroupAndClusterRoleBinding,
   708  			kubeadmconstants.SuperAdminKubeConfigFileName)
   709  	}
   710  
   711  	// Once the CRB is in place, start using the admin.conf client.
   712  	return adminClient, nil
   713  }
   714
View as plain text