Source file src/k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane/manifests_test.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5  Copyright 2016 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package controlplane
    21  
    22  import (
    23  	"fmt"
    24  	"os"
    25  	"path/filepath"
    26  	"reflect"
    27  	"sort"
    28  	"strings"
    29  	"testing"
    30  
    31  	"github.com/lithammer/dedent"
    32  
    33  	v1 "k8s.io/api/core/v1"
    34  	"k8s.io/apimachinery/pkg/util/sets"
    35  
    36  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    37  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    38  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
    39  	pkiutiltesting "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil/testing"
    40  	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
    41  	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
    42  )
    43  
    44  const (
    45  	testCertsDir = "/var/lib/certs"
    46  )
    47  
    48  var cpVersion = kubeadmconstants.MinimumControlPlaneVersion.WithPreRelease("beta.2").String()
    49  
    50  func TestGetStaticPodSpecs(t *testing.T) {
    51  
    52  	// Creates a Cluster Configuration
    53  	cfg := &kubeadmapi.ClusterConfiguration{
    54  		KubernetesVersion: "v1.9.0",
    55  		Scheduler: kubeadmapi.ControlPlaneComponent{ExtraEnvs: []kubeadmapi.EnvVar{
    56  			{
    57  				EnvVar: v1.EnvVar{Name: "Foo", Value: "Bar"},
    58  			},
    59  		}},
    60  	}
    61  
    62  	// Executes GetStaticPodSpecs
    63  	specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{}, []kubeadmapi.EnvVar{})
    64  
    65  	var tests = []struct {
    66  		name          string
    67  		staticPodName string
    68  		env           []v1.EnvVar
    69  	}{
    70  		{
    71  			name:          "KubeAPIServer",
    72  			staticPodName: kubeadmconstants.KubeAPIServer,
    73  		},
    74  		{
    75  			name:          "KubeControllerManager",
    76  			staticPodName: kubeadmconstants.KubeControllerManager,
    77  		},
    78  		{
    79  			name:          "KubeScheduler",
    80  			staticPodName: kubeadmconstants.KubeScheduler,
    81  			env:           []v1.EnvVar{{Name: "Foo", Value: "Bar"}},
    82  		},
    83  	}
    84  
    85  	for _, tc := range tests {
    86  		t.Run(tc.name, func(t *testing.T) {
    87  			// assert the spec for the staticPodName exists
    88  			if spec, ok := specs[tc.staticPodName]; ok {
    89  				// Assert each specs refers to the right pod
    90  				if spec.Spec.Containers[0].Name != tc.staticPodName {
    91  					t.Errorf("getKubeConfigSpecs spec for %s contains pod %s, expects %s", tc.staticPodName, spec.Spec.Containers[0].Name, tc.staticPodName)
    92  				}
    93  				if tc.env != nil {
    94  					if !reflect.DeepEqual(spec.Spec.Containers[0].Env, tc.env) {
    95  						t.Errorf("expected env: %v, got: %v", tc.env, spec.Spec.Containers[0].Env)
    96  					}
    97  				}
    98  			} else {
    99  				t.Errorf("getStaticPodSpecs didn't create spec for %s ", tc.staticPodName)
   100  			}
   101  		})
   102  	}
   103  }
   104  
   105  func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
   106  
   107  	var tests = []struct {
   108  		name       string
   109  		components []string
   110  	}{
   111  		{
   112  			name:       "KubeAPIServer KubeAPIServer KubeScheduler",
   113  			components: []string{kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler},
   114  		},
   115  		{
   116  			name:       "KubeAPIServer",
   117  			components: []string{kubeadmconstants.KubeAPIServer},
   118  		},
   119  		{
   120  			name:       "KubeControllerManager",
   121  			components: []string{kubeadmconstants.KubeControllerManager},
   122  		},
   123  		{
   124  			name:       "KubeScheduler",
   125  			components: []string{kubeadmconstants.KubeScheduler},
   126  		},
   127  	}
   128  
   129  	for _, test := range tests {
   130  		t.Run(test.name, func(t *testing.T) {
   131  			// Create temp folder for the test case
   132  			tmpdir := testutil.SetupTempDir(t)
   133  			defer os.RemoveAll(tmpdir)
   134  
   135  			// Creates a Cluster Configuration
   136  			cfg := &kubeadmapi.ClusterConfiguration{
   137  				KubernetesVersion: "v1.9.0",
   138  			}
   139  
   140  			// Execute createStaticPodFunction
   141  			manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
   142  			err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, false /* isDryRun */, test.components...)
   143  			if err != nil {
   144  				t.Errorf("Error executing createStaticPodFunction: %v", err)
   145  				return
   146  			}
   147  
   148  			// Assert expected files are there
   149  			testutil.AssertFilesCount(t, manifestPath, len(test.components))
   150  
   151  			for _, fileName := range test.components {
   152  				testutil.AssertFileExists(t, manifestPath, fileName+".yaml")
   153  			}
   154  		})
   155  	}
   156  }
   157  
   158  func TestCreateStaticPodFilesWithPatches(t *testing.T) {
   159  	// Create temp folder for the test case
   160  	tmpdir := testutil.SetupTempDir(t)
   161  	defer os.RemoveAll(tmpdir)
   162  
   163  	// Creates a Cluster Configuration
   164  	cfg := &kubeadmapi.ClusterConfiguration{
   165  		KubernetesVersion: "v1.9.0",
   166  	}
   167  
   168  	patchesPath := filepath.Join(tmpdir, "patch-files")
   169  	err := os.MkdirAll(patchesPath, 0777)
   170  	if err != nil {
   171  		t.Fatalf("Couldn't create %s", patchesPath)
   172  	}
   173  
   174  	patchString := dedent.Dedent(`
   175  	metadata:
   176  	  annotations:
   177  	    patched: "true"
   178  	`)
   179  
   180  	err = os.WriteFile(filepath.Join(patchesPath, kubeadmconstants.KubeAPIServer+".yaml"), []byte(patchString), 0644)
   181  	if err != nil {
   182  		t.Fatalf("WriteFile returned unexpected error: %v", err)
   183  	}
   184  
   185  	// Execute createStaticPodFunction with patches
   186  	manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
   187  	err = CreateStaticPodFiles(manifestPath, patchesPath, cfg, &kubeadmapi.APIEndpoint{}, false /* isDryRun */, kubeadmconstants.KubeAPIServer)
   188  	if err != nil {
   189  		t.Errorf("Error executing createStaticPodFunction: %v", err)
   190  		return
   191  	}
   192  
   193  	pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer)))
   194  	if err != nil {
   195  		t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
   196  		return
   197  	}
   198  
   199  	if _, ok := pod.ObjectMeta.Annotations["patched"]; !ok {
   200  		t.Errorf("Patches were not applied to %s", kubeadmconstants.KubeAPIServer)
   201  	}
   202  }
   203  
   204  func TestGetAPIServerCommand(t *testing.T) {
   205  	var tests = []struct {
   206  		name     string
   207  		cfg      *kubeadmapi.ClusterConfiguration
   208  		endpoint *kubeadmapi.APIEndpoint
   209  		expected []string
   210  	}{
   211  		{
   212  			name: "testing defaults",
   213  			cfg: &kubeadmapi.ClusterConfiguration{
   214  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   215  				CertificatesDir: testCertsDir,
   216  			},
   217  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   218  			expected: []string{
   219  				"kube-apiserver",
   220  				"--enable-admission-plugins=NodeRestriction",
   221  				"--service-cluster-ip-range=bar",
   222  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   223  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   224  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   225  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   226  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   227  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   228  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   229  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   230  				"--enable-bootstrap-token-auth=true",
   231  				"--secure-port=123",
   232  				"--allow-privileged=true",
   233  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   234  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   235  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   236  				"--requestheader-username-headers=X-Remote-User",
   237  				"--requestheader-group-headers=X-Remote-Group",
   238  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   239  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   240  				"--requestheader-allowed-names=front-proxy-client",
   241  				"--authorization-mode=Node,RBAC",
   242  				"--advertise-address=1.2.3.4",
   243  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   244  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   245  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   246  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   247  			},
   248  		},
   249  		{
   250  			name: "ipv6 advertise address",
   251  			cfg: &kubeadmapi.ClusterConfiguration{
   252  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   253  				CertificatesDir: testCertsDir,
   254  			},
   255  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
   256  			expected: []string{
   257  				"kube-apiserver",
   258  				"--enable-admission-plugins=NodeRestriction",
   259  				"--service-cluster-ip-range=bar",
   260  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   261  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   262  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   263  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   264  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   265  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   266  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   267  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   268  				"--enable-bootstrap-token-auth=true",
   269  				fmt.Sprintf("--secure-port=%d", 123),
   270  				"--allow-privileged=true",
   271  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   272  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   273  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   274  				"--requestheader-username-headers=X-Remote-User",
   275  				"--requestheader-group-headers=X-Remote-Group",
   276  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   277  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   278  				"--requestheader-allowed-names=front-proxy-client",
   279  				"--authorization-mode=Node,RBAC",
   280  				"--advertise-address=2001:db8::1",
   281  				fmt.Sprintf("--etcd-servers=https://[::1]:%d", kubeadmconstants.EtcdListenClientPort),
   282  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   283  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   284  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   285  			},
   286  		},
   287  		{
   288  			name: "an external etcd with custom ca, certs and keys",
   289  			cfg: &kubeadmapi.ClusterConfiguration{
   290  				Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   291  				Etcd: kubeadmapi.Etcd{
   292  					External: &kubeadmapi.ExternalEtcd{
   293  						Endpoints: []string{"https://[2001:abcd:bcda::1]:2379", "https://[2001:abcd:bcda::2]:2379"},
   294  						CAFile:    "fuz",
   295  						CertFile:  "fiz",
   296  						KeyFile:   "faz",
   297  					},
   298  				},
   299  				CertificatesDir: testCertsDir,
   300  			},
   301  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
   302  			expected: []string{
   303  				"kube-apiserver",
   304  				"--enable-admission-plugins=NodeRestriction",
   305  				"--service-cluster-ip-range=bar",
   306  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   307  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   308  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   309  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   310  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   311  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   312  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   313  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   314  				fmt.Sprintf("--secure-port=%d", 123),
   315  				"--allow-privileged=true",
   316  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   317  				"--enable-bootstrap-token-auth=true",
   318  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   319  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   320  				"--requestheader-username-headers=X-Remote-User",
   321  				"--requestheader-group-headers=X-Remote-Group",
   322  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   323  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   324  				"--requestheader-allowed-names=front-proxy-client",
   325  				"--authorization-mode=Node,RBAC",
   326  				"--advertise-address=2001:db8::1",
   327  				"--etcd-servers=https://[2001:abcd:bcda::1]:2379,https://[2001:abcd:bcda::2]:2379",
   328  				"--etcd-cafile=fuz",
   329  				"--etcd-certfile=fiz",
   330  				"--etcd-keyfile=faz",
   331  			},
   332  		},
   333  		{
   334  			name: "an insecure etcd",
   335  			cfg: &kubeadmapi.ClusterConfiguration{
   336  				Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   337  				Etcd: kubeadmapi.Etcd{
   338  					External: &kubeadmapi.ExternalEtcd{
   339  						Endpoints: []string{"http://[::1]:2379", "http://[::1]:2380"},
   340  					},
   341  				},
   342  				CertificatesDir: testCertsDir,
   343  			},
   344  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
   345  			expected: []string{
   346  				"kube-apiserver",
   347  				"--enable-admission-plugins=NodeRestriction",
   348  				"--service-cluster-ip-range=bar",
   349  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   350  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   351  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   352  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   353  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   354  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   355  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   356  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   357  				fmt.Sprintf("--secure-port=%d", 123),
   358  				"--allow-privileged=true",
   359  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   360  				"--enable-bootstrap-token-auth=true",
   361  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   362  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   363  				"--requestheader-username-headers=X-Remote-User",
   364  				"--requestheader-group-headers=X-Remote-Group",
   365  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   366  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   367  				"--requestheader-allowed-names=front-proxy-client",
   368  				"--authorization-mode=Node,RBAC",
   369  				"--advertise-address=2001:db8::1",
   370  				"--etcd-servers=http://[::1]:2379,http://[::1]:2380",
   371  			},
   372  		},
   373  		{
   374  			name: "test APIServer.ExtraArgs works as expected",
   375  			cfg: &kubeadmapi.ClusterConfiguration{
   376  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   377  				CertificatesDir: testCertsDir,
   378  				APIServer: kubeadmapi.APIServer{
   379  					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
   380  						ExtraArgs: []kubeadmapi.Arg{
   381  							{Name: "service-cluster-ip-range", Value: "baz"},
   382  							{Name: "advertise-address", Value: "9.9.9.9"},
   383  							{Name: "audit-policy-file", Value: "/etc/config/audit.yaml"},
   384  							{Name: "audit-log-path", Value: "/var/log/kubernetes"},
   385  						},
   386  					},
   387  				},
   388  			},
   389  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   390  			expected: []string{
   391  				"kube-apiserver",
   392  				"--enable-admission-plugins=NodeRestriction",
   393  				"--service-cluster-ip-range=baz",
   394  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   395  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   396  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   397  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   398  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   399  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   400  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   401  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   402  				"--enable-bootstrap-token-auth=true",
   403  				"--secure-port=123",
   404  				"--allow-privileged=true",
   405  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   406  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   407  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   408  				"--requestheader-username-headers=X-Remote-User",
   409  				"--requestheader-group-headers=X-Remote-Group",
   410  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   411  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   412  				"--requestheader-allowed-names=front-proxy-client",
   413  				"--authorization-mode=Node,RBAC",
   414  				"--advertise-address=9.9.9.9",
   415  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   416  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   417  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   418  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   419  				"--audit-policy-file=/etc/config/audit.yaml",
   420  				"--audit-log-path=/var/log/kubernetes",
   421  			},
   422  		},
   423  		{
   424  			name: "authorization-mode extra-args ABAC",
   425  			cfg: &kubeadmapi.ClusterConfiguration{
   426  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   427  				CertificatesDir: testCertsDir,
   428  				APIServer: kubeadmapi.APIServer{
   429  					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
   430  						ExtraArgs: []kubeadmapi.Arg{
   431  							{Name: "authorization-mode", Value: kubeadmconstants.ModeABAC},
   432  						},
   433  					},
   434  				},
   435  			},
   436  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   437  			expected: []string{
   438  				"kube-apiserver",
   439  				"--enable-admission-plugins=NodeRestriction",
   440  				"--service-cluster-ip-range=bar",
   441  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   442  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   443  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   444  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   445  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   446  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   447  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   448  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   449  				"--enable-bootstrap-token-auth=true",
   450  				"--secure-port=123",
   451  				"--allow-privileged=true",
   452  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   453  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   454  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   455  				"--requestheader-username-headers=X-Remote-User",
   456  				"--requestheader-group-headers=X-Remote-Group",
   457  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   458  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   459  				"--requestheader-allowed-names=front-proxy-client",
   460  				"--authorization-mode=ABAC",
   461  				"--advertise-address=1.2.3.4",
   462  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   463  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   464  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   465  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   466  			},
   467  		},
   468  		{
   469  			name: "authorization-mode extra-args Webhook",
   470  			cfg: &kubeadmapi.ClusterConfiguration{
   471  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   472  				CertificatesDir: testCertsDir,
   473  				APIServer: kubeadmapi.APIServer{
   474  					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
   475  						ExtraArgs: []kubeadmapi.Arg{
   476  							{Name: "authorization-mode", Value: strings.Join([]string{
   477  								kubeadmconstants.ModeNode,
   478  								kubeadmconstants.ModeRBAC,
   479  								kubeadmconstants.ModeWebhook,
   480  							}, ",")},
   481  						},
   482  					},
   483  				},
   484  			},
   485  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   486  			expected: []string{
   487  				"kube-apiserver",
   488  				"--enable-admission-plugins=NodeRestriction",
   489  				"--service-cluster-ip-range=bar",
   490  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   491  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   492  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   493  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   494  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   495  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   496  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   497  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   498  				"--enable-bootstrap-token-auth=true",
   499  				"--secure-port=123",
   500  				"--allow-privileged=true",
   501  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   502  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   503  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   504  				"--requestheader-username-headers=X-Remote-User",
   505  				"--requestheader-group-headers=X-Remote-Group",
   506  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   507  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   508  				"--requestheader-allowed-names=front-proxy-client",
   509  				"--authorization-mode=Node,RBAC,Webhook",
   510  				"--advertise-address=1.2.3.4",
   511  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   512  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   513  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   514  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   515  			},
   516  		},
   517  		{
   518  			name: "authorization-config extra-args",
   519  			cfg: &kubeadmapi.ClusterConfiguration{
   520  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   521  				CertificatesDir: testCertsDir,
   522  				APIServer: kubeadmapi.APIServer{
   523  					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
   524  						ExtraArgs: []kubeadmapi.Arg{
   525  							{Name: "authorization-config", Value: "/path/to/authorization/config/file"},
   526  						},
   527  					},
   528  				},
   529  			},
   530  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   531  			expected: []string{
   532  				"kube-apiserver",
   533  				"--enable-admission-plugins=NodeRestriction",
   534  				"--service-cluster-ip-range=bar",
   535  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   536  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   537  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   538  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   539  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   540  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   541  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   542  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   543  				"--enable-bootstrap-token-auth=true",
   544  				"--secure-port=123",
   545  				"--allow-privileged=true",
   546  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   547  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   548  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   549  				"--requestheader-username-headers=X-Remote-User",
   550  				"--requestheader-group-headers=X-Remote-Group",
   551  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   552  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   553  				"--requestheader-allowed-names=front-proxy-client",
   554  				"--authorization-config=/path/to/authorization/config/file",
   555  				"--advertise-address=1.2.3.4",
   556  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   557  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   558  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   559  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   560  			},
   561  		},
   562  		{
   563  			// Note that we do not block it at this level but api server would fail to start.
   564  			name: "authorization-config and authorization-mode extra-args",
   565  			cfg: &kubeadmapi.ClusterConfiguration{
   566  				Networking:      kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"},
   567  				CertificatesDir: testCertsDir,
   568  				APIServer: kubeadmapi.APIServer{
   569  					ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
   570  						ExtraArgs: []kubeadmapi.Arg{
   571  							{Name: "authorization-config", Value: "/path/to/authorization/config/file"},
   572  							{Name: "authorization-mode", Value: strings.Join([]string{
   573  								kubeadmconstants.ModeNode,
   574  								kubeadmconstants.ModeRBAC,
   575  								kubeadmconstants.ModeWebhook,
   576  							}, ",")},
   577  						},
   578  					},
   579  				},
   580  			},
   581  			endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
   582  			expected: []string{
   583  				"kube-apiserver",
   584  				"--enable-admission-plugins=NodeRestriction",
   585  				"--service-cluster-ip-range=bar",
   586  				"--service-account-key-file=" + filepath.Join(testCertsDir, "sa.pub"),
   587  				"--service-account-signing-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   588  				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
   589  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   590  				"--tls-cert-file=" + filepath.Join(testCertsDir, "apiserver.crt"),
   591  				"--tls-private-key-file=" + filepath.Join(testCertsDir, "apiserver.key"),
   592  				"--kubelet-client-certificate=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.crt"),
   593  				"--kubelet-client-key=" + filepath.Join(testCertsDir, "apiserver-kubelet-client.key"),
   594  				"--enable-bootstrap-token-auth=true",
   595  				"--secure-port=123",
   596  				"--allow-privileged=true",
   597  				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
   598  				"--proxy-client-cert-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.crt"),
   599  				"--proxy-client-key-file=" + filepath.FromSlash("/var/lib/certs/front-proxy-client.key"),
   600  				"--requestheader-username-headers=X-Remote-User",
   601  				"--requestheader-group-headers=X-Remote-Group",
   602  				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
   603  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   604  				"--requestheader-allowed-names=front-proxy-client",
   605  				"--authorization-config=/path/to/authorization/config/file",
   606  				"--authorization-mode=Node,RBAC,Webhook",
   607  				"--advertise-address=1.2.3.4",
   608  				fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
   609  				"--etcd-cafile=" + filepath.Join(testCertsDir, "etcd/ca.crt"),
   610  				"--etcd-certfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.crt"),
   611  				"--etcd-keyfile=" + filepath.Join(testCertsDir, "apiserver-etcd-client.key"),
   612  			},
   613  		},
   614  	}
   615  
   616  	for _, rt := range tests {
   617  		t.Run(rt.name, func(t *testing.T) {
   618  			actual := getAPIServerCommand(rt.cfg, rt.endpoint)
   619  			sort.Strings(actual)
   620  			sort.Strings(rt.expected)
   621  			if !reflect.DeepEqual(actual, rt.expected) {
   622  				errorDiffArguments(t, rt.name, actual, rt.expected)
   623  			}
   624  		})
   625  	}
   626  }
   627  
   628  func errorDiffArguments(t *testing.T, name string, actual, expected []string) {
   629  	expectedShort := removeCommon(expected, actual)
   630  	actualShort := removeCommon(actual, expected)
   631  	t.Errorf(
   632  		"[%s] failed getAPIServerCommand:\nexpected:\n%v\nsaw:\n%v"+
   633  			"\nexpectedShort:\n%v\nsawShort:\n%v\n",
   634  		name, expected, actual,
   635  		expectedShort, actualShort)
   636  }
   637  
   638  // removeCommon removes common items from left list
   639  // makes compairing two cmdline (with lots of arguments) easier
   640  func removeCommon(left, right []string) []string {
   641  	origSet := sets.New(left...)
   642  	origSet.Delete(right...)
   643  	return sets.List(origSet)
   644  }
   645  
   646  func TestGetControllerManagerCommand(t *testing.T) {
   647  	var tests = []struct {
   648  		name     string
   649  		cfg      *kubeadmapi.ClusterConfiguration
   650  		expected []string
   651  	}{
   652  		{
   653  			name: "custom cluster name for " + cpVersion,
   654  			cfg: &kubeadmapi.ClusterConfiguration{
   655  				KubernetesVersion: cpVersion,
   656  				CertificatesDir:   testCertsDir,
   657  				ClusterName:       "some-other-cluster-name",
   658  			},
   659  			expected: []string{
   660  				"kube-controller-manager",
   661  				"--bind-address=127.0.0.1",
   662  				"--leader-elect=true",
   663  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   664  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   665  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   666  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   667  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   668  				"--use-service-account-credentials=true",
   669  				"--controllers=*,bootstrapsigner,tokencleaner",
   670  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   671  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   672  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   673  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   674  				"--cluster-name=some-other-cluster-name",
   675  			},
   676  		},
   677  		{
   678  			name: "custom certs dir for " + cpVersion,
   679  			cfg: &kubeadmapi.ClusterConfiguration{
   680  				CertificatesDir:   testCertsDir,
   681  				KubernetesVersion: cpVersion,
   682  			},
   683  			expected: []string{
   684  				"kube-controller-manager",
   685  				"--bind-address=127.0.0.1",
   686  				"--leader-elect=true",
   687  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   688  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   689  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   690  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   691  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   692  				"--use-service-account-credentials=true",
   693  				"--controllers=*,bootstrapsigner,tokencleaner",
   694  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   695  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   696  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   697  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   698  			},
   699  		},
   700  		{
   701  			name: "custom cluster-cidr for " + cpVersion,
   702  			cfg: &kubeadmapi.ClusterConfiguration{
   703  				Networking:        kubeadmapi.Networking{PodSubnet: "10.0.1.15/16", DNSDomain: "cluster.local"},
   704  				CertificatesDir:   testCertsDir,
   705  				KubernetesVersion: cpVersion,
   706  			},
   707  			expected: []string{
   708  				"kube-controller-manager",
   709  				"--bind-address=127.0.0.1",
   710  				"--leader-elect=true",
   711  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   712  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   713  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   714  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   715  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   716  				"--use-service-account-credentials=true",
   717  				"--controllers=*,bootstrapsigner,tokencleaner",
   718  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   719  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   720  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   721  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   722  				"--allocate-node-cidrs=true",
   723  				"--cluster-cidr=10.0.1.15/16",
   724  			},
   725  		},
   726  		{
   727  			name: "custom service-cluster-ip-range for " + cpVersion,
   728  			cfg: &kubeadmapi.ClusterConfiguration{
   729  				Networking: kubeadmapi.Networking{
   730  					PodSubnet:     "10.0.1.15/16",
   731  					ServiceSubnet: "172.20.0.0/24",
   732  					DNSDomain:     "cluster.local",
   733  				},
   734  				CertificatesDir:   testCertsDir,
   735  				KubernetesVersion: cpVersion,
   736  			},
   737  			expected: []string{
   738  				"kube-controller-manager",
   739  				"--bind-address=127.0.0.1",
   740  				"--leader-elect=true",
   741  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   742  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   743  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   744  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   745  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   746  				"--use-service-account-credentials=true",
   747  				"--controllers=*,bootstrapsigner,tokencleaner",
   748  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   749  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   750  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   751  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   752  				"--allocate-node-cidrs=true",
   753  				"--cluster-cidr=10.0.1.15/16",
   754  				"--service-cluster-ip-range=172.20.0.0/24",
   755  			},
   756  		},
   757  		{
   758  			name: "custom extra-args for " + cpVersion,
   759  			cfg: &kubeadmapi.ClusterConfiguration{
   760  				Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16", DNSDomain: "cluster.local"},
   761  				ControllerManager: kubeadmapi.ControlPlaneComponent{
   762  					ExtraArgs: []kubeadmapi.Arg{{Name: "node-cidr-mask-size", Value: "20"}},
   763  				},
   764  				CertificatesDir:   testCertsDir,
   765  				KubernetesVersion: cpVersion,
   766  			},
   767  			expected: []string{
   768  				"kube-controller-manager",
   769  				"--bind-address=127.0.0.1",
   770  				"--leader-elect=true",
   771  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   772  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   773  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   774  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   775  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   776  				"--use-service-account-credentials=true",
   777  				"--controllers=*,bootstrapsigner,tokencleaner",
   778  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   779  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   780  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   781  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   782  				"--allocate-node-cidrs=true",
   783  				"--cluster-cidr=10.0.1.15/16",
   784  				"--node-cidr-mask-size=20",
   785  			},
   786  		},
   787  		{
   788  			name: "custom IPv6 networking for " + cpVersion,
   789  			cfg: &kubeadmapi.ClusterConfiguration{
   790  				Networking: kubeadmapi.Networking{
   791  					PodSubnet:     "2001:db8::/64",
   792  					ServiceSubnet: "fd03::/112",
   793  					DNSDomain:     "cluster.local",
   794  				},
   795  
   796  				CertificatesDir:   testCertsDir,
   797  				KubernetesVersion: cpVersion,
   798  			},
   799  			expected: []string{
   800  				"kube-controller-manager",
   801  				"--bind-address=127.0.0.1",
   802  				"--leader-elect=true",
   803  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   804  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   805  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   806  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   807  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   808  				"--use-service-account-credentials=true",
   809  				"--controllers=*,bootstrapsigner,tokencleaner",
   810  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   811  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   812  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   813  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   814  				"--allocate-node-cidrs=true",
   815  				"--cluster-cidr=2001:db8::/64",
   816  				"--service-cluster-ip-range=fd03::/112",
   817  			},
   818  		},
   819  		{
   820  			name: "IPv6 networking custom extra-args for " + cpVersion,
   821  			cfg: &kubeadmapi.ClusterConfiguration{
   822  				Networking: kubeadmapi.Networking{
   823  					PodSubnet:     "2001:db8::/64",
   824  					ServiceSubnet: "fd03::/112",
   825  					DNSDomain:     "cluster.local",
   826  				},
   827  				ControllerManager: kubeadmapi.ControlPlaneComponent{
   828  					ExtraArgs: []kubeadmapi.Arg{{Name: "allocate-node-cidrs", Value: "false"}},
   829  				},
   830  				CertificatesDir:   testCertsDir,
   831  				KubernetesVersion: cpVersion,
   832  			},
   833  			expected: []string{
   834  				"kube-controller-manager",
   835  				"--bind-address=127.0.0.1",
   836  				"--leader-elect=true",
   837  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   838  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   839  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   840  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   841  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   842  				"--use-service-account-credentials=true",
   843  				"--controllers=*,bootstrapsigner,tokencleaner",
   844  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   845  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   846  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   847  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   848  				"--allocate-node-cidrs=false",
   849  				"--cluster-cidr=2001:db8::/64",
   850  				"--service-cluster-ip-range=fd03::/112",
   851  			},
   852  		},
   853  		{
   854  			name: "dual-stack networking for " + cpVersion,
   855  			cfg: &kubeadmapi.ClusterConfiguration{
   856  				Networking: kubeadmapi.Networking{
   857  					PodSubnet:     "2001:db8::/64,10.1.0.0/16",
   858  					ServiceSubnet: "fd03::/112,192.168.0.0/16",
   859  					DNSDomain:     "cluster.local",
   860  				},
   861  				CertificatesDir:   testCertsDir,
   862  				KubernetesVersion: cpVersion,
   863  			},
   864  			expected: []string{
   865  				"kube-controller-manager",
   866  				"--bind-address=127.0.0.1",
   867  				"--leader-elect=true",
   868  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   869  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   870  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   871  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   872  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   873  				"--use-service-account-credentials=true",
   874  				"--controllers=*,bootstrapsigner,tokencleaner",
   875  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   876  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   877  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   878  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   879  				"--allocate-node-cidrs=true",
   880  				"--cluster-cidr=2001:db8::/64,10.1.0.0/16",
   881  				"--service-cluster-ip-range=fd03::/112,192.168.0.0/16",
   882  			},
   883  		},
   884  		{
   885  			name: "dual-stack networking custom extra-args for " + cpVersion,
   886  			cfg: &kubeadmapi.ClusterConfiguration{
   887  				Networking: kubeadmapi.Networking{
   888  					PodSubnet: "10.0.1.15/16,2001:db8::/64",
   889  					DNSDomain: "cluster.local",
   890  				},
   891  				ControllerManager: kubeadmapi.ControlPlaneComponent{
   892  					ExtraArgs: []kubeadmapi.Arg{
   893  						{Name: "node-cidr-mask-size-ipv4", Value: "20"},
   894  						{Name: "node-cidr-mask-size-ipv6", Value: "80"},
   895  					},
   896  				},
   897  				CertificatesDir:   testCertsDir,
   898  				KubernetesVersion: cpVersion,
   899  			},
   900  			expected: []string{
   901  				"kube-controller-manager",
   902  				"--bind-address=127.0.0.1",
   903  				"--leader-elect=true",
   904  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   905  				"--root-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   906  				"--service-account-private-key-file=" + filepath.Join(testCertsDir, "sa.key"),
   907  				"--cluster-signing-cert-file=" + filepath.Join(testCertsDir, "ca.crt"),
   908  				"--cluster-signing-key-file=" + filepath.Join(testCertsDir, "ca.key"),
   909  				"--use-service-account-credentials=true",
   910  				"--controllers=*,bootstrapsigner,tokencleaner",
   911  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   912  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   913  				"--client-ca-file=" + filepath.Join(testCertsDir, "ca.crt"),
   914  				"--requestheader-client-ca-file=" + filepath.Join(testCertsDir, "front-proxy-ca.crt"),
   915  				"--allocate-node-cidrs=true",
   916  				"--cluster-cidr=10.0.1.15/16,2001:db8::/64",
   917  				"--node-cidr-mask-size-ipv4=20",
   918  				"--node-cidr-mask-size-ipv6=80",
   919  			},
   920  		},
   921  	}
   922  
   923  	for _, rt := range tests {
   924  		t.Run(rt.name, func(t *testing.T) {
   925  			actual := getControllerManagerCommand(rt.cfg)
   926  			sort.Strings(actual)
   927  			sort.Strings(rt.expected)
   928  			if !reflect.DeepEqual(actual, rt.expected) {
   929  				errorDiffArguments(t, rt.name, actual, rt.expected)
   930  			}
   931  		})
   932  	}
   933  }
   934  
   935  func TestGetControllerManagerCommandExternalCA(t *testing.T) {
   936  	tests := []struct {
   937  		name            string
   938  		cfg             *kubeadmapi.InitConfiguration
   939  		caKeyPresent    bool
   940  		expectedArgFunc func(dir string) []string
   941  	}{
   942  		{
   943  			name: "caKeyPresent-false for " + cpVersion,
   944  			cfg: &kubeadmapi.InitConfiguration{
   945  				LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
   946  				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
   947  					KubernetesVersion: cpVersion,
   948  					Networking:        kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
   949  				},
   950  			},
   951  			caKeyPresent: false,
   952  			expectedArgFunc: func(tmpdir string) []string {
   953  				return []string{
   954  					"kube-controller-manager",
   955  					"--bind-address=127.0.0.1",
   956  					"--leader-elect=true",
   957  					"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   958  					"--root-ca-file=" + filepath.Join(tmpdir, "ca.crt"),
   959  					"--service-account-private-key-file=" + filepath.Join(tmpdir, "sa.key"),
   960  					"--cluster-signing-cert-file=",
   961  					"--cluster-signing-key-file=",
   962  					"--use-service-account-credentials=true",
   963  					"--controllers=*,bootstrapsigner,tokencleaner",
   964  					"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   965  					"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   966  					"--client-ca-file=" + filepath.Join(tmpdir, "ca.crt"),
   967  					"--requestheader-client-ca-file=" + filepath.Join(tmpdir, "front-proxy-ca.crt"),
   968  				}
   969  			},
   970  		},
   971  		{
   972  			name: "caKeyPresent true for " + cpVersion,
   973  			cfg: &kubeadmapi.InitConfiguration{
   974  				LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
   975  				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
   976  					KubernetesVersion: cpVersion,
   977  					Networking:        kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
   978  				},
   979  			},
   980  			caKeyPresent: true,
   981  			expectedArgFunc: func(tmpdir string) []string {
   982  				return []string{
   983  					"kube-controller-manager",
   984  					"--bind-address=127.0.0.1",
   985  					"--leader-elect=true",
   986  					"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   987  					"--root-ca-file=" + filepath.Join(tmpdir, "ca.crt"),
   988  					"--service-account-private-key-file=" + filepath.Join(tmpdir, "sa.key"),
   989  					"--cluster-signing-cert-file=" + filepath.Join(tmpdir, "ca.crt"),
   990  					"--cluster-signing-key-file=" + filepath.Join(tmpdir, "ca.key"),
   991  					"--use-service-account-credentials=true",
   992  					"--controllers=*,bootstrapsigner,tokencleaner",
   993  					"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   994  					"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "controller-manager.conf"),
   995  					"--client-ca-file=" + filepath.Join(tmpdir, "ca.crt"),
   996  					"--requestheader-client-ca-file=" + filepath.Join(tmpdir, "front-proxy-ca.crt"),
   997  				}
   998  			},
   999  		},
  1000  	}
  1001  
  1002  	for _, test := range tests {
  1003  		t.Run(test.name, func(t *testing.T) {
  1004  			pkiutiltesting.Reset()
  1005  
  1006  			// Create temp folder for the test case
  1007  			tmpdir := testutil.SetupTempDir(t)
  1008  			defer os.RemoveAll(tmpdir)
  1009  			test.cfg.CertificatesDir = tmpdir
  1010  
  1011  			if err := certs.CreatePKIAssets(test.cfg); err != nil {
  1012  				t.Errorf("failed creating pki assets: %v", err)
  1013  			}
  1014  
  1015  			// delete ca.key and front-proxy-ca.key if test.caKeyPresent is false
  1016  			if !test.caKeyPresent {
  1017  				if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil {
  1018  					t.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err)
  1019  				}
  1020  				if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil {
  1021  					t.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err)
  1022  				}
  1023  			}
  1024  
  1025  			actual := getControllerManagerCommand(&test.cfg.ClusterConfiguration)
  1026  			expected := test.expectedArgFunc(tmpdir)
  1027  			sort.Strings(actual)
  1028  			sort.Strings(expected)
  1029  			if !reflect.DeepEqual(actual, expected) {
  1030  				errorDiffArguments(t, test.name, actual, expected)
  1031  			}
  1032  		})
  1033  	}
  1034  }
  1035  
  1036  func TestGetSchedulerCommand(t *testing.T) {
  1037  	var tests = []struct {
  1038  		name     string
  1039  		cfg      *kubeadmapi.ClusterConfiguration
  1040  		expected []string
  1041  	}{
  1042  		{
  1043  			name: "scheduler defaults",
  1044  			cfg:  &kubeadmapi.ClusterConfiguration{},
  1045  			expected: []string{
  1046  				"kube-scheduler",
  1047  				"--bind-address=127.0.0.1",
  1048  				"--leader-elect=true",
  1049  				"--kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "scheduler.conf"),
  1050  				"--authentication-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "scheduler.conf"),
  1051  				"--authorization-kubeconfig=" + filepath.Join(kubeadmconstants.KubernetesDir, "scheduler.conf"),
  1052  			},
  1053  		},
  1054  	}
  1055  
  1056  	for _, rt := range tests {
  1057  		t.Run(rt.name, func(t *testing.T) {
  1058  			actual := getSchedulerCommand(rt.cfg)
  1059  			sort.Strings(actual)
  1060  			sort.Strings(rt.expected)
  1061  			if !reflect.DeepEqual(actual, rt.expected) {
  1062  				errorDiffArguments(t, rt.name, actual, rt.expected)
  1063  			}
  1064  		})
  1065  	}
  1066  }
  1067  
  1068  func TestGetAuthzModes(t *testing.T) {
  1069  	var tests = []struct {
  1070  		name     string
  1071  		authMode []string
  1072  		expected string
  1073  	}{
  1074  		{
  1075  			name:     "default if empty",
  1076  			authMode: []string{},
  1077  			expected: "Node,RBAC",
  1078  		},
  1079  		{
  1080  			name:     "default non empty",
  1081  			authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeRBAC},
  1082  			expected: "Node,RBAC",
  1083  		},
  1084  		{
  1085  			name:     "single unspecified returning default",
  1086  			authMode: []string{"FooAuthzMode"},
  1087  			expected: "Node,RBAC",
  1088  		},
  1089  		{
  1090  			name:     "multiple ignored",
  1091  			authMode: []string{kubeadmconstants.ModeNode, "foo", kubeadmconstants.ModeRBAC, "bar"},
  1092  			expected: "Node,RBAC",
  1093  		},
  1094  		{
  1095  			name:     "single mode",
  1096  			authMode: []string{kubeadmconstants.ModeAlwaysDeny},
  1097  			expected: "AlwaysDeny",
  1098  		},
  1099  		{
  1100  			name:     "multiple special order",
  1101  			authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeWebhook, kubeadmconstants.ModeRBAC, kubeadmconstants.ModeABAC},
  1102  			expected: "Node,Webhook,RBAC,ABAC",
  1103  		},
  1104  	}
  1105  
  1106  	for _, rt := range tests {
  1107  		t.Run(rt.name, func(t *testing.T) {
  1108  			actual := getAuthzModes(strings.Join(rt.authMode, ","))
  1109  			if actual != rt.expected {
  1110  				t.Errorf("failed getAuthzModes:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
  1111  			}
  1112  		})
  1113  	}
  1114  }
  1115  
  1116  func TestIsValidAuthzMode(t *testing.T) {
  1117  	var tests = []struct {
  1118  		mode  string
  1119  		valid bool
  1120  	}{
  1121  		{
  1122  			mode:  "Node",
  1123  			valid: true,
  1124  		},
  1125  		{
  1126  			mode:  "RBAC",
  1127  			valid: true,
  1128  		},
  1129  		{
  1130  			mode:  "ABAC",
  1131  			valid: true,
  1132  		},
  1133  		{
  1134  			mode:  "AlwaysAllow",
  1135  			valid: true,
  1136  		},
  1137  		{
  1138  			mode:  "Webhook",
  1139  			valid: true,
  1140  		},
  1141  		{
  1142  			mode:  "AlwaysDeny",
  1143  			valid: true,
  1144  		},
  1145  		{
  1146  			mode:  "Foo",
  1147  			valid: false,
  1148  		},
  1149  	}
  1150  
  1151  	for _, rt := range tests {
  1152  		t.Run(rt.mode, func(t *testing.T) {
  1153  			isValid := isValidAuthzMode(rt.mode)
  1154  			if isValid != rt.valid {
  1155  				t.Errorf("failed isValidAuthzMode:\nexpected:\n%v\nsaw:\n%v", rt.valid, isValid)
  1156  			}
  1157  		})
  1158  	}
  1159  }
  1160  
  1161  func TestCompareAuthzModes(t *testing.T) {
  1162  	var tests = []struct {
  1163  		name   string
  1164  		modesA []string
  1165  		modesB []string
  1166  		equal  bool
  1167  	}{
  1168  		{
  1169  			name:   "modes match",
  1170  			modesA: []string{"a", "b", "c"},
  1171  			modesB: []string{"a", "b", "c"},
  1172  			equal:  true,
  1173  		},
  1174  		{
  1175  			name:   "modes order does not match",
  1176  			modesA: []string{"a", "c", "b"},
  1177  			modesB: []string{"a", "b", "c"},
  1178  		},
  1179  		{
  1180  			name:   "modes do not match; A has less modes",
  1181  			modesA: []string{"a", "b"},
  1182  			modesB: []string{"a", "b", "c"},
  1183  		},
  1184  		{
  1185  			name:   "modes do not match; B has less modes",
  1186  			modesA: []string{"a", "b", "c"},
  1187  			modesB: []string{"a", "b"},
  1188  		},
  1189  	}
  1190  
  1191  	for _, rt := range tests {
  1192  		t.Run(rt.name, func(t *testing.T) {
  1193  			equal := compareAuthzModes(rt.modesA, rt.modesB)
  1194  			if equal != rt.equal {
  1195  				t.Errorf("failed compareAuthzModes:\nexpected:\n%v\nsaw:\n%v", rt.equal, equal)
  1196  			}
  1197  		})
  1198  	}
  1199  }
  1200  

View as plain text