local_test.go

Documentation: k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5  Copyright 2017 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package etcd
    21  
    22  import (
    23  	"fmt"
    24  	"os"
    25  	"path"
    26  	"path/filepath"
    27  	"reflect"
    28  	"sort"
    29  	"testing"
    30  
    31  	"github.com/google/go-cmp/cmp"
    32  	"github.com/lithammer/dedent"
    33  
    34  	v1 "k8s.io/api/core/v1"
    35  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    36  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    37  	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
    38  	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
    39  	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
    40  )
    41  
    42  func TestGetEtcdPodSpec(t *testing.T) {
    43  	// Creates a ClusterConfiguration
    44  	cfg := &kubeadmapi.ClusterConfiguration{
    45  		KubernetesVersion: "v1.7.0",
    46  		Etcd: kubeadmapi.Etcd{
    47  			Local: &kubeadmapi.LocalEtcd{
    48  				DataDir: "/var/lib/etcd",
    49  				ExtraEnvs: []kubeadmapi.EnvVar{
    50  					{
    51  						EnvVar: v1.EnvVar{Name: "Foo", Value: "Bar"},
    52  					},
    53  				},
    54  			},
    55  		},
    56  	}
    57  	endpoint := &kubeadmapi.APIEndpoint{}
    58  
    59  	// Executes GetEtcdPodSpec
    60  	spec := GetEtcdPodSpec(cfg, endpoint, "", []etcdutil.Member{})
    61  
    62  	// Assert each specs refers to the right pod
    63  	if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd {
    64  		t.Errorf("getKubeConfigSpecs spec for etcd contains pod %s, expects %s", spec.Spec.Containers[0].Name, kubeadmconstants.Etcd)
    65  	}
    66  	env := []v1.EnvVar{{Name: "Foo", Value: "Bar"}}
    67  	if !reflect.DeepEqual(spec.Spec.Containers[0].Env, env) {
    68  		t.Errorf("expected env: %v, got: %v", env, spec.Spec.Containers[0].Env)
    69  	}
    70  }
    71  
    72  func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
    73  	// Create temp folder for the test case
    74  	tmpdir := testutil.SetupTempDir(t)
    75  	defer os.RemoveAll(tmpdir)
    76  
    77  	var tests = []struct {
    78  		cfg              *kubeadmapi.ClusterConfiguration
    79  		expectedError    bool
    80  		expectedManifest string
    81  	}{
    82  		{
    83  			cfg: &kubeadmapi.ClusterConfiguration{
    84  				KubernetesVersion: "v1.7.0",
    85  				Etcd: kubeadmapi.Etcd{
    86  					Local: &kubeadmapi.LocalEtcd{
    87  						DataDir: tmpdir + "/etcd",
    88  					},
    89  				},
    90  			},
    91  			expectedError: false,
    92  			expectedManifest: fmt.Sprintf(`apiVersion: v1
    93  kind: Pod
    94  metadata:
    95    annotations:
    96      kubeadm.kubernetes.io/etcd.advertise-client-urls: https://:2379
    97    creationTimestamp: null
    98    labels:
    99      component: etcd
   100      tier: control-plane
   101    name: etcd
   102    namespace: kube-system
   103  spec:
   104    containers:
   105    - command:
   106      - etcd
   107      - --advertise-client-urls=https://:2379
   108      - --cert-file=etcd/server.crt
   109      - --client-cert-auth=true
   110      - --data-dir=%s/etcd
   111      - --experimental-initial-corrupt-check=true
   112      - --experimental-watch-progress-notify-interval=5s
   113      - --initial-advertise-peer-urls=https://:2380
   114      - --initial-cluster==https://:2380
   115      - --key-file=etcd/server.key
   116      - --listen-client-urls=https://127.0.0.1:2379,https://:2379
   117      - --listen-metrics-urls=http://127.0.0.1:2381
   118      - --listen-peer-urls=https://:2380
   119      - --name=
   120      - --peer-cert-file=etcd/peer.crt
   121      - --peer-client-cert-auth=true
   122      - --peer-key-file=etcd/peer.key
   123      - --peer-trusted-ca-file=etcd/ca.crt
   124      - --snapshot-count=10000
   125      - --trusted-ca-file=etcd/ca.crt
   126      image: /etcd:%s
   127      imagePullPolicy: IfNotPresent
   128      livenessProbe:
   129        failureThreshold: 8
   130        httpGet:
   131          host: 127.0.0.1
   132          path: /health?exclude=NOSPACE&serializable=true
   133          port: 2381
   134          scheme: HTTP
   135        initialDelaySeconds: 10
   136        periodSeconds: 10
   137        timeoutSeconds: 15
   138      name: etcd
   139      resources:
   140        requests:
   141          cpu: 100m
   142          memory: 100Mi
   143      startupProbe:
   144        failureThreshold: 24
   145        httpGet:
   146          host: 127.0.0.1
   147          path: /health?serializable=false
   148          port: 2381
   149          scheme: HTTP
   150        initialDelaySeconds: 10
   151        periodSeconds: 10
   152        timeoutSeconds: 15
   153      volumeMounts:
   154      - mountPath: %s/etcd
   155        name: etcd-data
   156      - mountPath: /etcd
   157        name: etcd-certs
   158    hostNetwork: true
   159    priority: 2000001000
   160    priorityClassName: system-node-critical
   161    securityContext:
   162      seccompProfile:
   163        type: RuntimeDefault
   164    volumes:
   165    - hostPath:
   166        path: /etcd
   167        type: DirectoryOrCreate
   168      name: etcd-certs
   169    - hostPath:
   170        path: %s/etcd
   171        type: DirectoryOrCreate
   172      name: etcd-data
   173  status: {}
   174  `, tmpdir, kubeadmconstants.DefaultEtcdVersion, tmpdir, tmpdir),
   175  		},
   176  		{
   177  			cfg: &kubeadmapi.ClusterConfiguration{
   178  				KubernetesVersion: "v1.7.0",
   179  				Etcd: kubeadmapi.Etcd{
   180  					External: &kubeadmapi.ExternalEtcd{
   181  						Endpoints: []string{
   182  							"https://etcd-instance:2379",
   183  						},
   184  						CAFile:   "/etc/kubernetes/pki/etcd/ca.crt",
   185  						CertFile: "/etc/kubernetes/pki/etcd/apiserver-etcd-client.crt",
   186  						KeyFile:  "/etc/kubernetes/pki/etcd/apiserver-etcd-client.key",
   187  					},
   188  				},
   189  			},
   190  			expectedError: true,
   191  		},
   192  	}
   193  
   194  	for _, test := range tests {
   195  		// Execute createStaticPodFunction
   196  		manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
   197  		err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", "", test.cfg, &kubeadmapi.APIEndpoint{}, false /* IsDryRun */)
   198  
   199  		if !test.expectedError {
   200  			if err != nil {
   201  				t.Errorf("CreateLocalEtcdStaticPodManifestFile failed when not expected: %v", err)
   202  			}
   203  			// Assert expected files are there
   204  			testutil.AssertFilesCount(t, manifestPath, 1)
   205  			testutil.AssertFileExists(t, manifestPath, kubeadmconstants.Etcd+".yaml")
   206  			manifestBytes, err := os.ReadFile(path.Join(manifestPath, kubeadmconstants.Etcd+".yaml"))
   207  			if err != nil {
   208  				t.Errorf("failed to load generated manifest file: %v", err)
   209  			}
   210  			if test.expectedManifest != string(manifestBytes) {
   211  				t.Errorf(
   212  					"File created by CreateLocalEtcdStaticPodManifestFile is not as expected. Diff: \n%s",
   213  					cmp.Diff(string(manifestBytes), test.expectedManifest),
   214  				)
   215  			}
   216  		} else {
   217  			testutil.AssertError(t, err, "etcd static pod manifest cannot be generated for cluster using external etcd")
   218  		}
   219  	}
   220  }
   221  
   222  func TestCreateLocalEtcdStaticPodManifestFileWithPatches(t *testing.T) {
   223  	// Create temp folder for the test case
   224  	tmpdir := testutil.SetupTempDir(t)
   225  	defer os.RemoveAll(tmpdir)
   226  
   227  	// Creates a Cluster Configuration
   228  	cfg := &kubeadmapi.ClusterConfiguration{
   229  		KubernetesVersion: "v1.7.0",
   230  		Etcd: kubeadmapi.Etcd{
   231  			Local: &kubeadmapi.LocalEtcd{
   232  				DataDir: tmpdir + "/etcd",
   233  			},
   234  		},
   235  	}
   236  
   237  	patchesPath := filepath.Join(tmpdir, "patch-files")
   238  	err := os.MkdirAll(patchesPath, 0777)
   239  	if err != nil {
   240  		t.Fatalf("Couldn't create %s", patchesPath)
   241  	}
   242  
   243  	patchString := dedent.Dedent(`
   244  	metadata:
   245  	  annotations:
   246  	    patched: "true"
   247  	`)
   248  
   249  	err = os.WriteFile(filepath.Join(patchesPath, kubeadmconstants.Etcd+".yaml"), []byte(patchString), 0644)
   250  	if err != nil {
   251  		t.Fatalf("WriteFile returned unexpected error: %v", err)
   252  	}
   253  
   254  	manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
   255  	err = CreateLocalEtcdStaticPodManifestFile(manifestPath, patchesPath, "", cfg, &kubeadmapi.APIEndpoint{}, false /* IsDryRun */)
   256  	if err != nil {
   257  		t.Errorf("Error executing createStaticPodFunction: %v", err)
   258  		return
   259  	}
   260  
   261  	pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, kubeadmconstants.Etcd+".yaml"))
   262  	if err != nil {
   263  		t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
   264  		return
   265  	}
   266  	if pod.Spec.DNSPolicy != "" {
   267  		t.Errorf("DNSPolicy should be empty but: %v", pod.Spec.DNSPolicy)
   268  	}
   269  
   270  	if _, ok := pod.ObjectMeta.Annotations["patched"]; !ok {
   271  		t.Errorf("Patches were not applied to %s", kubeadmconstants.Etcd)
   272  	}
   273  }
   274  
   275  func TestGetEtcdCommand(t *testing.T) {
   276  	var tests = []struct {
   277  		name             string
   278  		advertiseAddress string
   279  		nodeName         string
   280  		extraArgs        []kubeadmapi.Arg
   281  		initialCluster   []etcdutil.Member
   282  		expected         []string
   283  	}{
   284  		{
   285  			name:             "Default args - with empty etcd initial cluster",
   286  			advertiseAddress: "1.2.3.4",
   287  			nodeName:         "foo",
   288  			expected: []string{
   289  				"etcd",
   290  				"--name=foo",
   291  				"--experimental-initial-corrupt-check=true",
   292  				"--experimental-watch-progress-notify-interval=5s",
   293  				fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort),
   294  				fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort),
   295  				fmt.Sprintf("--advertise-client-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort),
   296  				fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   297  				fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   298  				"--data-dir=/var/lib/etcd",
   299  				"--cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerCertName),
   300  				"--key-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerKeyName),
   301  				"--trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   302  				"--client-cert-auth=true",
   303  				"--peer-cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerCertName),
   304  				"--peer-key-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerKeyName),
   305  				"--peer-trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   306  				"--snapshot-count=10000",
   307  				"--peer-client-cert-auth=true",
   308  				fmt.Sprintf("--initial-cluster=foo=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   309  			},
   310  		},
   311  		{
   312  			name:             "Default args - With an existing etcd cluster",
   313  			advertiseAddress: "1.2.3.4",
   314  			nodeName:         "foo",
   315  			initialCluster: []etcdutil.Member{
   316  				{Name: "foo", PeerURL: fmt.Sprintf("https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort)}, // NB. the joining etcd instance should be part of the initialCluster list
   317  				{Name: "bar", PeerURL: fmt.Sprintf("https://5.6.7.8:%d", kubeadmconstants.EtcdListenPeerPort)},
   318  			},
   319  			expected: []string{
   320  				"etcd",
   321  				"--name=foo",
   322  				"--experimental-initial-corrupt-check=true",
   323  				"--experimental-watch-progress-notify-interval=5s",
   324  				fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort),
   325  				fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort),
   326  				fmt.Sprintf("--advertise-client-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort),
   327  				fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   328  				fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   329  				"--data-dir=/var/lib/etcd",
   330  				"--cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerCertName),
   331  				"--key-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerKeyName),
   332  				"--trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   333  				"--client-cert-auth=true",
   334  				"--peer-cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerCertName),
   335  				"--peer-key-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerKeyName),
   336  				"--peer-trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   337  				"--snapshot-count=10000",
   338  				"--peer-client-cert-auth=true",
   339  				"--initial-cluster-state=existing",
   340  				fmt.Sprintf("--initial-cluster=foo=https://1.2.3.4:%d,bar=https://5.6.7.8:%d", kubeadmconstants.EtcdListenPeerPort, kubeadmconstants.EtcdListenPeerPort),
   341  			},
   342  		},
   343  		{
   344  			name:             "Extra args",
   345  			advertiseAddress: "1.2.3.4",
   346  			nodeName:         "bar",
   347  			extraArgs: []kubeadmapi.Arg{
   348  				{Name: "listen-client-urls", Value: "https://10.0.1.10:2379"},
   349  				{Name: "advertise-client-urls", Value: "https://10.0.1.10:2379"},
   350  			},
   351  			expected: []string{
   352  				"etcd",
   353  				"--name=bar",
   354  				"--experimental-initial-corrupt-check=true",
   355  				"--experimental-watch-progress-notify-interval=5s",
   356  				"--listen-client-urls=https://10.0.1.10:2379",
   357  				fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort),
   358  				"--advertise-client-urls=https://10.0.1.10:2379",
   359  				fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   360  				fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   361  				"--data-dir=/var/lib/etcd",
   362  				"--cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerCertName),
   363  				"--key-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerKeyName),
   364  				"--trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   365  				"--client-cert-auth=true",
   366  				"--peer-cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerCertName),
   367  				"--peer-key-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerKeyName),
   368  				"--peer-trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   369  				"--snapshot-count=10000",
   370  				"--peer-client-cert-auth=true",
   371  				fmt.Sprintf("--initial-cluster=bar=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort),
   372  			},
   373  		},
   374  		{
   375  			name:             "IPv6 advertise address",
   376  			advertiseAddress: "2001:db8::3",
   377  			nodeName:         "foo",
   378  			expected: []string{
   379  				"etcd",
   380  				"--name=foo",
   381  				"--experimental-initial-corrupt-check=true",
   382  				"--experimental-watch-progress-notify-interval=5s",
   383  				fmt.Sprintf("--listen-client-urls=https://[::1]:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort),
   384  				fmt.Sprintf("--listen-metrics-urls=http://[::1]:%d", kubeadmconstants.EtcdMetricsPort),
   385  				fmt.Sprintf("--advertise-client-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort),
   386  				fmt.Sprintf("--listen-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort),
   387  				fmt.Sprintf("--initial-advertise-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort),
   388  				"--data-dir=/var/lib/etcd",
   389  				"--cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerCertName),
   390  				"--key-file=" + filepath.FromSlash(kubeadmconstants.EtcdServerKeyName),
   391  				"--trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   392  				"--client-cert-auth=true",
   393  				"--peer-cert-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerCertName),
   394  				"--peer-key-file=" + filepath.FromSlash(kubeadmconstants.EtcdPeerKeyName),
   395  				"--peer-trusted-ca-file=" + filepath.FromSlash(kubeadmconstants.EtcdCACertName),
   396  				"--snapshot-count=10000",
   397  				"--peer-client-cert-auth=true",
   398  				fmt.Sprintf("--initial-cluster=foo=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort),
   399  			},
   400  		},
   401  	}
   402  
   403  	for _, rt := range tests {
   404  		t.Run(rt.name, func(t *testing.T) {
   405  			endpoint := &kubeadmapi.APIEndpoint{
   406  				AdvertiseAddress: rt.advertiseAddress,
   407  			}
   408  			cfg := &kubeadmapi.ClusterConfiguration{
   409  				Etcd: kubeadmapi.Etcd{
   410  					Local: &kubeadmapi.LocalEtcd{
   411  						DataDir:   "/var/lib/etcd",
   412  						ExtraArgs: rt.extraArgs,
   413  					},
   414  				},
   415  			}
   416  			actual := getEtcdCommand(cfg, endpoint, rt.nodeName, rt.initialCluster)
   417  			sort.Strings(actual)
   418  			sort.Strings(rt.expected)
   419  			if !reflect.DeepEqual(actual, rt.expected) {
   420  				t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
   421  			}
   422  		})
   423  	}
   424  }
   425
View as plain text